From ab2f2ea1b47fcc79c9f9e718aa9033e9b4411029 Mon Sep 17 00:00:00 2001 From: KittenML <224667585+KittenML@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:59:20 +0000 Subject: [PATCH 1/8] Add KittenTTS2 CPU inference, Python bindings and wheel builds Assisted-by: Codex --- .github/workflows/ai-issues.yml | 89 + .github/workflows/bench.yml.disabled | 304 +++ .github/workflows/build-3rd-party.yml | 57 + .../workflows/build-and-test-snapdragon.yml | 148 ++ .github/workflows/build-android.yml | 149 ++ .github/workflows/build-apple.yml | 264 +++ .github/workflows/build-cache.yml | 145 ++ .github/workflows/build-cann.yml | 104 + .github/workflows/build-cmake-pkg.yml | 51 + .github/workflows/build-cpu.yml | 215 ++ .github/workflows/build-cross.yml | 317 +++ .github/workflows/build-cuda-ubuntu.yml | 134 ++ .github/workflows/build-cuda-windows.yml | 162 ++ .github/workflows/build-ibm.yml | 150 ++ .github/workflows/build-msys.yml | 70 + .github/workflows/build-opencl.yml | 82 + .github/workflows/build-openvino.yml | 169 ++ .github/workflows/build-riscv.yml | 188 ++ .github/workflows/build-rpc.yml | 66 + .github/workflows/build-sanitize.yml | 86 + .github/workflows/build-self-hosted.yml | 389 ++++ .github/workflows/build-sycl.yml | 141 ++ .github/workflows/build-virtgpu.yml | 50 + .github/workflows/build-vulkan.yml | 135 ++ .github/workflows/build-wasm.yml | 90 + .github/workflows/build-webgpu.yml | 155 ++ .github/workflows/check-vendor.yml | 52 + .github/workflows/close-issue.yml | 28 + .github/workflows/code-style.yml | 51 + .github/workflows/copilot-setup-steps.yml | 56 + .github/workflows/docker.yml | 529 +++++ .github/workflows/editorconfig.yml | 24 + .github/workflows/gguf-publish.yml | 44 + .github/workflows/hip-quality-check.yml | 86 + .github/workflows/labeler.yml | 17 + .github/workflows/pre-tokenizer-hashes.yml | 45 + .../workflows/python-check-requirements.yml | 33 + .github/workflows/python-lint.yml | 36 + .github/workflows/python-type-check.yml | 43 + .github/workflows/python-wheels.yml | 60 + .github/workflows/release.yml | 1731 +++++++++++++++++ .github/workflows/server-sanitize.yml | 112 ++ .github/workflows/server-self-hosted.yml | 222 +++ .github/workflows/server.yml | 181 ++ .github/workflows/ui-build-self-hosted.yml | 36 + .github/workflows/ui-build.yml | 48 + .github/workflows/ui-publish.yml | 75 + .github/workflows/ui-self-hosted.yml | 125 ++ .github/workflows/ui.yml | 151 ++ .github/workflows/update-ops-docs.yml | 44 + .github/workflows/winget.yml | 44 + .gitignore | 11 + MANIFEST.in | 18 + README.md | 73 +- common/download.cpp | 3 + common/download.h | 1 + packaging/README.md | 52 + packaging/prepare_wheel.py | 10 + packaging/repair_wheel.py | 26 + packaging/smoke_wheel.py | 66 + pyproject.toml | 106 +- python/kitten_tts_cpp/__init__.py | 5 + python/kitten_tts_cpp/model.py | 283 +++ setup.py | 103 + tools/kitten-tts/CMakeLists.txt | 35 +- tools/kitten-tts/README.md | 28 +- tools/kitten-tts/assets.cpp | 3 +- tools/kitten-tts/decoder.cpp | 6 +- tools/kitten-tts/engine.cpp | 174 ++ tools/kitten-tts/engine.h | 23 + tools/kitten-tts/main.cpp | 130 +- tools/kitten-tts/python.cpp | 32 + tools/kitten-tts/test_python.py | 115 ++ 73 files changed, 8883 insertions(+), 203 deletions(-) create mode 100644 .github/workflows/ai-issues.yml create mode 100644 .github/workflows/bench.yml.disabled create mode 100644 .github/workflows/build-3rd-party.yml create mode 100644 .github/workflows/build-and-test-snapdragon.yml create mode 100644 .github/workflows/build-android.yml create mode 100644 .github/workflows/build-apple.yml create mode 100644 .github/workflows/build-cache.yml create mode 100644 .github/workflows/build-cann.yml create mode 100644 .github/workflows/build-cmake-pkg.yml create mode 100644 .github/workflows/build-cpu.yml create mode 100644 .github/workflows/build-cross.yml create mode 100644 .github/workflows/build-cuda-ubuntu.yml create mode 100644 .github/workflows/build-cuda-windows.yml create mode 100644 .github/workflows/build-ibm.yml create mode 100644 .github/workflows/build-msys.yml create mode 100644 .github/workflows/build-opencl.yml create mode 100644 .github/workflows/build-openvino.yml create mode 100644 .github/workflows/build-riscv.yml create mode 100644 .github/workflows/build-rpc.yml create mode 100644 .github/workflows/build-sanitize.yml create mode 100644 .github/workflows/build-self-hosted.yml create mode 100644 .github/workflows/build-sycl.yml create mode 100644 .github/workflows/build-virtgpu.yml create mode 100644 .github/workflows/build-vulkan.yml create mode 100644 .github/workflows/build-wasm.yml create mode 100644 .github/workflows/build-webgpu.yml create mode 100644 .github/workflows/check-vendor.yml create mode 100644 .github/workflows/close-issue.yml create mode 100644 .github/workflows/code-style.yml create mode 100644 .github/workflows/copilot-setup-steps.yml create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/editorconfig.yml create mode 100644 .github/workflows/gguf-publish.yml create mode 100644 .github/workflows/hip-quality-check.yml create mode 100644 .github/workflows/labeler.yml create mode 100644 .github/workflows/pre-tokenizer-hashes.yml create mode 100644 .github/workflows/python-check-requirements.yml create mode 100644 .github/workflows/python-lint.yml create mode 100644 .github/workflows/python-type-check.yml create mode 100644 .github/workflows/python-wheels.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/server-sanitize.yml create mode 100644 .github/workflows/server-self-hosted.yml create mode 100644 .github/workflows/server.yml create mode 100644 .github/workflows/ui-build-self-hosted.yml create mode 100644 .github/workflows/ui-build.yml create mode 100644 .github/workflows/ui-publish.yml create mode 100644 .github/workflows/ui-self-hosted.yml create mode 100644 .github/workflows/ui.yml create mode 100644 .github/workflows/update-ops-docs.yml create mode 100644 .github/workflows/winget.yml create mode 100644 MANIFEST.in create mode 100644 packaging/README.md create mode 100644 packaging/prepare_wheel.py create mode 100644 packaging/repair_wheel.py create mode 100644 packaging/smoke_wheel.py create mode 100644 python/kitten_tts_cpp/__init__.py create mode 100644 python/kitten_tts_cpp/model.py create mode 100644 setup.py create mode 100644 tools/kitten-tts/engine.cpp create mode 100644 tools/kitten-tts/engine.h create mode 100644 tools/kitten-tts/python.cpp create mode 100644 tools/kitten-tts/test_python.py diff --git a/.github/workflows/ai-issues.yml b/.github/workflows/ai-issues.yml new file mode 100644 index 0000000..c762901 --- /dev/null +++ b/.github/workflows/ai-issues.yml @@ -0,0 +1,89 @@ +name: AI review (issues) + +on: + issues: + types: [opened] + +jobs: + find-related: + if: github.event.action == 'opened' + runs-on: [self-hosted, opencode] + + permissions: + contents: read + issues: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Find related + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENCODE_PERMISSION: | + { + "bash": { + "*": "deny", + "gh issue view*": "allow", + "gh issue list*": "allow", + "gh issue comment*": "allow", + "gh search issues*": "allow" + }, + "webfetch": "deny" + } + run: | + rm AGENTS.md + rm CLAUDE.md + + timeout 5m opencode run -m llama.cpp-dgx/ai-review-issues-find-similar --thinking "A new issue has been created: + + Issue number: ${{ github.event.issue.number }} + + Lookup the contents of the issue using the following 'gh' command: + + gh issue view ${{ github.event.issue.number }} --json title,body,url,number + + Next, perform the following task and then post a SINGLE comment (if needed). + + --- + + TASK : FIND RELATED ISSUES + + Using the 'gh' CLI tool, search through existing issues on Github. + Find related or similar issues to the newly created one and list them. + Do not list the new issue itself (it is #${{ github.event.issue.number }}). + + Consider: + 1. Similar titles or descriptions + 2. Same error messages or symptoms + 3. Related functionality or components + 4. Similar feature requests + + --- + + POSTING YOUR COMMENT: + + Based on your findings, post a SINGLE comment on issue #${{ github.event.issue.number }}. Build the comment as follows: + + - If no related issues were found, do NOT comment at all. + - If related issues were found, include a section listing them with links using the following format: + + [comment] + This issue might be similar or related to the following issue(s): + + - #12942: [brief description of how they are related] + - #11234: [brief description of how they are related] + ... + + _This comment was auto-generated locally using **$GA_ENGINE** on **$GA_MACHINE**_ + [/comment] + + Remember: + - Do not include the comment tags in your actual comment. + - Post at most ONE comment combining all findings. + - If you didn't find issues that are related enough, post nothing. + - You have access only to the 'gh' CLI tool - don't try to use other tools. + - If the output from a tool call is too long, try to limit down the search. + " diff --git a/.github/workflows/bench.yml.disabled b/.github/workflows/bench.yml.disabled new file mode 100644 index 0000000..f2d7e16 --- /dev/null +++ b/.github/workflows/bench.yml.disabled @@ -0,0 +1,304 @@ +# TODO: there have been some issues with the workflow, so disabling for now +# https://github.com/ggml-org/llama.cpp/issues/7893 +# +# Benchmark +name: Benchmark + +on: + workflow_dispatch: + inputs: + gpu-series: + description: 'Azure GPU series to run with' + required: true + type: choice + options: + - Standard_NC4as_T4_v3 + - Standard_NC24ads_A100_v4 + - Standard_NC80adis_H100_v5 + sha: + description: 'Commit SHA1 to build' + required: false + type: string + duration: + description: 'Duration of the bench' + type: string + default: 10m + + push: + branches: + - master + paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'tools/server/*.h*', 'tools/server/*.cpp'] + pull_request_target: + types: [opened, synchronize, reopened] + paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'tools/server/*.h*', 'tools/server/*.cpp'] + schedule: + - cron: '04 2 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}-${{ github.event.inputs.sha }} + cancel-in-progress: true + +jobs: + bench-server-baseline: + runs-on: Standard_NC4as_T4_v3 + env: + RUNNER_LABEL: Standard_NC4as_T4_v3 # FIXME Do not find a way to not duplicate it + N_USERS: 8 + DURATION: 10m + + strategy: + matrix: + model: [phi-2] + ftype: [q4_0, q8_0, f16] + include: + - model: phi-2 + ftype: q4_0 + pr_comment_enabled: "true" + + if: | + inputs.gpu-series == 'Standard_NC4as_T4_v3' + || github.event_name == 'pull_request_target' + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Install python env + id: pipenv + run: | + cd tools/server/bench + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + + - name: Prometheus + id: install_prometheus + run: | + wget --quiet https://github.com/prometheus/prometheus/releases/download/v2.51.0/prometheus-2.51.0.linux-amd64.tar.gz + tar xzf prometheus*.tar.gz --strip-components=1 + ./prometheus --config.file=tools/server/bench/prometheus.yml & + while ! nc -z localhost 9090; do + sleep 0.1 + done + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.21' + + - name: Install k6 and xk6-sse + id: k6_installation + run: | + cd tools/server/bench + go install go.k6.io/xk6/cmd/xk6@latest + xk6 build master \ + --with github.com/phymbert/xk6-sse + + - name: Build + id: cmake_build + run: | + set -eux + cmake -B build \ + -DGGML_NATIVE=OFF \ + -DLLAMA_BUILD_SERVER=ON \ + -DLLAMA_CUBLAS=ON \ + -DCUDAToolkit_ROOT=/usr/local/cuda \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ + -DCMAKE_CUDA_ARCHITECTURES=75 \ + -DLLAMA_FATAL_WARNINGS=OFF \ + -DLLAMA_ALL_WARNINGS=OFF \ + -DCMAKE_BUILD_TYPE=Release; + cmake --build build --config Release -j $(nproc) --target llama-server + + - name: Download the dataset + id: download_dataset + run: | + cd tools/server/bench + wget --quiet https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json + + - name: Server bench + id: server_bench + env: + HEAD_REF: ${{ github.head_ref || github.ref_name }} + run: | + set -eux + + cd tools/server/bench + source venv/bin/activate + python bench.py \ + --runner-label ${{ env.RUNNER_LABEL }} \ + --name ${{ github.job }} \ + --branch $HEAD_REF \ + --commit ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha }} \ + --scenario script.js \ + --duration ${{ github.event.inputs.duration || env.DURATION }} \ + --hf-repo ggml-org/models \ + --hf-file ${{ matrix.model }}/ggml-model-${{ matrix.ftype }}.gguf \ + --model-path-prefix /models \ + --parallel ${{ env.N_USERS }} \ + -ngl 33 \ + --batch-size 2048 \ + --ubatch-size 256 \ + --ctx-size 16384 \ + --n-prompts 1000 \ + --max-prompt-tokens 1024 \ + --max-tokens 2048 + + cat results.github.env >> $GITHUB_ENV + + # Remove dataset as we do not want it in the artefact + rm ShareGPT_V3_unfiltered_cleaned_split.json + + - uses: actions/upload-artifact@v4 + with: + name: bench-server-${{ github.job }}-${{ env.RUNNER_LABEL }}-${{ matrix.model }}-${{ matrix.ftype }} + compression-level: 9 + path: | + tools/server/bench/*.jpg + tools/server/bench/*.json + tools/server/bench/*.log + + - name: Commit status + uses: Sibz/github-status-action@v1 + with: + authToken: ${{secrets.GITHUB_TOKEN}} + sha: ${{ inputs.sha || github.event.pull_request.head.sha || github.sha }} + context: bench-server-${{ github.job }}-${{ env.RUNNER_LABEL }}-${{ matrix.model }}-${{ matrix.ftype }} + description: | + ${{ env.BENCH_RESULTS }} + state: 'success' + + - name: Upload benchmark images + uses: devicons/public-upload-to-imgur@v2.2.2 + continue-on-error: true # Important as it looks unstable: 503 + id: imgur_step + with: + client_id: ${{secrets.IMGUR_CLIENT_ID}} + path: | + tools/server/bench/prompt_tokens_seconds.jpg + tools/server/bench/predicted_tokens_seconds.jpg + tools/server/bench/kv_cache_usage_ratio.jpg + tools/server/bench/requests_processing.jpg + + - name: Extract mermaid + id: set_mermaid + run: | + set -eux + + cd tools/server/bench + PROMPT_TOKENS_SECONDS=$(cat prompt_tokens_seconds.mermaid) + echo "PROMPT_TOKENS_SECONDS<> $GITHUB_ENV + echo "$PROMPT_TOKENS_SECONDS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + PREDICTED_TOKENS_SECONDS=$(cat predicted_tokens_seconds.mermaid) + echo "PREDICTED_TOKENS_SECONDS<> $GITHUB_ENV + echo "$PREDICTED_TOKENS_SECONDS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + KV_CACHE_USAGE_RATIO=$(cat kv_cache_usage_ratio.mermaid) + echo "KV_CACHE_USAGE_RATIO<> $GITHUB_ENV + echo "$KV_CACHE_USAGE_RATIO" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + REQUESTS_PROCESSING=$(cat requests_processing.mermaid) + echo "REQUESTS_PROCESSING<> $GITHUB_ENV + echo "$REQUESTS_PROCESSING" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Extract image url + id: extract_image_url + continue-on-error: true + run: | + set -eux + + echo "IMAGE_O=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[0] }}" >> $GITHUB_ENV + echo "IMAGE_1=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[1] }}" >> $GITHUB_ENV + echo "IMAGE_2=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[2] }}" >> $GITHUB_ENV + echo "IMAGE_3=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[3] }}" >> $GITHUB_ENV + + - name: Comment PR + uses: mshick/add-pr-comment@v2 + id: comment_pr + if: ${{ github.event.pull_request != '' && matrix.pr_comment_enabled == 'true' }} + with: + message-id: bench-server-${{ github.job }}-${{ env.RUNNER_LABEL }}-${{ matrix.model }}-${{ matrix.ftype }} + message: | +

+ + 📈 **llama.cpp server** for _${{ github.job }}_ on _${{ env.RUNNER_LABEL }}_ for `${{ matrix.model }}`-`${{ matrix.ftype }}`: **${{ env.BENCH_ITERATIONS}} iterations** 🚀 + +

+ +
+ + Expand details for performance related PR only + + - Concurrent users: ${{ env.N_USERS }}, duration: ${{ github.event.inputs.duration || env.DURATION }} + - HTTP request : avg=${{ env.HTTP_REQ_DURATION_AVG }}ms p(95)=${{ env.HTTP_REQ_DURATION_P_95_ }}ms fails=${{ env.HTTP_REQ_FAILED_PASSES }}, finish reason: stop=${{ env.LLAMACPP_COMPLETIONS_STOP_RATE_PASSES }} truncated=${{ env.LLAMACPP_COMPLETIONS_TRUNCATED_RATE_PASSES }} + - Prompt processing (pp): avg=${{ env.LLAMACPP_PROMPT_PROCESSING_SECOND_AVG }}tk/s p(95)=${{ env.LLAMACPP_PROMPT_PROCESSING_SECOND_P_95_ }}tk/s + - Token generation (tg): avg=${{ env.LLAMACPP_TOKENS_SECOND_AVG }}tk/s p(95)=${{ env.LLAMACPP_TOKENS_SECOND_P_95_ }}tk/s + - ${{ env.BENCH_GRAPH_XLABEL }} + + +

+ + prompt_tokens_seconds + +

+ + More + + ```mermaid + ${{ env.PROMPT_TOKENS_SECONDS }} + ``` + +
+ + predicted_tokens_seconds + +
+ More + + ```mermaid + ${{ env.PREDICTED_TOKENS_SECONDS }} + ``` + +
+ +

+ +
+ + Details + +

+ + kv_cache_usage_ratio + +

+ More + + ```mermaid + ${{ env.KV_CACHE_USAGE_RATIO }} + ``` + +
+ + requests_processing + +
+ More + + ```mermaid + ${{ env.REQUESTS_PROCESSING }} + ``` + +
+ +

+
+
diff --git a/.github/workflows/build-3rd-party.yml b/.github/workflows/build-3rd-party.yml new file mode 100644 index 0000000..82e53db --- /dev/null +++ b/.github/workflows/build-3rd-party.yml @@ -0,0 +1,57 @@ +name: CI (3rd-party) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-3rd-party.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ubuntu-24-llguidance: + runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libssl-dev + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_LLGUIDANCE=ON + cmake --build build --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + diff --git a/.github/workflows/build-and-test-snapdragon.yml b/.github/workflows/build-and-test-snapdragon.yml new file mode 100644 index 0000000..3e857d4 --- /dev/null +++ b/.github/workflows/build-and-test-snapdragon.yml @@ -0,0 +1,148 @@ +name: CI (snapdragon) + +on: + workflow_dispatch: + push: + branches: + - master + paths: + - '.github/workflows/build-and-test-snapdragon.yml' + - 'ggml/include/ggml-hexagon.h' + - 'ggml/src/ggml-hexagon/**' + - 'docs/backend/snapdragon/**' + - 'scripts/snapdragon/**' + - 'CMakePresets.json' + + pull_request: + types: [opened, synchronize, reopened] + paths: + - '.github/workflows/build-and-test-snapdragon.yml' + - 'ggml/include/ggml-hexagon.h' + - 'ggml/src/ggml-hexagon/**' + - 'docs/backend/snapdragon/**' + - 'scripts/snapdragon/**' + - 'CMakePresets.json' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + android-ndk-snapdragon: + runs-on: ubuntu-latest + container: + image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7' + defaults: + run: + shell: bash + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + lfs: false + + - name: Build Llama.CPP for Snapdragon Android + id: build_llama_cpp_snapdragon_android + run: | + cp docs/backend/snapdragon/CMakeUserPresets.json . + cmake --preset arm64-android-snapdragon-release -B build + cmake --build build + cmake --install build --prefix pkg-snapdragon/llama.cpp + + - name: Upload Llama.CPP Snapdragon Android Build Artifact + if: ${{ always() && steps.build_llama_cpp_snapdragon_android.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: llama-cpp-android-arm64-snapdragon + path: pkg-snapdragon/llama.cpp + + linux-iot-snapdragon: + runs-on: ubuntu-latest + container: + image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7' + defaults: + run: + shell: bash + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + lfs: false + + - name: Build Llama.CPP for Snapdragon Linux IoT + id: build_llama_cpp_snapdragon_linux + run: | + cp docs/backend/snapdragon/CMakeUserPresets.json . + cmake --preset arm64-linux-snapdragon-release -B build-snapdragon -DGGML_OPENCL=ON + cmake --build build-snapdragon -j $(nproc) + cmake --install build-snapdragon --prefix pkg-snapdragon/llama.cpp + + - name: Upload Llama.CPP Snapdragon Linux IoT Build Artifact + if: ${{ always() && steps.build_llama_cpp_snapdragon_linux.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: llama-cpp-linux-arm64-snapdragon + path: pkg-snapdragon/llama.cpp + + test-snapdragon-qdc: + name: Test on QDC Device (${{ matrix.device }}) + needs: [android-ndk-snapdragon, linux-iot-snapdragon] + runs-on: ubuntu-24.04-arm + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + device: [SM8750, SM8850, QCS9075M] + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Download build artifact + uses: actions/download-artifact@v7 + with: + name: ${{ startsWith(matrix.device, 'QCS') && 'llama-cpp-linux-arm64-snapdragon' || 'llama-cpp-android-arm64-snapdragon' }} + path: pkg-snapdragon/llama.cpp + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.x' + cache: pip + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y curl unzip + + - name: Install QDC SDK wheel + run: | + curl -fSL -o qdc_sdk.zip https://softwarecenter.qualcomm.com/api/download/software/tools/Qualcomm_Device_Cloud_SDK/All/0.2.3/qualcomm_device_cloud_sdk-0.2.3.zip + unzip qdc_sdk.zip -d qdc_sdk + pip install qdc_sdk/qualcomm_device_cloud_sdk-0.2.3-py3-none-any.whl + + - name: Check QDC API key + id: check_secret + env: + QDC_API_KEY: ${{ secrets.QDC_API_KEY }} + run: echo "has-qdc-key=${{ env.QDC_API_KEY != '' }}" >> "$GITHUB_OUTPUT" + + - name: Run QDC tests (${{ matrix.device }}) + if: steps.check_secret.outputs.has-qdc-key == 'true' + run: | + python scripts/snapdragon/qdc/run_qdc_jobs.py \ + --test all \ + --pkg-dir pkg-snapdragon/llama.cpp \ + --model-url "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" \ + --device ${{ matrix.device }} \ + ${{ startsWith(matrix.device, 'QCS') && '--retries 2 --retry-delay 300' || '' }} + env: + QDC_API_KEY: ${{ secrets.QDC_API_KEY }} + + - name: Cleanup + if: always() + run: rm -rf pkg-snapdragon qdc_sdk qdc_sdk.zip diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml new file mode 100644 index 0000000..a05248e --- /dev/null +++ b/.github/workflows/build-android.yml @@ -0,0 +1,149 @@ +name: CI (android) + +on: + workflow_dispatch: + push: + branches: + - master + paths: + - '.github/workflows/build-android.yml' + - '**/CMakeLists.txt' + - '**/.cmake' + - '**/*.h' + - '**/*.hpp' + - '**/*.c' + - '**/*.cpp' + + pull_request: + types: [opened, synchronize, reopened] + paths: + - '.github/workflows/build-android.yml' + - 'examples/llama.android/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + default: + runs-on: ubuntu-latest + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + lfs: false + + - name: Set up JDK + uses: actions/setup-java@v5 + with: + java-version: 17 + distribution: zulu + + - name: Setup Android SDK + uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 + with: + log-accepted-android-sdk-licenses: false + + - name: Build + run: | + cd examples/llama.android + ./gradlew build --no-daemon + + ndk: + runs-on: ubuntu-latest + container: + image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3' + defaults: + run: + shell: bash + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + lfs: false + + - name: Dependencies + run: | + apt-get update + apt-get install -y build-essential + + - name: Build + id: ndk_build + run: | + cmake -D ANDROID_ABI=arm64-v8a -D ANDROID_PLATFORM=android-31 -D CMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -D GGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm -G Ninja -D LLAMA_OPENSSL=OFF -D GGML_OPENMP=OFF -B build + cmake --build build + cmake --install build --prefix pkg-adb/llama.cpp + + - name: Upload Android Build Artifact + if: ${{ always() && steps.ndk_build.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: llama-cpp-android-arm64-cpu + path: pkg-adb/llama.cpp + + arm64: + runs-on: ubuntu-latest + + env: + NDK_VERSION: "29.0.14206865" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # note : disabled to spare some cache space (https://github.com/ggml-org/llama.cpp/pull/23789) + # for some reason, the ccache does not improve the build time in this case + # example: + # cache off: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78160400831 + # cache on: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394 + # + #- name: ccache + # uses: ggml-org/ccache-action@v1.2.21 + # with: + # key: android-ubuntu-arm64 + # evict-old-files: 1d + # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Set up JDK + uses: actions/setup-java@v5 + with: + java-version: 17 + distribution: temurin + + - name: Setup Android SDK + uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 + with: + log-accepted-android-sdk-licenses: false + + - name: Install NDK + run: | + sdkmanager "ndk;${{ env.NDK_VERSION }}" + echo "ANDROID_NDK=${ANDROID_SDK_ROOT}/ndk/${{ env.NDK_VERSION }}" >> $GITHUB_ENV + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-28 \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DGGML_RPC=ON + time cmake --build build --config Release -j $(nproc) diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.yml new file mode 100644 index 0000000..289e514 --- /dev/null +++ b/.github/workflows/build-apple.yml @@ -0,0 +1,264 @@ +name: CI (apple) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-apple.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.swift', + '**/*.m', + '**/*.metal' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-apple.yml', + 'ggml/src/ggml-metal/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + macos-latest-arm64: + runs-on: macos-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: apple-arm64 + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + run: | + sysctl -a + cmake -B build \ + -DCMAKE_BUILD_RPATH="@loader_path" \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DGGML_METAL_EMBED_LIBRARY=OFF \ + -DGGML_METAL_SHADER_DEBUG=ON \ + -DGGML_RPC=ON \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 + time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) + leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1 + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main -E "test-llama-archs" --verbose --timeout 900 + + macos-latest-x64: + runs-on: macos-15-intel + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: apple-x64 + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + run: | + sysctl -a + # Metal is disabled due to intermittent failures with Github runners not having a GPU: + # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313 + cmake -B build \ + -DCMAKE_BUILD_RPATH="@loader_path" \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DGGML_METAL=OFF \ + -DGGML_RPC=ON \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 + time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + + macos-latest-ios-xcode: + runs-on: macos-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Xcode + uses: ggml-org/setup-xcode@v1 + with: + xcode-version: latest-stable + + - name: Build + id: cmake_build + run: | + sysctl -a + cmake -B build -G Xcode \ + -DGGML_METAL_EMBED_LIBRARY=ON \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \ + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + + - name: xcodebuild for swift package + id: xcodebuild + run: | + ./build-xcframework.sh + + - name: Upload xcframework artifact + uses: actions/upload-artifact@v6 + with: + name: llama-xcframework + path: build-apple/llama.xcframework/ + retention-days: 1 + + - name: Build Xcode project + run: | + xcodebuild -downloadPlatform iOS + xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build + + macos-latest-tvos: + runs-on: macos-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # TODO: this likely does not do anything - if yes, remove it + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: apple-tvos + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + run: | + sysctl -a + cmake -B build -G Xcode \ + -DGGML_METAL_EMBED_LIBRARY=ON \ + -DLLAMA_BUILD_COMMON=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_SYSTEM_NAME=tvOS \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \ + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + + macos-latest-visionos: + runs-on: macos-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # TODO: this likely does not do anything - if yes, remove it + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: apple-visionos + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + run: | + sysctl -a + cmake -B build -G Xcode \ + -DGGML_METAL_EMBED_LIBRARY=ON \ + -DLLAMA_BUILD_COMMON=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_SYSTEM_NAME=visionOS \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \ + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + + macos-latest-swift: + runs-on: macos-latest + needs: macos-latest-ios-xcode + + strategy: + matrix: + destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS'] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # TODO: this likely does not do anything - if yes, remove it + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: apple-swift + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Download xcframework artifact + uses: actions/download-artifact@v7 + with: + name: llama-xcframework + path: build-apple/llama.xcframework/ + + - name: Build llama.cpp with CMake + id: cmake_build + run: | + sysctl -a + cmake -B build -G Xcode \ + -DGGML_METAL_EMBED_LIBRARY=ON \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml new file mode 100644 index 0000000..327f719 --- /dev/null +++ b/.github/workflows/build-cache.yml @@ -0,0 +1,145 @@ +name: Build Actions Cache + +on: + workflow_dispatch: # allows manual triggering + schedule: + - cron: '0 * * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + ubuntu-24-vulkan-cache: + runs-on: ubuntu-24.04 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Get latest Vulkan SDK version + id: vulkan_sdk_version + run: | + echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV" + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-sdk + with: + path: ./vulkan_sdk + key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }} + + - name: Setup Vulkan SDK + if: steps.cache-sdk.outputs.cache-hit != 'true' + uses: ./.github/actions/linux-setup-vulkan + with: + path: ./vulkan_sdk + version: ${{ env.VULKAN_SDK_VERSION }} + + #ubuntu-24-spacemit-cache: + # runs-on: ubuntu-24.04 + + # env: + # # Make sure this is in sync with build-linux-cross.yml + # SPACEMIT_IME_TOOLCHAIN_VERSION: "1.1.2" + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v6 + + # - name: Setup Cache + # uses: actions/cache@v5 + # id: cache-toolchain + # with: + # path: ./spacemit_toolchain + # key: cache-gha-spacemit-ime-toolchain-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}-${{ runner.os }} + + # - name: Setup SpacemiT Toolchain + # if: steps.cache-toolchain.outputs.cache-hit != 'true' + # uses: ./.github/actions/linux-setup-spacemit + # with: + # path: ./spacemit_toolchain + # version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }} + + ubuntu-24-openvino-cache: + runs-on: ubuntu-24.04 + + env: + # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-openvino + with: + path: ./openvino_toolkit + key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }} + + - name: Setup OpenVINO Toolkit + if: steps.cache-openvino.outputs.cache-hit != 'true' + uses: ./.github/actions/linux-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + windows-2022-openvino-cache: + runs-on: windows-2022 + + env: + # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-openvino + with: + path: ./openvino_toolkit + key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }} + + - name: Setup OpenVINO Toolkit + if: steps.cache-openvino.outputs.cache-hit != 'true' + uses: ./.github/actions/windows-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + windows-2022-rocm-cache: + runs-on: windows-2022 + + env: + # Make sure this is in sync with build.yml + HIPSDK_INSTALLER_VERSION: "26.Q1" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-rocm + with: + path: C:\Program Files\AMD\ROCm + key: cache-gha-rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }} + + - name: Setup ROCm + if: steps.cache-rocm.outputs.cache-hit != 'true' + uses: ./.github/actions/windows-setup-rocm + with: + version: ${{ env.HIPSDK_INSTALLER_VERSION }} diff --git a/.github/workflows/build-cann.yml b/.github/workflows/build-cann.yml new file mode 100644 index 0000000..6d76ed4 --- /dev/null +++ b/.github/workflows/build-cann.yml @@ -0,0 +1,104 @@ +name: CI (cann) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-cann.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-cann.yml', + 'ggml/src/ggml-cann/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: +# TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23705) +# in order to enable it again, we have to provision dedicated runners to run it +# openEuler-latest-cann: +# defaults: +# run: +# shell: bash -el {0} +# strategy: +# matrix: +# arch: [x86, aarch64] +# chip_type: ['910b', '310p'] +# build: ['Release'] +# use_acl_graph: ['on', 'off'] +# exclude: +# # 310P does not support USE_ACL_GRAPH=on +# - chip_type: '310p' +# use_acl_graph: 'on' +# runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} +# steps: +# - name: Checkout +# uses: actions/checkout@v6 +# with: +# fetch-depth: 0 +# +# - name: Free up disk space +# uses: ggml-org/free-disk-space@v1.3.1 +# with: +# tool-cache: true +# +# - name: Set container image +# id: cann-image +# run: | +# image="ascendai/cann:${{ matrix.chip_type == '910b' && '8.5.0-910b-openeuler24.03-py3.11' || '8.5.0-310p-openeuler24.03-py3.11' }}" +# echo "image=${image}" >> "${GITHUB_OUTPUT}" +# +# - name: Pull container image +# run: docker pull "${{ steps.cann-image.outputs.image }}" +# +# - name: Build +# env: +# BUILD_TYPE: ${{ matrix.build }} +# SOC_TYPE: ascend${{ matrix.chip_type }} +# USE_ACL_GRAPH: ${{ matrix.use_acl_graph }} +# run: | +# HOST_UID=$(id -u) +# HOST_GID=$(id -g) +# +# docker run --rm \ +# -v "${PWD}:/workspace" \ +# -w /workspace \ +# -e SOC_TYPE=${SOC_TYPE} \ +# -e BUILD_TYPE=${BUILD_TYPE} \ +# -e USE_ACL_GRAPH=${USE_ACL_GRAPH} \ +# "${{ steps.cann-image.outputs.image }}" \ +# bash -lc ' +# set -e +# yum install -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs git gcc gcc-c++ make cmake openssl-devel +# yum clean all && rm -rf /var/cache/yum +# git config --global --add safe.directory "/workspace" +# export LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/lib64:${ASCEND_TOOLKIT_HOME}/$(uname -m)-linux/devlib/:${LD_LIBRARY_PATH} +# cmake -S . -B build \ +# -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ +# -DGGML_CANN=on \ +# -DSOC_TYPE=${SOC_TYPE} \ +# -DUSE_ACL_GRAPH=${USE_ACL_GRAPH} +# cmake --build build -j $(nproc) +# +# chown -R '"${HOST_UID}"':'"${HOST_GID}"' /workspace/build +# ' diff --git a/.github/workflows/build-cmake-pkg.yml b/.github/workflows/build-cmake-pkg.yml new file mode 100644 index 0000000..5becff0 --- /dev/null +++ b/.github/workflows/build-cmake-pkg.yml @@ -0,0 +1,51 @@ +name: Build relocatable cmake package +on: + workflow_dispatch: + workflow_call: + +jobs: + linux: + runs-on: [self-hosted, Linux, CPU] + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Build + run: | + PREFIX="$(pwd)"/inst + cmake -S . -B build \ + -DCMAKE_PREFIX_PATH="$PREFIX" \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DCMAKE_BUILD_TYPE=Release + cmake --build build --config Release + cmake --install build --prefix "$PREFIX" --config Release + + export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake + tclsh <<'EOF' + set build(commit) [string trim [exec git rev-parse --short HEAD]] + set build(number) [string trim [exec git rev-list --count HEAD]] + set build(version) "0.0.$build(number)" + + set llamaconfig [read [open "$env(LLAMA_CONFIG)" r]] + set checks [list "set\\(LLAMA_VERSION \\s+$build(version)\\)" \ + "set\\(LLAMA_BUILD_COMMIT\\s+$build(commit)\\)" \ + "set\\(LLAMA_BUILD_NUMBER\\s+$build(number)\\)"] + + puts -nonewline "Checking llama-config.cmake version... " + foreach check $checks { + if {![regexp -expanded -- $check $llamaconfig]} { + puts "\"$check\" failed!" + exit 1 + } + } + puts "success." + EOF + + cd examples/simple-cmake-pkg + cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake + cmake --build build diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml new file mode 100644 index 0000000..30b07ce --- /dev/null +++ b/.github/workflows/build-cpu.yml @@ -0,0 +1,215 @@ +name: CI (cpu) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-cpu.yml', + '.github/workflows/build-cmake-pkg.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-cpu.yml', + '.github/workflows/build-cmake-pkg.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + build-cmake-pkg: + uses: ./.github/workflows/build-cmake-pkg.yml + + ubuntu: + strategy: + matrix: + include: + - build: 'x64' + os: ubuntu-22.04 + - build: 'arm64' + os: ubuntu-24.04-arm + + runs-on: ${{ matrix.os }} + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: cpu-${{ matrix.os }} + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build Dependencies + id: build_depends + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + python3 python3-pip python3-dev python3-wheel \ + libjpeg-dev build-essential libssl-dev \ + git-lfs + + - name: Toolchain workaround (GCC 14) + if: ${{ contains(matrix.os, 'ubuntu-24.04') }} + run: | + sudo apt-get install -y gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + + - name: Python Dependencies + id: python_depends + run: | + export PIP_BREAK_SYSTEM_PACKAGES="1" + python3 -m pip install --upgrade pip setuptools + pip3 install ./gguf-py + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DGGML_RPC=ON + time cmake --build build --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + + - name: Test llama2c conversion + id: llama2c_test + run: | + cd build + echo "Fetch tokenizer" + wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin + echo "Fetch llama2c model" + wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin + ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf + ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 + + windows: + runs-on: windows-2025 + + env: + OPENBLAS_VERSION: 0.3.23 + SDE_VERSION: 9.33.0-2024-01-07 + VULKAN_VERSION: 1.4.357.0 + + strategy: + matrix: + include: + - build: 'x64-cpu-static' + arch: 'x64' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF' + - build: 'x64-openblas' + arch: 'x64' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"' + - build: 'x64-vulkan' + arch: 'x64' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_VULKAN=ON' + - build: 'arm64' + arch: 'arm64' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON' + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: cpu-windows-2025-${{ matrix.build }} + variant: ccache + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Download OpenBLAS + id: get_openblas + if: ${{ matrix.build == 'x64-openblas' }} + run: | + curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip" + curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE" + mkdir $env:RUNNER_TEMP/openblas + tar.exe -xvf $env:RUNNER_TEMP/openblas.zip -C $env:RUNNER_TEMP/openblas + $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath) + $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim())) + $lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe') + & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll + + - name: Install Vulkan SDK + id: get_vulkan + if: ${{ matrix.build == 'x64-vulkan' }} + run: | + curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe" + & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install + Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}" + Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin" + + - name: Install Ninja + id: install_ninja + run: | + choco install ninja + + - name: Build + id: cmake_build + run: | + cmake -S . -B build ${{ matrix.defines }} ` + -DLLAMA_BUILD_BORINGSSL=ON + cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} + + - name: Add libopenblas.dll + id: add_libopenblas_dll + if: ${{ matrix.build == 'x64-openblas' }} + run: | + cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll + cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt + + - name: Test + id: cmake_test + if: ${{ matrix.arch == 'x64' }} + run: | + cd build + ctest -L main -C Release --verbose --timeout 900 + + # TODO: disabled for now, consider adding tests for all CPU variants instead + # - name: Test (Intel SDE) + # id: cmake_test_sde + # if: ${{ matrix.build == 'avx512-x64' && env.HAS_AVX512F == '0' }} # use Intel SDE for AVX-512 emulation + # run: | + # curl.exe -o $env:RUNNER_TEMP/sde.tar.xz -L "https://downloadmirror.intel.com/813591/sde-external-${env:SDE_VERSION}-win.tar.xz" + # # for some weird reason windows tar doesn't like sde tar.xz + # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar.xz + # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar + # $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe) + # cd build + # $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1 + # & $sde -future -- ctest -L main -C Release --verbose --timeout 900 diff --git a/.github/workflows/build-cross.yml b/.github/workflows/build-cross.yml new file mode 100644 index 0000000..eef78b6 --- /dev/null +++ b/.github/workflows/build-cross.yml @@ -0,0 +1,317 @@ +name: CI (cross) +on: + # only manual triggers due to low-importance of the workflows + # TODO: for regular runs, provision dedicated self-hosted runners + workflow_dispatch: + push: + branches: + - master + paths: [ + '.github/workflows/build-cross.yml', + 'ggml/src/spacemit/*', + 'ggml/src/arch/loongarch/*' + ] + # run once every week + schedule: + - cron: '0 0 * * 0' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + + +jobs: + # ubuntu-24-riscv64-cpu-cross: + # runs-on: ubuntu-24.04 + + # steps: + # - uses: actions/checkout@v6 + # - name: Setup Riscv + # run: | + # sudo dpkg --add-architecture riscv64 + + # # Add arch-specific repositories for non-amd64 architectures + # cat << EOF | sudo tee /etc/apt/sources.list.d/riscv64-ports.list + # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe + # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe + # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe + # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe + # EOF + + # sudo apt-get update || true ;# Prevent failure due to missing URLs. + + # sudo apt-get install -y --no-install-recommends \ + # build-essential \ + # gcc-14-riscv64-linux-gnu \ + # g++-14-riscv64-linux-gnu + + # - name: Build + # run: | + # cmake -B build -DLLAMA_OPENSSL=OFF \ + # -DCMAKE_BUILD_TYPE=Release \ + # -DGGML_OPENMP=OFF \ + # -DLLAMA_BUILD_EXAMPLES=ON \ + # -DLLAMA_BUILD_TOOLS=ON \ + # -DLLAMA_BUILD_TESTS=OFF \ + # -DCMAKE_SYSTEM_NAME=Linux \ + # -DCMAKE_SYSTEM_PROCESSOR=riscv64 \ + # -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ + # -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \ + # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + # -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \ + # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \ + # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ + # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH + + # cmake --build build --config Release -j $(nproc) + + # ubuntu-24-riscv64-vulkan-cross: + # runs-on: ubuntu-24.04 + + # steps: + # - uses: actions/checkout@v6 + # - name: Setup Riscv + # run: | + # sudo dpkg --add-architecture riscv64 + + # # Add arch-specific repositories for non-amd64 architectures + # cat << EOF | sudo tee /etc/apt/sources.list.d/riscv64-ports.list + # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe + # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe + # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe + # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe + # EOF + + # sudo apt-get update || true ;# Prevent failure due to missing URLs. + + # sudo apt-get install -y --no-install-recommends \ + # build-essential \ + # glslc \ + # gcc-14-riscv64-linux-gnu \ + # g++-14-riscv64-linux-gnu \ + # libvulkan-dev:riscv64 + + # - name: Build + # run: | + # cmake -B build -DLLAMA_OPENSSL=OFF \ + # -DCMAKE_BUILD_TYPE=Release \ + # -DGGML_VULKAN=ON \ + # -DGGML_OPENMP=OFF \ + # -DLLAMA_BUILD_EXAMPLES=ON \ + # -DLLAMA_BUILD_TOOLS=ON \ + # -DLLAMA_BUILD_TESTS=OFF \ + # -DCMAKE_SYSTEM_NAME=Linux \ + # -DCMAKE_SYSTEM_PROCESSOR=riscv64 \ + # -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ + # -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \ + # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + # -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \ + # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \ + # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ + # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH + + # cmake --build build --config Release -j $(nproc) + + # ubuntu-24-arm64-vulkan-cross: + # runs-on: ubuntu-24.04 + + # steps: + # - uses: actions/checkout@v6 + # - name: Setup Arm64 + # run: | + # sudo dpkg --add-architecture arm64 + + # # Add arch-specific repositories for non-amd64 architectures + # cat << EOF | sudo tee /etc/apt/sources.list.d/arm64-ports.list + # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe + # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe + # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe + # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe + # EOF + + # sudo apt-get update || true ;# Prevent failure due to missing URLs. + + # sudo apt-get install -y --no-install-recommends \ + # build-essential \ + # glslc \ + # crossbuild-essential-arm64 \ + # libvulkan-dev:arm64 + + # - name: Build + # run: | + # cmake -B build -DLLAMA_OPENSSL=OFF \ + # -DCMAKE_BUILD_TYPE=Release \ + # -DGGML_VULKAN=ON \ + # -DGGML_OPENMP=OFF \ + # -DLLAMA_BUILD_EXAMPLES=ON \ + # -DLLAMA_BUILD_TOOLS=ON \ + # -DLLAMA_BUILD_TESTS=OFF \ + # -DCMAKE_SYSTEM_NAME=Linux \ + # -DCMAKE_SYSTEM_PROCESSOR=aarch64 \ + # -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \ + # -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ \ + # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + # -DCMAKE_FIND_ROOT_PATH=/usr/lib/aarch64-linux-gnu \ + # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \ + # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ + # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH + + # cmake --build build --config Release -j $(nproc) + + debian-13-loongarch64-cpu-cross: + runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671 + + steps: + - uses: actions/checkout@v6 + - name: Setup LoongArch + run: | + rm -f /etc/apt/sources.list.d/* + cat << EOF | tee /etc/apt/sources.list.d/debian-ports.list + deb http://snapshot.debian.org/archive/debian/20250515T202920Z/ trixie main + EOF + ( echo 'quiet "true";'; \ + echo 'APT::Get::Assume-Yes "true";'; \ + echo 'APT::Install-Recommends "false";'; \ + echo 'Acquire::Check-Valid-Until "false";'; \ + echo 'Acquire::Retries "5";'; \ + ) > /etc/apt/apt.conf.d/99snapshot-repos + + apt-get update + apt-get install -y ca-certificates debian-ports-archive-keyring cmake git zip + dpkg --add-architecture loong64 + + # Add arch-specific repositories for non-amd64 architectures + cat << EOF | tee /etc/apt/sources.list.d/loong64-ports.list + deb [arch=loong64] http://snapshot.debian.org/archive/debian-ports/20250515T194251Z/ sid main + EOF + + apt-get update || true ;# Prevent failure due to missing URLs. + + apt-get install -y --no-install-recommends \ + build-essential \ + gcc-14-loongarch64-linux-gnu \ + g++-14-loongarch64-linux-gnu + + - name: Build + run: | + cmake -B build -DLLAMA_OPENSSL=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_EXAMPLES=ON \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR=loongarch64 \ + -DCMAKE_C_COMPILER=loongarch64-linux-gnu-gcc-14 \ + -DCMAKE_CXX_COMPILER=loongarch64-linux-gnu-g++-14 \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_FIND_ROOT_PATH=/usr/lib/loongarch64-linux-gnu \ + -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \ + -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ + -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH + + cmake --build build --config Release -j $(nproc) + + debian-13-loongarch64-vulkan-cross: + runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671 + + steps: + - uses: actions/checkout@v6 + - name: Setup LoongArch + run: | + rm -f /etc/apt/sources.list.d/* + cat << EOF | tee /etc/apt/sources.list.d/debian-ports.list + deb http://snapshot.debian.org/archive/debian/20250515T202920Z/ trixie main + EOF + ( echo 'quiet "true";'; \ + echo 'APT::Get::Assume-Yes "true";'; \ + echo 'APT::Install-Recommends "false";'; \ + echo 'Acquire::Check-Valid-Until "false";'; \ + echo 'Acquire::Retries "5";'; \ + ) > /etc/apt/apt.conf.d/99snapshot-repos + + apt-get update + apt-get install -y ca-certificates debian-ports-archive-keyring cmake git zip + dpkg --add-architecture loong64 + + # Add arch-specific repositories for non-amd64 architectures + cat << EOF | tee /etc/apt/sources.list.d/loong64-ports.list + deb [arch=loong64] http://snapshot.debian.org/archive/debian-ports/20250515T194251Z/ sid main + EOF + + apt-get update || true ;# Prevent failure due to missing URLs. + + apt-get install -y --no-install-recommends \ + build-essential \ + glslc \ + spirv-headers \ + gcc-14-loongarch64-linux-gnu \ + g++-14-loongarch64-linux-gnu \ + libvulkan-dev:loong64 + + - name: Build + run: | + cmake -B build -DLLAMA_OPENSSL=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_VULKAN=ON \ + -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_EXAMPLES=ON \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR=loongarch64 \ + -DCMAKE_C_COMPILER=loongarch64-linux-gnu-gcc-14 \ + -DCMAKE_CXX_COMPILER=loongarch64-linux-gnu-g++-14 \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_FIND_ROOT_PATH=/usr/lib/loongarch64-linux-gnu \ + -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \ + -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ + -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH + + cmake --build build --config Release -j $(nproc) + + ubuntu-24-riscv64-cpu-spacemit-ime-cross: + runs-on: ubuntu-24.04 + + env: + # Make sure this is in sync with build-cache.yml + SPACEMIT_IME_TOOLCHAIN_VERSION: "1.2.4" + + steps: + - uses: actions/checkout@v6 + + #- name: Use SpacemiT Toolchain Cache + # uses: actions/cache@v5 + # id: cache-toolchain + # with: + # path: ./spacemit_toolchain + # key: cache-gha-spacemit-ime-toolchain-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}-${{ runner.os }} + + - name: Setup SpacemiT Toolchain + #if: steps.cache-toolchain.outputs.cache-hit != 'true' + uses: ./.github/actions/linux-setup-spacemit + with: + path: ./spacemit_toolchain + version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }} + + - name: Build + run: | + export RISCV_ROOT_PATH=${PWD}/spacemit_toolchain + cmake -B build -DLLAMA_OPENSSL=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_BUILD_EXAMPLES=ON \ + -DGGML_CPU_REPACK=OFF \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DGGML_CPU_RISCV64_SPACEMIT=ON \ + -DGGML_RVV=ON \ + -DGGML_RV_ZVFH=ON \ + -DGGML_RV_ZFH=ON \ + -DGGML_RV_ZICBOP=ON \ + -DGGML_RV_ZIHINTPAUSE=ON \ + -DGGML_RV_ZBA=ON \ + -DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake + + cmake --build build --config Release -j $(nproc) diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml new file mode 100644 index 0000000..6271b22 --- /dev/null +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -0,0 +1,134 @@ +name: CI (CUDA, ubuntu) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-cuda-ubuntu.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cu', + '**/*.cuh' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-cuda-ubuntu.yml', + 'ggml/src/ggml-cuda/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + cuda: + runs-on: ubuntu-24.04 + container: nvidia/cuda:12.6.2-devel-ubuntu24.04 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Install dependencies + env: + DEBIAN_FRONTEND: noninteractive + run: | + apt update + apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: cuda-ubuntu-24.04-cuda + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build with CMake + # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project + run: | + cmake -S . -B build -G Ninja \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_ARCHITECTURES=89-real \ + -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \ + -DGGML_NATIVE=OFF \ + -DGGML_CUDA=ON \ + -DGGML_CUDA_CUB_3DOT2=ON + cmake --build build + + hip: + runs-on: ubuntu-22.04 + container: rocm/dev-ubuntu-22.04:6.1.2 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: cuda-ubuntu-22.04-hip + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build with native CMake HIP support + id: cmake_build + run: | + cmake -B build -S . \ + -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ + -DGGML_HIP_ROCWMMA_FATTN=ON \ + -DGPU_TARGETS="gfx1030" \ + -DGGML_HIP=ON + cmake --build build --config Release -j $(nproc) + + musa: + runs-on: ubuntu-22.04 + container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + apt-get update + apt-get install -y build-essential git cmake libssl-dev + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: cuda-ubuntu-22.04-musa + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build with native CMake MUSA support + id: cmake_build + run: | + cmake -B build -S . \ + -DGGML_MUSA=ON + time cmake --build build --config Release -j $(nproc) diff --git a/.github/workflows/build-cuda-windows.yml b/.github/workflows/build-cuda-windows.yml new file mode 100644 index 0000000..e9e9414 --- /dev/null +++ b/.github/workflows/build-cuda-windows.yml @@ -0,0 +1,162 @@ +name: CI (CUDA, windows) + +# TODO: this workflow is only triggered manually because it is very heavy on the CI +# when we provision dedicated windows runners, we can enable it for pushes too +# note: running this workflow manually will populate the ccache for the release builds +# this can be used before merging a PR to speed up the release workflow +on: + workflow_dispatch: # allows manual triggering + +# note: this will run in queue with the release workflow +concurrency: + group: release + queue: max + +env: + GH_TOKEN: ${{ github.token }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + cuda: + runs-on: windows-2022 + + permissions: + actions: write + + strategy: + matrix: + cuda: ['12.4', '13.3'] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-windows-2022-x64-cuda-${{ matrix.cuda }} + + - name: Install Cuda Toolkit + uses: ./.github/actions/windows-setup-cuda + with: + cuda_version: ${{ matrix.cuda }} + + - name: Install Ninja + id: install_ninja + run: | + choco install ninja + + - name: Build + id: cmake_build + shell: cmd + # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + cmake -S . -B build -G "Ninja Multi-Config" ^ + -DLLAMA_BUILD_SERVER=ON ^ + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DGGML_NATIVE=OFF ^ + -DGGML_BACKEND_DL=ON ^ + -DGGML_CPU_ALL_VARIANTS=ON ^ + -DGGML_CUDA=ON ^ + -DGGML_RPC=ON ^ + -DGGML_CUDA_CUB_3DOT2=ON + set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1 + cmake --build build --config Release -j %NINJA_JOBS% -t ggml + cmake --build build --config Release + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-windows-2022-x64-cuda-${{ matrix.cuda }} + + hip: + runs-on: windows-2022 + + permissions: + actions: write + + env: + # Make sure this is in sync with build-cache.yml + HIPSDK_INSTALLER_VERSION: "26.Q1" + + strategy: + matrix: + include: + # sync with release.yml + - name: "radeon" + gpu_targets: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Grab rocWMMA package + id: grab_rocwmma + run: | + curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.2.0.70201-81~24.04_amd64.deb" + 7z x rocwmma.deb + 7z x data.tar + + - name: Use ROCm Installation Cache + uses: actions/cache@v5 + id: cache-rocm + with: + path: C:\Program Files\AMD\ROCm + key: cache-gha-rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }} + + - name: Setup ROCm + if: steps.cache-rocm.outputs.cache-hit != 'true' + uses: ./.github/actions/windows-setup-rocm + with: + version: ${{ env.HIPSDK_INSTALLER_VERSION }} + + - name: Verify ROCm + id: verify + run: | + # Find and test ROCm installation + $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1 + if (-not $clangPath) { + Write-Error "ROCm installation not found" + exit 1 + } + & $clangPath.FullName --version + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + # TODO: this build does not match the build in release.yml, so we use a different cache key + # ideally, the builds should match, similar to the CUDA build above so that we would be able + # to populate the ccache for the release with manual runs of this workflow + #key: release-windows-2022-x64-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ matrix.name }} + key: cuda-windows-2022-x64-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ matrix.name }} + + - name: Build + id: cmake_build + run: | + $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path) + $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}" + cmake -G "Unix Makefiles" -B build -S . ` + -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" ` + -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" ` + -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-7.2.1/include/" ` + -DCMAKE_BUILD_TYPE=Release ` + -DLLAMA_BUILD_BORINGSSL=ON ` + -DROCM_DIR="${env:HIP_PATH}" ` + -DGGML_HIP=ON ` + -DGGML_HIP_ROCWMMA_FATTN=ON ` + -DGPU_TARGETS="gfx1100" ` + -DGGML_RPC=ON + cmake --build build -j ${env:NUMBER_OF_PROCESSORS} + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + #key: release-windows-2022-x64-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ matrix.name }} + key: cuda-windows-2022-x64-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ matrix.name }} diff --git a/.github/workflows/build-ibm.yml b/.github/workflows/build-ibm.yml new file mode 100644 index 0000000..d2e4f3c --- /dev/null +++ b/.github/workflows/build-ibm.yml @@ -0,0 +1,150 @@ +name: CI (ibm) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-ibm.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-ibm.yml', + 'ggml/src/ggml-cpu/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + + ubuntu-24-s390x: + runs-on: ubuntu-24.04-s390x + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Build Dependencies + id: build_depends + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + python3 python3-pip python3-dev python3-wheel \ + libjpeg-dev build-essential libssl-dev \ + git-lfs + + - name: Toolchain workaround (GCC 14) + run: | + sudo apt-get install -y gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + + - name: Python Dependencies + id: python_depends + run: | + export PIP_BREAK_SYSTEM_PACKAGES="1" + python3 -m pip install --upgrade pip setuptools + pip3 install ./gguf-py + + - name: Swap Endianness + id: endianness + run: | + for f in models/*.gguf; do + echo YES | python3 gguf-py/gguf/scripts/gguf_convert_endian.py $f big + done + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DGGML_RPC=ON + time cmake --build build --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + + - name: Test llama2c (s390x) + id: llama2c_test_s390x + run: | + cd build + echo "Fetch llama2c big-endian model" + wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf + ./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 + + ubuntu-24-ppc64le: + runs-on: ubuntu-24.04-ppc64le + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Build Dependencies + id: build_depends + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + python3 python3-pip python3-dev python3-wheel \ + libjpeg-dev build-essential libssl-dev \ + git-lfs + + - name: Toolchain workaround (GCC 14) + run: | + sudo apt-get install -y gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + + - name: Python Dependencies + id: python_depends + run: | + export PIP_BREAK_SYSTEM_PACKAGES="1" + python3 -m pip install --upgrade pip setuptools + pip3 install ./gguf-py + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DGGML_RPC=ON + time cmake --build build --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + + - name: Test llama2c conversion + id: llama2c_test + run: | + cd build + echo "Fetch tokenizer" + wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin + echo "Fetch llama2c model" + wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin + ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf + ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 diff --git a/.github/workflows/build-msys.yml b/.github/workflows/build-msys.yml new file mode 100644 index 0000000..15c55cf --- /dev/null +++ b/.github/workflows/build-msys.yml @@ -0,0 +1,70 @@ +name: CI (msys) + +on: + # only manual triggers due to low-importance of the workflows + # TODO: for regular runs, provision dedicated self-hosted runners + workflow_dispatch: + # run once every week + schedule: + - cron: '0 0 * * 0' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + windows-msys2: + runs-on: windows-2025 + + strategy: + fail-fast: false + matrix: + include: + - { sys: UCRT64, env: ucrt-x86_64, compiler: gcc, build: Release } + - { sys: CLANG64, env: clang-x86_64, compiler: clang, build: Release } + + steps: + - name: Clone + uses: actions/checkout@v6 + + #- name: ccache + # uses: ggml-org/ccache-action@v1.2.16 + # with: + # key: msys-windows-2025-x64 + # variant: ccache + # evict-old-files: 1d + # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Setup ${{ matrix.sys }} + uses: msys2/setup-msys2@cafece8e6baf9247cf9b1bf95097b0b983cc558d # v2 + with: + update: true + msystem: ${{matrix.sys}} + install: >- + mingw-w64-${{matrix.env}}-${{matrix.compiler}} + mingw-w64-${{matrix.env}}-cmake + mingw-w64-${{matrix.env}}-openblas + + - name: Build using CMake + shell: msys2 {0} + run: | + cmake -B build + cmake --build build --config ${{ matrix.build }} -j $(nproc) + + - name: Clean after building using CMake + shell: msys2 {0} + run: | + rm -rf build + + - name: Build using CMake w/ OpenBLAS + shell: msys2 {0} + run: | + cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS + cmake --build build --config ${{ matrix.build }} -j $(nproc) diff --git a/.github/workflows/build-opencl.yml b/.github/workflows/build-opencl.yml new file mode 100644 index 0000000..251b1f8 --- /dev/null +++ b/.github/workflows/build-opencl.yml @@ -0,0 +1,82 @@ +name: CI (opencl) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-opencl.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cl' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-opencl.yml', + 'ggml/src/ggml-opencl/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + windows-2025-opencl-adreno: + runs-on: windows-2025 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: opencl-windows-2025-x64 + variant: ccache + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Install Ninja + id: install_ninja + run: | + choco install ninja + + - name: Install OpenCL Headers and Libs + id: install_opencl + run: | + git clone https://github.com/KhronosGroup/OpenCL-Headers + cd OpenCL-Headers + cmake -B build ` + -DBUILD_TESTING=OFF ` + -DOPENCL_HEADERS_BUILD_TESTING=OFF ` + -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF ` + -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release" + cmake --build build --target install + git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader + cd OpenCL-ICD-Loader + cmake -B build-arm64-release ` + -A arm64 ` + -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" ` + -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release" + cmake --build build-arm64-release --target install --config release + + - name: Build + id: cmake_build + run: | + cmake -S . -B build -G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON -DLLAMA_BUILD_BORINGSSL=ON + cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml new file mode 100644 index 0000000..938cde3 --- /dev/null +++ b/.github/workflows/build-openvino.yml @@ -0,0 +1,169 @@ +name: CI (openvino) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-openvino.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-openvino.yml', + 'ggml/src/ggml-openvino/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ubuntu-24-openvino: + runs-on: [self-hosted, Linux, Intel, OpenVINO] + + env: + # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install -y build-essential libssl-dev libtbb12 cmake ninja-build python3-pip + sudo apt-get install -y ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd + + - name: Setup OpenVINO Toolkit + uses: ./.github/actions/linux-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + - name: Install OpenVINO dependencies + run: | + cd ./openvino_toolkit + chmod +x ./install_dependencies/install_openvino_dependencies.sh + echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh + + - name: Build + id: cmake_build + run: | + source ./openvino_toolkit/setupvars.sh + cmake -B build/ReleaseOV -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_OPENVINO=ON + time cmake --build build/ReleaseOV --config Release --parallel + + - name: Test (CPU) + id: cmake_test_cpu + # TODO: fix and re-enable the `test-llama-archs` test below + run: | + cd ${{ github.workspace }} + ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 2000 + + - name: Test (GPU) + id: cmake_test_gpu + # TODO: fix and re-enable the `test-llama-archs` test below + run: | + cd ${{ github.workspace }} + export GGML_OPENVINO_DEVICE=GPU + ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 3000 + + openvino-windows-2022: + runs-on: windows-2022 + + env: + # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: openvino-windows-2022 + variant: ccache + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-openvino + with: + path: ./openvino_toolkit + key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }} + + - name: Setup OpenVINO Toolkit + if: steps.cache-openvino.outputs.cache-hit != 'true' + uses: ./.github/actions/windows-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + - name: Install OpenCL using vcpkg + shell: powershell + run: | + git clone https://github.com/microsoft/vcpkg C:\vcpkg + C:\vcpkg\bootstrap-vcpkg.bat + C:\vcpkg\vcpkg install opencl + + - name: Build + id: cmake_build + shell: cmd + run: | + REM Find extracted OpenVINO folder dynamically + for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i + + if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" ( + echo ERROR: OpenVINOConfig.cmake not found + exit /b 1 + ) + + call "%OPENVINO_ROOT%\setupvars.bat" + + cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^ + -A x64 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DGGML_OPENVINO=ON ^ + -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake + + cmake --build build\ReleaseOV --config Release -- /m + + - name: Test (CPU) + id: cmake_test_cpu + shell: cmd + # TODO: fix and re-enable the `test-llama-archs` test below + run: | + REM Find extracted OpenVINO folder dynamically + for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i + call "%OPENVINO_ROOT%\setupvars.bat" + + cd build + ctest --test-dir ReleaseOV -L main -E "test-llama-archs" -C Release --verbose --timeout 3000 diff --git a/.github/workflows/build-riscv.yml b/.github/workflows/build-riscv.yml new file mode 100644 index 0000000..7061537 --- /dev/null +++ b/.github/workflows/build-riscv.yml @@ -0,0 +1,188 @@ +name: CI (riscv) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-riscv.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-riscv.yml', + 'ggml/src/ggml-cpu/arch/riscv/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ubuntu-cpu-riscv64-native: + runs-on: ubuntu-24.04-riscv + + steps: + - name: Install dependencies + run: | + # Install necessary packages + sudo apt-get update + sudo apt-get install -y libssl-dev + + # Set gcc-14 and g++-14 as the default compilers + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 + + git lfs install + + - name: Check environment + run: | + uname -a + gcc --version + g++ --version + ldd --version + cmake --version + rustc --version + env + echo "nproc=$(nproc)" + + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation + #- name: ccache + # uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1 + # with: + # key: riscv-ubuntu-native + # evict-old-files: 1d + # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_EXAMPLES=ON \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_TESTS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DGGML_RPC=ON \ + -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ + -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 + + time cmake --build build --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + + - name: Test llama2c conversion + id: llama2c_test + run: | + cd build + echo "Fetch tokenizer" + wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin + echo "Fetch llama2c model" + wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin + ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf + ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 + + ubuntu-riscv64-native-sanitizer: + runs-on: ubuntu-24.04-riscv + + continue-on-error: true + + strategy: + matrix: + sanitizer: [ADDRESS, THREAD, UNDEFINED] + build_type: [Debug] + + steps: + - name: Install dependencies + run: | + # Set gcc-14 and g++-14 as the default compilers + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 + + git lfs install + + - name: GCC version check + run: | + gcc --version + g++ --version + + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation + #- name: ccache + # uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1 + # with: + # key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }} + # evict-old-files: 1d + # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + if: ${{ matrix.sanitizer != 'THREAD' }} + run: | + cmake -B build \ + -DLLAMA_OPENSSL=OFF \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DGGML_OPENMP=ON \ + -DLLAMA_BUILD_EXAMPLES=ON \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ + -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ + -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 + + cmake --build build --config ${{ matrix.build_type }} -j $(nproc) + + - name: Build (no OpenMP) + id: cmake_build_no_openmp + if: ${{ matrix.sanitizer == 'THREAD' }} + run: | + cmake -B build \ + -DLLAMA_OPENSSL=OFF \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_EXAMPLES=ON \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ + -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ + -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 + + cmake --build build --config ${{ matrix.build_type }} -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 diff --git a/.github/workflows/build-rpc.yml b/.github/workflows/build-rpc.yml new file mode 100644 index 0000000..d04dc37 --- /dev/null +++ b/.github/workflows/build-rpc.yml @@ -0,0 +1,66 @@ +name: CI (rpc) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-rpc.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-rpc.yml', + 'ggml/src/ggml-rpc/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ubuntu-24-rpc: + runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + + continue-on-error: true + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libssl-dev ninja-build + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_RPC=ON + time cmake --build build --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose diff --git a/.github/workflows/build-sanitize.yml b/.github/workflows/build-sanitize.yml new file mode 100644 index 0000000..e242abc --- /dev/null +++ b/.github/workflows/build-sanitize.yml @@ -0,0 +1,86 @@ +name: CI (sanitize) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-sanitize.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ctest: + runs-on: [self-hosted, X64, CPU, Linux] + + continue-on-error: true + + strategy: + matrix: + sanitizer: [ADDRESS, THREAD, UNDEFINED] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings + - name: Build (undefined) + id: cmake_build_undefined + if: ${{ matrix.sanitizer == 'UNDEFINED' }} + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ + -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON + + cmake --build build --config Debug -j $(nproc) + + - name: Build + id: cmake_build + if: ${{ matrix.sanitizer == 'ADDRESS' }} + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ + -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON + + cmake --build build --config RelWithDebInfo -j $(nproc) + + - name: Build (no OpenMP) + id: cmake_build_no_openmp + if: ${{ matrix.sanitizer == 'THREAD' }} + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ + -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON \ + -DGGML_OPENMP=OFF + + cmake --build build --config RelWithDebInfo -j $(nproc) + + - name: Test + id: cmake_test + # skip run in Debug - very slow + if: ${{ matrix.sanitizer != 'UNDEFINED' }} + run: | + cd build + ctest -L main -E tokenizer --verbose --timeout 900 diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml new file mode 100644 index 0000000..441a897 --- /dev/null +++ b/.github/workflows/build-self-hosted.yml @@ -0,0 +1,389 @@ +name: CI (self-hosted) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-self-hosted.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cu', + '**/*.cuh', + '**/*.swift', + '**/*.m', + '**/*.metal', + '**/*.comp', + '**/*.glsl', + '**/*.wgsl' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-self-hosted.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cu', + '**/*.cuh', + '**/*.swift', + '**/*.m', + '**/*.metal', + '**/*.comp', + '**/*.glsl', + '**/*.wgsl' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + gpu-cuda: + runs-on: [self-hosted, Linux, NVIDIA] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + nvidia-smi + GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-vulkan-nvidia-cm: + runs-on: [self-hosted, Linux, NVIDIA] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-vulkan-nvidia-cm2: + runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-webgpu-nvidia: + runs-on: [self-hosted, Linux, NVIDIA, X64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dawn Dependency + id: dawn-depends + run: | + DAWN_VERSION="v20260317.182325" + DAWN_OWNER="google" + DAWN_REPO="dawn" + DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release" + echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + curl -L -o artifact.tar.gz \ + "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + mkdir dawn + tar -xvf artifact.tar.gz -C dawn --strip-components=1 + + - name: Test + id: ggml-ci + run: | + GG_BUILD_WEBGPU=1 \ + GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \ + GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + # TODO: provision AMX-compatible machine + #cpu-amx: + # runs-on: [self-hosted, Linux, CPU, AMX] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v6 + + # - name: Test + # id: ggml-ci + # run: | + # bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + # TODO: provision AMD GPU machine + # amd-vulkan: + # runs-on: [self-hosted, Linux, AMD] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v6 + + # - name: Test + # id: ggml-ci + # run: | + # vulkaninfo --summary + # GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + # TODO: provision AMD GPU machine + # amd-rocm: + # runs-on: [self-hosted, Linux, AMD] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v6 + + # - name: Test + # id: ggml-ci + # run: | + # amd-smi static + # GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-metal: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-webgpu-apple: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dawn Dependency + id: dawn-depends + run: | + DAWN_VERSION="v20260317.182325" + DAWN_OWNER="google" + DAWN_REPO="dawn" + DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release" + echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + curl -L -o artifact.tar.gz \ + "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + mkdir dawn + tar -xvf artifact.tar.gz -C dawn --strip-components=1 + + - name: Test + id: ggml-ci + run: | + GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-vulkan-apple: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-vulkan-intel-linux: + runs-on: [self-hosted, Linux, Intel] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-vulkan-intel-windows: + runs-on: [self-hosted, Windows, X64, Intel] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}" + env: + MSYSTEM: UCRT64 + CHERE_INVOKING: 1 + PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }} + run: | + vulkaninfo --summary + # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create + # a valid python environment for testing + LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp + + gpu-openvino-low-perf: + runs-on: [self-hosted, Linux, Intel, OpenVINO] + + env: + # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup OpenVINO Toolkit + uses: ./.github/actions/linux-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + - name: Install OpenVINO dependencies + run: | + cd ./openvino_toolkit + chmod +x ./install_dependencies/install_openvino_dependencies.sh + echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh + + - name: Test + id: ggml-ci + run: | + source ./openvino_toolkit/setupvars.sh + GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + cpu-x64-high-perf: + runs-on: [self-hosted, Linux, X64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + cpu-arm64-high-perf-graviton4: + runs-on: ah-ubuntu_22_04-c8g_8x + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + set -euxo pipefail + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \ + apt-get install -y \ + build-essential \ + python3-venv \ + gpg \ + wget \ + time \ + git-lfs + + git lfs install + + # install the latest cmake + sudo install -d /usr/share/keyrings + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \ + | gpg --dearmor \ + | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \ + | sudo tee /etc/apt/sources.list.d/kitware.list + sudo apt-get update + sudo apt-get install -y cmake + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + cpu-arm64-graviton4-kleidiai: + runs-on: ah-ubuntu_22_04-c8g_8x + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + set -euxo pipefail + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \ + apt-get install -y \ + build-essential \ + python3-venv \ + gpg \ + wget \ + time \ + git-lfs + + git lfs install + + # install the latest cmake + sudo install -d /usr/share/keyrings + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \ + | gpg --dearmor \ + | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \ + | sudo tee /etc/apt/sources.list.d/kitware.list + sudo apt-get update + sudo apt-get install -y cmake + + - name: Test + id: ggml-ci + run: | + GG_BUILD_KLEIDIAI=1 \ + GG_BUILD_EXTRA_TESTS_0=1 \ + bash ./ci/run.sh ./tmp/results ./tmp/mnt diff --git a/.github/workflows/build-sycl.yml b/.github/workflows/build-sycl.yml new file mode 100644 index 0000000..deb0e54 --- /dev/null +++ b/.github/workflows/build-sycl.yml @@ -0,0 +1,141 @@ +name: CI (sycl) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-sycl.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-sycl.yml', + 'ggml/src/ggml-sycl/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ubuntu-24-sycl: + strategy: + matrix: + build: [fp32, fp16] + include: + - build: fp32 + fp16: OFF + - build: fp16 + fp16: ON + + runs-on: ubuntu-24.04 + + env: + ONEAPI_ROOT: /opt/intel/oneapi/ + ONEAPI_INSTALLER_VERSION: "2025.3.3" + LEVEL_ZERO_VERSION: "1.28.2" + LEVEL_ZERO_UBUNTU_VERSION: "u24.04" + + continue-on-error: true + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Download & Install oneAPI + shell: bash + run: | + cd /tmp + wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh + sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept + + - name: Install Level Zero SDK + shell: bash + run: | + cd /tmp + wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb + wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb + sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: sycl-ubuntu-24-${{ matrix.build }} + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + run: | + source /opt/intel/oneapi/setvars.sh + cmake -B build \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DLLAMA_OPENSSL=OFF \ + -DGGML_NATIVE=OFF \ + -DGGML_SYCL_F16=${{ matrix.fp16 }} + time cmake --build build --config Release -j $(nproc) + + windows-latest-sycl: + runs-on: windows-2022 + + defaults: + run: + shell: bash + + env: + WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe + WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel + LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip + ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI" + ONEAPI_INSTALLER_VERSION: "2025.3.3" + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Download & Install oneAPI + shell: bash + run: | + scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL + + - name: Install Level Zero SDK + shell: pwsh + run: | + Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile "level-zero-win-sdk.zip" + Expand-Archive -Path "level-zero-win-sdk.zip" -DestinationPath "C:/level-zero-sdk" -Force + "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: sycl-windows-latest + variant: ccache + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + # TODO: add ssl support ; we will also need to modify win-build-sycl.bat to accept user-specified args + + - name: Build + id: cmake_build + run: examples/sycl/win-build-sycl.bat diff --git a/.github/workflows/build-virtgpu.yml b/.github/workflows/build-virtgpu.yml new file mode 100644 index 0000000..5b74059 --- /dev/null +++ b/.github/workflows/build-virtgpu.yml @@ -0,0 +1,50 @@ +name: CI (virtgpu) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-virtgpu.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-virtgpu.yml', + 'ggml/src/ggml-virtgpu/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + ubuntu-24-virtgpu: + runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install -y build-essential libdrm-dev pkg-config libssl-dev + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DGGML_VIRTGPU=ON \ + -DGGML_VIRTGPU_BACKEND=ON + cmake --build build --config Release -j $(nproc) diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml new file mode 100644 index 0000000..0111380 --- /dev/null +++ b/.github/workflows/build-vulkan.yml @@ -0,0 +1,135 @@ +name: CI (vulkan) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-vulkan.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.comp', + '**/*.glsl' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-vulkan.yml', + 'ggml/src/ggml-vulkan/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ubuntu-arm64: + runs-on: ubuntu-24.04-arm + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: vulkan-ubuntu-24.04-arm-new + variant: ccache + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Configure + id: cmake_configure + run: | + cmake -B build \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_VULKAN=ON + + - name: Build + id: cmake_build + run: | + time cmake --build build -j $(nproc) + + ubuntu-llvmpipe: + runs-on: ubuntu-24.04 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + sudo add-apt-repository -y ppa:kisak/kisak-mesa + sudo apt-get update -y + sudo apt-get install -y build-essential mesa-vulkan-drivers libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev + + - name: Get latest Vulkan SDK version + id: vulkan_sdk_version + run: | + echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV" + + - name: Use Vulkan SDK Cache + uses: actions/cache@v5 + id: cache-sdk + with: + path: ./vulkan_sdk + key: cache-gha-vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }} + + - name: Setup Vulkan SDK + if: steps.cache-sdk.outputs.cache-hit != 'true' + uses: ./.github/actions/linux-setup-vulkan + with: + path: ./vulkan_sdk + version: ${{ env.VULKAN_SDK_VERSION }} + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: vulkan-ubuntu-24.04-llvmpipe + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + run: | + source ./vulkan_sdk/setup-env.sh + cmake -B build \ + -DGGML_NATIVE=OFF \ + -DGGML_VULKAN=ON + cmake --build build --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + export GGML_VK_VISIBLE_DEVICES=0 + export GGML_VK_DISABLE_F16=1 + export GGML_VK_DISABLE_COOPMAT=1 + # This is using llvmpipe and runs slower than other backends + # test-backend-ops is too slow on llvmpipe, skip it + ctest -L main -E test-backend-ops --verbose --timeout 900 diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml new file mode 100644 index 0000000..aa7ae88 --- /dev/null +++ b/.github/workflows/build-wasm.yml @@ -0,0 +1,90 @@ +name: CI (wasm) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-wasm.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.wgsl', + '**/*.tmpl', + 'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-wasm.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.wgsl', + '**/*.tmpl', + 'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ubuntu-webgpu: + runs-on: ubuntu-24.04-arm + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: webgpu-ubuntu-24.04-arm-wasm + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Install Emscripten + run: | + git clone https://github.com/emscripten-core/emsdk.git + cd emsdk + ./emsdk install latest + ./emsdk activate latest + + - name: Fetch emdawnwebgpu + run: | + DAWN_TAG="v20260317.182325" + EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip" + echo "Downloading ${EMDAWN_PKG}" + curl -L -o emdawn.zip \ + "https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}" + unzip emdawn.zip + + - name: Build WASM WebGPU + run: | + source emsdk/emsdk_env.sh + emcmake cmake -B build-wasm \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_WEBGPU=ON \ + -DGGML_OPENMP=OFF \ + -DLLAMA_OPENSSL=OFF \ + -DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkg + + time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc) diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml new file mode 100644 index 0000000..ed73c18 --- /dev/null +++ b/.github/workflows/build-webgpu.yml @@ -0,0 +1,155 @@ +name: CI (webgpu) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/build-webgpu.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.wgsl', + '**/*.tmpl', + 'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/build-webgpu.yml', + 'ggml/src/ggml-webgpu/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + format: + runs-on: ubuntu-24.04 + + steps: + - name: Clone + uses: actions/checkout@v6 + + - name: Install clang-format 22 + run: | + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | + sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null + sudo add-apt-repository -y \ + "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main" + sudo apt-get update + sudo apt-get install -y clang-format-22 + + - name: Check formatting + run: | + find ggml/src/ggml-webgpu \ + -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.h' \) \ + -print0 | + xargs -0 clang-format-22 --dry-run --Werror + + macos: + runs-on: macos-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: webgpu-macos-latest + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Dawn Dependency + id: dawn-depends + run: | + DAWN_VERSION="v20260317.182325" + DAWN_OWNER="google" + DAWN_REPO="dawn" + DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release" + echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + curl -L -o artifact.tar.gz \ + "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + mkdir dawn + tar -xvf artifact.tar.gz -C dawn --strip-components=1 + + - name: Build + id: cmake_build + run: | + export CMAKE_PREFIX_PATH=dawn + cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF + time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + + ubuntu: + runs-on: ubuntu-24.04 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: webgpu-ubuntu-24.04 + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Dependencies + id: depends + run: | + sudo add-apt-repository -y ppa:kisak/kisak-mesa + sudo apt-get update -y + sudo apt-get install -y build-essential mesa-vulkan-drivers \ + libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev + + - name: Dawn Dependency + id: dawn-depends + run: | + sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev + DAWN_VERSION="v20260317.182325" + DAWN_OWNER="google" + DAWN_REPO="dawn" + DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release" + echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + curl -L -o artifact.tar.gz \ + "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + mkdir dawn + tar -xvf artifact.tar.gz -C dawn --strip-components=1 + + - name: Build + id: cmake_build + run: | + export Dawn_DIR=dawn/lib64/cmake/Dawn + cmake -B build \ + -DGGML_WEBGPU=ON + time cmake --build build --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + # This is using llvmpipe and runs slower than other backends + # test-backend-ops is too slow on llvmpipe, skip it + ctest -L main -E test-backend-ops --verbose --timeout 900 diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml new file mode 100644 index 0000000..015629f --- /dev/null +++ b/.github/workflows/check-vendor.yml @@ -0,0 +1,52 @@ +name: Check vendor + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + 'vendor/**', + 'scripts/sync_vendor.py' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + 'vendor/**', + 'scripts/sync_vendor.py' + ] + +jobs: + check-vendor: + runs-on: [self-hosted, fast] + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.x' + + - name: Run vendor sync + run: | + set -euo pipefail + python3 scripts/sync_vendor.py + + - name: Check for changes + run: | + set -euo pipefail + # detect modified or untracked files + changed=$(git status --porcelain --untracked-files=all || true) + if [ -n "$changed" ]; then + echo "Vendor sync modified files:" + echo "$changed" | awk '{ print $2 }' | sed '/^$/d' + echo "Failing because vendor files mismatch. Please update scripts/sync_vendor.py" + exit 1 + else + echo "Vendor files are up-to-date." + fi diff --git a/.github/workflows/close-issue.yml b/.github/workflows/close-issue.yml new file mode 100644 index 0000000..4698cee --- /dev/null +++ b/.github/workflows/close-issue.yml @@ -0,0 +1,28 @@ +name: Close inactive issues +on: + schedule: + - cron: "42 0 * * *" + +# Fine-grant permission +# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token +permissions: + issues: write + +jobs: + close-issues: + runs-on: ubuntu-slim + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v10 + with: + exempt-issue-labels: "refactoring,help wanted,good first issue,research 🔬,bug,roadmap,security" + days-before-issue-stale: 30 + days-before-issue-close: 14 + stale-issue-label: "stale" + close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." + days-before-pr-stale: -1 + days-before-pr-close: -1 + operations-per-run: 10000 + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml new file mode 100644 index 0000000..50b598b --- /dev/null +++ b/.github/workflows/code-style.yml @@ -0,0 +1,51 @@ +name: Code Style Checker + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + pull_request: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + model-naming: + runs-on: [self-hosted, fast] + steps: + - uses: actions/checkout@v6 + - name: Check model naming conventions + run: | + python3 - << 'EOF' + import re, os, sys + + pairs = re.findall( + r'case\s+(LLM_ARCH_\w+)\s*:\s*\n\s+return new (llama_model_\w+)\s*\(', + open("src/llama-model.cpp").read()) + + errors = [] + for arch, cls in pairs: + suffix = arch[len("LLM_ARCH_"):] + csuffix = cls[len("llama_model_"):] + fname = csuffix.replace("_", "-") + ".cpp" + + if not re.fullmatch(r'[A-Z][A-Z0-9_]*', suffix): + errors.append(f"{arch}: suffix not upper snake case, example: LLM_ARCH_MY_MODEL") + + if not re.fullmatch(r'[a-z][a-z0-9_]*', csuffix): + errors.append(f"{arch}: class suffix not lower snake case, example: llama_model_my_model") + + elif suffix.lower() != csuffix: + errors.append(f"{arch}: arch/class name mismatch, expected class 'llama_model_{suffix.lower()}' but got '{cls}'") + + elif not os.path.isfile(f"src/models/{fname}"): + errors.append(f"{arch}: expects model file name to be src/models/{fname}, but not found") + + if errors: + print('\n'.join(f" - {e}" for e in errors)); sys.exit(1) + print(f"OK: {len(pairs)} mappings validated.") + EOF diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 0000000..6f648ba --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,56 @@ +name: "Copilot Setup Steps" + +# Automatically run the setup steps when they are changed to allow for easy validation, and +# allow manual testing through the repository's "Actions" tab +on: + workflow_dispatch: + push: + paths: + - .github/workflows/copilot-setup-steps.yml + pull_request: + paths: + - .github/workflows/copilot-setup-steps.yml + +jobs: + # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot. + copilot-setup-steps: + runs-on: ubuntu-latest + + # Set the permissions to the lowest permissions possible needed for your steps. + # Copilot will be given its own token for its operations. + permissions: + # If you want to clone the repository as part of your setup steps, for example to install dependencies, you'll need the `contents: read` permission. If you don't clone the repository in your setup steps, Copilot will do this for you automatically after the steps complete. + contents: read + + # You can define any steps you want, and they will run before the agent starts. + # If you do not check out your code, Copilot will do this for you. + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: copilot-setup-steps + evict-old-files: 1d + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libssl-dev + # Install git-clang-format script for formatting only changed code + wget -O /tmp/git-clang-format https://raw.githubusercontent.com/llvm/llvm-project/release/18.x/clang/tools/clang-format/git-clang-format + sudo cp /tmp/git-clang-format /usr/local/bin/git-clang-format + sudo chmod +x /usr/local/bin/git-clang-format + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + python3 -m venv .venv + source .venv/bin/activate + pip install -r requirements/requirements-all.txt -r tools/server/tests/requirements.txt diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..afe4b7c --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,529 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, but the action may change without warning. + +name: Publish Docker image + +on: + workflow_dispatch: # allows manual triggering + inputs: + skip_s390x: + description: "Skip the s390x build target (useful for fast test runs that do not need the IBM Z runner)" + type: boolean + default: false + schedule: + # Rebuild daily rather than on every push because it is expensive + - cron: '12 4 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +# Fine-grant permission +# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token +permissions: + packages: write + +jobs: + create_tag: + name: Create and push git tag + runs-on: ubuntu-slim + permissions: + contents: write + outputs: + source_tag: ${{ steps.srctag.outputs.name }} + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Determine source tag name + id: srctag + uses: ./.github/actions/get-tag-name + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + + - name: Create and push git tag + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git tag ${{ steps.srctag.outputs.name }} || exit 0 + git push origin ${{ steps.srctag.outputs.name }} || exit 0 + + build_ui: + name: Build UI + needs: create_tag + uses: ./.github/workflows/ui-build.yml + with: + hf_ui_version: ${{ needs.create_tag.outputs.source_tag }} + + prepare_matrices: + name: Prepare Docker matrices + runs-on: ubuntu-24.04 + outputs: + build_matrix: ${{ steps.matrices.outputs.build_matrix }} + merge_matrix: ${{ steps.matrices.outputs.merge_matrix }} + + steps: + - name: Generate build and merge matrices + id: matrices + shell: bash + env: + SKIP_S390X: ${{ inputs.skip_s390x || 'false' }} + run: | + set -euo pipefail + + # Keep all build targets in one place and derive merge targets from it. + cat > build-matrix.json <<'JSON' + [ + { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, + { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" }, + { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true }, + { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, + { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, + { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, + { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, + { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, + { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" }, + { "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, + { "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" } + ] + JSON + + if [ "${SKIP_S390X}" = "true" ]; then + jq 'map(select(.platforms != "linux/s390x"))' build-matrix.json > build-matrix.json.tmp + mv build-matrix.json.tmp build-matrix.json + fi + + BUILD_MATRIX="$(jq -c . build-matrix.json)" + MERGE_MATRIX="$(jq -c ' + reduce .[] as $entry ({}; .[$entry.tag] |= ( + . // { + tag: $entry.tag, + arches: [], + full: false, + light: false, + server: false + } + | .full = (.full or ($entry.full // false)) + | .light = (.light or ($entry.light // false)) + | .server = (.server or ($entry.server // false)) + | .arches += [($entry.platforms | sub("^linux/"; ""))] + )) + # Backward compatibility: s390x tags are aliases of cpu for the linux/s390x platform. + | if (has("cpu") and (((.cpu.arches // []) | index("s390x")) != null)) then + . + { + s390x: { + tag: "s390x", + arches: ["s390x"], + full: .cpu.full, + light: .cpu.light, + server: .cpu.server + } + } + else + . + end + | [.[] | .arches = (.arches | unique | sort | join(" "))] + ' build-matrix.json)" + + echo "build_matrix=$BUILD_MATRIX" >> "$GITHUB_OUTPUT" + echo "merge_matrix=$MERGE_MATRIX" >> "$GITHUB_OUTPUT" + + push_to_registry: + name: Push Docker image to Docker Registry + needs: [prepare_matrices, create_tag, build_ui] + + runs-on: ${{ matrix.config.runs_on }} + strategy: + fail-fast: false + matrix: + config: ${{ fromJSON(needs.prepare_matrices.outputs.build_matrix) }} + steps: + - name: Check out the repo + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ needs.create_tag.outputs.source_tag }} + + - name: Download prebuilt UI + if: ${{ matrix.config.prebuilt_ui == true }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ui-build + path: tools/ui/dist + + - name: Set up QEMU + if: ${{ contains(matrix.config.platforms, 'linux/amd64') }} + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 + with: + image: tonistiigi/binfmt:qemu-v10.2.1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + + - name: Log in to Docker Registry + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Determine image metadata + id: meta + shell: bash + run: | + set -euo pipefail + + REPO_OWNER="${GITHUB_REPOSITORY_OWNER@L}" # to lower case + REPO_NAME="${{ github.event.repository.name }}" + IMAGE_REPO="ghcr.io/${REPO_OWNER}/${REPO_NAME}" + PREFIX="${IMAGE_REPO}:" + PLATFORM="${{ matrix.config.platforms }}" + ARCH_SUFFIX="${PLATFORM#linux/}" + + # list all tags possible + tags="${{ matrix.config.tag }}" + for tag in $tags; do + if [[ "$tag" == "cpu" ]]; then + TYPE="" + else + TYPE="-$tag" + fi + CACHETAG="${PREFIX}buildcache${TYPE}-${ARCH_SUFFIX}" + done + + SAFE_TAGS="$(echo "$tags" | tr ' ' '_')" + + echo "image_repo=$IMAGE_REPO" >> $GITHUB_OUTPUT + echo "arch_suffix=$ARCH_SUFFIX" >> $GITHUB_OUTPUT + echo "cache_output_tag=$CACHETAG" >> $GITHUB_OUTPUT + echo "digest_artifact_suffix=${SAFE_TAGS}-${ARCH_SUFFIX}" >> $GITHUB_OUTPUT + echo "cache_output_tag=$CACHETAG" # print out for debugging + env: + GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}' + + - name: Get build date + id: build_date + run: echo "date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_OUTPUT + + - name: Free Disk Space (Ubuntu) + if: ${{ matrix.config.free_disk_space == true }} + uses: ggml-org/free-disk-space@v1.3.1 + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Build and push Full Docker image by digest + id: build_full + if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.full == true }} + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 + with: + context: . + platforms: ${{ matrix.config.platforms }} + outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true + file: ${{ matrix.config.dockerfile }} + target: full + provenance: false + build-args: | + BUILD_DATE=${{ steps.build_date.outputs.date }} + APP_VERSION=${{ needs.create_tag.outputs.source_tag }} + APP_REVISION=${{ steps.checkout.outputs.commit }} + IMAGE_URL=${{ github.server_url }}/${{ github.repository }} + IMAGE_SOURCE=${{ github.server_url }}/${{ github.repository }} + ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} + ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} + annotations: | + manifest:org.opencontainers.image.created=${{ steps.build_date.outputs.date }} + manifest:org.opencontainers.image.version=${{ needs.create_tag.outputs.source_tag }} + manifest:org.opencontainers.image.revision=${{ steps.checkout.outputs.commit }} + manifest:org.opencontainers.image.title=llama.cpp + manifest:org.opencontainers.image.description=LLM inference in C/C++ + manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }} + manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + # using github experimental cache + #cache-from: type=gha + #cache-to: type=gha,mode=max + # return to this if the experimental github cache is having issues + #cache-to: type=local,dest=/tmp/.buildx-cache + #cache-from: type=local,src=/tmp/.buildx-cache + # using registry cache (no storage limit) + cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} + cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max + + - name: Build and push Light Docker image by digest + id: build_light + if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.light == true }} + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 + with: + context: . + platforms: ${{ matrix.config.platforms }} + outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true + file: ${{ matrix.config.dockerfile }} + target: light + provenance: false + build-args: | + BUILD_DATE=${{ steps.build_date.outputs.date }} + APP_VERSION=${{ needs.create_tag.outputs.source_tag }} + APP_REVISION=${{ steps.checkout.outputs.commit }} + IMAGE_URL=${{ github.server_url }}/${{ github.repository }} + IMAGE_SOURCE=${{ github.server_url }}/${{ github.repository }} + ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} + ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} + annotations: | + manifest:org.opencontainers.image.created=${{ steps.build_date.outputs.date }} + manifest:org.opencontainers.image.version=${{ needs.create_tag.outputs.source_tag }} + manifest:org.opencontainers.image.revision=${{ steps.checkout.outputs.commit }} + manifest:org.opencontainers.image.title=llama.cpp + manifest:org.opencontainers.image.description=LLM inference in C/C++ + manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }} + manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + # using github experimental cache + #cache-from: type=gha + #cache-to: type=gha,mode=max + # return to this if the experimental github cache is having issues + #cache-to: type=local,dest=/tmp/.buildx-cache + #cache-from: type=local,src=/tmp/.buildx-cache + # using registry cache (no storage limit) + cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} + cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max + + - name: Build and push Server Docker image by digest + id: build_server + if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.server == true }} + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 + with: + context: . + platforms: ${{ matrix.config.platforms }} + outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true + file: ${{ matrix.config.dockerfile }} + target: server + provenance: false + build-args: | + BUILD_DATE=${{ steps.build_date.outputs.date }} + APP_VERSION=${{ needs.create_tag.outputs.source_tag }} + APP_REVISION=${{ steps.checkout.outputs.commit }} + IMAGE_URL=${{ github.server_url }}/${{ github.repository }} + IMAGE_SOURCE=${{ github.server_url }}/${{ github.repository }} + ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} + ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} + annotations: | + manifest:org.opencontainers.image.created=${{ steps.build_date.outputs.date }} + manifest:org.opencontainers.image.version=${{ needs.create_tag.outputs.source_tag }} + manifest:org.opencontainers.image.revision=${{ steps.checkout.outputs.commit }} + manifest:org.opencontainers.image.title=llama.cpp + manifest:org.opencontainers.image.description=LLM inference in C/C++ + manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }} + manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + # using github experimental cache + #cache-from: type=gha + #cache-to: type=gha,mode=max + # return to this if the experimental github cache is having issues + #cache-to: type=local,dest=/tmp/.buildx-cache + #cache-from: type=local,src=/tmp/.buildx-cache + # using registry cache (no storage limit) + cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} + cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max + + - name: Export digest metadata + shell: bash + run: | + set -euo pipefail + + TAGS="${{ matrix.config.tag }}" + ARCH_SUFFIX="${{ steps.meta.outputs.arch_suffix }}" + DIGEST_FILE="/tmp/digests/${{ steps.meta.outputs.digest_artifact_suffix }}.tsv" + mkdir -p /tmp/digests + + add_digest_rows() { + local image_type="$1" + local digest="$2" + + if [[ -z "$digest" ]]; then + echo "Missing digest for image_type=${image_type}" >&2 + exit 1 + fi + + for tag in $TAGS; do + printf '%s\t%s\t%s\t%s\n' "$tag" "$ARCH_SUFFIX" "$image_type" "$digest" >> "$DIGEST_FILE" + done + } + + if [[ "${{ matrix.config.full }}" == "true" ]]; then + add_digest_rows "full" "${{ steps.build_full.outputs.digest }}" + fi + + if [[ "${{ matrix.config.light }}" == "true" ]]; then + add_digest_rows "light" "${{ steps.build_light.outputs.digest }}" + fi + + if [[ "${{ matrix.config.server }}" == "true" ]]; then + add_digest_rows "server" "${{ steps.build_server.outputs.digest }}" + fi + + - name: Upload digest metadata + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: digests-${{ steps.meta.outputs.digest_artifact_suffix }} + path: /tmp/digests/${{ steps.meta.outputs.digest_artifact_suffix }}.tsv + if-no-files-found: error + + merge_arch_tags: + name: Create shared tags from digests + needs: [prepare_matrices, push_to_registry, create_tag] + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + config: ${{ fromJSON(needs.prepare_matrices.outputs.merge_matrix) }} + + steps: + - name: Check out the repo + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Get build date + id: build_date + run: echo "date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_OUTPUT + + - name: Download digest metadata + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: digests-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + + - name: Log in to Docker Registry + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create tags from digests + shell: bash + run: | + set -euo pipefail + + REPO_OWNER="${GITHUB_REPOSITORY_OWNER@L}" # to lower case + REPO_NAME="${{ github.event.repository.name }}" + IMAGE_REPO="ghcr.io/${REPO_OWNER}/${REPO_NAME}" + PREFIX="${IMAGE_REPO}:" + SRC_TAG="${{ needs.create_tag.outputs.source_tag }}" + BUILD_DATE="${{ steps.build_date.outputs.date }}" + COMMIT_SHA="${{ steps.checkout.outputs.commit }}" + TAGS="${{ matrix.config.tag }}" + ARCHES="${{ matrix.config.arches }}" + DIGEST_GLOB="/tmp/digests/*.tsv" + + if ! ls ${DIGEST_GLOB} >/dev/null 2>&1; then + echo "No digest metadata found in /tmp/digests" >&2 + exit 1 + fi + + if [[ -z "$SRC_TAG" ]]; then + echo "Missing source tag from create_tag" >&2 + exit 1 + fi + + find_digest() { + local tag_name="$1" + local arch="$2" + local image_type="$3" + local digest + + digest="$(awk -F '\t' -v t="$tag_name" -v a="$arch" -v i="$image_type" '$1 == t && $2 == a && $3 == i { print $4; exit }' ${DIGEST_GLOB})" + + # Backward compatibility: s390x tags are aliases of cpu for the linux/s390x platform. + if [[ -z "$digest" && "$tag_name" == "s390x" && "$arch" == "s390x" ]]; then + digest="$(awk -F '\t' -v t="cpu" -v a="$arch" -v i="$image_type" '$1 == t && $2 == a && $3 == i { print $4; exit }' ${DIGEST_GLOB})" + fi + + if [[ -z "$digest" ]]; then + echo "Missing digest for tag=${tag_name} arch=${arch} image_type=${image_type}" >&2 + exit 1 + fi + + echo "$digest" + } + + create_manifest_tags() { + local image_type="$1" + local tag_name="$2" + local suffix="$3" + + local merged_tag="${PREFIX}${image_type}${suffix}" + local merged_versioned_tag="${merged_tag}-${SRC_TAG}" + + local refs=() + + for arch in $ARCHES; do + local digest + digest="$(find_digest "$tag_name" "$arch" "$image_type")" + refs+=("${IMAGE_REPO}@${digest}") + done + + local annotations=( + --annotation "index:org.opencontainers.image.created=${BUILD_DATE}" + --annotation "index:org.opencontainers.image.version=${SRC_TAG}" + --annotation "index:org.opencontainers.image.revision=${COMMIT_SHA}" + --annotation "index:org.opencontainers.image.title=llama.cpp" + --annotation "index:org.opencontainers.image.description=LLM inference in C/C++" + --annotation "index:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}" + --annotation "index:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}" + ) + + echo "Creating ${merged_tag} from ${refs[*]}" + docker buildx imagetools create "${annotations[@]}" --tag "${merged_tag}" "${refs[@]}" + + echo "Creating ${merged_versioned_tag} from ${refs[*]}" + docker buildx imagetools create "${annotations[@]}" --tag "${merged_versioned_tag}" "${refs[@]}" + } + + for tag in $TAGS; do + if [[ "$tag" == "cpu" ]]; then + TYPE="" + else + TYPE="-$tag" + fi + + if [[ "${{ matrix.config.full }}" == "true" ]]; then + create_manifest_tags "full" "$tag" "$TYPE" + fi + + if [[ "${{ matrix.config.light }}" == "true" ]]; then + create_manifest_tags "light" "$tag" "$TYPE" + fi + + if [[ "${{ matrix.config.server }}" == "true" ]]; then + create_manifest_tags "server" "$tag" "$TYPE" + fi + done + env: + GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}' diff --git a/.github/workflows/editorconfig.yml b/.github/workflows/editorconfig.yml new file mode 100644 index 0000000..59159cd --- /dev/null +++ b/.github/workflows/editorconfig.yml @@ -0,0 +1,24 @@ +name: EditorConfig Checker + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + pull_request: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + editorconfig: + runs-on: [self-hosted, fast] + steps: + - uses: actions/checkout@v6 + - uses: editorconfig-checker/action-editorconfig-checker@840e866d93b8e032123c23bac69dece044d4d84c # v2.2.0 + with: + version: v3.0.3 + - run: editorconfig-checker diff --git a/.github/workflows/gguf-publish.yml b/.github/workflows/gguf-publish.yml new file mode 100644 index 0000000..fb8eab3 --- /dev/null +++ b/.github/workflows/gguf-publish.yml @@ -0,0 +1,44 @@ +# This workflow will upload a Python Package using Twine when a GGUF release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +# See `gguf-py/README.md` for how to make a release. + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + workflow_dispatch: + push: + # Pattern matched against refs/tags + tags: + - 'gguf-v*' # Push events to every version tag + + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + pip-install: poetry==2.4.0 + - name: Install dependencies + run: | + cd gguf-py + poetry install + + - name: Build package + run: cd gguf-py && poetry build + - name: Publish package + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + packages-dir: gguf-py/dist diff --git a/.github/workflows/hip-quality-check.yml b/.github/workflows/hip-quality-check.yml new file mode 100644 index 0000000..5d23f01 --- /dev/null +++ b/.github/workflows/hip-quality-check.yml @@ -0,0 +1,86 @@ +name: HIP quality check + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/hip-quality-check.yml', + '**/*.cu', + '**/*.cuh', + 'ggml/src/ggml-hip/CMakeLists.txt', + 'ggml/src/ggml-cuda/vendors/hip.h', + 'scripts/hip/gcn-cdna-vgpr-check.py' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/hip-quality-check.yml', + '**/*.cu', + '**/*.cuh', + 'ggml/src/ggml-hip/CMakeLists.txt', + 'ggml/src/ggml-cuda/vendors/hip.h', + 'scripts/hip/gcn-cdna-vgpr-check.py' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + ubuntu-22-hip-quality-check: + runs-on: ubuntu-22.04 + container: rocm/dev-ubuntu-22.04:7.2.1 + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: hip-quality-check-ubuntu-22.04 + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build with Werror + id: cmake_build + run: | + cmake -B build -S . \ + -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ + -DGPU_TARGETS=gfx942 \ + -DGGML_HIP=ON \ + -DGGML_HIP_EXPORT_METRICS=Off \ + -DCMAKE_HIP_FLAGS="-Werror -Wno-tautological-compare" \ + -DCMAKE_BUILD_TYPE=Release + cd build + make -j $(nproc) + + - name: Check for major VGPR spills + id: vgpr_check + run: | + cmake -B build -S . \ + -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ + -DGPU_TARGETS=gfx908 \ + -DGGML_HIP=ON \ + -DGGML_HIP_EXPORT_METRICS=On \ + -DCMAKE_HIP_FLAGS="" \ + -DCMAKE_BUILD_TYPE=Release + cd build + make -j $(nproc) 2>&1 | tee metrics.log | grep -v 'Rpass-analysis=kernel-resource-usage\|remark:\|^$' + python3 ../scripts/hip/gcn-cdna-vgpr-check.py metrics.log diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000..eab20c6 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,17 @@ +name: "Pull Request Labeler" +on: +- pull_request_target + +jobs: + labeler: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-slim + steps: + - uses: actions/checkout@v6 + with: + repository: "ggml-org/llama.cpp" + - uses: actions/labeler@v6 + with: + configuration-path: '.github/labeler.yml' diff --git a/.github/workflows/pre-tokenizer-hashes.yml b/.github/workflows/pre-tokenizer-hashes.yml new file mode 100644 index 0000000..3e440b6 --- /dev/null +++ b/.github/workflows/pre-tokenizer-hashes.yml @@ -0,0 +1,45 @@ +name: Check Pre-Tokenizer Hashes + +on: + push: + paths: + - 'conversion/base.py' + - 'convert_hf_to_gguf_update.py' + pull_request: + paths: + - 'conversion/base.py' + - 'convert_hf_to_gguf_update.py' + +jobs: + pre-tokenizer-hashes: + runs-on: [self-hosted, fast] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + python3 -m venv .venv + .venv/bin/pip install -r requirements/requirements-convert_hf_to_gguf_update.txt + + - name: Update pre-tokenizer hashes + run: | + cp conversion/base.py /tmp + .venv/bin/python convert_hf_to_gguf_update.py --check-missing + + - name: Check if committed pre-tokenizer hashes matches generated version + run: | + if ! diff -q conversion/base.py /tmp/base.py; then + echo "Model pre-tokenizer hashes (in conversion/base.py) do not match generated hashes (from convert_hf_to_gguf_update.py)." + echo "To fix: run ./convert_hf_to_gguf_update.py and commit the updated conversion/base.py along with your changes" + echo "Differences found:" + diff conversion/base.py /tmp/base.py || true + exit 1 + fi + echo "Model pre-tokenizer hashes are up to date." diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml new file mode 100644 index 0000000..2c7fab4 --- /dev/null +++ b/.github/workflows/python-check-requirements.yml @@ -0,0 +1,33 @@ +name: Python check requirements.txt + +on: + push: + paths: + - '.github/workflows/python-check-requirements.yml' + - 'scripts/check-requirements.sh' + - 'convert*.py' + - '**/requirements*.txt' + pull_request: + paths: + - '.github/workflows/python-check-requirements.yml' + - 'scripts/check-requirements.sh' + - 'convert*.py' + - '**/requirements*.txt' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + python-check-requirements: + runs-on: [self-hosted, CPU, fast] + name: check-requirements + steps: + - name: Check out source repository + uses: actions/checkout@v6 + - name: Set up Python environment + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Run check-requirements.sh script + run: bash scripts/check-requirements.sh diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml new file mode 100644 index 0000000..0424f37 --- /dev/null +++ b/.github/workflows/python-lint.yml @@ -0,0 +1,36 @@ +name: flake8 Lint + +on: + push: + branches: + - master + paths: [ + '.github/workflows/python-lint.yml', + '**/*.py' + ] + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/python-lint.yml', + '**/*.py' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + flake8-lint: + runs-on: [self-hosted, fast] + name: Lint + steps: + - name: Check out source repository + uses: actions/checkout@v6 + - name: Set up Python environment + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: flake8 Lint + uses: py-actions/flake8@84ec6726560b6d5bd68f2a5bed83d62b52bb50ba # v2 + with: + plugins: "flake8-no-print" diff --git a/.github/workflows/python-type-check.yml b/.github/workflows/python-type-check.yml new file mode 100644 index 0000000..14edb1a --- /dev/null +++ b/.github/workflows/python-type-check.yml @@ -0,0 +1,43 @@ +name: Python Type-Check + +on: + push: + paths: + - '.github/workflows/python-type-check.yml' + - 'ty.toml' + - '**.py' + - '**/requirements*.txt' + # - 'pyrightconfig.json' + pull_request: + paths: + - '.github/workflows/python-type-check.yml' + - 'ty.toml' + - '**.py' + - '**/requirements*.txt' + # - 'pyrightconfig.json' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + python-type-check: + runs-on: [self-hosted, fast] + name: python type-check + steps: + - name: Check out source repository + uses: actions/checkout@v6 + - name: Set up Python environment + uses: actions/setup-python@v6 + with: + python-version: "3.11" + pip-install: -r requirements/requirements-all.txt ty==0.0.35 + # - name: Type-check with Pyright + # uses: jakebailey/pyright-action@v2 + # with: + # version: 1.1.382 + # level: warning + # warnings: true + - name: Type-check with ty + run: | + ty check --output-format=github diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml new file mode 100644 index 0000000..14551c7 --- /dev/null +++ b/.github/workflows/python-wheels.yml @@ -0,0 +1,60 @@ +name: Python wheels + +on: + workflow_dispatch: + inputs: + model_test: + description: Run real W4 speech synthesis too (requires HF_TOKEN for private models) + type: boolean + default: false + +permissions: + contents: read + +jobs: + wheels: + name: ${{ matrix.platform }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x86_64 + runner: ubuntu-24.04 + arch: x86_64 + - platform: linux-arm64 + runner: ubuntu-24.04-arm + arch: aarch64 + - platform: macos-arm64 + runner: macos-14 + arch: arm64 + - platform: windows-x86_64 + runner: windows-2022 + arch: AMD64 + - platform: windows-arm64-experimental + runner: windows-11-arm + arch: ARM64 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.KITTEN_SUBMODULE_TOKEN || github.token }} + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Install wheel builder + run: python -m pip install cibuildwheel==4.3.0 + - name: Build and test wheels + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.arch }} + KITTEN_WHEEL_NETWORK_TEST: '1' + KITTEN_WHEEL_MODEL_TEST: ${{ inputs.model_test && '1' || '0' }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + CIBW_ENVIRONMENT_PASS_LINUX: HF_TOKEN KITTEN_WHEEL_MODEL_TEST KITTEN_WHEEL_NETWORK_TEST + - uses: actions/upload-artifact@v4 + with: + name: kitten-tts-cpp-${{ matrix.platform }} + path: wheelhouse/*.whl + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f587ed9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,1731 @@ +name: Release + +on: + workflow_dispatch: # allows manual triggering + inputs: + create_release: + description: 'Create new release' + required: true + type: boolean + push: + branches: + - master + paths: [ + '.github/workflows/release.yml', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cu', + '**/*.cuh', + '**/*.swift', + '**/*.m', + '**/*.metal', + '**/*.comp', + '**/*.glsl' + ] + +env: + GH_TOKEN: ${{ github.token }} + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON" + +# note: run this workflow one at a time for better cache reuse +concurrency: + group: release + queue: max + +jobs: + check-release: + runs-on: ubuntu-slim + + outputs: + should_release: ${{ steps.check.outputs.should_release }} + + steps: + - id: check + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "should_release=true" >> $GITHUB_OUTPUT + elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/master" ]]; then + if echo "$COMMIT_MESSAGE" | grep -q '\[no release\]'; then + echo "should_release=false" >> $GITHUB_OUTPUT + else + echo "should_release=true" >> $GITHUB_OUTPUT + fi + else + echo "should_release=false" >> $GITHUB_OUTPUT + fi + + get-version: + runs-on: ubuntu-slim + outputs: + ui_version: ${{ steps.version.outputs.ui_version }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - id: version + run: | + # Resolve UI version: BUILD_NUMBER from cmake/build-info.cmake > git hash + epoch > fallback + version="" + if grep -q "BUILD_NUMBER" cmake/build-info.cmake; then + build_number=$(grep "set(BUILD_NUMBER" cmake/build-info.cmake | grep -oP '\d+') + if [ -n "$build_number" ] && [ "$build_number" -gt 0 ]; then + version="b${build_number}" + fi + fi + if [ -z "$version" ]; then + version=$(git rev-parse --short HEAD)-$(date +%s) + fi + echo "ui_version=${version}" >> $GITHUB_OUTPUT + + macos-cpu: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + strategy: + matrix: + include: + - build: 'arm64' + arch: 'arm64' + os: macos-26 + defines: "-DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3" + # TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23780) + # in order to enable it again, we have to provision dedicated runners to run it + #- build: 'arm64-kleidiai' + # arch: 'arm64' + # os: macos-14 + # defines: "-DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 -DGGML_CPU_KLEIDIAI=ON" + - build: 'x64' + arch: 'x64' + os: macos-15-intel + # Metal is disabled on x64 due to intermittent failures with Github runners not having a GPU: + # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313 + defines: "-DGGML_METAL=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3" + + runs-on: ${{ matrix.os }} + + permissions: + actions: write + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-${{ matrix.os }}-${{ matrix.arch }} + + - name: Build + id: cmake_build + run: | + sysctl -a + cmake -B build \ + ${{ matrix.defines }} \ + -DCMAKE_INSTALL_RPATH='@loader_path' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-${{ matrix.os }}-${{ matrix.arch }} + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz -s ",^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz + name: llama-bin-macos-${{ matrix.build }}.tar.gz + + ubuntu-cpu: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + strategy: + matrix: + include: + - build: 'x64' + os: ubuntu-22.04 + - build: 'arm64' + os: ubuntu-24.04-arm + - build: 's390x' + os: ubuntu-24.04-s390x + + runs-on: ${{ matrix.os }} + + permissions: + actions: write + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libssl-dev + + - name: Toolchain workaround (GCC 14) + if: ${{ contains(matrix.os, 'ubuntu-24.04') }} + run: | + sudo apt-get install -y gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + + - name: ccache + if: ${{ matrix.build != 's390x' }} + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-${{ matrix.os }}-cpu + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + + - name: ccache-clear + if: ${{ matrix.build != 's390x' }} + uses: ./.github/actions/ccache-clear + with: + key: release-${{ matrix.os }}-cpu + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-${{ matrix.build }}.tar.gz + + ubuntu-vulkan: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + strategy: + matrix: + include: + - build: 'x64' + os: ubuntu-22.04 + - build: 'arm64' + os: ubuntu-24.04-arm + + runs-on: ${{ matrix.os }} + + permissions: + actions: write + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Dependencies + id: depends + run: | + if [[ "${{ matrix.os }}" =~ "ubuntu-22.04" ]]; then + wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add - + sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list + sudo apt-get update -y + sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev + else + sudo apt-get update -y + sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + fi + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-${{ matrix.os }}-vulkan + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_VULKAN=ON \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-${{ matrix.os }}-vulkan + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz + + android-arm64: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: ubuntu-latest + + #permissions: + # actions: write + + env: + NDK_VERSION: "29.0.14206865" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Set up JDK + uses: actions/setup-java@v5 + with: + java-version: 17 + distribution: temurin + + - name: Setup Android SDK + uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 + with: + log-accepted-android-sdk-licenses: false + + - name: Install NDK + run: | + sdkmanager "ndk;${{ env.NDK_VERSION }}" + echo "ANDROID_NDK=${ANDROID_SDK_ROOT}/ndk/${{ env.NDK_VERSION }}" >> $GITHUB_ENV + + # note : disabled to spare some cache space (https://github.com/ggml-org/llama.cpp/pull/23789) + # for some reason, the ccache does not improve the build time in this case + # example: + # cache off: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78160400831 + # cache on: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394 + # + #- name: ccache + # uses: ggml-org/ccache-action@v1.2.21 + # with: + # key: release-android-arm64 + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-28 \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + + #- name: ccache-clear + # uses: ./.github/actions/ccache-clear + # with: + # key: release-android-arm64 + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz + name: llama-bin-android-arm64.tar.gz + + ubuntu-24-openvino: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: ubuntu-24.04 + + permissions: + actions: write + + outputs: + openvino_version: ${{ steps.openvino_version.outputs.value }} + + env: + # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" + + steps: + - name: Set OpenVINO version output + id: openvino_version + run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT + + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-ubuntu-24.04-openvino-release-no-preset-v1 + + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential libssl-dev libtbb12 cmake ninja-build python3-pip + sudo apt install ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd + + - name: Use OpenVINO Toolkit Cache + uses: actions/cache@v5 + id: cache-openvino + with: + path: ./openvino_toolkit + key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }} + + - name: Setup OpenVINO Toolkit + if: steps.cache-openvino.outputs.cache-hit != 'true' + uses: ./.github/actions/linux-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + - name: Install OpenVINO dependencies + run: | + cd ./openvino_toolkit + chmod +x ./install_dependencies/install_openvino_dependencies.sh + echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh + + - name: Build + id: cmake_build + run: | + source ./openvino_toolkit/setupvars.sh + cmake -B build/ReleaseOV -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_OPENVINO=ON \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ + ${{ env.CMAKE_ARGS }} + cmake --build build/ReleaseOV --config Release --parallel + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-ubuntu-24.04-openvino-release-no-preset-v1 + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + dest=./build/ReleaseOV/bin + OPENVINO_ROOT=./openvino_toolkit + ov_lib="$OPENVINO_ROOT/runtime/lib/intel64" + + # Bundle OpenVINO runtime libs + TBB. Binaries built with RPATH=$ORIGIN + # load these siblings without setupvars.sh / LD_LIBRARY_PATH. + cp -P "$ov_lib"/libopenvino.so* \ + "$ov_lib"/libopenvino_c.so* \ + "$ov_lib"/libopenvino_*_plugin.so \ + "$ov_lib"/libopenvino_intel_npu_compiler*.so \ + "$OPENVINO_ROOT"/runtime/3rdparty/tbb/lib/*.so* \ + "$dest" + cp -P /usr/lib/x86_64-linux-gnu/libOpenCL.so.1* "$dest" 2>/dev/null || true + cp "$ov_lib"/cache.json "$dest" 2>/dev/null || true + + # OpenVINO licensing + cp -r "$OPENVINO_ROOT"/docs/licensing "$dest"/openvino-licensing + + cp LICENSE "$dest" + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C "$dest" . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz + name: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz + + windows-openvino: + needs: [check-release] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: windows-2022 + + outputs: + openvino_version: ${{ steps.openvino_version.outputs.value }} + + env: + # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" + + steps: + - name: Set OpenVINO version output + id: openvino_version + shell: bash + run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT + + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-windows-2022-openvino + variant: ccache + evict-old-files: 1d + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-openvino + with: + path: ./openvino_toolkit + key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }} + + - name: Setup OpenVINO Toolkit + if: steps.cache-openvino.outputs.cache-hit != 'true' + uses: ./.github/actions/windows-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + - name: Install OpenCL using vcpkg + shell: powershell + run: | + git clone https://github.com/microsoft/vcpkg C:\vcpkg + C:\vcpkg\bootstrap-vcpkg.bat + C:\vcpkg\vcpkg install opencl + + - name: Build + id: cmake_build + shell: cmd + run: | + REM Find extracted OpenVINO folder dynamically + for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i + + if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" ( + echo ERROR: OpenVINOConfig.cmake not found + exit /b 1 + ) + + call "%OPENVINO_ROOT%\setupvars.bat" + + cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^ + -A x64 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DGGML_OPENVINO=ON ^ + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^ + ${{ env.CMAKE_ARGS }} + + cmake --build build\ReleaseOV --config Release -- /m + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-windows-2022-openvino + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + shell: powershell + run: | + # Locate the extracted OpenVINO toolkit root (same pattern as the Build step). + $OPENVINO_ROOT = (Get-ChildItem -Directory openvino_toolkit | Select-Object -First 1).FullName + if (-not $OPENVINO_ROOT) { + Write-Error "OpenVINO toolkit folder not found under .\openvino_toolkit" + exit 1 + } + + $dest = ".\build\ReleaseOV\bin\Release" + + $ovBin = Join-Path $OPENVINO_ROOT 'runtime\bin\intel64\Release' + Copy-Item -Path (Join-Path $ovBin '*.dll') -Destination $dest -Force + Copy-Item -Path (Join-Path $ovBin 'cache.json') -Destination $dest -Force + + $tbbBin = Join-Path $OPENVINO_ROOT 'runtime\3rdparty\tbb\bin' + Copy-Item -Path (Join-Path $tbbBin 'tbb*.dll') -Destination $dest -Force + + # OpenVINO licensing + $licensingDest = Join-Path $dest 'openvino-licensing' + New-Item -ItemType Directory -Force -Path $licensingDest | Out-Null + Copy-Item -Path (Join-Path $OPENVINO_ROOT 'docs\licensing\*') -Destination $licensingDest -Recurse -Force + + Copy-Item LICENSE $dest + 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip $dest\* + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip + name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip + + windows-cpu: + needs: [check-release] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: windows-2025-vs2026 + + permissions: + actions: write + + strategy: + matrix: + include: + - arch: 'x64' + - arch: 'arm64' + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Install Ninja + run: | + choco install ninja + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu + + - name: Build + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }} + cmake -S . -B build -G "Ninja Multi-Config" ^ + -D CMAKE_TOOLCHAIN_FILE=cmake/${{ matrix.arch }}-windows-llvm.cmake ^ + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DGGML_NATIVE=OFF ^ + -DGGML_BACKEND_DL=ON ^ + -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^ + -DGGML_OPENMP=ON ^ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu + + - name: Pack artifacts + id: pack_artifacts + run: | + Copy-Item "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Redist\MSVC\14.51.36231\debug_nonredist\${{ matrix.arch }}\Microsoft.VC145.OpenMP.LLVM\libomp140.${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }}.dll" .\build\bin\Release\ + 7z a -snl llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\* + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-bin-win-cpu-${{ matrix.arch }}.zip + name: llama-bin-win-cpu-${{ matrix.arch }}.zip + + windows: + needs: [check-release] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: windows-2025 + + permissions: + actions: write + + env: + OPENBLAS_VERSION: 0.3.23 + VULKAN_VERSION: 1.4.357.0 + + strategy: + matrix: + include: + - backend: 'vulkan' + arch: 'x64' + defines: '-DGGML_VULKAN=ON' + target: 'ggml-vulkan' + - backend: 'opencl-adreno' + arch: 'arm64' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON' + target: 'ggml-opencl' + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Install Vulkan SDK + id: get_vulkan + if: ${{ matrix.backend == 'vulkan' }} + run: | + curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe" + & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install + Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}" + Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin" + + - name: Install Ninja + id: install_ninja + run: | + choco install ninja + + # TODO: these jobs need to use llvm toolchain in order to utilize the ccache + #- name: ccache + # uses: ggml-org/ccache-action@v1.2.21 + # with: + # key: release-windows-2025-${{ matrix.arch }}-${{ matrix.backend }} + + - name: Install OpenCL Headers and Libs + id: install_opencl + if: ${{ matrix.backend == 'opencl-adreno' && matrix.arch == 'arm64' }} + run: | + git clone https://github.com/KhronosGroup/OpenCL-Headers + cd OpenCL-Headers + cmake -B build ` + -DBUILD_TESTING=OFF ` + -DOPENCL_HEADERS_BUILD_TESTING=OFF ` + -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF ` + -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release" + cmake --build build --target install + git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader + cd OpenCL-ICD-Loader + cmake -B build-arm64-release ` + -A arm64 ` + -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" ` + -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release" + cmake --build build-arm64-release --target install --config release + + - name: Build + id: cmake_build + run: | + cmake -S . -B build ${{ matrix.defines }} -DGGML_NATIVE=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_BUILD_BORINGSSL=ON + cmake --build build --config Release --target ${{ matrix.target }} + + #- name: ccache-clear + # uses: ./.github/actions/ccache-clear + # with: + # key: release-windows-2025-${{ matrix.arch }}-${{ matrix.backend }} + + - name: Pack artifacts + id: pack_artifacts + run: | + 7z a -snl llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip .\build\bin\Release\${{ matrix.target }}.dll + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip + name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip + + windows-cuda: + needs: [check-release] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: windows-2022 + + permissions: + actions: write + + strategy: + matrix: + cuda: ['12.4', '13.3'] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Install Cuda Toolkit + uses: ./.github/actions/windows-setup-cuda + with: + cuda_version: ${{ matrix.cuda }} + + - name: Install Ninja + id: install_ninja + run: | + choco install ninja + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-windows-2022-x64-cuda-${{ matrix.cuda }} + + - name: Build + id: cmake_build + shell: cmd + # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + cmake -S . -B build -G "Ninja Multi-Config" ^ + -DGGML_BACKEND_DL=ON ^ + -DGGML_NATIVE=OFF ^ + -DGGML_CPU=OFF ^ + -DGGML_CUDA=ON ^ + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DGGML_CUDA_CUB_3DOT2=ON + set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1 + cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-windows-2022-x64-cuda-${{ matrix.cuda }} + + - name: Pack artifacts + id: pack_artifacts + run: | + 7z a -snl llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip .\build\bin\Release\ggml-cuda.dll + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + name: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + + - name: Copy and pack Cuda runtime + run: | + echo "Cuda install location: ${{ env.CUDA_PATH }}" + $dst='.\build\bin\cudart\' + robocopy "${{env.CUDA_PATH}}\bin" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll + robocopy "${{env.CUDA_PATH}}\lib" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll + robocopy "${{env.CUDA_PATH}}\bin\x64" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll + 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip $dst\* + + - name: Upload Cuda runtime + uses: actions/upload-artifact@v6 + with: + path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip + + windows-sycl: + needs: [check-release] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: windows-2022 + + defaults: + run: + shell: bash + + env: + WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe + WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel + LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip + ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI" + ONEAPI_INSTALLER_VERSION: "2025.3.3" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Download & Install oneAPI + shell: bash + run: | + scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL + + - name: Install Level Zero SDK + shell: pwsh + run: | + Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile "level-zero-win-sdk.zip" + Expand-Archive -Path "level-zero-win-sdk.zip" -DestinationPath "C:/level-zero-sdk" -Force + "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-windows-2022-x64-sycl + + - name: Build + id: cmake_build + shell: cmd + run: | + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force + cmake -G "Ninja" -B build ^ + -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DGGML_BACKEND_DL=ON -DBUILD_SHARED_LIBS=ON ^ + -DGGML_CPU=OFF -DGGML_SYCL=ON ^ + -DLLAMA_BUILD_BORINGSSL=ON + cmake --build build --target ggml-sycl -j %NUMBER_OF_PROCESSORS% + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-windows-2022-x64-sycl + + - name: Build the release package + id: pack_artifacts + run: | + echo "cp oneAPI running time dll files in ${{ env.ONEAPI_ROOT }} to ./build/bin" + + cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_sycl_blas.5.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_core.2.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.2.dll" ./build/bin + + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_opencl.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_win_proxy_loader.dll" ./build/bin + ZE_LOADER_DLL=$(find "${{ env.ONEAPI_ROOT }}" "$LEVEL_ZERO_V1_SDK_PATH" -iname ze_loader.dll -print -quit 2>/dev/null || true) + if [ -n "$ZE_LOADER_DLL" ]; then + echo "Using Level Zero loader: $ZE_LOADER_DLL" + cp "$ZE_LOADER_DLL" ./build/bin + else + echo "Level Zero loader DLL not found in oneAPI or SDK; relying on system driver/runtime" + fi + + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl8.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/svml_dispmd.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libmmd.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libiomp5md.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl-ls.exe" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libsycl-fallback-bfloat16.spv" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libsycl-native-bfloat16.spv" ./build/bin + + cp "${{ env.ONEAPI_ROOT }}/dnnl/latest/bin/dnnl.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/tbb/latest/bin/tbb12.dll" ./build/bin + + cp "${{ env.ONEAPI_ROOT }}/tcm/latest/bin/tcm.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/tcm/latest/bin/libhwloc-15.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/umf/latest/bin/umf.dll" ./build/bin + + echo "cp oneAPI running time dll files to ./build/bin done" + 7z a -snl llama-bin-win-sycl-x64.zip ./build/bin/* + + - name: Upload the release package + uses: actions/upload-artifact@v6 + with: + path: llama-bin-win-sycl-x64.zip + name: llama-bin-win-sycl-x64.zip + + ubuntu-24-sycl: + needs: [check-release] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + strategy: + matrix: + build: [fp32, fp16] + include: + - build: fp32 + fp16: OFF + - build: fp16 + fp16: ON + + runs-on: ubuntu-24.04 + + env: + ONEAPI_ROOT: /opt/intel/oneapi/ + ONEAPI_INSTALLER_VERSION: "2025.3.3" + LEVEL_ZERO_VERSION: "1.28.2" + LEVEL_ZERO_UBUNTU_VERSION: "u24.04" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Download & Install oneAPI + shell: bash + run: | + cd /tmp + wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh + sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept + + - name: Install Level Zero SDK + shell: bash + run: | + cd /tmp + wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb + wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb + sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-ubuntu-24.04-sycl-${{ matrix.build }} + + - name: Build + id: cmake_build + run: | + source /opt/intel/oneapi/setvars.sh + cmake -B build \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DLLAMA_OPENSSL=OFF \ + -DGGML_NATIVE=OFF \ + -DGGML_SYCL_F16=${{ matrix.fp16 }} + time cmake --build build --config Release -j $(nproc) + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-ubuntu-24.04-sycl-${{ matrix.build }} + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz + name: llama-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz + + ubuntu-22-rocm: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: ubuntu-22.04 + + permissions: + actions: write + + strategy: + matrix: + include: + - ROCM_VERSION: "7.2.1" + gpu_targets: "gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1151;gfx1150;gfx1200;gfx1201" + build: 'x64' + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Free up disk space + uses: ggml-org/free-disk-space@v1.3.1 + with: + tool-cache: true + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} + + - name: Dependencies + id: depends + run: | + sudo apt install -y build-essential git cmake wget + + - name: Setup Legacy ROCm + if: matrix.ROCM_VERSION == '7.2.1' + id: legacy_env + run: | + sudo mkdir --parents --mode=0755 /etc/apt/keyrings + wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \ + gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null + + sudo tee /etc/apt/sources.list.d/rocm.list << EOF + deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ matrix.ROCM_VERSION }} jammy main + EOF + + sudo tee /etc/apt/preferences.d/rocm-pin-600 << EOF + Package: * + Pin: release o=repo.radeon.com + Pin-Priority: 600 + EOF + + sudo apt update + sudo apt-get install -y libssl-dev rocm-hip-sdk + + - name: Setup TheRock + if: matrix.ROCM_VERSION != '7.2.1' + id: therock_env + run: | + wget https://repo.amd.com/rocm/tarball/therock-dist-linux-gfx1151-${{ matrix.ROCM_VERSION }}.tar.gz + mkdir install + tar -xf *.tar.gz -C install + export ROCM_PATH=$(pwd)/install + echo ROCM_PATH=$ROCM_PATH >> $GITHUB_ENV + echo PATH=$PATH:$ROCM_PATH/bin >> $GITHUB_ENV + echo LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/llvm/lib:$ROCM_PATH/lib/rocprofiler-systems >> $GITHUB_ENV + + - name: Build with native CMake HIP support + id: cmake_build + run: | + cmake -B build -S . \ + -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGPU_TARGETS="${{ matrix.gpu_targets }}" \ + -DGGML_HIP=ON \ + -DHIP_PLATFORM=amd \ + -DGGML_HIP_ROCWMMA_FATTN=ON \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Get ROCm short version + run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz + + windows-hip: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: windows-2022 + + permissions: + actions: write + + env: + HIPSDK_INSTALLER_VERSION: "26.Q1" + + strategy: + matrix: + include: + - name: "radeon" + gpu_targets: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Grab rocWMMA package + id: grab_rocwmma + run: | + curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.2.0.70201-81~24.04_amd64.deb" + 7z x rocwmma.deb + 7z x data.tar + + - name: Cache ROCm Installation + id: cache-rocm + uses: actions/cache@v5 + with: + path: C:\Program Files\AMD\ROCm + key: cache-gha-rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }} + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-windows-2022-x64-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ matrix.name }} + + - name: Install ROCm + if: steps.cache-rocm.outputs.cache-hit != 'true' + id: depends + run: | + $ErrorActionPreference = "Stop" + write-host "Downloading AMD HIP SDK Installer" + Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-Win11-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe" + write-host "Installing AMD HIP SDK" + $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -PassThru + $completed = $proc.WaitForExit(600000) + if (-not $completed) { + Write-Error "ROCm installation timed out after 10 minutes. Killing the process" + $proc.Kill() + exit 1 + } + if ($proc.ExitCode -ne 0) { + Write-Error "ROCm installation failed with exit code $($proc.ExitCode)" + exit 1 + } + write-host "Completed AMD HIP SDK installation" + + - name: Verify ROCm + id: verify + run: | + # Find and test ROCm installation + $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1 + if (-not $clangPath) { + Write-Error "ROCm installation not found" + exit 1 + } + & $clangPath.FullName --version + + - name: Build + id: cmake_build + run: | + $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path) + $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}" + cmake -G "Unix Makefiles" -B build -S . ` + -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" ` + -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" ` + -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-7.2.1/include/ -Wno-ignored-attributes -Wno-nested-anon-types" ` + -DCMAKE_BUILD_TYPE=Release ` + -DGGML_BACKEND_DL=ON ` + -DGGML_NATIVE=OFF ` + -DGGML_CPU=OFF ` + -DGPU_TARGETS="${{ matrix.gpu_targets }}" ` + -DGGML_HIP_ROCWMMA_FATTN=ON ` + -DGGML_HIP=ON ` + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} ` + -DLLAMA_BUILD_BORINGSSL=ON + cmake --build build --target ggml-hip -j ${env:NUMBER_OF_PROCESSORS} + md "build\bin\rocblas\library\" + md "build\bin\hipblaslt\library" + cp "${env:HIP_PATH}\bin\libhipblas.dll" "build\bin\" + cp "${env:HIP_PATH}\bin\libhipblaslt.dll" "build\bin\" + cp "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\" + cp "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\" + cp "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\" + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-windows-2022-x64-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ matrix.name }} + + - name: Pack artifacts + id: pack_artifacts + run: | + 7z a -snl llama-bin-win-hip-${{ matrix.name }}-x64.zip .\build\bin\* + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-bin-win-hip-${{ matrix.name }}-x64.zip + name: llama-bin-win-hip-${{ matrix.name }}-x64.zip + + ios-xcode: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + runs-on: macos-26 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Xcode + run: | + sudo xcode-select -s /Applications/Xcode_26.4.app + + - name: Build + id: cmake_build + run: | + sysctl -a + cmake -B build -G Xcode \ + -DGGML_METAL_EMBED_LIBRARY=ON \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=16.0 \ + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO + + - name: xcodebuild for swift package + id: xcodebuild + run: | + ./build-xcframework.sh + + - name: Build Xcode project + run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + # Zip file is required for Swift Package Manager, which does not support tar.gz for binary targets. + # For more details, see https://developer.apple.com/documentation/xcode/distributing-binary-frameworks-as-swift-packages + zip -r -y llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-xcframework.zip + name: llama-${{ steps.tag.outputs.name }}-xcframework.zip + +# TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23705) +# in order to enable it again, we have to provision dedicated runners to run it +# openEuler-cann: +# strategy: +# matrix: +# include: +# # 910b with aclgraph (both architectures) +# - arch: x86 +# chip_type: '910b' +# build: 'Release' +# use_acl_graph: 'on' +# - arch: aarch64 +# chip_type: '910b' +# build: 'Release' +# use_acl_graph: 'on' +# # 310p without aclgraph (both architectures) +# - arch: x86 +# chip_type: '310p' +# build: 'Release' +# use_acl_graph: 'off' +# - arch: aarch64 +# chip_type: '310p' +# build: 'Release' +# use_acl_graph: 'off' +# runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} +# steps: +# - name: Checkout +# uses: actions/checkout@v6 +# with: +# fetch-depth: 0 +# +# - name: Free up disk space +# uses: ggml-org/free-disk-space@v1.3.1 +# with: +# tool-cache: true +# +# - name: Set container image +# id: cann-image +# run: | +# image="ascendai/cann:${{ matrix.chip_type == '910b' && '8.5.0-910b-openeuler24.03-py3.11' || '8.5.0-310p-openeuler24.03-py3.11' }}" +# echo "image=${image}" >> "${GITHUB_OUTPUT}" +# +# - name: Pull container image +# run: docker pull "${{ steps.cann-image.outputs.image }}" +# +# - name: Build +# env: +# BUILD_TYPE: ${{ matrix.build }} +# SOC_TYPE: ascend${{ matrix.chip_type }} +# USE_ACL_GRAPH: ${{ matrix.use_acl_graph }} +# run: | +# HOST_UID=$(id -u) +# HOST_GID=$(id -g) +# +# docker run --rm \ +# -v "${PWD}:/workspace" \ +# -w /workspace \ +# -e SOC_TYPE=${SOC_TYPE} \ +# -e BUILD_TYPE=${BUILD_TYPE} \ +# -e USE_ACL_GRAPH=${USE_ACL_GRAPH} \ +# "${{ steps.cann-image.outputs.image }}" \ +# bash -lc ' +# set -e +# yum install -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs git gcc gcc-c++ make cmake openssl-devel +# yum clean all && rm -rf /var/cache/yum +# git config --global --add safe.directory "/workspace" +# export LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/lib64:${ASCEND_TOOLKIT_HOME}/$(uname -m)-linux/devlib/:${LD_LIBRARY_PATH} +# cmake -S . -B build \ +# -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ +# -DGGML_CANN=on \ +# -DSOC_TYPE=${SOC_TYPE} \ +# -DUSE_ACL_GRAPH=${USE_ACL_GRAPH} +# cmake --build build -j $(nproc) +# +# chown -R '"${HOST_UID}"':'"${HOST_GID}"' /workspace/build +# ' +# +# - name: Determine tag name +# id: tag +# uses: ./.github/actions/get-tag-name +# +# - name: Pack artifacts +# run: | +# cp LICENSE ./build/bin/ +# tar -czvf llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . +# +# - name: Upload artifacts +# uses: actions/upload-artifact@v6 +# with: +# path: llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz +# name: llama-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz + + ui-build: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + uses: ./.github/workflows/ui-build.yml + with: + hf_ui_version: ${{ needs.get-version.outputs.ui_version }} + + release: + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + + # Fine-grant permission + # https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token + permissions: + contents: write # for creating release + + runs-on: ubuntu-slim + + needs: + - get-version + - windows + - windows-cpu + - windows-cuda + #- windows-sycl + - windows-hip + - windows-openvino + - ubuntu-22-rocm + - ubuntu-cpu + - ubuntu-vulkan + - ubuntu-24-openvino + #- ubuntu-24-sycl + - android-arm64 + - macos-cpu + - ios-xcode + #- openEuler-cann + - ui-build + + outputs: + tag_name: ${{ steps.tag.outputs.name }} + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Download artifacts + id: download-artifact + uses: actions/download-artifact@v7 + with: + path: ./artifact + merge-multiple: true + + - name: Move artifacts + id: move_artifacts + run: | + mkdir -p release + + echo "Adding CPU backend files to existing zips..." + for arch in x64 arm64; do + cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip" + temp_dir=$(mktemp -d) + echo "Extracting CPU backend for $arch..." + unzip "$cpu_zip" -d "$temp_dir" + + echo "Adding CPU files to $arch zips..." + for target_zip in artifact/llama-bin-win-*-${arch}.zip; do + if [[ "$target_zip" == "$cpu_zip" ]]; then + continue + fi + echo "Adding CPU backend to $(basename "$target_zip")" + realpath_target_zip=$(realpath "$target_zip") + (cd "$temp_dir" && zip -r "$realpath_target_zip" .) + done + + rm -rf "$temp_dir" + done + + echo "Renaming and moving zips to release..." + for zip_file in artifact/llama-bin-win-*.zip; do + base_name=$(basename "$zip_file" .zip) + zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip" + echo "Moving $zip_file to release/$zip_name" + mv "$zip_file" "release/$zip_name" + done + + echo "Moving other artifacts..." + mv -v artifact/*.zip release + mv -v artifact/*.tar.gz release + + - name: Download UI build + id: download_ui + uses: actions/download-artifact@v7 + with: + name: ui-build + path: ./ui-dist + + - name: Package UI + id: package_ui + run: | + tar -czvf release/llama-${{ steps.tag.outputs.name }}-ui.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./ui-dist . + + - name: Create release + id: create_release + uses: ggml-org/action-create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.tag.outputs.name }} + body: | +
+ + ${{ github.event.head_commit.message }} + +
+ + **Website:** + - + + **macOS/iOS:** + - [macOS Apple Silicon (arm64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.tar.gz) + - macOS Apple Silicon (arm64, KleidiAI enabled) [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23780) + - [macOS Intel (x64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-x64.tar.gz) + - [iOS XCFramework](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-xcframework.zip) + + **Linux:** + - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz) + - [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz) + - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) + - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) + - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) + - [Ubuntu x64 (ROCm 7.2)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.2-x64.tar.gz) + - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz) + - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz) + - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz) + + **Android:** + - [Android arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz) + + **Windows:** + - [Windows x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-x64.zip) + - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip) + - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip) + - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip) + - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip) + - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip) + - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip) + - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip) + - [Windows x64 (HIP)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-hip-radeon-x64.zip) + + **openEuler:** + - [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23705) + - openEuler x86 (310p) + - openEuler x86 (910b, ACL Graph) + - openEuler aarch64 (310p) + - openEuler aarch64 (910b, ACL Graph) + + **UI:** + - [UI](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-ui.tar.gz) + + - name: Upload release + id: upload_release + uses: actions/github-script@v8 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const path = require('path'); + const fs = require('fs'); + const release_id = '${{ steps.create_release.outputs.id }}'; + for (let file of await fs.readdirSync('./release')) { + if (path.extname(file) === '.zip' || file.endsWith('.tar.gz')) { + console.log('uploadReleaseAsset', file); + await github.rest.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release_id, + name: file, + data: await fs.readFileSync(`./release/${file}`) + }); + } + } + + ui-publish: + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + + needs: + - release + + uses: ./.github/workflows/ui-publish.yml + with: + version_tag: ${{ needs.release.outputs.tag_name }} + secrets: + hf_token: ${{ secrets.HF_TOKEN_UI_STATIC_OUTPUT }} diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml new file mode 100644 index 0000000..c0817cb --- /dev/null +++ b/.github/workflows/server-sanitize.yml @@ -0,0 +1,112 @@ +name: Server (sanitize) + +on: + workflow_dispatch: # allows manual triggering + inputs: + sha: + description: 'Commit SHA1 to build' + required: false + type: string + slow_tests: + description: 'Run slow tests' + required: true + type: boolean + push: + branches: + - master + paths: [ + '.github/workflows/server-sanitize.yml', + '**/CMakeLists.txt', + '**/Makefile', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + 'tools/server/**.*' + ] + +env: + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + LLAMA_ARG_LOG_VERBOSITY: 10 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + server: + runs-on: [self-hosted, CPU, Linux, llama-server] + + strategy: + matrix: + sanitizer: [ADDRESS, UNDEFINED] # THREAD is very slow + build_type: [RelWithDebInfo] + fail-fast: false + + steps: + #- name: Dependencies + # id: depends + # run: | + # sudo apt-get update + # sudo apt-get -y install \ + # build-essential \ + # xxd \ + # git \ + # cmake \ + # curl \ + # wget \ + # language-pack-en \ + # libssl-dev + + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DGGML_SCHED_NO_REALLOC=ON \ + -DGGML_SANITIZE_ADDRESS=${{ matrix.sanitizer == 'ADDRESS' }} \ + -DGGML_SANITIZE_THREAD=${{ matrix.sanitizer == 'THREAD' }} \ + -DGGML_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }} \ + -DLLAMA_SANITIZE_ADDRESS=${{ matrix.sanitizer == 'ADDRESS' }} \ + -DLLAMA_SANITIZE_THREAD=${{ matrix.sanitizer == 'THREAD' }} \ + -DLLAMA_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }} + cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server + + - name: Python setup + id: setup_python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + pip-install: -r tools/server/tests/requirements.txt + + - name: Tests + id: server_integration_tests + if: ${{ (!matrix.disabled_on_pr || !github.event.pull_request) }} + run: | + cd tools/server/tests + export ${{ matrix.extra_args }} + pytest -v -x -m "not slow" + + - name: Slow tests + id: server_integration_tests_slow + if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }} + run: | + cd tools/server/tests + export ${{ matrix.extra_args }} + SLOW_TESTS=1 pytest -v -x diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml new file mode 100644 index 0000000..249f389 --- /dev/null +++ b/.github/workflows/server-self-hosted.yml @@ -0,0 +1,222 @@ +name: Server (self-hosted) + +on: + workflow_dispatch: # allows manual triggering + inputs: + sha: + description: 'Commit SHA1 to build' + required: false + type: string + slow_tests: + description: 'Run slow tests' + required: true + type: boolean + push: + branches: + - master + paths: [ + '.github/workflows/server-self-hosted.yml', + '**/CMakeLists.txt', + '**/Makefile', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cu', + '**/*.swift', + '**/*.m', + 'tools/server/**.*' + ] + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + LLAMA_ARG_LOG_VERBOSITY: 10 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + server-metal: + runs-on: [self-hosted, llama-server, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Build + id: cmake_build + run: | + cmake -B build -DGGML_SCHED_NO_REALLOC=ON + cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) --target llama-server + + - name: Python setup + id: setup_python + run: | + cd tools/server/tests + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + + - name: Tests (GPUx1) + id: server_integration_tests + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + pytest -v -x -m "not slow" + + - name: Tests (GPUx1, backend-sampling) + id: server_integration_tests_backend_sampling + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + export LLAMA_ARG_BACKEND_SAMPLING=1 + pytest -v -x -m "not slow" + + - name: Tests (GPUx2) + id: server_integration_tests_gpu2 + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + export GGML_METAL_DEVICES=2 + pytest -v -x -m "not slow" + + - name: Tests (GPUx2, backend-sampling) + id: server_integration_tests_gpu2_backend_sampling + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1 + pytest -v -x -m "not slow" + + server-cuda: + runs-on: [self-hosted, llama-server, Linux, NVIDIA] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Build + id: cmake_build + run: | + cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON + cmake --build build --config Release -j $(nproc) --target llama-server + + - name: Python setup + id: setup_python + run: | + cd tools/server/tests + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + + - name: Tests (GPUx1) + id: server_integration_tests + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + pytest -v -x -m "not slow" + + - name: Tests (GPUx1, backend-sampling) + id: server_integration_tests_backend_sampling + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + export LLAMA_ARG_BACKEND_SAMPLING=1 + pytest -v -x -m "not slow" + + - name: Tests (GPUx2) + id: server_integration_tests_gpu2 + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + export GGML_CUDA_DEVICES=2 + pytest -v -x -m "not slow" + + - name: Tests (GPUx2, backend-sampling) + id: server_integration_tests_gpu2_backend_sampling + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1 + pytest -v -x -m "not slow" + + server-kleidiai: + runs-on: ah-ubuntu_22_04-c8g_8x + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Dependencies + id: depends + run: | + set -euxo pipefail + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \ + apt-get install -y \ + build-essential \ + libssl-dev \ + python3-venv \ + gpg \ + wget \ + time \ + git-lfs + + git lfs install + + # install the latest cmake + sudo install -d /usr/share/keyrings + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \ + | gpg --dearmor \ + | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \ + | sudo tee /etc/apt/sources.list.d/kitware.list + sudo apt-get update + sudo apt-get install -y cmake + + - name: Build + id: cmake_build + run: | + cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON + cmake --build build --config Release -j $(nproc) --target llama-server + + - name: Python setup + id: setup_python + run: | + cd tools/server/tests + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + + - name: Tests + id: server_integration_tests + if: ${{ !github.event.pull_request }} + run: | + cd tools/server/tests + source venv/bin/activate + pytest -v -x -m "not slow" diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml new file mode 100644 index 0000000..5a02cc1 --- /dev/null +++ b/.github/workflows/server.yml @@ -0,0 +1,181 @@ +name: Server + +on: + workflow_dispatch: # allows manual triggering + inputs: + sha: + description: 'Commit SHA1 to build' + required: false + type: string + slow_tests: + description: 'Run slow tests' + required: true + type: boolean + push: + branches: + - master + paths: [ + '.github/workflows/server.yml', + '**/CMakeLists.txt', + '**/Makefile', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cu', + '**/*.swift', + '**/*.m', + 'tools/server/**.*' + ] + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/server.yml', + '**/CMakeLists.txt', + '**/Makefile', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cu', + '**/*.swift', + '**/*.m', + 'tools/server/**.*' + ] + +env: + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + LLAMA_ARG_LOG_VERBOSITY: 10 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + ubuntu: + runs-on: ubuntu-24.04-arm + + steps: + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get -y install \ + build-essential \ + xxd \ + git \ + cmake \ + curl \ + wget \ + language-pack-en \ + libssl-dev + + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: server-ubuntu-24.04-arm + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DGGML_SCHED_NO_REALLOC=ON + cmake --build build --config Release -j $(nproc) --target llama-server + + - name: Python setup + id: setup_python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + pip-install: -r tools/server/tests/requirements.txt + + - name: Tests + id: server_integration_tests + run: | + cd tools/server/tests + pytest -v -x -m "not slow" + + - name: Slow tests + id: server_integration_tests_slow + if: ${{ github.event.schedule || github.event.inputs.slow_tests == 'true' }} + run: | + cd tools/server/tests + SLOW_TESTS=1 pytest -v -x + + - name: Tests (Backend sampling) + id: server_integration_tests_backend_sampling + run: | + cd tools/server/tests + export LLAMA_ARG_BACKEND_SAMPLING=1 + pytest -v -x -m "not slow" + + - name: Slow tests (Backend sampling) + id: server_integration_tests_slow_backend_sampling + if: ${{ github.event.schedule || github.event.inputs.slow_tests == 'true' }} + run: | + cd tools/server/tests + export LLAMA_ARG_BACKEND_SAMPLING=1 + SLOW_TESTS=1 pytest -v -x + + windows: + runs-on: windows-2025 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: server-windows-2025-x64 + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Build + id: cmake_build + shell: cmd + run: | + cmake -B build -G "Ninja Multi-Config" ^ + -DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DGGML_SCHED_NO_REALLOC=ON + set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1 + cmake --build build --config Release -j %NINJA_JOBS% --target llama-server + + - name: Python setup + id: setup_python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + pip-install: -r tools/server/tests/requirements.txt + + - name: Tests + id: server_integration_tests + run: | + cd tools/server/tests + $env:PYTHONIOENCODING = ":replace" + pytest -v -x -m "not slow" + + - name: Slow tests + id: server_integration_tests_slow + if: ${{ github.event.schedule || github.event.inputs.slow_tests == 'true' }} + run: | + cd tools/server/tests + $env:SLOW_TESTS = "1" + pytest -v -x diff --git a/.github/workflows/ui-build-self-hosted.yml b/.github/workflows/ui-build-self-hosted.yml new file mode 100644 index 0000000..7b7f8b6 --- /dev/null +++ b/.github/workflows/ui-build-self-hosted.yml @@ -0,0 +1,36 @@ +name: UI Build (self-hosted) + +on: + workflow_call: + +jobs: + build: + runs-on: [self-hosted, fast] + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Install dependencies + run: npm ci + working-directory: tools/ui + + - name: Build application + run: npm run build + working-directory: tools/ui + + - name: Upload built UI + uses: actions/upload-artifact@v6 + with: + name: ui-build + path: tools/ui/dist/ + retention-days: 1 diff --git a/.github/workflows/ui-build.yml b/.github/workflows/ui-build.yml new file mode 100644 index 0000000..85642f3 --- /dev/null +++ b/.github/workflows/ui-build.yml @@ -0,0 +1,48 @@ +name: UI Build + +on: + workflow_call: + inputs: + hf_ui_version: + description: 'Version string for version.json (e.g. 12345)' + required: false + type: string + +jobs: + build: + runs-on: ubuntu-slim + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Install dependencies + run: npm ci + working-directory: tools/ui + + - name: Build application + env: + HF_UI_VERSION: ${{ inputs.hf_ui_version || '' }} + LLAMA_BUILD_NUMBER: ${{ inputs.hf_ui_version || 'b0000' }} + run: npm run build + working-directory: tools/ui + + - name: Run PWA unit tests (versioned build output) + run: npx vitest --project=unit --run tests/unit/pwa.spec.ts + working-directory: tools/ui + + - name: Upload built UI + uses: actions/upload-artifact@v6 + with: + name: ui-build + path: tools/ui/dist/ + retention-days: 1 diff --git a/.github/workflows/ui-publish.yml b/.github/workflows/ui-publish.yml new file mode 100644 index 0000000..99a6d84 --- /dev/null +++ b/.github/workflows/ui-publish.yml @@ -0,0 +1,75 @@ +name: UI Publish + +on: + workflow_call: + inputs: + version_tag: + description: 'Version tag to publish under (e.g., b1234)' + required: true + type: string + secrets: + hf_token: + description: 'Hugging Face token with write access' + required: true + +jobs: + build: + name: Build static output + uses: ./.github/workflows/ui-build.yml + + publish: + name: Publish UI Static Output + needs: build + runs-on: ubuntu-slim + + permissions: + contents: read + + env: + HF_BUCKET_NAME: ${{ vars.HF_BUCKET_UI_STATIC_OUTPUT }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Download UI build artifact + uses: actions/download-artifact@v7 + with: + name: ui-build + path: tools/ui/dist/ + + - name: Create distribution archive + run: | + tar -czf dist.tar.gz -C tools/ui/dist . + sha256sum dist.tar.gz > dist.tar.gz.sha256 + mv dist.tar.gz dist.tar.gz.sha256 tools/ui/dist/ + + - name: Install Hugging Face Hub CLI + run: pip install -U huggingface_hub + + - name: Authenticate with Hugging Face + run: hf auth login --token ${{ secrets.hf_token }} + + - name: Sync built files to Hugging Face bucket (version tag) + run: | + # Upload the built files to the Hugging Face bucket under the release version + hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet + + - name: Sync built files to Hugging Face bucket (latest) + run: | + # Also upload to the 'latest' directory for fallback downloads + hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet + + - name: Verify upload + run: | + # List the files in the bucket to verify the upload + hf buckets list hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} -R -h + + - name: Clean up root-level files + run: | + # Clean up any old root-level files from previous non-versioned deployments + hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/index.html --yes 2>/dev/null || true + hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/bundle.js --yes 2>/dev/null || true + hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/bundle.css --yes 2>/dev/null || true diff --git a/.github/workflows/ui-self-hosted.yml b/.github/workflows/ui-self-hosted.yml new file mode 100644 index 0000000..79d7800 --- /dev/null +++ b/.github/workflows/ui-self-hosted.yml @@ -0,0 +1,125 @@ +name: UI (self-hosted) + +# these are the same as ui.yml, but with self-hosted runners +# the jobs are lighter because they don't need to install Node.js or Playwright browsers +# the runner has pre-installed Playwright browsers for @playwright/test (1.56.1) at /ms-playwright/ + +on: + workflow_dispatch: + inputs: + sha: + description: 'Commit SHA1 to build' + required: false + type: string + push: + branches: + - master + paths: [ + '.github/workflows/ui-self-hosted.yml', + '.github/workflows/ui-build-self-hosted.yml', + 'tools/ui/**.*', + 'tools/server/tests/**.*' + ] + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ui-self-hosted.yml', + '.github/workflows/ui-build-self-hosted.yml', + 'tools/ui/**.*', + 'tools/server/tests/**.*' + ] + +env: + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + LLAMA_ARG_LOG_VERBOSITY: 10 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + ui-build: + name: Build static output + uses: ./.github/workflows/ui-build-self-hosted.yml + + ui-checks: + name: Checks + needs: ui-build + runs-on: [self-hosted, PLAYWRIGHT] + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Install dependencies + id: setup + run: npm ci + working-directory: tools/ui + + - name: Download built UI artifacts + uses: actions/download-artifact@v6 + with: + name: ui-build + path: tools/ui/dist/ + + - name: Run type checking + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run check + working-directory: tools/ui + + - name: Run linting + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run lint + working-directory: tools/ui + + - name: Run Client tests + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run test:client + working-directory: tools/ui + + - name: Run Unit tests + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run test:unit + working-directory: tools/ui + + e2e-tests: + name: E2E Tests + needs: ui-build + runs-on: [self-hosted, PLAYWRIGHT] + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Install dependencies + id: setup + run: npm ci + working-directory: tools/ui + + - name: Download built UI artifacts + uses: actions/download-artifact@v6 + with: + name: ui-build + path: tools/ui/dist/ + + - name: Build Storybook + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run build-storybook + working-directory: tools/ui + + - name: Run UI tests + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run test:ui -- --testTimeout=60000 + working-directory: tools/ui + + - name: Run E2E tests + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run test:e2e + working-directory: tools/ui diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml new file mode 100644 index 0000000..fa99a0c --- /dev/null +++ b/.github/workflows/ui.yml @@ -0,0 +1,151 @@ +name: UI + +on: + workflow_dispatch: + inputs: + sha: + description: 'Commit SHA1 to build' + required: false + type: string + push: + branches: + - master + paths: [ + '.github/workflows/ui.yml', + '.github/workflows/ui-build.yml', + 'tools/ui/**.*', + 'tools/server/tests/**.*' + ] + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ui.yml', + '.github/workflows/ui-build.yml', + 'tools/ui/**.*', + 'tools/server/tests/**.*' + ] + +env: + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + LLAMA_ARG_LOG_VERBOSITY: 10 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + ui-build: + name: Build static output + uses: ./.github/workflows/ui-build.yml + + ui-checks: + name: Checks + needs: ui-build + runs-on: ubuntu-24.04 + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Setup Node.js + id: node + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Download built UI artifacts + uses: actions/download-artifact@v6 + with: + name: ui-build + path: tools/ui/dist/ + + - name: Install dependencies + id: setup + if: ${{ steps.node.conclusion == 'success' }} + run: npm ci + working-directory: tools/ui + + - name: Run type checking + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run check + working-directory: tools/ui + + - name: Run linting + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npm run lint + working-directory: tools/ui + + - name: Install Playwright browsers + id: playwright + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npx playwright install --with-deps + working-directory: tools/ui + + - name: Run Client tests + if: ${{ always() && steps.playwright.conclusion == 'success' }} + run: npm run test:client + working-directory: tools/ui + + - name: Run Unit tests (uses pre-built dist/ from ui-build) + if: ${{ always() && steps.playwright.conclusion == 'success' }} + run: npm run test:unit + working-directory: tools/ui + + e2e-tests: + name: E2E Tests + needs: ui-build + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Setup Node.js + id: node + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Install dependencies + id: setup + if: ${{ steps.node.conclusion == 'success' }} + run: npm ci + working-directory: tools/ui + + - name: Download built UI artifacts (reuses ui-build) + uses: actions/download-artifact@v6 + with: + name: ui-build + path: tools/ui/dist/ + + - name: Install Playwright browsers + id: playwright + if: ${{ always() && steps.setup.conclusion == 'success' }} + run: npx playwright install --with-deps + working-directory: tools/ui + + - name: Build Storybook + if: ${{ always() && steps.playwright.conclusion == 'success' }} + run: npm run build-storybook + working-directory: tools/ui + + - name: Run UI tests + if: ${{ always() && steps.playwright.conclusion == 'success' }} + run: npm run test:ui -- --testTimeout=60000 + working-directory: tools/ui + + - name: Run E2E tests (uses pre-built dist/ from ui-build) + if: ${{ always() && steps.playwright.conclusion == 'success' }} + run: npm run test:e2e + working-directory: tools/ui diff --git a/.github/workflows/update-ops-docs.yml b/.github/workflows/update-ops-docs.yml new file mode 100644 index 0000000..6e8bc1a --- /dev/null +++ b/.github/workflows/update-ops-docs.yml @@ -0,0 +1,44 @@ +name: Update Operations Documentation + +on: + push: + paths: + - '.github/workflows/update-ops-docs.yml' + - 'docs/ops.md' + - 'docs/ops/**' + - 'scripts/create_ops_docs.py' + pull_request: + paths: + - '.github/workflows/update-ops-docs.yml' + - 'docs/ops.md' + - 'docs/ops/**' + - 'scripts/create_ops_docs.py' + +jobs: + update-ops-docs: + runs-on: [self-hosted, fast, ARM64] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.x' + + - name: Generate operations documentation to temporary file + run: | + mkdir -p /tmp/ops_check + ./scripts/create_ops_docs.py /tmp/ops_check/ops.md + + - name: Check if docs/ops.md matches generated version + run: | + if ! diff -q docs/ops.md /tmp/ops_check/ops.md; then + echo "Operations documentation (docs/ops.md) is not up to date with the backend CSV files." + echo "To fix: run ./scripts/create_ops_docs.py and commit the updated docs/ops.md along with your changes" + echo "Differences found:" + diff docs/ops.md /tmp/ops_check/ops.md || true + exit 1 + fi + echo "Operations documentation is up to date." diff --git a/.github/workflows/winget.yml b/.github/workflows/winget.yml new file mode 100644 index 0000000..69e24f9 --- /dev/null +++ b/.github/workflows/winget.yml @@ -0,0 +1,44 @@ +name: Update Winget Package + +on: + workflow_dispatch: # allows manual triggering + schedule: + - cron: '28 5 * * *' # Update every day at 5:28 UTC + +jobs: + update: + name: Update Winget Package + runs-on: ubuntu-latest + if: github.repository_owner == 'ggml-org' + + steps: + - name: Install cargo binstall + uses: cargo-bins/cargo-binstall@268643a6b5ea099f5718ee5cd3ff7dc89a5eb49b + + - name: Install komac + run: | + cargo binstall komac@2.16.0 -y + + - name: Find latest release + id: find_latest_release + uses: actions/github-script@v8 + with: + script: | + const { data: releases } = await github.rest.repos.listReleases({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + const { tag_name: version, assets: assets } = releases.find(({assets}) => assets.find(asset => asset.name.includes('win-vulkan'))); + const { browser_download_url: asset_url } = assets.find(asset => asset.name.includes('win-vulkan')); + console.log("Latest release:", version); + core.setOutput('VERSION', version); + core.setOutput('ASSETURL', asset_url); + + - name: Update manifest + run: | + echo "Updating manifest..." + komac update --version ${{ steps.find_latest_release.outputs.VERSION }} \ + --urls "${{ steps.find_latest_release.outputs.ASSETURL }}" \ + --token ${{ secrets.WINGET_GITHUB_TOKEN }} \ + --submit \ + ggml.llamacpp diff --git a/.gitignore b/.gitignore index 9b58961..0e43a13 100644 --- a/.gitignore +++ b/.gitignore @@ -153,3 +153,14 @@ a.out.* AGENTS.local.md .pi/SYSTEM.md + +# Python native package +/dist/ +/*.egg-info/ +/python/*.egg-info/ +/python/kitten_tts_cpp/data/ +/python/kitten_tts_cpp/*.pyd + +/wheelhouse/ +/python/kitten_tts_cpp/_build_info.json +/python/kitten_tts_cpp/licenses/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..218d6ee --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,18 @@ +include CMakeLists.txt LICENSE README.md requirements.txt +recursive-include cmake * +recursive-include common * +recursive-include ggml * +recursive-include include * +recursive-include src * +recursive-include tools * +recursive-include scripts * +prune tools/ui/node_modules +recursive-include vendor * +recursive-include python *.py +prune vendor/kitten-text-processing/.git +prune vendor/kitten-text-processing/build +prune vendor/kitten-text-processing/dist +prune vendor/kitten-text-processing/.venv +global-exclude __pycache__ *.py[cod] *.so *.a *.o .git + +recursive-include packaging *.py *.md diff --git a/README.md b/README.md index 4fd2abd..f8df6d2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Kitten-TTS 2 C++ +# KittenTTS2 C++ CPU-focused C++ inference for KittenTTS2, with built-in voices, expression controls, and prepared reference voices. The runtime downloads prepared model assets and runs synthesis natively in C++. Users do not need to run the Python exporter. @@ -13,9 +13,43 @@ The speech language model runs through llama.cpp/GGML, text normalization uses t - Default S3 and optional `student_w4` / `student_w8` decoder exports. - Mono float32 WAV output at 24 kHz. +## Python interface + +Precompiled wheel builds are configured for Linux x86-64/ARM64, Apple Silicon macOS, Windows x86-64, and experimental Windows ARM64. See the [wheel matrix and installation guide](packaging/README.md). These builds remain local configuration until the workflow is pushed and run; cross-platform artifacts have not been published. + +The `py-interface` branch includes a native Python package with a KittenTTS-style API. `setup.py` invokes CMake during installation and bundles the normalizer grammar data. Install a CPU PyTorch build first, then build against that same installation: + +```sh +git submodule update --init vendor/kitten-text-processing +python -m pip install torch --index-url https://download.pytorch.org/whl/cpu +python -m pip install setuptools wheel cmake numpy soundfile +python -m pip install . --no-build-isolation +``` + +A C++17 compiler and OpenSSL development headers are required for HTTPS downloads (`libssl-dev` on Debian/Ubuntu). Set `CMAKE_BUILD_PARALLEL_LEVEL=8` to control build parallelism. `CMAKE_ARGS` accepts extra CMake flags, such as `-DOPENSSL_ROOT_DIR=/path/to/openssl`. Builds optimize for the local CPU; use `CMAKE_ARGS="-DGGML_NATIVE=OFF"` when building a wheel for other machines. Source-built wheels require the exact PyTorch release used at build time and the linked system OpenSSL libraries. The portable CI wheels instead link LibreSSL statically. + +```python +from kitten_tts_cpp import KittenTTS # KittenTTS2 and get_model are also available + +with KittenTTS("KittenML/kitten-tts-2", decoder="student_w4", threads=8) as model: + print(model.available_voices) + audio = model.generate("Hello again.", voice="Bruno", advanced={"seed": 1234}) + model.generate_to_file("Hello again.", "hello.wav", voice="Bruno") + for chunk in model.generate_stream("A longer passage goes here.", voice="Bruno"): + pass # Each chunk is a float32 NumPy array at model.sample_rate. +``` + +Weights and the decoder stay loaded between calls. The GIL is released during native loading and inference. Native inference calls are serialized because the TorchScript decoder uses process-wide random and thread settings. Streaming yields untrimmed sentence chunks as they are decoded; close the iterator when stopping early. `generate` applies the same chunk joins as the CLI. `last_report` contains token IDs and generation timings. + +The constructor also accepts `cache_dir`, `hf_token`, `revision`, `offline`, `decoder_threads`, `repack`, `model_path` (a local GGUF override), `data_dir`, and `ca_file` (a custom HTTPS trust bundle). A local prepared asset directory can replace the repository ID. Generation supports the Python KittenTTS2 preset, sampling, normalization, reference-prompt, and `advanced` options, including `use_emotion`. + +Only KittenTTS2 CPU inference and `weights="packed"` are supported. This package uses the separate `kitten_tts_cpp` import name so it can coexist with `kittenml`. Raw-audio `reference=` cloning is not implemented; prepare a reference with `prepare_reference.py` and select its voice name. It does not provide the older ONNX models or guarantee identical sampled output to Python's original PyTorch backend. + +For development, `python setup.py build_ext --inplace` builds the extension; set `PYTHONPATH=python` to import it directly. `python setup.py bdist_wheel` produces an installable wheel, and `python setup.py sdist` includes the native sources and normalizer submodule contents. + ## Build -Requires a C++20 compiler, CMake, and CPU LibTorch. A CPU PyTorch installation can provide LibTorch's headers and libraries. Use the same LibTorch version for export and inference, with compatible torch/torchaudio versions for asset preparation. +Requires a C++17 compiler, CMake, and CPU LibTorch. A CPU PyTorch installation can provide LibTorch's headers and libraries. Use the same LibTorch version for export and inference, with compatible torch/torchaudio versions for asset preparation. ```sh git submodule update --init vendor/kitten-text-processing @@ -31,7 +65,6 @@ cmake -S . -B build \ -DLLAMA_BUILD_EXAMPLES=OFF \ -DCMAKE_PREFIX_PATH="$(python -c 'import torch; print(torch.utils.cmake_prefix_path)')" - cmake --build build --target kitten-tts -j 8 ``` @@ -42,8 +75,8 @@ The normalizer prepares its grammar data using Python at build time. To use exis The runtime fetches `config.json` from `KittenML/kitten-tts-2` first, then downloads its GGUF, selected TorchScript decoder, and matching voice conditioning. Assets are cached under `$XDG_CACHE_HOME/kitten-tts` (or `~/.cache/kitten-tts`). ```sh -build/bin/kitten-tts --text 'One day, a little girl named Lily found a needle in her room.' --output hello.wav -build/bin/kitten-tts --decoder student_w4 --text 'One day, a little girl named Lily found a needle in her room.' --output w4.wav +build/bin/kitten-tts --text 'Hello from Kitten T T S.' --output hello.wav +build/bin/kitten-tts --decoder student_w4 --text 'Hello again.' --output w4.wav ``` Use `--repo OWNER/REPO` to select another KittenTTS2 repository, `--revision COMMIT` to pin a model version, `--cache-dir DIR` to change the cache location, and `--offline` to use cached files without network access. `HF_TOKEN` supports private repositories. `--download-only --decoder student_w4` prepares the cache without synthesis. Only assets needed for the selected mode are downloaded. @@ -78,11 +111,39 @@ build/bin/kitten-tts \ Use `--help` for sampling, repetition penalties, chunking, and decoder thread settings. `--tokens-only` skips waveform decoding; `--repeat 3` benchmarks repeated synthesis with the model and decoder kept loaded. +## Python parity and CPU performance + +The implementation is checked against `kittenml.kittentts2`: + +| Check | Recorded result | +| --- | --- | +| Text normalization and chunking | 228 cases passed | +| Sampling rules | 12 cases passed | +| Audio joins | 3 cases passed | +| Packed weights | All 310 tensors reconstruct the FP16 reference exactly | +| Native waveform decoding | Sample-for-sample agreement with Python in tested cases for all three decoders | +| Packed LM logits | Close numerical agreement with the Python reference | + +On a Xeon Platinum 8462Y+ with eight pinned CPU cores, the AMX path reduced a 256-token forced-sequence workload from **6.40 to 5.60 seconds** (about **14% higher throughput**). Prompt evaluation fell from **1.09 to 0.58 seconds**. TQ2_1 reached **45.7 generated tokens/second**, comparable to Q4_0 at **44.3** in the same test. These are medians of three warmed runs, including prompt evaluation and sampling but excluding model loading. Performance varies with hardware and workload. + +Run the differential checks with: + +```sh +python tools/kitten-tts/check_parity.py \ + --binary build/bin/kitten-tts \ + --python-package ../KittenTTS \ + --repo ../kitten-tts-2 --assets models/kitten2 \ + --model models/kitten2/model-tq2_1.gguf \ + --lm --decoder +``` + +Measurements and source revisions are recorded in [validation.json](tools/kitten-tts/validation.json). + ## Current scope Sampled output is not bit-identical to Python: BF16 arithmetic, GGML activation quantization, and sampling RNGs differ. Matching seeds do not guarantee matching generated tokens across runtimes. -Voice cloning requires an offline Python preparation step and a reference transcript. The runtime is a batch CLI; Python's streaming API is not implemented. Waveform decoding currently depends on CPU LibTorch. +Voice cloning requires an offline Python preparation step and a reference transcript. The Python interface supports sentence-chunk streaming; sample-by-sample streaming is not implemented. Waveform decoding currently depends on CPU LibTorch. ## Credits diff --git a/common/download.cpp b/common/download.cpp index 44c6cea..da88417 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -298,6 +298,9 @@ static int common_download_file_single_online(const std::string & url, } auto [cli, parts] = common_http_client(url); +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (!opts.ca_cert_file.empty()) cli.set_ca_cert_path(opts.ca_cert_file); +#endif httplib::Headers headers; for (const auto & h : opts.headers) { diff --git a/common/download.h b/common/download.h index 9a03f5e..88a63b2 100644 --- a/common/download.h +++ b/common/download.h @@ -61,6 +61,7 @@ struct common_download_opts { bool download_dflash = false; bool download_dspark = false; common_download_callback * callback = nullptr; + std::string ca_cert_file; }; struct common_download_task { diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 0000000..63de25b --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,52 @@ +# Precompiled wheels + +The local `py-interface` branch contains a manually triggered wheel workflow. It builds and tests wheels as GitHub Actions artifacts; it does not publish to PyPI or create a release. The workflow has not been run remotely yet. + +| Target | Python versions | Runner | Status | +| --- | --- | --- | --- | +| Linux x86-64, glibc 2.28+ | 3.10-3.14 | ubuntu-24.04 + manylinux_2_28 | Configured; local x86-64 build tested separately | +| Linux ARM64, glibc 2.28+ | 3.10-3.14 | ubuntu-24.04-arm + manylinux_2_28 | Configured; not yet runner-tested | +| macOS 14+, Apple Silicon | 3.10-3.14 | macos-14 | Configured; not yet runner-tested | +| Windows x86-64 | 3.10-3.14 | windows-2022 | Configured; not yet runner-tested | +| Windows ARM64 | 3.11-3.13 | windows-11-arm + ClangCL | Experimental; not yet runner-tested | + +These are 23 CPython builds. This is not universal architecture support: Intel macOS, 32-bit platforms, Alpine/musl, mobile, WebAssembly, PyPy, and free-threaded Python are outside this matrix. The pinned PyTorch 2.13.0 distribution does not provide compatible wheels for most of these targets. The CPU index has s390x builds, but this project does not yet have an s390x build/test runner. + +Sources for the matrix: [PyTorch CPU wheels](https://download.pytorch.org/whl/cpu/torch/), [cibuildwheel configuration](https://cibuildwheel.pypa.io/en/stable/options/), and [GitHub runner availability](https://docs.github.com/en/actions/reference/runners/github-hosted-runners). + +## Building + +After the workflow is available in GitHub, run **Python wheels** manually. If the normalizer repository is private, provide `KITTEN_SUBMODULE_TOKEN` with read access to both repositories. Download the per-platform artifacts after the jobs pass. No workflow has been pushed or triggered as part of this implementation. + +Every wheel is installed into a separate test environment. The smoke test checks the Torch ABI version, packaged grammar data, native normalization, TorchScript loading/decoding, NumPy output, and an HTTPS config download without model credentials. Select the `model_test` workflow input for an additional real W4 generation on every wheel; private models require an `HF_TOKEN` Actions secret. A full release should pass that test on each platform first. + +To build a single target locally: + +```sh +python -m pip install cibuildwheel==4.3.0 +python -m cibuildwheel --only cp311-manylinux_x86_64 --output-dir wheelhouse +``` + +Linux cibuildwheel builds require Docker or Podman. macOS and Windows builds run on those operating systems with their compiler toolchains. Cross-platform wheel filenames cannot be obtained by renaming an existing wheel. + +For a native portable build without containers: + +```sh +KITTEN_PORTABLE=1 CMAKE_ARGS='-DLLAMA_BUILD_LIBRESSL=ON' \ + python setup.py build --build-base build-portable bdist_wheel +``` + +This produces a wheel for the build host. On Linux, use `packaging/repair_wheel.py` with auditwheel/patchelf installed to repair and assign its actual glibc compatibility tag. A wheel built on a newer host cannot simply be labeled manylinux_2_28. The local CPython 3.11 test produced a manylinux_2_35_x86_64 wheel, not the CI matrix's manylinux_2_28 target. + +## Installing an artifact + +Install the matching CPU PyTorch runtime, then the wheel for your platform and Python version: + +```sh +python -m pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cpu +python -m pip install /path/to/kitten_tts_cpp-0.1.0---.whl +``` + +No compiler, CMake, or exporter is needed to install the wheel. Model assets are still downloaded from the model repository on first use. The wheel includes the normalizer data and statically linked LibreSSL. It uses the separately installed PyTorch libraries rather than bundling another copy. Repair tools handle other native dependencies. HTTPS uses the certifi CA bundle by default; `SSL_CERT_FILE` or the constructor's `ca_file` parameter can select a custom trust bundle. This is why the wheel requires the exact PyTorch release it was compiled against and checks it before loading the native extension. + +Portable builds disable host-specific instruction flags, including AVX/AVX2/AVX-512 and AMX on x86, and use baseline GGML kernels. They prioritize compatibility and may be slower than local CPU-optimized builds. LibTorch retains its own CPU optimizations. Rebuild from source without `KITTEN_PORTABLE=1` for native CPU optimizations; quantized arithmetic can differ between kernel paths. diff --git a/packaging/prepare_wheel.py b/packaging/prepare_wheel.py new file mode 100644 index 0000000..a01a77e --- /dev/null +++ b/packaging/prepare_wheel.py @@ -0,0 +1,10 @@ +"""Install the same CPU Torch version for the build and the resulting wheel.""" +import subprocess +import sys + +subprocess.run([sys.executable, '-m', 'pip', 'install', 'torch==2.13.0', + '--index-url', 'https://download.pytorch.org/whl/cpu'], check=True) +subprocess.run([sys.executable, '-m', 'pip', 'install', 'setuptools>=68', 'wheel', + 'cmake>=3.21', 'numpy>=1.26', 'soundfile>=0.12', 'certifi>=2024.7.4'], check=True) +if sys.platform == 'win32': + subprocess.run([sys.executable, '-m', 'pip', 'install', 'delvewheel>=1.9'], check=True) diff --git a/packaging/repair_wheel.py b/packaging/repair_wheel.py new file mode 100644 index 0000000..4019221 --- /dev/null +++ b/packaging/repair_wheel.py @@ -0,0 +1,26 @@ +"""Repair system dependencies, leaving LibTorch supplied by the pinned torch wheel.""" +import importlib.util +import os +from pathlib import Path +import subprocess +import sys + +wheel, destination = sys.argv[1:] +torch_lib = Path(importlib.util.find_spec('torch').origin).parent / 'lib' +env = os.environ.copy() +if sys.platform == 'linux': + env['LD_LIBRARY_PATH'] = str(torch_lib) + os.pathsep + env.get('LD_LIBRARY_PATH', '') + command = ['auditwheel', 'repair', '-w', destination] + for library in sorted(torch_lib.glob('*.so*')): + command += ['--exclude', library.name] + command.append(wheel) +elif sys.platform == 'darwin': + command = ['delocate-wheel', '--require-archs', 'arm64', '--exclude', '/torch/lib/', + '-w', destination, wheel] +elif sys.platform == 'win32': + excluded = ';'.join(p.name for p in sorted(torch_lib.glob('*.dll'))) + command = [sys.executable, '-m', 'delvewheel', 'repair', '--add-path', str(torch_lib), + '--exclude', excluded, '-w', destination, wheel] +else: + raise RuntimeError(f'Unsupported wheel platform: {sys.platform}') +subprocess.run(command, env=env, check=True) diff --git a/packaging/smoke_wheel.py b/packaging/smoke_wheel.py new file mode 100644 index 0000000..c8bf278 --- /dev/null +++ b/packaging/smoke_wheel.py @@ -0,0 +1,66 @@ +"""Exercise an installed wheel without downloading model weights or importing source code.""" +import json +import os +from pathlib import Path +import tempfile + +import numpy as np +import torch +import kitten_tts_cpp +from kitten_tts_cpp import _native + +package = Path(kitten_tts_cpp.__file__).parent +info = json.loads((package / '_build_info.json').read_text()) +assert info['portable'], info +assert info['torch_version'] == torch.__version__.split('+')[0] +assert (package / 'data/en/tagger.fst').is_file() +assert (package / 'licenses/vendor_kitten-text-processing_LICENSE').is_file() + + +class Decoder(torch.nn.Module): + def forward(self, tokens, reference, features, embedding): + return tokens.float().flatten() / 10000 + features.sum() * 0 + embedding.sum() * 0 + reference.sum() * 0 + + +with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + inputs = (torch.tensor([[1, 2]]), torch.tensor([[0]]), torch.zeros(1, 1, 80), torch.zeros(1, 2)) + torch.jit.trace(Decoder(), inputs).save(str(root / 'decoder.pt')) + (root / 'config.json').write_text(json.dumps({ + 'type': 'KITTEN2', 'default_voice': 'Test', 'token_map': { + 'audio_id_base': 10, 'num_audio_tokens': 6561, + 'speech_start_id': 1, 'speech_end_id': 2, 'text_start_id': 3, + 'start_id': 4, 'stop_id': 5, + }, + })) + (root / 'voices.json').write_text(json.dumps({'Test': { + 'prompt_token': [[0]], 'prompt_feat': [[[0.0] * 80]], 'embedding': [[0.0, 0.0]], + }})) + tokens = root / 'tokens.json' + tokens.write_text('[1, 2]') + model = _native.Engine({'--assets': str(root), '--decode-tokens': str(tokens), + '--data': str(package / 'data'), '--threads': '1'}) + assert 'twelve' in model.normalize('It costs $12.').lower() + audio, report = model.generate({'--seed': '1234'}) + assert audio.dtype == np.float32 and audio.shape == (5,) + np.testing.assert_allclose(audio, np.array([1, 2, 4299, 4299, 4299], dtype=np.float32) / 10000) + assert json.loads(report)['samples'] == 5 + del model +print('Installed wheel: import, ABI, grammar, TorchScript decoding, and NumPy output passed') + +if os.environ.get('KITTEN_WHEEL_NETWORK_TEST') == '1': + with tempfile.TemporaryDirectory() as cache: + try: + kitten_tts_cpp.KittenTTS('gpt2', cache_dir=cache) + except RuntimeError: + pass + config = Path(cache) / 'gpt2/main/config.json' + assert config.is_file(), 'Native HTTPS config download failed' + assert json.loads(config.read_text())['model_type'] == 'gpt2' + print('Native HTTPS config download passed') + +if os.environ.get('KITTEN_WHEEL_MODEL_TEST') == '1': + with kitten_tts_cpp.KittenTTS(decoder='student_w4', threads=2) as model: + audio = model.generate('Hello from Kitten T T S.', max_new_tokens=100, advanced={'seed': 1234}) + assert audio.dtype == np.float32 and audio.size > 0 and np.isfinite(audio).all() + print('Real W4 model synthesis passed') diff --git a/pyproject.toml b/pyproject.toml index 46cf68c..eef2c32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,67 +1,41 @@ -[project] -name = "llama-cpp-scripts" -description = "Scripts that ship with llama.cpp" -keywords = ["ggml", "gguf", "llama.cpp"] -version = "0.0.0" -dynamic = ["classifiers"] -readme = "README.md" -authors = [{name = "GGML", email = "ggml@ggml.ai"}] -requires-python = '>=3.10,<3.15' -dependencies = [ - 'numpy (>=1.26.4,<3.0.0)', - 'sentencepiece (>=0.1.98,<0.3.0)', - 'transformers (==4.57.6)', - 'protobuf (>=4.21.0,<5.0.0)', - 'torch (>=2.6.0,<3.0.0)', - 'gguf @ ./gguf-py', -] -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", -] - -[project.urls] -homepage = "https://ggml.ai" -repository = "https://github.com/ggml-org/llama.cpp" - -[project.scripts] -llama-convert-hf-to-gguf = "convert_hf_to_gguf:main" -llama-convert-lora-to-gguf = "convert_lora_to_gguf:main" -llama-convert-llama-ggml-to-gguf = "convert_llama_ggml_to_gguf:main" -llama-ggml-vk-generate-shaders = "ggml_vk_generate_shaders:main" - -[tool.poetry] -packages = [ - { include = "*.py", from = "." }, - { include = "conversion", from = "." }, -] - -[tool.poetry.dependencies] -torch = [ - { version = "==2.11.0", source = "pypi", markers = "sys_platform == 'darwin'" }, - { version = "==2.11.0+cpu", source = "pytorch", markers = "sys_platform == 'linux'" }, - { version = "==2.11.0", source = "pypi", markers = "sys_platform == 'win32'" }, -] - -[tool.poetry.group.dev.dependencies] -pytest = "~=8.3.3" - -# Force wheel + cpu -# For discussion and context see https://github.com/python-poetry/poetry#6409 -[[tool.poetry.source]] -name = "pytorch" -url = "https://download.pytorch.org/whl/cpu" -priority = "explicit" - -[tool.uv.sources] -torch = { index = "pytorch" } - -[[tool.uv.index]] -name = "pytorch" -url = "https://download.pytorch.org/whl/cpu" -explicit = true - [build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" +requires = ["setuptools>=68", "wheel", "cmake>=3.21", "torch==2.13.0"] +build-backend = "setuptools.build_meta" + +[tool.cibuildwheel] +build = "cp3{10,11,12,13,14}-*" +skip = ["*-musllinux_*", "*-manylinux_i686", "*-manylinux_ppc64le", "*-manylinux_s390x", "*-manylinux_armv7l", "*-manylinux_riscv64", "*-win32", "*-macosx_x86_64", "*-macosx_universal2", "cp310-win_arm64", "cp314-win_arm64"] +build-frontend = { name = "pip", args = ["--no-build-isolation"] } +before-build = "python {project}/packaging/prepare_wheel.py" +repair-wheel-command = "python {project}/packaging/repair_wheel.py {wheel} {dest_dir}" +test-command = "python {project}/packaging/smoke_wheel.py" +test-requires = ["numpy", "soundfile"] + +[tool.cibuildwheel.environment] +KITTEN_PORTABLE = "1" +CMAKE_BUILD_PARALLEL_LEVEL = "2" +CMAKE_ARGS = "-DLLAMA_BUILD_LIBRESSL=ON" +PIP_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cpu" + +[tool.cibuildwheel.linux] +archs = ["x86_64", "aarch64"] +manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" + +[tool.cibuildwheel.macos] +archs = ["arm64"] + +[tool.cibuildwheel.macos.environment] +MACOSX_DEPLOYMENT_TARGET = "14.0" +KITTEN_PORTABLE = "1" +CMAKE_BUILD_PARALLEL_LEVEL = "2" +CMAKE_ARGS = "-DLLAMA_BUILD_LIBRESSL=ON" +PIP_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cpu" + +[tool.cibuildwheel.windows] +archs = ["AMD64", "ARM64"] + +[[tool.cibuildwheel.overrides]] +select = "*-win_arm64" +inherit.environment = "append" +environment = { CMAKE_ARGS = "-DLLAMA_BUILD_LIBRESSL=ON -T ClangCL -A ARM64" } diff --git a/python/kitten_tts_cpp/__init__.py b/python/kitten_tts_cpp/__init__.py new file mode 100644 index 0000000..5bde481 --- /dev/null +++ b/python/kitten_tts_cpp/__init__.py @@ -0,0 +1,5 @@ +"""KittenTTS2's Python interface backed by the CPU C++ runtime.""" +from .model import KittenTTS2, KittenTTS, get_model + +__version__ = '0.1.0' +__all__ = ['KittenTTS', 'KittenTTS2', 'get_model'] diff --git a/python/kitten_tts_cpp/model.py b/python/kitten_tts_cpp/model.py new file mode 100644 index 0000000..ac0e753 --- /dev/null +++ b/python/kitten_tts_cpp/model.py @@ -0,0 +1,283 @@ +import json +import math +import numbers +import os +from pathlib import Path +import queue +import threading + +import numpy as np +import certifi +import torch # Load LibTorch's shared dependencies before the extension. + +_build_info = Path(__file__).with_name('_build_info.json') +if _build_info.is_file(): + _required_torch = json.loads(_build_info.read_text())['torch_version'] + if torch.__version__.split('+')[0] != _required_torch: + raise ImportError(f'This kitten-tts-cpp wheel requires torch=={_required_torch}; install that version or rebuild from source') + +from . import _native + +_ADVANCED = { + 'seed': None, 'repetition_penalty': 1.1, 'repetition_window': None, + 'token_run_penalty': 1.3, 'token_run_grace': 10, 'chunk_chars': None, + 'chunk_min_chars': None, 'chunk_gap_s': 0.16, 'use_emotion': None, +} +_OPTIONS = { + 'seed': 'seed', 'repetition_penalty': 'repetition-penalty', + 'repetition_window': 'repetition-window', 'token_run_penalty': 'run-penalty', + 'token_run_grace': 'run-grace', 'chunk_chars': 'chunk-chars', + 'chunk_min_chars': 'chunk-min-chars', 'chunk_gap_s': 'chunk-gap', + 'use_emotion': 'use-emotion', +} + + +def _integer(name, value, minimum=0, maximum=2147483647): + if isinstance(value, bool) or not isinstance(value, numbers.Integral) or not minimum <= value <= maximum: + raise ValueError(f'{name} must be an integer in {minimum}..{maximum}') + + +def _finite(name, value): + if isinstance(value, bool) or not isinstance(value, numbers.Real) or not math.isfinite(value): + raise ValueError(f'{name} must be a finite number') + + +class KittenTTS2: + """Persistent CPU model. Uses the same native engine as the kitten-tts CLI. + + model_name is a Hugging Face repository or a local native asset directory. + Raw recording cloning and Python checkpoint weight formats are not supported. + """ + + def __init__(self, model_name='KittenML/kitten-tts-2', cache_dir=None, + device=None, hf_token=None, decoder=None, weights='packed', *, + threads=8, decoder_threads=None, revision=None, offline=False, + model_path=None, data_dir=None, repack=True, ca_file=None): + if device not in (None, 'cpu', torch.device('cpu')): + raise ValueError('kitten-tts-cpp supports device="cpu" only') + if weights != 'packed': + raise ValueError('Only weights="packed" is supported; use model_path for a custom GGUF') + _integer('threads', threads, 1) + decoder_threads = threads if decoder_threads is None else decoder_threads + _integer('decoder_threads', decoder_threads, 1) + args = {'--threads': str(threads), '--decoder-threads': str(decoder_threads), + '--data': str(data_dir or Path(__file__).parent / 'data'), + '--ca-file': str(ca_file or os.environ.get('SSL_CERT_FILE') or certifi.where())} + model_name = os.fspath(model_name) + if Path(model_name).is_dir(): + if revision is not None: + raise ValueError('revision applies only to a Hugging Face repository') + args['--assets'] = model_name + else: + args['--repo'] = model_name + if revision is not None: + args['--revision'] = revision + for key, value in [('cache-dir', cache_dir), ('decoder', decoder), + ('hf-token', hf_token), ('model', model_path)]: + if value is not None: + args['--' + key] = str(value) + if offline: + args['--offline'] = '1' + if not repack: + args['--no-repack'] = '1' + self._streams_lock = threading.Lock() + self._stops = set() + self._lock = threading.RLock() + self._engine = _native.Engine(args) + metadata = json.loads(self._engine.metadata()) + self.config = metadata['config'] + self._voices = metadata['voices'] + self.device = 'cpu' + self.decoder = decoder or self.config.get('cpp_decoder', self.config.get('default_decoder', 'default')) + self.last_report = None + + @property + def sample_rate(self): + return self.config.get('sample_rate', 24000) + + @property + def available_voices(self): + return list(self._voices) + + @property + def available_decoders(self): + return list(self.config.get('cpp', {}).get('decoders', {self.decoder: None})) + + @property + def available_weights(self): + return ['packed'] + + @property + def advanced_options(self): + result = dict(_ADVANCED) + generation = self.config.get('generation', {}) + for name, fallback in [('repetition_window', 0), ('chunk_chars', 380), ('chunk_min_chars', 130)]: + result[name] = generation.get(name, fallback) + return result + + def _options(self, text, voice=None, reference=None, reference_text=None, + preset=None, max_new_tokens=1000, normalize=True, + use_reference_prompt=None, temperature=None, top_k=None, + top_p=None, min_p=None, advanced=None): + if reference is not None and voice is not None: + raise ValueError('give either voice= or reference=, not both') + if reference_text is not None and reference is None: + raise ValueError('reference_text requires reference') + if reference is not None: + raise NotImplementedError('Raw-audio cloning requires offline prepare_reference.py; select the prepared voice by name') + if not isinstance(text, str): + raise TypeError('text must be a string') + if not text.strip(): + raise ValueError('no text to speak') + voice = voice or self.config.get('default_voice', 'Bruno') + if voice not in self._voices: + raise ValueError(f'unknown voice: {voice}') + preset = preset or self.config.get('default_preset', 'stable') + if preset not in self.config['decode_presets']: + raise ValueError(f'unknown preset: {preset}') + _integer('max_new_tokens', max_new_tokens, 1) + opts = dict(self.advanced_options) + if advanced is not None: + unknown = set(advanced) - set(opts) + if unknown: + raise ValueError(f'unknown advanced options: {sorted(unknown)}') + opts.update({key: value for key, value in advanced.items() if value is not None}) + for name in ('chunk_chars', 'chunk_min_chars'): + if opts[name] is None: + opts[name] = self.advanced_options[name] + if opts['repetition_window'] is None: + opts['repetition_window'] = 0 + for name, minimum in [('seed', 0), ('repetition_window', 0), ('token_run_grace', 1), ('chunk_chars', 1), ('chunk_min_chars', 0)]: + if opts[name] is not None: + _integer(name, opts[name], minimum) + for name in ('repetition_penalty', 'token_run_penalty', 'chunk_gap_s'): + _finite(name, opts[name]) + if opts['repetition_penalty'] <= 0 or opts['token_run_penalty'] < 0 or not 0 <= opts['chunk_gap_s'] <= 60: + raise ValueError('invalid repetition, run penalty, or chunk gap') + for name, value in [('normalize', normalize), ('use_reference_prompt', use_reference_prompt), ('use_emotion', opts['use_emotion'])]: + if (value is None and name != 'normalize') or isinstance(value, bool): + continue + raise ValueError(f'{name} must be a boolean' + ('' if name == 'normalize' else ' or None')) + if top_k is not None: + _integer('top_k', top_k) + for name, value in [('temperature', temperature), ('top_p', top_p), ('min_p', min_p)]: + if value is not None: + _finite(name, value) + if temperature is not None and temperature <= 0: + raise ValueError('temperature must be positive') + if top_p is not None and not 0 < top_p <= 1: + raise ValueError('top_p must be in (0, 1]') + if min_p is not None and not 0 <= min_p <= 1: + raise ValueError('min_p must be in [0, 1]') + args = {'--text': text, '--voice': voice, '--preset': preset, '--max-tokens': str(max_new_tokens)} + if not normalize: + args['--no-normalize'] = '1' + if use_reference_prompt is not None: + args['--use-reference'] = str(int(use_reference_prompt)) + for name, value in [('temperature', temperature), ('top-k', top_k), ('top-p', top_p), ('min-p', min_p)]: + if value is not None: + args['--' + name] = str(value) + for name, value in opts.items(): + if value is not None: + args['--' + _OPTIONS[name]] = str(int(value)) if isinstance(value, bool) else str(value) + return args + + def _run(self, args, callback=None): + with self._lock: + if self._engine is None: + raise RuntimeError('model is closed') + audio, report = self._engine.generate(args, callback) + self.last_report = json.loads(report) + return audio + + def generate(self, text, voice=None, reference=None, reference_text=None, + preset=None, max_new_tokens=1000, normalize=True, + use_reference_prompt=None, temperature=None, top_k=None, + top_p=None, min_p=None, advanced=None): + """Return a mono float32 NumPy array at sample_rate.""" + args = self._options(text, voice, reference, reference_text, preset, max_new_tokens, + normalize, use_reference_prompt, temperature, top_k, top_p, min_p, advanced) + return self._run(args) + + def generate_stream(self, text, voice=None, reference=None, reference_text=None, + preset=None, max_new_tokens=1000, normalize=True, + use_reference_prompt=None, temperature=None, top_k=None, + top_p=None, min_p=None, advanced=None): + """Yield untrimmed audio chunks as decoded, like KittenTTS2's Python API. + + Close the iterator if stopping early. Cancellation completes the current chunk. + """ + args = self._options(text, voice, reference, reference_text, preset, max_new_tokens, + normalize, use_reference_prompt, temperature, top_k, top_p, min_p, advanced) + messages = queue.Queue() + stopped = threading.Event() + + def put(kind, value): + if stopped.is_set(): + return False + messages.put((kind, value)) + return True + + def worker(): + try: + self._run(args, lambda audio: put('audio', audio)) + except Exception as exc: + put('error', exc) + finally: + put('done', None) + + with self._streams_lock: + self._stops.add(stopped) + thread = threading.Thread(target=worker, name='kitten-tts-cpp-stream', daemon=True) + thread.start() + try: + while True: + try: + kind, value = messages.get(timeout=0.05) + except queue.Empty: + if stopped.is_set(): + break + continue + if kind == 'done': + break + if kind == 'error': + raise value + yield value + finally: + stopped.set() + thread.join() + with self._streams_lock: + self._stops.discard(stopped) + + def generate_to_file(self, text, output_path, **kwargs): + import soundfile as sf + audio = self.generate(text, **kwargs) + sf.write(str(output_path), audio, self.sample_rate) + return str(output_path) + + def normalize_text(self, text): + with self._lock: + if self._engine is None: + raise RuntimeError('model is closed') + return self._engine.normalize(text) + + def close(self): + with self._streams_lock: + for stopped in self._stops: + stopped.set() + with self._lock: + self._engine = None + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + +KittenTTS = KittenTTS2 + + +def get_model(model_name='KittenML/kitten-tts-2', cache_dir=None, device=None, hf_token=None, + decoder=None, weights='packed', **kwargs): + return KittenTTS2(model_name, cache_dir, device, hf_token, decoder, weights, **kwargs) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d15eae5 --- /dev/null +++ b/setup.py @@ -0,0 +1,103 @@ +"""Build the KittenTTS2 CPU extension with CMake during package installation.""" +import os +import importlib.util +import importlib.metadata +import json +from pathlib import Path +import shlex +import shutil +import subprocess +import sys + +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext + +ROOT = Path(__file__).resolve().parent +TORCH_VERSION = importlib.metadata.version('torch').split('+')[0] + + +class CMakeBuild(build_ext): + def run(self): + super().run() + if self.inplace: + for directory in ('data', 'licenses'): + source = Path(self.build_lib) / 'kitten_tts_cpp' / directory + destination = Path(self.get_ext_fullpath('kitten_tts_cpp._native')).resolve().parent / directory + if source.resolve() != destination.resolve(): + shutil.copytree(source, destination, dirs_exist_ok=True) + shutil.copyfile(Path(self.build_lib) / 'kitten_tts_cpp/_build_info.json', + Path(self.get_ext_fullpath('kitten_tts_cpp._native')).resolve().parent / '_build_info.json') + + def build_extension(self, ext): + import torch + if not (ROOT / 'vendor/kitten-text-processing/CMakeLists.txt').is_file(): + raise RuntimeError('Initialize vendor/kitten-text-processing with git submodule update --init before installing') + output = Path(self.get_ext_fullpath(ext.name)).resolve().parent + build = Path(self.build_temp).resolve() / 'kitten' + build.mkdir(parents=True, exist_ok=True) + output.mkdir(parents=True, exist_ok=True) + if importlib.util.find_spec('cmake'): + cmake = [sys.executable, '-m', 'cmake'] + elif shutil.which('cmake'): + cmake = [shutil.which('cmake')] + else: + raise RuntimeError('CMake is required; install it with pip install cmake') + args = [ + '-DCMAKE_BUILD_TYPE=Release', '-DBUILD_SHARED_LIBS=OFF', + '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', '-DBUILD_TESTING=OFF', + '-DLLAMA_BUILD_COMMON=ON', '-DLLAMA_BUILD_TOOLS=ON', + '-DLLAMA_BUILD_KITTEN_TTS=ON', '-DKITTEN_BUILD_PYTHON=ON', + '-DLLAMA_BUILD_TESTS=OFF', '-DLLAMA_BUILD_EXAMPLES=OFF', + '-DLLAMA_BUILD_SERVER=OFF', '-DLLAMA_BUILD_MTMD=OFF', '-DLLAMA_BUILD_APP=OFF', + '-DGGML_CUDA=OFF', '-DGGML_METAL=OFF', '-DGGML_BACKEND_DL=OFF', + '-DGGML_NATIVE=ON', '-DLLAMA_OPENSSL=ON', + f'-DPython3_EXECUTABLE={sys.executable}', + f'-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}', + f'-DKITTEN_PYTHON_OUTPUT_DIR={output}', + ] + if (Path(sys.base_prefix) / 'include/openssl/ssl.h').is_file(): + args.append(f'-DOPENSSL_ROOT_DIR={sys.base_prefix}') + args += shlex.split(os.environ.get('CMAKE_ARGS', '')) + portable = os.environ.get('KITTEN_PORTABLE', '0') == '1' + if portable: + args += ['-DGGML_NATIVE=OFF', '-DGGML_OPENMP=OFF'] + for feature in ('SSE42', 'AVX', 'AVX2', 'AVX_VNNI', 'AVX512', 'AVX512_VBMI', + 'AVX512_VNNI', 'AVX512_BF16', 'FMA', 'F16C', 'BMI2', + 'AMX_TILE', 'AMX_INT8', 'AMX_BF16'): + args.append(f'-DGGML_{feature}=OFF') + subprocess.run([*cmake, '-S', str(ROOT), '-B', str(build), *args], check=True) + subprocess.run([*cmake, '--build', str(build), '--config', 'Release', '--target', '_native', + '--parallel', os.environ.get('CMAKE_BUILD_PARALLEL_LEVEL', '4')], check=True) + data = build / 'tools/kitten-tts/text-processing/data' + if not (data / 'en/tagger.fst').is_file(): + raise RuntimeError('Native normalizer grammar data was not generated') + shutil.copytree(data, output / 'data', dirs_exist_ok=True) + (output / '_build_info.json').write_text(json.dumps({ + 'torch_version': TORCH_VERSION, 'portable': portable, + }, indent=2) + '\n') + licenses = output / 'licenses' + licenses.mkdir(exist_ok=True) + for relative in ('LICENSE', 'tools/kitten-tts/LICENSE-KITTENTTS', + 'vendor/kitten-text-processing/LICENSE', + 'vendor/kitten-text-processing/LICENSE-SACREMOSES', + 'vendor/kitten-text-processing/NOTICE'): + shutil.copyfile(ROOT / relative, licenses / relative.replace('/', '_')) + for backend in ('libressl', 'boringssl'): + for filename in ('COPYING', 'LICENSE'): + source = build / '_deps' / f'{backend}-src' / filename + if source.is_file(): + shutil.copyfile(source, licenses / f'{backend}-{filename}') + + +setup( + name='kitten-tts-cpp', version='0.1.0', + description='CPU KittenTTS2 inference with native C++ bindings', + long_description=(ROOT / 'README.md').read_text(), long_description_content_type='text/markdown', + author='KittenML', url='https://github.com/KittenML/kitten-tts-2-cpp', + python_requires='>=3.10', + packages=['kitten_tts_cpp'], package_dir={'': 'python'}, + ext_modules=[Extension('kitten_tts_cpp._native', sources=[])], + cmdclass={'build_ext': CMakeBuild}, + install_requires=['numpy>=1.26', f'torch=={TORCH_VERSION}', 'soundfile>=0.12', 'certifi>=2024.7.4'], + include_package_data=True, zip_safe=False, +) diff --git a/tools/kitten-tts/CMakeLists.txt b/tools/kitten-tts/CMakeLists.txt index 125e91b..a308197 100644 --- a/tools/kitten-tts/CMakeLists.txt +++ b/tools/kitten-tts/CMakeLists.txt @@ -8,15 +8,18 @@ add_library(kitten-decoder STATIC decoder.cpp) target_include_directories(kitten-decoder PRIVATE ${PROJECT_SOURCE_DIR}/vendor/nlohmann) target_link_libraries(kitten-decoder PUBLIC ${TORCH_LIBRARIES}) target_compile_features(kitten-decoder PUBLIC cxx_std_17) -add_executable(kitten-tts main.cpp assets.cpp) +add_library(kitten-engine STATIC engine.cpp assets.cpp) +target_include_directories(kitten-engine PUBLIC ${PROJECT_SOURCE_DIR}/vendor/nlohmann) +target_link_libraries(kitten-engine PUBLIC llama-common llama kitten-text kitten-decoder) +add_executable(kitten-tts main.cpp) target_include_directories(kitten-tts PRIVATE ${PROJECT_SOURCE_DIR}/vendor/nlohmann) -target_link_libraries(kitten-tts PRIVATE llama-common llama kitten-text kitten-decoder) +target_link_libraries(kitten-tts PRIVATE kitten-engine) if(KITTEN_DATA_DIR) set(kitten_data "${KITTEN_DATA_DIR}") else() set(kitten_data "${CMAKE_CURRENT_BINARY_DIR}/text-processing/data") endif() -target_compile_definitions(kitten-tts PRIVATE KITTEN_DEFAULT_DATA="${kitten_data}") +target_compile_definitions(kitten-engine PUBLIC KITTEN_DEFAULT_DATA="${kitten_data}") set(KITTEN_REFERENCE_PACKAGE "" CACHE PATH "KittenTTS Python checkout for differential tests") set(KITTEN_TEST_PYTHON "python3" CACHE FILEPATH "Python with KittenTTS2 conversion dependencies") @@ -36,3 +39,29 @@ if(LLAMA_BUILD_TESTS) set_tests_properties(kitten.assets PROPERTIES TIMEOUT 120) endif() endif() + +option(KITTEN_BUILD_PYTHON "Build the Python extension" OFF) +if(KITTEN_BUILD_PYTHON) + get_target_property(http_definitions cpp-httplib INTERFACE_COMPILE_DEFINITIONS) + if(NOT "CPPHTTPLIB_OPENSSL_SUPPORT" IN_LIST http_definitions) + message(FATAL_ERROR "The Python package requires HTTPS. Install OpenSSL development headers or configure LLAMA_BUILD_BORINGSSL/LLAMA_BUILD_LIBRESSL.") + endif() + find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module) + Python3_add_library(_native MODULE WITH_SOABI python.cpp) + target_include_directories(_native PRIVATE ${TORCH_INCLUDE_DIRS}) + target_link_libraries(_native PRIVATE kitten-engine) + set_target_properties(_native PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${KITTEN_PYTHON_OUTPUT_DIR}" + RUNTIME_OUTPUT_DIRECTORY "${KITTEN_PYTHON_OUTPUT_DIR}") + foreach(config DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + set_target_properties(_native PROPERTIES + LIBRARY_OUTPUT_DIRECTORY_${config} "${KITTEN_PYTHON_OUTPUT_DIR}" + RUNTIME_OUTPUT_DIRECTORY_${config} "${KITTEN_PYTHON_OUTPUT_DIR}") + endforeach() + if(APPLE) + set_target_properties(_native PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_RPATH "@loader_path/../torch/lib") + elseif(UNIX) + target_link_options(_native PRIVATE "-Wl,--exclude-libs,ALL") + set_target_properties(_native PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_RPATH "$ORIGIN/../torch/lib") + endif() +endif() diff --git a/tools/kitten-tts/README.md b/tools/kitten-tts/README.md index 24ffd0d..6744b21 100644 --- a/tools/kitten-tts/README.md +++ b/tools/kitten-tts/README.md @@ -17,14 +17,19 @@ Alternatively, set `KITTEN_DATA_DIR` to an already prepared grammar directory. ```sh git submodule update --init vendor/kitten-text-processing -python -m cmake -S . -B build \ + +cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON \ -DGGML_CUDA=OFF -DGGML_METAL=OFF \ -DLLAMA_BUILD_KITTEN_TTS=ON \ - -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ -DLLAMA_BUILD_EXAMPLES=OFF \ -DCMAKE_PREFIX_PATH="$(python -c 'import torch; print(torch.utils.cmake_prefix_path)')" -python -m cmake --build build --target kitten-tts -j 8 + +cmake --build build --target kitten-tts -j 8 ``` The GitHub normalizer URL was unavailable during initial setup. This workspace's @@ -185,8 +190,7 @@ The same seed does not produce the same sampled tokens in both implementations. FP16 is available when minimizing LM numerical differences matters more than speed. All 310 packed GGUF tensors were verified to reconstruct the FP16 reference exactly, including all 196 packed transformer matrices. -The runtime is a batch CLI, not a drop-in implementation of the Python streaming -API. The decoder currently depends on LibTorch rather than a GGML S3 implementation. +The CLI runs batch synthesis. The Python binding supports sentence-chunk streaming through the same native engine. The decoder currently depends on LibTorch rather than a GGML S3 implementation. Additional format checks: @@ -212,3 +216,17 @@ Decoder exports come from the Python package's S3 implementation and preserve it ResembleAI, Alibaba/CosyVoice and Matcha-TTS provenance. The normalizer retains its own license and notices in the submodule. The underlying llama.cpp fork retains its MIT license. + +## Python binding development + +The package at `python/kitten_tts_cpp` wraps `kitten::engine` through a pybind11 extension. The CLI and binding share model loading, token generation, waveform decoding, and audio joins. `setup.py` builds a CPU extension using the pybind11 headers shipped with PyTorch, links the project libraries statically, and bundles the normalizer grammars. LibTorch remains a dependency supplied by the installed PyTorch package. + +```sh +python setup.py build_ext --inplace +PYTHONPATH=python \ + KITTEN_TEST_ASSETS=models/kitten2-student-w4 \ + KITTEN_TEST_GGUF=models/kitten2/model-tq2_1.gguf \ + python tools/kitten-tts/test_python.py +``` + +The integration tests cover actual native inference, CLI token/audio parity, deterministic repeated calls, file output, input validation, and streaming cancellation. Run `test_assets.py` and `check_parity.py` as well after modifying the shared engine. Source distributions include the normalizer source/data, so building from a source archive does not require Git submodule access. diff --git a/tools/kitten-tts/assets.cpp b/tools/kitten-tts/assets.cpp index ee38713..d3c00bc 100644 --- a/tools/kitten-tts/assets.cpp +++ b/tools/kitten-tts/assets.cpp @@ -33,7 +33,8 @@ assets resolve_assets(const std::map & args) { fs::path root = local ? fs::path(args.at("--assets")) : fs::path(get("--cache-dir", env("XDG_CACHE_HOME", env("HOME", ".") + "/.cache") + "/kitten-tts")) / repo / cache_revision; common_download_opts opts; opts.offline = args.count("--offline"); - opts.bearer_token = env("HF_TOKEN"); + opts.bearer_token = get("--hf-token", env("HF_TOKEN")); + opts.ca_cert_file = get("--ca-file", env("SSL_CERT_FILE")); std::string base = env("HF_ENDPOINT", "https://huggingface.co") + "/" + repo + "/resolve/" + revision + "/"; auto fetch = [&](const std::string & file) { check_path(file); diff --git a/tools/kitten-tts/decoder.cpp b/tools/kitten-tts/decoder.cpp index 86c801b..98d0dab 100644 --- a/tools/kitten-tts/decoder.cpp +++ b/tools/kitten-tts/decoder.cpp @@ -3,12 +3,12 @@ #include #include namespace kitten { -struct decoder::impl { torch::jit::Module module; explicit impl(const std::string & path): module(torch::jit::load(path,torch::kCPU)) { module.eval(); } }; +struct decoder::impl { int threads; torch::jit::Module module; explicit impl(const std::string & path, int n_threads): threads(n_threads), module(torch::jit::load(path,torch::kCPU)) { module.eval(); } }; decoder::decoder(const std::string & path,int threads,int seed) { - at::set_num_threads(threads); at::manual_seed(seed); p=std::make_unique(path); + at::set_num_threads(threads); at::manual_seed(seed); p=std::make_unique(path,threads); } decoder::~decoder() = default; -void decoder::seed(int value) { at::manual_seed(value); } +void decoder::seed(int value) { at::set_num_threads(p->threads); at::manual_seed(value); } std::vector decoder::decode(const std::vector & ids,const std::string & conditioning_json) { if(ids.empty()) return {}; c10::InferenceMode guard; diff --git a/tools/kitten-tts/engine.cpp b/tools/kitten-tts/engine.cpp new file mode 100644 index 0000000..6d9de2f --- /dev/null +++ b/tools/kitten-tts/engine.cpp @@ -0,0 +1,174 @@ +#include "engine.h" +#include "llama.h" +#include "text.h" +#include "sampling.h" +#include "decoder.h" +#include +#include +#include +#include + +namespace kitten { +using clock_type = std::chrono::steady_clock; +static json read_json(const std::string & path) { + std::ifstream f(path); if(!f) throw std::runtime_error("cannot open "+path); json j; f>>j;return j; +} +// LibTorch decoder threads and random state are process-wide. +static std::mutex inference_mutex; +static std::vector tokenize(const llama_vocab * vocab,const std::string & text) { + int n=llama_tokenize(vocab,text.data(),text.size(),nullptr,0,false,true); + if(n==0) return {}; + std::vector ids(-n); n=llama_tokenize(vocab,text.data(),text.size(),ids.data(),ids.size(),false,true); + if(n<0) throw std::runtime_error("tokenization failed"); + ids.resize(n);return ids; +} +struct batch { + llama_batch b; + batch(int n,int dim=0): b(llama_batch_init(n,dim,1)) {} + ~batch(){llama_batch_free(b);} + void position(int i,int pos,bool logits) {b.pos[i]=pos;b.n_seq_id[i]=1;b.seq_id[i][0]=0;b.logits[i]=logits;} +}; + +struct engine::impl { + std::map defaults; + kitten::assets assets; + json config, voices; + std::unique_ptr normalizer; + std::unique_ptr model{nullptr, llama_model_free}; + std::unique_ptr decoder; +}; +engine::engine(const std::map & args): p(std::make_unique()) { + std::lock_guard lock(inference_mutex); + auto get=[&](const std::string & key,const std::string & fallback){auto it=args.find(key);return it==args.end()?fallback:it->second;}; + p->defaults=args; + p->defaults.erase("--hf-token"); + int threads=std::stoi(get("--threads","8")), decoder_threads=std::stoi(get("--decoder-threads",std::to_string(threads))); + if(threads<1 || decoder_threads<1) throw std::invalid_argument("thread counts must be positive"); + p->assets=resolve_assets(args); + p->config=p->assets.config; + p->voices=read_json(p->assets.voices); + p->normalizer=std::make_unique(get("--data",KITTEN_DEFAULT_DATA)); + static std::once_flag initialized; + std::call_once(initialized, [] { ggml_backend_load_all(); llama_backend_init(); }); + if(!args.count("--decode-tokens")) { + auto mp=llama_model_default_params();mp.n_gpu_layers=0;mp.use_extra_bufts=!args.count("--no-repack"); + p->model.reset(llama_model_load_from_file(p->assets.model.c_str(),mp)); + if(!p->model) throw std::runtime_error("cannot load language model"); + token_map tm(p->config.at("token_map")); + if(tm.base+tm.count>llama_vocab_n_tokens(llama_model_get_vocab(p->model.get()))) throw std::runtime_error("token map exceeds vocabulary"); + } + if(!p->assets.decoder.empty()) p->decoder=std::make_unique(p->assets.decoder,decoder_threads,0); +} +engine::~engine() = default; +json engine::metadata() const { + json names=json::array(); for(auto it=p->voices.begin();it!=p->voices.end();++it) names.push_back(it.key()); + return {{"config",p->config},{"voices",names}}; +} +std::string engine::normalize(const std::string & text) { + std::lock_guard lock(inference_mutex); + return kitten::normalize(text,*p->normalizer,true); +} +generation_result engine::generate(const std::map & options, + const std::function &)> & on_chunk) { + std::lock_guard lock(inference_mutex); + auto args=p->defaults; for(const auto & pair:options) args[pair.first]=pair.second; + auto get=[&](const std::string & key,const std::string & fallback){auto it=args.find(key);return it==args.end()?fallback:it->second;}; + const auto & config=p->config; + const auto & voices=p->voices; + auto & model=p->model; + auto & decoder=p->decoder; + auto & normalizer=*p->normalizer; + token_map tm(config.at("token_map")); + std::string voice=get("--voice",config.value("default_voice","Bruno")); + if(!voices.contains(voice)) throw std::invalid_argument("unknown voice: "+voice); + const auto & ref=voices.at(voice); + int threads=std::stoi(get("--threads","8")); + int seed=std::stoi(get("--seed",std::to_string(std::random_device{}() % 2147483647))), max_tokens=std::stoi(get("--max-tokens","1000")); + if(threads<1 || max_tokens<1) throw std::invalid_argument("thread and token counts must be positive"); + if(args.count("--decode-tokens")) { + auto tokens=read_json(args.at("--decode-tokens")).get>(); + tokens.erase(std::remove_if(tokens.begin(),tokens.end(),[&](int id){return id<0 || id>=tm.count;}),tokens.end()); + if(tokens.empty()) throw std::runtime_error("no valid codec tokens"); + if(!decoder) throw std::runtime_error("decoder is not loaded"); + decoder->seed(seed); + auto begin=clock_type::now(); + auto audio=decoder->decode(tokens,ref.dump()); + json report={{"samples",audio.size()},{"decoder_seconds",std::chrono::duration(clock_type::now()-begin).count()}}; + return {std::move(audio),std::move(report)}; + } + if(!model) throw std::runtime_error("language model is not loaded"); + auto gen=config.value("generation",json::object()); + int chunk_chars=std::stoi(get("--chunk-chars",std::to_string(gen.value("chunk_chars",380)))); + int chunk_min=std::stoi(get("--chunk-min-chars",std::to_string(gen.value("chunk_min_chars",130)))); + float gap=std::stof(get("--chunk-gap","0.16")); + if(!std::isfinite(gap) || gap<0 || gap>60) throw std::runtime_error("invalid chunk gap"); + auto spoken=kitten::normalize(args.at("--text"),normalizer,!args.count("--no-normalize")); + auto chunks=kitten::split(spoken,chunk_chars,chunk_min); + if(chunks.empty()) throw std::runtime_error("no text to speak"); + auto preset=config.at("decode_presets").at(get("--preset",config.value("default_preset","stable"))); + kitten::sampling settings; + settings.temperature=std::stof(get("--temperature",std::to_string(preset.at("temperature").get()))); + settings.top_k=std::stoi(get("--top-k",std::to_string(preset.at("top_k").get()))); + settings.top_p=std::stof(get("--top-p",std::to_string(preset.at("top_p").get()))); + settings.min_p=std::stof(get("--min-p",std::to_string(preset.value("min_p",0.f)))); + settings.repetition=std::stof(get("--repetition-penalty","1.1"));settings.window=std::stoi(get("--repetition-window",std::to_string(gen.value("repetition_window",0)))); + settings.run_penalty=std::stof(get("--run-penalty","1.3"));settings.grace=std::stoi(get("--run-grace","10")); + if(!std::isfinite(settings.temperature) || settings.temperature<=0 || settings.top_k<0 || !(settings.top_p>0 && settings.top_p<=1) || !(settings.min_p>=0 && settings.min_p<=1) || !std::isfinite(settings.repetition) || settings.repetition<=0 || !std::isfinite(settings.run_penalty) || settings.run_penalty<0 || settings.grace<1) throw std::runtime_error("invalid sampling settings"); + auto vocab=llama_model_get_vocab(model.get());int n_vocab=llama_vocab_n_tokens(vocab); + auto speaker=ref.at("speaker").get>(); + if(speaker.size()!=size_t(llama_model_n_embd(model.get()))) throw std::runtime_error("speaker projection dimension mismatch"); + auto ref_text=tokenize(vocab,ref.at("transcript").get()); + auto ref_audio=ref.at("reference_tokens").get>(); + std::vector emotion; + if((args.count("--use-emotion") ? args.at("--use-emotion")=="1" : kitten::has_expression(args.at("--text"))) && gen.contains("emotion_control")) emotion=tokenize(vocab,gen.at("emotion_control")); + std::mt19937 rng(seed); + std::vector> waves; + json report={{"normalized",spoken},{"voice",voice},{"sample_rate",24000},{"chunks",json::array()}}; + std::ofstream logits_file; + if(args.count("--logits")) {logits_file.open(args.at("--logits"),std::ios::binary);if(!logits_file) throw std::runtime_error("cannot open logits output");} + std::vector forced; + if(args.count("--force-tokens")) {forced=read_json(args.at("--force-tokens")).get>();if(chunks.size()!=1) throw std::runtime_error("teacher forcing requires one chunk");} + report["iteration"]=1; report["seed"]=seed; + if(decoder) decoder->seed(seed); + double lm_seconds=0,decoder_seconds=0;size_t total_tokens=0; + for(auto & chunk:chunks) { + auto ids=kitten::prompt(tm,tokenize(vocab,chunk),ref_text,ref_audio,emotion,args.count("--use-reference") ? args.at("--use-reference")=="1" : (!args.count("--no-reference") && gen.value("use_reference_prompt",true))); + int budget=chunks.size()==1?max_tokens:std::max(200,std::min(max_tokens,int(kitten::characters(chunk)/20.0*25*1.8))); + if(args.count("--force-tokens")) budget=forced.size()+1; + auto cp=llama_context_default_params();cp.n_ctx=ids.size()+budget;cp.n_batch=512;cp.n_ubatch=512;cp.n_threads=threads;cp.n_threads_batch=threads;cp.no_perf=false; + std::unique_ptr ctx(llama_init_from_model(model.get(),cp),llama_free); + if(!ctx) throw std::runtime_error("cannot initialize context"); + auto begin=clock_type::now(); + { batch b(1,speaker.size());b.b.n_tokens=1;std::copy(speaker.begin(),speaker.end(),b.b.embd);b.position(0,0,false); + if(llama_decode(ctx.get(),b.b)) throw std::runtime_error("speaker prefill failed"); } + for(size_t offset=1;offset audio_ids,generated;bool ended=false; + for(int step=0;step(raw),n_vocab*sizeof(float)); + if(args.count("--force-tokens") && step==int(forced.size())) break; + std::vector scores(raw,raw+n_vocab);kitten::process(scores,history,ids.size(),tm,settings); + int token=args.count("--force-tokens")?forced.at(step):kitten::sample(scores,rng,args.count("--greedy")); + if(token<0 || token>=n_vocab) throw std::runtime_error("forced token out of range"); + generated.push_back(token);history.push_back(token); + if(!args.count("--force-tokens") && (token==tm.speech_end || token==tm.stop)) {ended=true;break;} + if(token>=tm.base && token(clock_type::now()-begin).count();lm_seconds+=elapsed;total_tokens+=generated.size(); + report["chunks"].push_back({{"text",chunk},{"prompt",ids},{"generated",generated},{"audio_tokens",audio_ids},{"terminated",ended},{"lm_seconds",elapsed}}); + llama_perf_context_print(ctx.get()); + if(decoder) {begin=clock_type::now();auto wave=decoder->decode(audio_ids,ref.dump());decoder_seconds+=std::chrono::duration(clock_type::now()-begin).count(); if(on_chunk && !wave.empty() && !on_chunk(wave)) throw std::runtime_error("generation cancelled"); waves.push_back(std::move(wave));} + } + report["lm_seconds"]=lm_seconds;report["decoder_seconds"]=decoder_seconds;report["generated_tokens"]=total_tokens; + generation_result result; + if(decoder) {result.audio=kitten::join(waves,gap);report["audio_seconds"]=result.audio.size()/24000.0;report["rtf"]=(lm_seconds+decoder_seconds)/(result.audio.size()/24000.0);} + result.report=std::move(report); + return result; +} +} diff --git a/tools/kitten-tts/engine.h b/tools/kitten-tts/engine.h new file mode 100644 index 0000000..87feb51 --- /dev/null +++ b/tools/kitten-tts/engine.h @@ -0,0 +1,23 @@ +#pragma once +#include "assets.h" +#include +#include +#include + +namespace kitten { +struct generation_result { + std::vector audio; + json report; +}; +class engine { + struct impl; + std::unique_ptr p; +public: + explicit engine(const std::map & options); + ~engine(); + json metadata() const; + std::string normalize(const std::string & text); + generation_result generate(const std::map & options, + const std::function &)> & on_chunk = {}); +}; +} diff --git a/tools/kitten-tts/main.cpp b/tools/kitten-tts/main.cpp index 39db846..74e0165 100644 --- a/tools/kitten-tts/main.cpp +++ b/tools/kitten-tts/main.cpp @@ -3,6 +3,7 @@ #include "sampling.h" #include "decoder.h" #include "assets.h" +#include "engine.h" #include #include #include @@ -14,9 +15,6 @@ #include using kitten::json; using clock_type = std::chrono::steady_clock; -static json read_json(const std::string & path) { - std::ifstream f(path); if(!f) throw std::runtime_error("cannot open "+path); json j; f>>j;return j; -} static void write_json(const std::string & path,const json & j) { std::ofstream f(path); f< & audio) { std::ofstream f(path,std::ios::binary); @@ -27,19 +25,6 @@ static void write_wav(const std::string & path,const std::vector & audio) for(float x:audio) {uint32_t bits; std::memcpy(&bits,&x,4);u32(bits);} if(!f) throw std::runtime_error("cannot write "+path); } -static std::vector tokenize(const llama_vocab * vocab,const std::string & text) { - int n=llama_tokenize(vocab,text.data(),text.size(),nullptr,0,false,true); - if(n==0) return {}; - std::vector ids(-n); n=llama_tokenize(vocab,text.data(),text.size(),ids.data(),ids.size(),false,true); - if(n<0) throw std::runtime_error("tokenization failed"); - ids.resize(n);return ids; -} -struct batch { - llama_batch b; - batch(int n,int dim=0): b(llama_batch_init(n,dim,1)) {} - ~batch(){llama_batch_free(b);} - void position(int i,int pos,bool logits) {b.pos[i]=pos;b.n_seq_id[i]=1;b.seq_id[i][0]=0;b.logits[i]=logits;} -}; int main(int argc,char ** argv) try { std::setlocale(LC_ALL,"C.UTF-8"); try { std::locale::global(std::locale("C.UTF-8")); } catch (const std::runtime_error &) {} @@ -72,8 +57,8 @@ int main(int argc,char ** argv) try { auto get=[&](const std::string & key,const std::string & fallback){auto it=args.find(key);return it==args.end()?fallback:it->second;}; const std::vector known={"--repo","--revision","--decoder","--cache-dir","--assets","--model","--text","--voice","--output","--data","--threads","--decoder-threads","--seed","--preset","--max-tokens","--temperature","--top-k","--top-p","--min-p","--repetition-penalty","--repetition-window","--run-penalty","--run-grace","--chunk-chars","--chunk-min-chars","--chunk-gap","--report","--logits","--force-tokens","--decode-tokens","--repeat"}; for(auto & a:args) if(std::find(flags.begin(),flags.end(),a.first)==flags.end() && std::find(known.begin(),known.end(),a.first)==known.end()) throw std::runtime_error("unknown option "+a.first); - kitten_text_processing::Normalizer normalizer(get("--data",KITTEN_DEFAULT_DATA)); if(args.count("--frontend")) { + kitten_text_processing::Normalizer normalizer(get("--data",KITTEN_DEFAULT_DATA)); std::string line; while(std::getline(std::cin,line)) { try { @@ -104,110 +89,17 @@ int main(int argc,char ** argv) try { return 0; } if(!args.count("--download-only") && !args.count("--text") && !args.count("--decode-tokens")) throw std::runtime_error("--text or --decode-tokens is required"); - auto assets=kitten::resolve_assets(args); - if(args.count("--download-only")) { std::cout<<"Assets ready\n"; return 0; } - auto config=assets.config; - kitten::token_map tm(config.at("token_map")); - auto voices=read_json(assets.voices); - std::string voice=get("--voice",config.value("default_voice","Bruno")); - if(!voices.contains(voice)) throw std::runtime_error("unknown voice: "+voice); - const auto & ref=voices.at(voice); - int threads=std::stoi(get("--threads","8")), decoder_threads=std::stoi(get("--decoder-threads",std::to_string(threads))); - int seed=std::stoi(get("--seed",std::to_string(std::random_device{}() % 2147483647))), max_tokens=std::stoi(get("--max-tokens","1000")); - if(threads<1 || decoder_threads<1 || max_tokens<1) throw std::runtime_error("thread and token counts must be positive"); - if (args.count("--decode-tokens")) { - auto tokens=read_json(args.at("--decode-tokens")).get>(); - tokens.erase(std::remove_if(tokens.begin(),tokens.end(),[&](int id){return id<0 || id>=tm.count;}),tokens.end()); - if(tokens.empty()) throw std::runtime_error("no valid codec tokens"); - kitten::decoder dec(assets.decoder,decoder_threads,seed); - auto begin=clock_type::now(); - auto audio=dec.decode(tokens,ref.dump()); - write_wav(get("--output","output.wav"),audio); - std::cout<(clock_type::now()-begin).count()}}.dump()<<'\n'; - return 0; - } - auto gen=config.value("generation",json::object()); - int chunk_chars=std::stoi(get("--chunk-chars",std::to_string(gen.value("chunk_chars",380)))); - int chunk_min=std::stoi(get("--chunk-min-chars",std::to_string(gen.value("chunk_min_chars",130)))); - float gap=std::stof(get("--chunk-gap","0.16")); - if(!std::isfinite(gap) || gap<0 || gap>60) throw std::runtime_error("invalid chunk gap"); - auto spoken=kitten::normalize(args.at("--text"),normalizer,!args.count("--no-normalize")); - auto chunks=kitten::split(spoken,chunk_chars,chunk_min); - if(chunks.empty()) throw std::runtime_error("no text to speak"); - auto preset=config.at("decode_presets").at(get("--preset",config.value("default_preset","stable"))); - kitten::sampling settings; - settings.temperature=std::stof(get("--temperature",std::to_string(preset.at("temperature").get()))); - settings.top_k=std::stoi(get("--top-k",std::to_string(preset.at("top_k").get()))); - settings.top_p=std::stof(get("--top-p",std::to_string(preset.at("top_p").get()))); - settings.min_p=std::stof(get("--min-p",std::to_string(preset.value("min_p",0.f)))); - settings.repetition=std::stof(get("--repetition-penalty","1.1"));settings.window=std::stoi(get("--repetition-window",std::to_string(gen.value("repetition_window",0)))); - settings.run_penalty=std::stof(get("--run-penalty","1.3"));settings.grace=std::stoi(get("--run-grace","10")); - if(!std::isfinite(settings.temperature) || settings.temperature<=0 || settings.top_k<0 || !(settings.top_p>0 && settings.top_p<=1) || !(settings.min_p>=0 && settings.min_p<=1) || !std::isfinite(settings.repetition) || settings.repetition<=0 || !std::isfinite(settings.run_penalty) || settings.run_penalty<0 || settings.grace<1) throw std::runtime_error("invalid sampling settings"); - ggml_backend_load_all();llama_backend_init(); - auto mp=llama_model_default_params();mp.n_gpu_layers=0;mp.use_extra_bufts=!args.count("--no-repack"); - std::unique_ptr model(llama_model_load_from_file(assets.model.c_str(),mp),llama_model_free); - if(!model) throw std::runtime_error("cannot load language model"); - auto vocab=llama_model_get_vocab(model.get());int n_vocab=llama_vocab_n_tokens(vocab); - if(tm.base+tm.count>n_vocab) throw std::runtime_error("token map exceeds vocabulary"); - auto speaker=ref.at("speaker").get>(); - if(speaker.size()!=size_t(llama_model_n_embd(model.get()))) throw std::runtime_error("speaker projection dimension mismatch"); - auto ref_text=tokenize(vocab,ref.at("transcript").get()); - auto ref_audio=ref.at("reference_tokens").get>(); - std::vector emotion; - if(kitten::has_expression(args.at("--text")) && gen.contains("emotion_control")) emotion=tokenize(vocab,gen.at("emotion_control")); - std::mt19937 rng(seed); - std::unique_ptr decoder; - if(!args.count("--tokens-only")) decoder=std::make_unique(assets.decoder,decoder_threads,seed); - std::vector> waves; - json report={{"normalized",spoken},{"voice",voice},{"sample_rate",24000},{"chunks",json::array()}}; - std::ofstream logits_file; - if(args.count("--logits")) {logits_file.open(args.at("--logits"),std::ios::binary);if(!logits_file) throw std::runtime_error("cannot open logits output");} - std::vector forced; - if(args.count("--force-tokens")) {forced=read_json(args.at("--force-tokens")).get>();if(chunks.size()!=1) throw std::runtime_error("teacher forcing requires one chunk");} + if(args.count("--download-only")) { kitten::resolve_assets(args); std::cout<<"Assets ready\n"; return 0; } + if(!args.count("--seed")) args["--seed"]=std::to_string(std::random_device{}() % 2147483647); + kitten::engine engine(args); int repeats=std::stoi(get("--repeat","1")); if(repeats<1 || repeats>100) throw std::runtime_error("repeat must be 1..100"); - for(int repeat=0;repeatseed(seed); - double lm_seconds=0,decoder_seconds=0;size_t total_tokens=0; - for(auto & chunk:chunks) { - auto ids=kitten::prompt(tm,tokenize(vocab,chunk),ref_text,ref_audio,emotion,!args.count("--no-reference") && gen.value("use_reference_prompt",true)); - int budget=chunks.size()==1?max_tokens:std::max(200,std::min(max_tokens,int(kitten::characters(chunk)/20.0*25*1.8))); - if(args.count("--force-tokens")) budget=forced.size()+1; - auto cp=llama_context_default_params();cp.n_ctx=ids.size()+budget;cp.n_batch=512;cp.n_ubatch=512;cp.n_threads=threads;cp.n_threads_batch=threads;cp.no_perf=false; - std::unique_ptr ctx(llama_init_from_model(model.get(),cp),llama_free); - if(!ctx) throw std::runtime_error("cannot initialize context"); - auto begin=clock_type::now(); - { batch b(1,speaker.size());b.b.n_tokens=1;std::copy(speaker.begin(),speaker.end(),b.b.embd);b.position(0,0,false); - if(llama_decode(ctx.get(),b.b)) throw std::runtime_error("speaker prefill failed"); } - for(size_t offset=1;offset audio_ids,generated;bool ended=false; - for(int step=0;step(raw),n_vocab*sizeof(float)); - if(args.count("--force-tokens") && step==int(forced.size())) break; - std::vector scores(raw,raw+n_vocab);kitten::process(scores,history,ids.size(),tm,settings); - int token=args.count("--force-tokens")?forced.at(step):kitten::sample(scores,rng,args.count("--greedy")); - if(token<0 || token>=n_vocab) throw std::runtime_error("forced token out of range"); - generated.push_back(token);history.push_back(token); - if(!args.count("--force-tokens") && (token==tm.speech_end || token==tm.stop)) {ended=true;break;} - if(token>=tm.base && token(clock_type::now()-begin).count();lm_seconds+=elapsed;total_tokens+=generated.size(); - report["chunks"].push_back({{"text",chunk},{"prompt",ids},{"generated",generated},{"audio_tokens",audio_ids},{"terminated",ended},{"lm_seconds",elapsed}}); - llama_perf_context_print(ctx.get()); - if(decoder) {begin=clock_type::now();waves.push_back(decoder->decode(audio_ids,ref.dump()));decoder_seconds+=std::chrono::duration(clock_type::now()-begin).count();} - } - report["lm_seconds"]=lm_seconds;report["decoder_seconds"]=decoder_seconds;report["generated_tokens"]=total_tokens; - if(decoder) {auto audio=kitten::join(waves,gap);write_wav(get("--output","output.wav"),audio);report["audio_seconds"]=audio.size()/24000.0;report["rtf"]=(lm_seconds+decoder_seconds)/(audio.size()/24000.0);} - if(args.count("--report")) write_json(args.at("--report"),report); - std::cout< +#include +#include + +namespace py = pybind11; +static py::array_t array(const std::vector & samples) { + py::array_t result(samples.size()); + std::copy(samples.begin(), samples.end(), result.mutable_data()); + return result; +} +PYBIND11_MODULE(_native, m) { + py::class_(m, "Engine") + .def(py::init([](const std::map & options) { + py::gil_scoped_release release; + return std::make_unique(options); + })) + .def("metadata", [](kitten::engine & self) { return self.metadata().dump(); }) + .def("normalize", &kitten::engine::normalize, py::call_guard()) + .def("generate", [](kitten::engine & self, const std::map & options, py::object callback) { + kitten::generation_result result; + { + py::gil_scoped_release release; + result = self.generate(options, callback.is_none() ? std::function &)>{} : + [&callback](const std::vector & samples) { + py::gil_scoped_acquire acquire; + return callback(array(samples)).cast(); + }); + } + return py::make_tuple(array(result.audio), result.report.dump()); + }, py::arg("options"), py::arg("callback") = py::none()); +} diff --git a/tools/kitten-tts/test_python.py b/tools/kitten-tts/test_python.py new file mode 100644 index 0000000..75f4aae --- /dev/null +++ b/tools/kitten-tts/test_python.py @@ -0,0 +1,115 @@ +"""Run with PYTHONPATH=python and KITTEN_TEST_ASSETS pointing to prepared assets.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import threading +import time +import unittest + +import numpy as np +import soundfile as sf + +from kitten_tts_cpp import KittenTTS, KittenTTS2, get_model + + +class InterfaceTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + assets = os.environ.get('KITTEN_TEST_ASSETS') + if not assets: + raise unittest.SkipTest('set KITTEN_TEST_ASSETS for real model tests') + cls.assets = str(Path(assets).resolve()) + cls.model_path = os.environ.get('KITTEN_TEST_GGUF') + cls.model = KittenTTS(cls.assets, model_path=cls.model_path, decoder='student_w4') + + @classmethod + def tearDownClass(cls): + cls.model.close() + + def test_metadata_and_validation(self): + self.assertIs(KittenTTS, KittenTTS2) + self.assertEqual(self.model.sample_rate, 24000) + self.assertIn('Bruno', self.model.available_voices) + self.assertIn('student_w4', self.model.available_decoders) + for kwargs in [dict(voice='missing'), dict(advanced={'typo': 1}), dict(advanced={'chunk_chars': 0}), + dict(temperature=float('nan')), dict(top_k=1.5), dict(reference_text='Hello'), + dict(max_new_tokens=0), dict(advanced={'use_emotion': 'false'})]: + with self.subTest(kwargs=kwargs), self.assertRaises(ValueError): + self.model.generate('Hello.', **kwargs) + with self.assertRaises(NotImplementedError): + self.model.generate('Hello.', reference='recording.wav') + with self.assertRaises(ValueError): + get_model(self.assets, device='cuda') + with self.assertRaises(ValueError): + get_model(self.assets, weights='emb4') + self.assertIn('twelve', self.model.normalize_text('It costs $12.')) + self.assertEqual(self.model._options('Hello.', advanced={'repetition_window': None})['--repetition-window'], + str(self.model.advanced_options['repetition_window'])) + + def test_cli_parity_and_repeat(self): + audio = self.model.generate('Hello again.', advanced={'seed': 1234}) + ids = self.model.last_report['chunks'][0]['generated'] + self.assertEqual(audio.dtype, np.float32) + self.assertEqual(audio.ndim, 1) + self.assertTrue(np.isfinite(audio).all()) + self.assertGreater(audio.size, 0) + np.testing.assert_array_equal(audio, self.model.generate('Hello again.', advanced={'seed': 1234})) + with tempfile.TemporaryDirectory() as d: + wav = Path(d) / 'cli.wav' + report = Path(d) / 'cli.json' + cmd = [os.environ.get('KITTEN_TEST_BINARY', 'build/bin/kitten-tts'), '--assets', self.assets, + '--decoder', 'student_w4', '--text', 'Hello again.', '--seed', '1234', '--threads', '8', + '--output', str(wav), '--report', str(report)] + if self.model_path: + cmd += ['--model', self.model_path] + subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + actual, rate = sf.read(wav, dtype='float32') + self.assertEqual(rate, self.model.sample_rate) + self.assertEqual(ids, json.loads(report.read_text())['chunks'][0]['generated']) + np.testing.assert_array_equal(audio, actual) + output = self.model.generate_to_file('Hello again.', Path(d) / 'python.wav', advanced={'seed': 1234}) + written, rate = sf.read(output, dtype='float32') + self.assertEqual(rate, 24000) + np.testing.assert_allclose(audio, written, atol=1/32768) + + def test_stream_and_cancellation(self): + kwargs = {'advanced': {'seed': 1234}} + whole = self.model.generate('Hello again.', **kwargs) + chunks = list(self.model.generate_stream('Hello again.', **kwargs)) + self.assertEqual(len(chunks), 1) + np.testing.assert_array_equal(whole, chunks[0]) + stream = self.model.generate_stream('Hello again. Welcome home. Nice to meet you.', + advanced={'seed': 1234, 'chunk_chars': 15, 'chunk_min_chars': 0}) + self.assertGreater(next(stream).size, 0) + stream.close() + np.testing.assert_array_equal(whole, self.model.generate('Hello again.', **kwargs)) + + def test_z_gil_and_close(self): + ticks = [] + stopped = threading.Event() + + def heartbeat(): + while not stopped.wait(0.005): + ticks.append(time.monotonic()) + + thread = threading.Thread(target=heartbeat) + thread.start() + try: + self.model.generate('Hello again.', advanced={'seed': 1234}) + finally: + stopped.set() + thread.join() + self.assertGreater(len(ticks), 10) + stream = self.model.generate_stream('Hello again. Welcome home. Nice to meet you.', + advanced={'seed': 1234, 'chunk_chars': 15, 'chunk_min_chars': 0}) + next(stream) + self.model.close() + stream.close() + with self.assertRaises(RuntimeError): + self.model.generate('Hello.') + + +if __name__ == '__main__': + unittest.main() From 88e2cd0e8495f73f5fb9ba42b611401e553d131e Mon Sep 17 00:00:00 2001 From: KittenML <224667585+KittenML@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:51:51 +0000 Subject: [PATCH 2/8] Build Python wheels on py-interface pushes Assisted-by: Codex --- .github/workflows/python-wheels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 14551c7..0be286b 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -1,6 +1,9 @@ name: Python wheels on: + push: + branches: + - py-interface workflow_dispatch: inputs: model_test: From 4392b0a0a21bdeb19560096171f02c2960db4522 Mon Sep 17 00:00:00 2001 From: KittenML <224667585+KittenML@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:06:49 +0000 Subject: [PATCH 3/8] Fix Torch wheel builds on macOS and Windows and report Linux ABI Assisted-by: Codex --- packaging/repair_wheel.py | 4 +++- tools/kitten-tts/CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packaging/repair_wheel.py b/packaging/repair_wheel.py index 4019221..6db8eb1 100644 --- a/packaging/repair_wheel.py +++ b/packaging/repair_wheel.py @@ -10,12 +10,14 @@ env = os.environ.copy() if sys.platform == 'linux': env['LD_LIBRARY_PATH'] = str(torch_lib) + os.pathsep + env.get('LD_LIBRARY_PATH', '') + subprocess.run(['auditwheel', 'show', wheel], env=env, check=True) command = ['auditwheel', 'repair', '-w', destination] for library in sorted(torch_lib.glob('*.so*')): command += ['--exclude', library.name] command.append(wheel) elif sys.platform == 'darwin': - command = ['delocate-wheel', '--require-archs', 'arm64', '--exclude', '/torch/lib/', + env['DYLD_LIBRARY_PATH'] = str(torch_lib) + os.pathsep + env.get('DYLD_LIBRARY_PATH', '') + command = [sys.executable, '-m', 'delocate.cmd.delocate_wheel', '--require-archs', 'arm64', '--exclude', '/torch/lib/', '-w', destination, wheel] elif sys.platform == 'win32': excluded = ';'.join(p.name for p in sorted(torch_lib.glob('*.dll'))) diff --git a/tools/kitten-tts/CMakeLists.txt b/tools/kitten-tts/CMakeLists.txt index a308197..e1acd65 100644 --- a/tools/kitten-tts/CMakeLists.txt +++ b/tools/kitten-tts/CMakeLists.txt @@ -7,7 +7,7 @@ target_compile_features(kitten-text PUBLIC cxx_std_17) add_library(kitten-decoder STATIC decoder.cpp) target_include_directories(kitten-decoder PRIVATE ${PROJECT_SOURCE_DIR}/vendor/nlohmann) target_link_libraries(kitten-decoder PUBLIC ${TORCH_LIBRARIES}) -target_compile_features(kitten-decoder PUBLIC cxx_std_17) +target_compile_features(kitten-decoder PUBLIC cxx_std_20) add_library(kitten-engine STATIC engine.cpp assets.cpp) target_include_directories(kitten-engine PUBLIC ${PROJECT_SOURCE_DIR}/vendor/nlohmann) target_link_libraries(kitten-engine PUBLIC llama-common llama kitten-text kitten-decoder) From aa70c056f451ad00d4287679af785c9d4b8a3875 Mon Sep 17 00:00:00 2001 From: KittenML <224667585+KittenML@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:13:21 +0000 Subject: [PATCH 4/8] Build Linux wheels against the manylinux 2.34 baseline Assisted-by: Codex --- packaging/README.md | 16 ++++++++-------- pyproject.toml | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packaging/README.md b/packaging/README.md index 63de25b..aae475d 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -1,14 +1,14 @@ # Precompiled wheels -The local `py-interface` branch contains a manually triggered wheel workflow. It builds and tests wheels as GitHub Actions artifacts; it does not publish to PyPI or create a release. The workflow has not been run remotely yet. +The `Python wheels` workflow runs on pushes to `py-interface` and supports manual dispatch. It builds and tests wheels as GitHub Actions artifacts; it does not publish to PyPI or create a release. | Target | Python versions | Runner | Status | | --- | --- | --- | --- | -| Linux x86-64, glibc 2.28+ | 3.10-3.14 | ubuntu-24.04 + manylinux_2_28 | Configured; local x86-64 build tested separately | -| Linux ARM64, glibc 2.28+ | 3.10-3.14 | ubuntu-24.04-arm + manylinux_2_28 | Configured; not yet runner-tested | -| macOS 14+, Apple Silicon | 3.10-3.14 | macos-14 | Configured; not yet runner-tested | -| Windows x86-64 | 3.10-3.14 | windows-2022 | Configured; not yet runner-tested | -| Windows ARM64 | 3.11-3.13 | windows-11-arm + ClangCL | Experimental; not yet runner-tested | +| Linux x86-64, glibc 2.34+ | 3.10-3.14 | ubuntu-24.04 + manylinux_2_34 | CI validation in progress; local x86-64 build tested | +| Linux ARM64, glibc 2.34+ | 3.10-3.14 | ubuntu-24.04-arm + manylinux_2_34 | CI validation in progress | +| macOS 14+, Apple Silicon | 3.10-3.14 | macos-14 | CI validation in progress | +| Windows x86-64 | 3.10-3.14 | windows-2022 | CI validation in progress | +| Windows ARM64 | 3.11-3.13 | windows-11-arm + ClangCL | Experimental; CI validation in progress | These are 23 CPython builds. This is not universal architecture support: Intel macOS, 32-bit platforms, Alpine/musl, mobile, WebAssembly, PyPy, and free-threaded Python are outside this matrix. The pinned PyTorch 2.13.0 distribution does not provide compatible wheels for most of these targets. The CPU index has s390x builds, but this project does not yet have an s390x build/test runner. @@ -16,7 +16,7 @@ Sources for the matrix: [PyTorch CPU wheels](https://download.pytorch.org/whl/cp ## Building -After the workflow is available in GitHub, run **Python wheels** manually. If the normalizer repository is private, provide `KITTEN_SUBMODULE_TOKEN` with read access to both repositories. Download the per-platform artifacts after the jobs pass. No workflow has been pushed or triggered as part of this implementation. +Push to `py-interface` to start the build. Manual dispatch is also available once the workflow file exists on the default branch; select `py-interface` as the run branch. If the normalizer repository is private, provide `KITTEN_SUBMODULE_TOKEN` with read access to both repositories. Download the per-platform artifacts after the jobs pass. Every wheel is installed into a separate test environment. The smoke test checks the Torch ABI version, packaged grammar data, native normalization, TorchScript loading/decoding, NumPy output, and an HTTPS config download without model credentials. Select the `model_test` workflow input for an additional real W4 generation on every wheel; private models require an `HF_TOKEN` Actions secret. A full release should pass that test on each platform first. @@ -36,7 +36,7 @@ KITTEN_PORTABLE=1 CMAKE_ARGS='-DLLAMA_BUILD_LIBRESSL=ON' \ python setup.py build --build-base build-portable bdist_wheel ``` -This produces a wheel for the build host. On Linux, use `packaging/repair_wheel.py` with auditwheel/patchelf installed to repair and assign its actual glibc compatibility tag. A wheel built on a newer host cannot simply be labeled manylinux_2_28. The local CPython 3.11 test produced a manylinux_2_35_x86_64 wheel, not the CI matrix's manylinux_2_28 target. +This produces a wheel for the build host. On Linux, use `packaging/repair_wheel.py` with auditwheel/patchelf installed to repair and assign its actual glibc compatibility tag. A wheel built on a newer host cannot simply be labeled manylinux_2_28. The local CPython 3.11 test produced a manylinux_2_35_x86_64 wheel, not the CI matrix's manylinux_2_34 target. ## Installing an artifact diff --git a/pyproject.toml b/pyproject.toml index eef2c32..3401586 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,8 @@ PIP_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cpu" [tool.cibuildwheel.linux] archs = ["x86_64", "aarch64"] -manylinux-x86_64-image = "manylinux_2_28" -manylinux-aarch64-image = "manylinux_2_28" +manylinux-x86_64-image = "manylinux_2_34" +manylinux-aarch64-image = "manylinux_2_34" [tool.cibuildwheel.macos] archs = ["arm64"] From 712c7f55ae9c0367107cee3b6c7aaf929cdc6d4c Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 21 Oct 2026 17:19:30 +0000 Subject: [PATCH 5/8] Penalise the silence token; guard top-p against non-finite logits Exempting silence (codec 4299) from the repetition penalty lets the main checkpoint stall in silence runs. Only speech_end and stop stay exempt, matching the Python sampler. Long silence runs could also drive the LM to all-NaN logits, after which top-p indexed an empty candidate list and segfaulted; skip top-p then so sample() reports 'no finite sampling candidates'. Co-Authored-By: Claude Opus 5.5 --- tools/kitten-tts/sampling.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/kitten-tts/sampling.h b/tools/kitten-tts/sampling.h index 8d20db0..cd56536 100644 --- a/tools/kitten-tts/sampling.h +++ b/tools/kitten-tts/sampling.h @@ -50,7 +50,9 @@ inline void process(std::vector & scores, const std::vector & histor std::vector seen(scores.size(),false); for(size_t i=begin;i & scores, const std::vector & histor if(s.top_p<1) { std::vector order; for(size_t i=0;i & scores, const std::vector & histor size_t id=order[i]; acc+=std::exp(double(scores[id]-peak))/sum; if(acc<=1.0-s.top_p) scores[id]=neg; } + } } if(s.min_p>0) { float cutoff=*std::max_element(scores.begin(),scores.end())+std::log(s.min_p); From a147ff0dc1b5ab037119adb0384745a998a4820c Mon Sep 17 00:00:00 2001 From: Kitten Date: Wed, 21 Oct 2026 21:35:22 +0000 Subject: [PATCH 6/8] Bump version to 0.2.0 The wheels published as 0.1.0 were built on 5 October, before the silence token was brought under the repetition penalty and top-p was guarded against non-finite logits. Generated output changes for a given seed, so this is a minor bump rather than a patch. Co-Authored-By: Claude Opus 5 --- python/kitten_tts_cpp/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/kitten_tts_cpp/__init__.py b/python/kitten_tts_cpp/__init__.py index 5bde481..f795e76 100644 --- a/python/kitten_tts_cpp/__init__.py +++ b/python/kitten_tts_cpp/__init__.py @@ -1,5 +1,5 @@ """KittenTTS2's Python interface backed by the CPU C++ runtime.""" from .model import KittenTTS2, KittenTTS, get_model -__version__ = '0.1.0' +__version__ = '0.2.0' __all__ = ['KittenTTS', 'KittenTTS2', 'get_model'] diff --git a/setup.py b/setup.py index d15eae5..3d0d6f7 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ def build_extension(self, ext): setup( - name='kitten-tts-cpp', version='0.1.0', + name='kitten-tts-cpp', version='0.2.0', description='CPU KittenTTS2 inference with native C++ bindings', long_description=(ROOT / 'README.md').read_text(), long_description_content_type='text/markdown', author='KittenML', url='https://github.com/KittenML/kitten-tts-2-cpp', From 2d03a653b06bf4ecf2e69ac117090bcd4017d4cd Mon Sep 17 00:00:00 2001 From: haaziq-sl Date: Tue, 20 Oct 2026 16:21:34 +0400 Subject: [PATCH 7/8] ggml, llama : repack TQ2_1 to Q2_0 at load time for GPU buffers No GPU backend has TQ2_1 kernels. When a TQ2_1 weight is assigned to a GPU buffer type that supports the op with Q2_0, create it as Q2_0 and repack it while loading. The repack is lossless: both formats store 2-bit codes with FP16 scales, and each 128-weight TQ2_1 scale is copied bit-exact onto two 64-weight Q2_0 blocks. CPU buffers are unchanged. Repacked tensors are not served from the file mapping. - ggml: ggml_repack_tq2_1_to_q2_0(), which rejects code 3 - llama: llama_model_params.gpu_repack (default true), load log lines for the repacked tensors and the repack time - llama-ext.h staging API: llama_model_n_tensors(), llama_model_get_tensor_by_index(), llama_get_graph_splits() and llama_get_flash_attn(), so tools can check weight placement, CPU fallbacks and the resolved flash attention setting - ggml-cuda: cache the device's integrated flag at registration instead of calling cudaGetDeviceProperties on every get_type call - tests: repack round trips and invalid codes in test-tq2_1, Q2_0 mul_mat cases at Qwen3-1.7B shapes in test-backend-ops, and test-tq2_1-load, which loads a TQ2_1 GGUF with all layers on the GPU, checks that every TQ2_1 weight is on the GPU as Q2_0 and dequantizes to the same values as the file, and that with gpu_repack off the weights stay on the CPU as TQ2_1 --- ggml/include/ggml.h | 4 + ggml/src/ggml-cuda/ggml-cuda.cu | 10 +- ggml/src/ggml-quants.c | 34 +++++++ include/llama.h | 1 + src/llama-context.cpp | 8 ++ src/llama-ext.h | 11 +++ src/llama-model-loader.cpp | 108 ++++++++++++++++++++- src/llama-model-loader.h | 12 +++ src/llama-model.cpp | 14 ++- src/llama.cpp | 1 + tests/CMakeLists.txt | 3 + tests/test-backend-ops.cpp | 7 ++ tests/test-tq2_1-load.cpp | 103 ++++++++++++++++++++ tests/test-tq2_1.cpp | 164 ++++++++++++++++++++++++++++++++ 14 files changed, 474 insertions(+), 6 deletions(-) create mode 100644 tests/test-tq2_1-load.cpp diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index dc2d2bc..99355a0 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -807,6 +807,10 @@ extern "C" { GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes); + // lossless TQ2_1 -> Q2_0 repack; dst holds nrows*ggml_row_size(GGML_TYPE_Q2_0, n_per_row) bytes + // returns false if a code is 3 or n_per_row is not a multiple of 256 + GGML_API bool ggml_repack_tq2_1_to_q2_0(const void * src, void * dst, int64_t nrows, int64_t n_per_row); + // main GGML_API struct ggml_context * ggml_init (struct ggml_init_params params); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 561ab7a..a8a4361 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4549,6 +4549,7 @@ struct ggml_backend_cuda_device_context { std::string description; std::string pci_bus_id; int op_offload_min_batch_size; + bool integrated; // cached at registration: cudaGetDeviceProperties is slow (1.5 ms per call under nsys) and get_type is called per tensor }; static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { @@ -4681,10 +4682,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend_dev_t dev) { ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *) dev->context; - cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); - - return prop.integrated + return ctx->integrated ? GGML_BACKEND_DEVICE_TYPE_IGPU : GGML_BACKEND_DEVICE_TYPE_GPU; } @@ -5395,6 +5393,10 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { } dev_ctx->op_offload_min_batch_size = min_batch_size; + cudaDeviceProp prop; + CUDA_CHECK(cudaGetDeviceProperties(&prop, physical_id)); + dev_ctx->integrated = prop.integrated; + ggml_backend_dev_t dev = new ggml_backend_device { /* .iface = */ ggml_backend_cuda_device_interface, /* .reg = */ ®, diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 3e0a3f1..d85c890 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -2421,6 +2421,40 @@ size_t quantize_tq2_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * ggml_row_size(GGML_TYPE_TQ2_1, n_per_row); } +// lossless: one TQ2_1 block -> four Q2_0 blocks; each 128-group scale is copied bit-exact to two Q2_0 blocks +bool ggml_repack_tq2_1_to_q2_0(const void * src, void * dst, int64_t nrows, int64_t n_per_row) { + if (nrows < 0 || n_per_row < 0 || n_per_row % QK_K != 0) { + return false; + } + const block_tq2_1 * x = (const block_tq2_1 *) src; + block_q2_0 * y = (block_q2_0 *) dst; + const int64_t nb = nrows * (n_per_row / QK_K); + for (int64_t i = 0; i < nb; ++i) { + for (int g = 0; g < 2; ++g) { + block_q2_0 * out = y + 4*i + 2*g; + out[0].d = x[i].d[g]; + out[1].d = x[i].d[g]; + for (int q = 0; q < 8; ++q) { + // byte t holds code l (elements g*128 + l*32 + 4*q + t) at bits 2*l + const uint8_t * b = x[i].qs + g*32 + 4*q; + uint32_t w = (uint32_t) b[0] | (uint32_t) b[1] << 8 | (uint32_t) b[2] << 16 | (uint32_t) b[3] << 24; + if (w & (w >> 1) & 0x55555555u) { + return false; // code 3 is invalid in TQ2_1 + } + // transpose the 4x4 matrix of 2-bit codes, so byte l holds code t at bits 2*t + uint32_t s = (w ^ (w >> 12)) & 0x0000F0F0u; + w ^= s ^ (s << 12); + s = (w ^ (w >> 6)) & 0x00CC00CCu; + w ^= s ^ (s << 6); + for (int l = 0; l < 4; ++l) { + out[l/2].qs[(l%2)*8 + q] = (uint8_t) (w >> (8*l)); + } + } + } + } + return true; +} + void quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; diff --git a/include/llama.h b/include/llama.h index fb24667..899801d 100644 --- a/include/llama.h +++ b/include/llama.h @@ -340,6 +340,7 @@ extern "C" { bool no_host; // bypass host buffer allowing extra buffers to be used bool no_alloc; // only load metadata and simulate memory allocations bool load_mtp; // whether to load MTP layers + bool gpu_repack; // repack TQ2_1 weights to Q2_0 (lossless) for GPUs that have no TQ2_1 kernels }; struct llama_sampler_seq_config { diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 19cca7d..d9680d2 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -4182,6 +4182,14 @@ llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * c return ctx->memory_breakdown(); } +int32_t llama_get_graph_splits(const struct llama_context * ctx) { + return ggml_backend_sched_get_n_splits(ctx->get_sched()); +} + +bool llama_get_flash_attn(const struct llama_context * ctx) { + return ctx->get_cparams().flash_attn; +} + llama_context * llama_get_ctx_other(struct llama_context * ctx) { return ctx->get_cparams().ctx_other; } diff --git a/src/llama-ext.h b/src/llama-ext.h index 35d6e58..bd088d9 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -88,8 +88,19 @@ LLAMA_API int32_t llama_model_n_devices(const struct llama_model * model); LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i); +// Model weight tensors by index, e.g. to check which buffer (device) holds each weight after load. +// Returns nullptr when i is out of range. +LLAMA_API int32_t llama_model_n_tensors (const struct llama_model * model); +LLAMA_API const struct ggml_tensor * llama_model_get_tensor_by_index(const struct llama_model * model, int32_t i); + LLAMA_API llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx); +// Backend splits of the last scheduled graph, e.g. to check that no op falls back to the CPU. +LLAMA_API int32_t llama_get_graph_splits(const struct llama_context * ctx); + +// Whether Flash Attention is enabled, after "auto" is resolved. +LLAMA_API bool llama_get_flash_attn(const struct llama_context * ctx); + // Set whether the context outputs nextn embeddings or not // If masked == true, output the embeddings only for the tokens with batch.logits != 0 // If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 2cd1b98..a083805 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1091,6 +1091,33 @@ struct ggml_tensor * llama_model_loader::create_tensor( return it->second.get(); }; + // set by buft_for_tensor when the TQ2_1 weight is created as Q2_0 and repacked at load time + bool repack = false; + + // use Q2_0 for a TQ2_1 weight if the GPU buffer type supports the op with Q2_0 + auto try_repack_q2_0 = [&](ggml_tensor * t_meta, ggml_op op, ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) -> bool { + if (!gpu_repack || files.empty() || t_meta->type != GGML_TYPE_TQ2_1 || !dev || ggml_backend_buft_is_host(buft)) { + return false; + } + const auto dev_type = ggml_backend_dev_type(dev); + if (dev_type != GGML_BACKEND_DEVICE_TYPE_GPU && dev_type != GGML_BACKEND_DEVICE_TYPE_IGPU) { + return false; + } + ggml_tensor t_q2_0 = *t_meta; + t_q2_0.type = GGML_TYPE_Q2_0; + t_q2_0.nb[0] = ggml_type_size(GGML_TYPE_Q2_0); + t_q2_0.nb[1] = ggml_row_size(GGML_TYPE_Q2_0, t_q2_0.ne[0]); + for (int i = 2; i < GGML_MAX_DIMS; i++) { + t_q2_0.nb[i] = t_q2_0.nb[i - 1]*t_q2_0.ne[i - 1]; + } + if (!weight_buft_supported(hparams, &t_q2_0, op, buft, dev)) { + return false; + } + *t_meta = t_q2_0; + repack = true; + return true; + }; + auto buft_for_tensor = [&](ggml_tensor * t_meta) -> ggml_backend_buffer_type_t { if (!t_meta) { if (flags & TENSOR_NOT_REQUIRED) { @@ -1185,6 +1212,10 @@ struct ggml_tensor * llama_model_loader::create_tensor( } } else { buft = overrides->buft; + auto * dev = ggml_backend_buft_get_device(buft); + if (t_meta->type == GGML_TYPE_TQ2_1 && dev && !weight_buft_supported(hparams, t_meta, op, buft, dev)) { + try_repack_q2_0(t_meta, op, dev, buft); + } } LLAMA_LOG_DEBUG("tensor %s (%zu MiB %s) buffer type overridden to %s\n", @@ -1201,6 +1232,18 @@ struct ggml_tensor * llama_model_loader::create_tensor( if (!buft) { throw std::runtime_error(format("failed to find a compatible buffer type for tensor %s", tn.str().c_str())); } + // the buffer types before buft do not support TQ2_1; use the first GPU one that supports Q2_0 + if (t_meta->type == GGML_TYPE_TQ2_1) { + for (const auto & [cur_dev, cur_buft] : *buft_list) { + if (cur_buft == buft) { + break; + } + if (try_repack_q2_0(t_meta, op, cur_dev, cur_buft)) { + buft = cur_buft; + break; + } + } + } } // avoid using a host buffer when using mmap @@ -1278,6 +1321,8 @@ struct ggml_tensor * llama_model_loader::create_tensor( } } + const size_t nbytes_file = ggml_nbytes(&t_meta); + ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta); if (buft == nullptr) { return nullptr; @@ -1298,12 +1343,20 @@ struct ggml_tensor * llama_model_loader::create_tensor( struct ggml_tensor * tensor = ggml_dup_tensor(ctx, &t_meta); ggml_set_name(tensor, ggml_get_name(&t_meta)); + // progress and the final unmap use file bytes if (duplicated) { - size_data += ggml_nbytes(&t_meta); + size_data += nbytes_file; } else { n_created++; } + if (repack) { + auto & stats = repacked[ggml_backend_buft_name(buft)]; + stats.n_tensors++; + stats.n_bytes_file += nbytes_file; + stats.n_bytes_repacked += ggml_nbytes(tensor); + } + return tensor; } @@ -1323,6 +1376,10 @@ void llama_model_loader::done_getting_tensors(bool partial) const { __func__, first_tensor_moved_name.c_str(), first_tensor_moved_type_name.c_str(), n_tensors_moved - 1, ggml_backend_buft_name(first_moved_from_buft), ggml_backend_buft_name(first_moved_to_buft)); } + for (const auto & [buft_name, stats] : repacked) { + LLAMA_LOG_INFO("%s: %s has no TQ2_1 kernels: %zu TQ2_1 tensors (%.2f MiB) will be repacked to Q2_0 (%.2f MiB) at load time (lossless)\n", + __func__, buft_name.c_str(), stats.n_tensors, stats.n_bytes_file/1024.0/1024.0, stats.n_bytes_repacked/1024.0/1024.0); + } } void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps) { @@ -1375,6 +1432,16 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void * } } +bool llama_model_loader::has_repacked_tensors(ggml_context * ctx) const { + for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) { + const auto * weight = get_weight(ggml_get_name(tensor)); + if (weight && weight->tensor->type != tensor->type) { + return true; + } + } + return false; +} + void llama_model_loader::load_data_for(struct ggml_tensor * cur) const { const auto & w = require_weight(ggml_get_name(cur)); @@ -1413,7 +1480,11 @@ bool llama_model_loader::load_all_data( GGML_ASSERT(size_data != 0 && "call init_mappings() first"); std::vector> read_buf; + std::vector> repack_buf; std::vector>> validation_result; + size_t n_repacked = 0; + int64_t t_repack_us = 0; + const char * repack_buf_name = nullptr; // 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives. // NVMe raid configurations might require more / larger buffers. @@ -1528,6 +1599,37 @@ bool llama_model_loader::load_all_data( size_t n_size = ggml_nbytes(cur); + if (cur->type != weight->tensor->type) { + // TQ2_1 in the file, Q2_0 in a GPU buffer (see create_tensor) + GGML_ASSERT(cur->type == GGML_TYPE_Q2_0 && weight->tensor->type == GGML_TYPE_TQ2_1); + GGML_ASSERT(cur->buffer != nullptr && cur->data != nullptr); + const size_t n_size_file = ggml_nbytes(weight->tensor); + const uint8_t * src = nullptr; + if (use_mmap) { + src = (const uint8_t *) mappings.at(weight->idx)->addr() + weight->offs; + } else { + const auto & file = files.at(weight->idx); + read_buf.resize(n_size_file); + file->seek(weight->offs, SEEK_SET); + file->read_raw(read_buf.data(), n_size_file); + src = (const uint8_t *) read_buf.data(); + } + if (check_tensors && !ggml_validate_row_data(weight->tensor->type, src, n_size_file)) { + throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur))); + } + const int64_t t_start_us = ggml_time_us(); + repack_buf.resize(n_size); + if (!ggml_repack_tq2_1_to_q2_0(src, repack_buf.data(), ggml_nrows(weight->tensor), weight->tensor->ne[0])) { + throw std::runtime_error(format("tensor '%s' has invalid TQ2_1 data (code 3), cannot repack it to Q2_0", ggml_get_name(cur))); + } + t_repack_us += ggml_time_us() - t_start_us; + n_repacked++; + repack_buf_name = ggml_backend_buffer_name(cur->buffer); + ggml_backend_tensor_set(cur, repack_buf.data(), 0, n_size); + size_done += n_size_file; + continue; + } + if (use_mmap) { const auto & mapping = mappings.at(weight->idx); ggml_backend_buffer_t buf_mmap = nullptr; @@ -1646,6 +1748,10 @@ bool llama_model_loader::load_all_data( } ggml_backend_free(upload_backend); + if (n_repacked > 0) { + LLAMA_LOG_INFO("%s: repacked %zu TQ2_1 tensors to Q2_0 for %s in %.1f ms (lossless)\n", __func__, n_repacked, repack_buf_name, t_repack_us/1000.0); + } + // check validation results bool validation_failed = false; for (auto & future : validation_result) { diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index d6b31c2..5eeb080 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -114,6 +114,15 @@ struct llama_model_loader { std::map ctx_map; + // TQ2_1 weights created as Q2_0 for a GPU buffer type that has no TQ2_1 kernels; load_all_data repacks them + bool gpu_repack = true; + struct repack_stats { + size_t n_tensors = 0; + size_t n_bytes_file = 0; + size_t n_bytes_repacked = 0; + }; + std::map repacked; // by buffer type name + // track tensors that had to be moved for debugging: size_t n_tensors_moved = 0; std::string first_tensor_moved_name; @@ -194,6 +203,9 @@ struct llama_model_loader { void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const; + // true if a tensor in ctx has a different type than in the file (repacked at load time) + bool has_repacked_tensors(ggml_context * ctx) const; + // for backwards compatibility, does not support ggml-backend void load_data_for(struct ggml_tensor * cur) const; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 34d55c9..3510266 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1568,7 +1568,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { bool is_default_buft = buft == ggml_backend_dev_buffer_type(dev); std::vector bufs; - if (ml.use_mmap && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) { + if (ml.use_mmap && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft && !ml.has_repacked_tensors(ctx)) { GGML_ASSERT(!ml.no_alloc); for (uint32_t idx = 0; idx < ml.files.size(); idx++) { // only the mmap region containing the tensors in the model is mapped to the backend buffer @@ -2456,6 +2456,7 @@ llama_model_params llama_model_default_params() { /*.no_host =*/ false, /*.no_alloc =*/ false, /*.load_mtp =*/ false, + /*.gpu_repack =*/ true, }; return result; @@ -2865,6 +2866,17 @@ ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int return model->devices[i].dev; } +int32_t llama_model_n_tensors(const struct llama_model * model) { + return (int32_t)model->tensors_by_name.size(); +} + +const ggml_tensor * llama_model_get_tensor_by_index(const struct llama_model * model, int32_t i) { + if (i < 0 || i >= (int32_t)model->tensors_by_name.size()) { + return nullptr; + } + return model->tensors_by_name[i].second; +} + // // llama_model_base // diff --git a/src/llama.cpp b/src/llama.cpp index d6e0bbf..8e880d9 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -306,6 +306,7 @@ static std::pair llama_model_load(struct gguf_context * meta try { llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode, params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides); + ml.gpu_repack = params.gpu_repack; ml.print_info(); std::unique_ptr model_ptr(llama_model_create(ml, params)); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cdf8fb9..383f604 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -296,6 +296,9 @@ if (NOT GGML_BACKEND_DL) llama_build_and_test(test-col2im-1d.cpp) endif() +# GPU load-time TQ2_1 -> Q2_0 repack; needs a model file, so not run by ctest: test-tq2_1-load model-tq2_1.gguf +llama_build(test-tq2_1-load.cpp) + # libmtmd set(LLAMA_TEST_NAME test-mtmd-c-api) llama_build_and_test(test-mtmd-c-api.c) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c0f8058..9c30a73 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8974,6 +8974,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 576, 512, 576, {1,1}, {1,1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1, 2048, 8192, {1, 1}, {1, 1})); + // Q2_0 at Qwen3-1.7B shapes (TQ2_1 weights are repacked to Q2_0 for GPUs): vector kernels for small n, MMQ tiles for large n + for (int64_t n : {1, 8, 9, 96, 128, 512}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q2_0, GGML_TYPE_F32, 2048, n, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q2_0, GGML_TYPE_F32, 2048, n, 6144, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q2_0, GGML_TYPE_F32, 6144, n, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q2_0, GGML_TYPE_F32, 1024, n, 2048, {1, 1}, {1, 1})); + } for (ggml_type type_a : all_types) { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1, 64, 256, {1, 1}, {1, 1})); } diff --git a/tests/test-tq2_1-load.cpp b/tests/test-tq2_1-load.cpp new file mode 100644 index 0000000..d4a59a4 --- /dev/null +++ b/tests/test-tq2_1-load.cpp @@ -0,0 +1,103 @@ +// Loads a TQ2_1 GGUF with all layers on the GPU and checks that every TQ2_1 weight is on the GPU as Q2_0 +// and dequantizes to the same values as the file. Needs a GPU build and a model file, so it is not run by ctest. +// usage: test-tq2_1-load model-tq2_1.gguf + +#include "llama.h" +#include "ggml.h" +#include "ggml-backend.h" +#include "gguf.h" +#include "../src/llama-ext.h" + +#include +#include +#include + +// GPU memory, not the GPU's pinned host buffer. Do not use is_host: CPU extra buffers (AMX, CPU_REPACK) do not set it. +static bool on_gpu(const ggml_tensor * t) { + if (!t->buffer) { + return false; + } + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(t->buffer); + ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft); + if (!dev) { + return false; + } + const enum ggml_backend_dev_type type = ggml_backend_dev_type(dev); + return (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) && buft != ggml_backend_dev_host_buffer_type(dev); +} + +static int check(const char * path, ggml_context * fctx, llama_load_mode load_mode, bool gpu_repack) { + llama_model_params mp = llama_model_default_params(); + mp.n_gpu_layers = 99; + mp.load_mode = load_mode; + mp.gpu_repack = gpu_repack; + llama_model * model = llama_model_load_from_file(path, mp); + if (!model) { + std::fprintf(stderr, "load failed (%s, gpu_repack %d)\n", llama_load_mode_name(load_mode), gpu_repack); + return 1; + } + + int n_tq2_1 = 0, n_ok = 0, n_bad = 0; + for (int32_t i = 0; i < llama_model_n_tensors(model); ++i) { + const ggml_tensor * t = llama_model_get_tensor_by_index(model, i); + const ggml_tensor * ft = ggml_get_tensor(fctx, ggml_get_name(t)); + if (!ft || ft->type != GGML_TYPE_TQ2_1) { + continue; + } + n_tq2_1++; + const bool gpu = on_gpu(t); + if (!gpu_repack) { + // without the repack, no GPU backend can run TQ2_1, so the weight must stay on the CPU unchanged + if (t->type == GGML_TYPE_TQ2_1 && !gpu) { n_ok++; } else { n_bad++; std::printf("NOT ON CPU AS TQ2_1: %s\n", ggml_get_name(t)); } + continue; + } + if (t->type != GGML_TYPE_Q2_0 || !gpu) { + n_bad++; + std::printf("NOT REPACKED TO A GPU BUFFER: %s (%s @ %s)\n", ggml_get_name(t), ggml_type_name(t->type), ggml_backend_buffer_name(t->buffer)); + continue; + } + const int64_t n = ggml_nelements(t); + std::vector bytes(ggml_nbytes(t)); + ggml_backend_tensor_get(t, bytes.data(), 0, bytes.size()); + std::vector got(n), want(n); + ggml_get_type_traits(GGML_TYPE_Q2_0)->to_float(bytes.data(), got.data(), n); + ggml_get_type_traits(GGML_TYPE_TQ2_1)->to_float(ft->data, want.data(), n); + if (std::memcmp(got.data(), want.data(), n*sizeof(float)) == 0) { n_ok++; } else { n_bad++; std::printf("MISMATCH: %s\n", ggml_get_name(t)); } + } + llama_model_free(model); + + std::printf("%s, gpu_repack %d: %d TQ2_1 weights, %d ok, %d failed\n", llama_load_mode_name(load_mode), gpu_repack, n_tq2_1, n_ok, n_bad); + return n_tq2_1 > 0 && n_bad == 0 ? 0 : 1; +} + +int main(int argc, char ** argv) { + if (argc != 2) { + std::fprintf(stderr, "usage: %s model-tq2_1.gguf\n", argv[0]); + return 2; + } + llama_backend_init(); + if (!llama_supports_gpu_offload()) { + std::fprintf(stderr, "no GPU backend: skipped\n"); + llama_backend_free(); + return 0; + } + + ggml_context * fctx = nullptr; + gguf_init_params gp = { /*no_alloc =*/ false, /*ctx =*/ &fctx }; + gguf_context * g = gguf_init_from_file(argv[1], gp); + if (!g) { + std::fprintf(stderr, "cannot read %s\n", argv[1]); + return 1; + } + + int failed = 0; + failed += check(argv[1], fctx, LLAMA_LOAD_MODE_MMAP, true); + failed += check(argv[1], fctx, LLAMA_LOAD_MODE_NONE, true); + failed += check(argv[1], fctx, LLAMA_LOAD_MODE_MMAP, false); + + gguf_free(g); + ggml_free(fctx); + llama_backend_free(); + std::printf("%s\n", failed == 0 ? "OK" : "FAILED"); + return failed == 0 ? 0 : 1; +} diff --git a/tests/test-tq2_1.cpp b/tests/test-tq2_1.cpp index b37b32a..be4cb6a 100644 --- a/tests/test-tq2_1.cpp +++ b/tests/test-tq2_1.cpp @@ -91,8 +91,172 @@ static void test_amx() { std::puts("TQ2_1 AMX: 8 matmul cases exactly match lossless Q4_0"); } +// element-wise reference: TQ2_1 element g*128 + l*32 + m is byte g*32 + m, bits 2*l; Q2_0 element j is byte j/4, bits 2*(j%4) +static void repack_naive(const block_tq2_1 * x, block_q2_0 * y, int64_t nb) { + std::memset(y, 0, 4*nb*sizeof(block_q2_0)); + for (int64_t i = 0; i < nb; ++i) { + for (int e = 0; e < QK_K; ++e) { + const int g = e/128, l = (e%128)/32, m = e%32; + const int code = (x[i].qs[g*32 + m] >> (2*l)) & 3; + block_q2_0 & out = y[4*i + e/QK2_0]; + const int j = e%QK2_0; + out.d = x[i].d[g]; + out.qs[j/4] |= code << (2*(j%4)); + } + } +} + +static uint32_t rng_state = 12345; +static uint32_t rng() { + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} + +// fills codes in {0,1,2}; mode 0: random, 1: all 0, 2: all 1, 3: all 2, 4: code (k+trial)%3 +static void fill_codes(block_tq2_1 * x, int64_t nb, int mode, int trial) { + for (int64_t i = 0; i < nb; ++i) { + for (int k = 0; k < QK_K/4; ++k) { + uint8_t b = 0; + for (int l = 0; l < 4; ++l) { + int c = 0; + switch (mode) { + case 0: c = rng()%3; break; + case 1: c = 0; break; + case 2: c = 1; break; + case 3: c = 2; break; + default: c = (k*4 + l + trial)%3; break; + } + b |= c << (2*l); + } + x[i].qs[k] = b; + } + } +} + +static bool fp16_is_finite(uint16_t h) { + return (h & 0x7c00) != 0x7c00; +} + +static void test_repack() { + const int64_t q2_0_row = ggml_row_size(GGML_TYPE_Q2_0, 256); + assert(q2_0_row == 4*(int64_t) sizeof(block_q2_0)); + assert(ggml_row_size(GGML_TYPE_TQ2_1, 256) == sizeof(block_tq2_1)); + + // edge scales: +-0, smallest subnormal, largest subnormal, smallest normal, 1, -1, largest finite, -largest finite + const uint16_t edge_scales[] = {0x0000, 0x8000, 0x0001, 0x03ff, 0x0400, 0x3c00, 0xbc00, 0x7bff, 0xfbff, 0x2e66}; + const int n_edge = sizeof(edge_scales)/sizeof(edge_scales[0]); + + int n_cases = 0; + for (int64_t n_per_row : {256, 512, 2048, 6144}) { + for (int64_t nrows : {1, 3, 8}) { + const int64_t nb = nrows*n_per_row/QK_K; + const int64_t n = nrows*n_per_row; + std::vector x(nb); + std::vector y(4*nb), y_ref(4*nb); + std::vector f_tq(n), f_q2(n); + for (int mode = 0; mode < 5; ++mode) { + for (int trial = 0; trial < 3; ++trial) { + fill_codes(x.data(), nb, mode, trial); + for (int64_t i = 0; i < nb; ++i) { + for (int g = 0; g < 2; ++g) { + uint16_t h; + if (trial == 0) { + h = edge_scales[(i*2 + g) % n_edge]; + } else { + do { h = rng() & 0xffff; } while (!fp16_is_finite(h)); + } + std::memcpy(&x[i].d[g], &h, sizeof(h)); + } + } + assert(ggml_validate_row_data(GGML_TYPE_TQ2_1, x.data(), nb*sizeof(block_tq2_1))); + std::memset(y.data(), 0xa5, y.size()*sizeof(block_q2_0)); + assert(ggml_repack_tq2_1_to_q2_0(x.data(), y.data(), nrows, n_per_row)); + repack_naive(x.data(), y_ref.data(), nb); + assert(std::memcmp(y.data(), y_ref.data(), y.size()*sizeof(block_q2_0)) == 0); + assert(ggml_validate_row_data(GGML_TYPE_Q2_0, y.data(), y.size()*sizeof(block_q2_0))); + dequantize_row_tq2_1(x.data(), f_tq.data(), n); + dequantize_row_q2_0(y.data(), f_q2.data(), n); + assert(std::memcmp(f_tq.data(), f_q2.data(), n*sizeof(float)) == 0); + ++n_cases; + } + } + } + } + + // each element on its own: one +1 code in a zero block, distinct scales per group + { + block_tq2_1 x; + block_q2_0 y[4]; + float f_tq[QK_K], f_q2[QK_K]; + x.d[0] = ggml_fp32_to_fp16(0.25f); + x.d[1] = ggml_fp32_to_fp16(-3.0f); + for (int e = 0; e < QK_K; ++e) { + std::memset(x.qs, 0x55, sizeof(x.qs)); // all codes 1 (value 0) + const int g = e/128, l = (e%128)/32, m = e%32; + x.qs[g*32 + m] ^= 3 << (2*l); // code 1 -> 2 (value +d) + assert(ggml_repack_tq2_1_to_q2_0(&x, y, 1, QK_K)); + dequantize_row_tq2_1(&x, f_tq, QK_K); + dequantize_row_q2_0(y, f_q2, QK_K); + assert(std::memcmp(f_tq, f_q2, sizeof(f_tq)) == 0); + for (int k = 0; k < QK_K; ++k) { + assert(f_q2[k] == (k == e ? (g == 0 ? 0.25f : -3.0f) : 0.0f)); + } + ++n_cases; + } + } + + // scale bits are copied unchanged for all 65536 fp16 patterns, also inf and nan + { + block_tq2_1 x; + block_q2_0 y[4]; + fill_codes(&x, 1, 0, 0); + for (uint32_t h = 0; h < 65536; ++h) { + const uint16_t h0 = (uint16_t) h, h1 = (uint16_t) (h ^ 0x8001); + std::memcpy(&x.d[0], &h0, 2); + std::memcpy(&x.d[1], &h1, 2); + assert(ggml_repack_tq2_1_to_q2_0(&x, y, 1, QK_K)); + assert(std::memcmp(&y[0].d, &h0, 2) == 0 && std::memcmp(&y[1].d, &h0, 2) == 0); + assert(std::memcmp(&y[2].d, &h1, 2) == 0 && std::memcmp(&y[3].d, &h1, 2) == 0); + } + } + + // code 3 at any position in any block is rejected + { + const int64_t nrows = 2, n_per_row = 512, nb = nrows*n_per_row/QK_K; + std::vector x(nb); + std::vector y(4*nb); + int n_rejected = 0; + for (int64_t i = 0; i < nb; ++i) { + for (int e = 0; e < QK_K; ++e) { + fill_codes(x.data(), nb, 0, 0); + x[i].d[0] = x[i].d[1] = ggml_fp32_to_fp16(1.0f); + const int g = e/128, l = (e%128)/32, m = e%32; + x[i].qs[g*32 + m] |= 3 << (2*l); + assert(!ggml_validate_row_data(GGML_TYPE_TQ2_1, x.data(), nb*sizeof(block_tq2_1))); + assert(!ggml_repack_tq2_1_to_q2_0(x.data(), y.data(), nrows, n_per_row)); + ++n_rejected; + } + } + assert(n_rejected == nb*QK_K); + } + + // rows that are not a multiple of 256 are rejected + { + block_tq2_1 x[2] = {}; + block_q2_0 y[8]; + assert(!ggml_repack_tq2_1_to_q2_0(x, y, 1, 128)); + assert(!ggml_repack_tq2_1_to_q2_0(x, y, 1, 320)); + assert(ggml_repack_tq2_1_to_q2_0(x, y, 0, 256)); + } + + std::printf("TQ2_1 -> Q2_0 repack: %d cases, dequantized output identical (memcmp), scale bits identical for all 65536 fp16 patterns, code 3 rejected at every position\n", n_cases); +} + int main() { ggml_cpu_init(); + test_repack(); ggml_context * ctx = ggml_init({1024*1024, nullptr, true}); const auto * traits = ggml_get_type_traits(GGML_TYPE_TQ2_1); const auto * cpu = ggml_get_type_traits_cpu(GGML_TYPE_TQ2_1); From fcafd9a7c2095d74e8de7f3d72f1e6452f827cbe Mon Sep 17 00:00:00 2001 From: haaziq-sl Date: Mon, 26 Oct 2026 15:20:50 +0400 Subject: [PATCH 8/8] kitten-tts: add CUDA inference, wheels and published decoder support Assisted-by: Codex --- .github/workflows/python-wheels.yml | 79 +++++++ MANIFEST.in | 2 +- README.md | 64 ++++-- packaging/README.md | 103 ++++++++- packaging/install_cuda_linux.sh | 9 + packaging/prepare_wheel.py | 9 +- packaging/repair_wheel.py | 21 +- packaging/smoke_wheel.py | 55 ++++- python/kitten_tts_cpp/__init__.py | 2 +- python/kitten_tts_cpp/model.py | 49 +++- setup.py | 44 +++- tools/kitten-tts/CMakeLists.txt | 7 +- tools/kitten-tts/README.md | 119 ++++++++-- tools/kitten-tts/assets.cpp | 8 +- tools/kitten-tts/decoder.cpp | 126 +++++++++- tools/kitten-tts/decoder.h | 13 +- tools/kitten-tts/engine.cpp | 303 ++++++++++++++++++++++++- tools/kitten-tts/export.py | 132 +++++++++-- tools/kitten-tts/main.cpp | 14 +- tools/kitten-tts/prepare_model_repo.py | 19 ++ tools/kitten-tts/test_assets.py | 34 +++ tools/kitten-tts/test_export.py | 36 +++ tools/kitten-tts/test_python.py | 13 ++ tools/kitten-tts/validation.json | 103 +++++++++ 24 files changed, 1243 insertions(+), 121 deletions(-) create mode 100755 packaging/install_cuda_linux.sh create mode 100644 tools/kitten-tts/test_export.py diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 0be286b..b34b62a 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -61,3 +61,82 @@ jobs: name: kitten-tts-cpp-${{ matrix.platform }} path: wheelhouse/*.whl if-no-files-found: error + + # CUDA builds of the same package, each compiled with the CUDA release of the PyTorch build whose CUDA runtime and + # cuBLAS it uses. One job per Python version keeps each build inside the job time limit. The runners have no GPU, + # so the smoke test checks import, CPU use and the "no CUDA device" error. + wheels-cuda: + name: ${{ matrix.platform }}-${{ matrix.python }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 330 + strategy: + fail-fast: false + matrix: + platform: [linux-x86_64-cuda, windows-x86_64-cuda] + python: [cp310, cp311, cp312, cp313, cp314] + include: + - platform: linux-x86_64-cuda + runner: ubuntu-24.04 + arch: x86_64 + tag: manylinux_x86_64 + artifact: linux-x86_64 + package: kitten-tts-cpp-cu12 + - platform: windows-x86_64-cuda + runner: windows-2022 + arch: AMD64 + tag: win_amd64 + artifact: windows-x86_64 + package: kitten-tts-cpp-cu13 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.KITTEN_SUBMODULE_TOKEN || github.token }} + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Free disk space + if: runner.os == 'Linux' + run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL + # The toolkit release that the Windows CUDA build of PyTorch is built with + - name: Install CUDA 13.0 (compiler, runtime, cuBLAS, NVRTC) + if: runner.os == 'Windows' + shell: pwsh + run: | + $installer = "$env:RUNNER_TEMP\cuda_13.0.3_windows_network.exe" + Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/13.0.3/network_installers/cuda_13.0.3_windows_network.exe -OutFile $installer + $process = Start-Process -FilePath $installer -ArgumentList '-s nvcc_13.0 crt_13.0 nvvm_13.0 nvptxcompiler_13.0 cudart_13.0 cublas_13.0 cublas_dev_13.0 nvrtc_13.0 nvrtc_dev_13.0 thrust_13.0 visual_studio_integration_13.0' -Wait -PassThru + if ($process.ExitCode -ne 0) { throw "CUDA installer failed with exit code $($process.ExitCode)" } + $cuda = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.0' + & "$cuda\bin\nvcc.exe" --version + "CUDA_PATH=$cuda" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "$cuda\bin" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + - uses: ilammy/msvc-dev-cmd@v1 + if: runner.os == 'Windows' + - name: Install wheel builder + run: python -m pip install cibuildwheel==4.3.0 + - name: Build and test wheels + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.tag }} + CIBW_ARCHS: ${{ matrix.arch }} + KITTEN_CUDA: '1' + CIBW_BEFORE_ALL_LINUX: sh {project}/packaging/install_cuda_linux.sh + # 4 vCPU and 16 GB runners: one compile job per vCPU. + CIBW_ENVIRONMENT_LINUX: >- + KITTEN_CUDA=1 KITTEN_PORTABLE=1 CMAKE_BUILD_PARALLEL_LEVEL=4 CMAKE_ARGS="-DLLAMA_BUILD_LIBRESSL=ON" + PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu129 + CUDACXX=/usr/local/cuda-12.8/bin/nvcc PATH=/usr/local/cuda-12.8/bin:$PATH + CIBW_ENVIRONMENT_WINDOWS: >- + KITTEN_CUDA=1 KITTEN_PORTABLE=1 CMAKE_BUILD_PARALLEL_LEVEL=4 CMAKE_GENERATOR=Ninja + CMAKE_ARGS="-DLLAMA_BUILD_LIBRESSL=ON" + PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu130 + KITTEN_WHEEL_NETWORK_TEST: '1' + KITTEN_WHEEL_MODEL_TEST: ${{ inputs.model_test && '1' || '0' }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + CIBW_ENVIRONMENT_PASS_LINUX: HF_TOKEN KITTEN_WHEEL_MODEL_TEST KITTEN_WHEEL_NETWORK_TEST + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.package }}-${{ matrix.artifact }}-${{ matrix.python }} + path: wheelhouse/*.whl + if-no-files-found: error diff --git a/MANIFEST.in b/MANIFEST.in index 218d6ee..d1334a6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,4 +15,4 @@ prune vendor/kitten-text-processing/dist prune vendor/kitten-text-processing/.venv global-exclude __pycache__ *.py[cod] *.so *.a *.o .git -recursive-include packaging *.py *.md +recursive-include packaging *.py *.md *.sh diff --git a/README.md b/README.md index f8df6d2..60be165 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # KittenTTS2 C++ -CPU-focused C++ inference for KittenTTS2, with built-in voices, expression controls, and prepared reference voices. The runtime downloads prepared model assets and runs synthesis natively in C++. Users do not need to run the Python exporter. +C++ inference for KittenTTS2 on the CPU, with an optional NVIDIA GPU (CUDA) path, built-in voices, expression controls, and prepared reference voices. The runtime downloads prepared model assets and runs synthesis natively in C++. Users do not need to run the Python exporter. -The speech language model runs through llama.cpp/GGML, text normalization uses the [kitten-text-processing](https://github.com/KittenML/kitten-text-processing) submodule, and the S3 waveform decoder uses CPU LibTorch. Only KittenTTS2 checkpoints are supported. +The speech language model runs through llama.cpp/GGML, text normalization uses the [kitten-text-processing](https://github.com/KittenML/kitten-text-processing) submodule, and the S3 waveform decoder uses LibTorch on the CPU or on the GPU. Only KittenTTS2 checkpoints are supported. ## Features @@ -15,18 +15,20 @@ The speech language model runs through llama.cpp/GGML, text normalization uses t ## Python interface -Precompiled wheel builds are configured for Linux x86-64/ARM64, Apple Silicon macOS, Windows x86-64, and experimental Windows ARM64. See the [wheel matrix and installation guide](packaging/README.md). These builds remain local configuration until the workflow is pushed and run; cross-platform artifacts have not been published. +Precompiled wheel builds are configured for Linux x86-64/ARM64, Apple Silicon macOS, Windows x86-64, and experimental Windows ARM64. See the [wheel matrix and installation guide](packaging/README.md). The `py-interface` branch includes a native Python package with a KittenTTS-style API. `setup.py` invokes CMake during installation and bundles the normalizer grammar data. Install a CPU PyTorch build first, then build against that same installation: ```sh git submodule update --init vendor/kitten-text-processing -python -m pip install torch --index-url https://download.pytorch.org/whl/cpu +python3 -m venv .venv +. .venv/bin/activate +python -m pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cpu python -m pip install setuptools wheel cmake numpy soundfile python -m pip install . --no-build-isolation ``` -A C++17 compiler and OpenSSL development headers are required for HTTPS downloads (`libssl-dev` on Debian/Ubuntu). Set `CMAKE_BUILD_PARALLEL_LEVEL=8` to control build parallelism. `CMAKE_ARGS` accepts extra CMake flags, such as `-DOPENSSL_ROOT_DIR=/path/to/openssl`. Builds optimize for the local CPU; use `CMAKE_ARGS="-DGGML_NATIVE=OFF"` when building a wheel for other machines. Source-built wheels require the exact PyTorch release used at build time and the linked system OpenSSL libraries. The portable CI wheels instead link LibreSSL statically. +A C++20 compiler and OpenSSL development headers are required for HTTPS downloads (`libssl-dev` on Debian/Ubuntu). Set `CMAKE_BUILD_PARALLEL_LEVEL=8` to control build parallelism. `CMAKE_ARGS` accepts extra CMake flags, such as `-DOPENSSL_ROOT_DIR=/path/to/openssl`. Builds optimize for the local CPU; use `CMAKE_ARGS="-DGGML_NATIVE=OFF"` when building a wheel for other machines. Source-built wheels require the exact PyTorch release used at build time and the linked system OpenSSL libraries. The portable CI wheels instead link LibreSSL statically. ```python from kitten_tts_cpp import KittenTTS # KittenTTS2 and get_model are also available @@ -39,17 +41,19 @@ with KittenTTS("KittenML/kitten-tts-2", decoder="student_w4", threads=8) as mode pass # Each chunk is a float32 NumPy array at model.sample_rate. ``` -Weights and the decoder stay loaded between calls. The GIL is released during native loading and inference. Native inference calls are serialized because the TorchScript decoder uses process-wide random and thread settings. Streaming yields untrimmed sentence chunks as they are decoded; close the iterator when stopping early. `generate` applies the same chunk joins as the CLI. `last_report` contains token IDs and generation timings. +For NVIDIA GPUs, install the GPU package instead (`kitten-tts-cpp-cu12` on Linux, `kitten-tts-cpp-cu13` on Windows) and pass `gpu_layers=99`; see the [GPU wheel guide](packaging/README.md#using-the-gpu). + +Weights and the decoder stay loaded between calls. The GIL is released during native loading and inference. Native inference calls are serialized because the TorchScript decoder uses process-wide random and thread settings. Streaming yields untrimmed sentence chunks as they are decoded; close the iterator when stopping early. `generate` applies the same chunk joins as the CLI. `last_report` contains token IDs, generation timings and the LM placement (`last_report["lm"]["placement"]`). The constructor also accepts `cache_dir`, `hf_token`, `revision`, `offline`, `decoder_threads`, `repack`, `model_path` (a local GGUF override), `data_dir`, and `ca_file` (a custom HTTPS trust bundle). A local prepared asset directory can replace the repository ID. Generation supports the Python KittenTTS2 preset, sampling, normalization, reference-prompt, and `advanced` options, including `use_emotion`. -Only KittenTTS2 CPU inference and `weights="packed"` are supported. This package uses the separate `kitten_tts_cpp` import name so it can coexist with `kittenml`. Raw-audio `reference=` cloning is not implemented; prepare a reference with `prepare_reference.py` and select its voice name. It does not provide the older ONNX models or guarantee identical sampled output to Python's original PyTorch backend. +Only KittenTTS2 and `weights="packed"` are supported; `device=` accepts only `"cpu"` (use `gpu_layers` and `decoder_device` for the GPU). This package uses the separate `kitten_tts_cpp` import name so it can coexist with `kittenml`. Raw-audio `reference=` cloning is not implemented; prepare a reference with `prepare_reference.py` and select its voice name. It does not provide the older ONNX models or guarantee identical sampled output to Python's original PyTorch backend. For development, `python setup.py build_ext --inplace` builds the extension; set `PYTHONPATH=python` to import it directly. `python setup.py bdist_wheel` produces an installable wheel, and `python setup.py sdist` includes the native sources and normalizer submodule contents. ## Build -Requires a C++17 compiler, CMake, and CPU LibTorch. A CPU PyTorch installation can provide LibTorch's headers and libraries. Use the same LibTorch version for export and inference, with compatible torch/torchaudio versions for asset preparation. +The build commands use a Linux shell. Requires a C++20 compiler, CMake, Python 3.10+, OpenSSL development files (`libssl-dev` on Debian/Ubuntu), and LibTorch 2.13.0. Use the CPU PyTorch environment from [Python interface](#python-interface); it supplies LibTorch. Publishers use a separate [export environment](tools/kitten-tts/README.md#export-assets-publishers-only). ```sh git submodule update --init vendor/kitten-text-processing @@ -70,9 +74,43 @@ cmake --build build --target kitten-tts -j 8 The normalizer prepares its grammar data using Python at build time. To use existing grammar data, configure with `-DKITTEN_DATA_DIR=/path/to/data`. +## GPU (CUDA) build + +For the recommended full GPU setup, install an NVIDIA driver, CUDA 12.8 (`nvcc` on `PATH`), and CMake 3.28+. From the repository root: + +```sh +python3 -m venv .venv-cuda +. .venv-cuda/bin/activate +python -m pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cu129 +git submodule update --init vendor/kitten-text-processing +cmake -S . -B build-cuda \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DGGML_CUDA=ON -DGGML_METAL=OFF \ + -DCMAKE_CUDA_ARCHITECTURES='75-real;86-real;89-real;120a-real' \ + -DLLAMA_BUILD_KITTEN_TTS=ON \ + -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DCMAKE_PREFIX_PATH="$(python -c 'import torch; print(torch.utils.cmake_prefix_path)')" +cmake --build build-cuda --target kitten-tts -j 8 +build-cuda/bin/kitten-tts --gpu-layers 99 --decoder-device cuda --timings --text 'Hello from the GPU.' --output gpu.wav +``` + +`--gpu-layers 99` runs the language model on the GPU; the default, 0, keeps the CPU path and its output unchanged. To run the S3 decoder on the GPU as well, add `--decoder-device cuda`: the runtime downloads the decoder's CUDA export (`default` and `student_w4`; `student_w8` has none) and needs a CUDA build of LibTorch ([how](tools/kitten-tts/README.md#cuda-waveform-decoder)). + +| Setup (RTX 4090) | Real-time factor (lower is faster) | GPU memory | +|---|---:|---:| +| Language model and decoder on the GPU (recommended) | 0.15 | about 2.2-2.9 GB | +| Language model on the GPU, decoder on the CPU | 0.41 | about 1.7 GB | +| CPU only | 1.28 | none | + +Running the decoder on the CPU, or only part of the language model on the GPU (`--allow-hybrid`), is supported but slower. Details: [GPU guide](tools/kitten-tts/README.md#gpu-cuda-build). + ## Download and run -The runtime fetches `config.json` from `KittenML/kitten-tts-2` first, then downloads its GGUF, selected TorchScript decoder, and matching voice conditioning. Assets are cached under `$XDG_CACHE_HOME/kitten-tts` (or `~/.cache/kitten-tts`). +The runtime fetches `config.json` from `KittenML/kitten-tts-2` first, then downloads its GGUF, selected TorchScript decoder, and matching voice conditioning. Assets are cached under `$XDG_CACHE_HOME/kitten-tts`, or `~/.cache/kitten-tts` when it is unset. `HOME` falls back to `USERPROFILE` on Windows. ```sh build/bin/kitten-tts --text 'Hello from Kitten T T S.' --output hello.wav @@ -85,9 +123,9 @@ The model repository must publish a `cpp` manifest and prepared files as describ The default GGUF uses this fork's `TQ2_1` format: 256 ternary weights packed into 64 bytes, plus two FP16 scales for the checkpoint's 128-weight groups. The exporter requires exact reconstruction. This reduces the model from 1.45 GB with Q4_0 to 1.03 GB. Embeddings remain FP16 and one-dimensional tensors remain FP32. GGML's quantized activation arithmetic can still introduce small logit differences. -`TQ2_1` requires this fork's CPU runtime; it is not standard `TQ2_0` and stock llama.cpp cannot load it. Use `--outtype ternary-q4_0` for the previous standard Q4_0 storage format, or `--outtype f16` for the reference export. +`TQ2_1` requires this fork; it is not standard `TQ2_0` and stock llama.cpp cannot load it. The fork has CPU kernels for it, and GPU builds repack it losslessly to Q2_0 at load time. Use `--outtype ternary-q4_0` for the previous standard Q4_0 storage format, or `--outtype f16` for the reference export. -On supported Intel CPUs, the runtime losslessly repacks TQ2_1 weights into the existing AMX Q4_0 layout at load time. The GGUF stays 1.03 GB, but runtime transformer-weight storage grows from about 357 MiB to 756 MiB. Use `--no-repack` to keep the compact CPU representation. CPUs without AMX use the existing compact kernels. +On supported Intel CPUs, the runtime losslessly repacks TQ2_1 weights into the existing AMX Q4_0 layout at load time. The GGUF stays 1.03 GB, but runtime transformer-weight storage grows from about 357 MiB to 756 MiB. Use `--no-repack` to keep the compact CPU representation; it affects only CPU buffers, not the GPU repack to Q2_0. CPUs without AMX use the existing compact kernels. See the [detailed guide](tools/kitten-tts/README.md) for FP16 exports, student decoders, reference voice preparation, and the normalizer submodule setup. @@ -121,7 +159,7 @@ The implementation is checked against `kittenml.kittentts2`: | Sampling rules | 12 cases passed | | Audio joins | 3 cases passed | | Packed weights | All 310 tensors reconstruct the FP16 reference exactly | -| Native waveform decoding | Sample-for-sample agreement with Python in tested cases for all three decoders | +| Native waveform decoding | Agreement with Python within numerical tolerances in tested cases | | Packed LM logits | Close numerical agreement with the Python reference | On a Xeon Platinum 8462Y+ with eight pinned CPU cores, the AMX path reduced a 256-token forced-sequence workload from **6.40 to 5.60 seconds** (about **14% higher throughput**). Prompt evaluation fell from **1.09 to 0.58 seconds**. TQ2_1 reached **45.7 generated tokens/second**, comparable to Q4_0 at **44.3** in the same test. These are medians of three warmed runs, including prompt evaluation and sampling but excluding model loading. Performance varies with hardware and workload. @@ -143,7 +181,7 @@ Measurements and source revisions are recorded in [validation.json](tools/kitten Sampled output is not bit-identical to Python: BF16 arithmetic, GGML activation quantization, and sampling RNGs differ. Matching seeds do not guarantee matching generated tokens across runtimes. -Voice cloning requires an offline Python preparation step and a reference transcript. The Python interface supports sentence-chunk streaming; sample-by-sample streaming is not implemented. Waveform decoding currently depends on CPU LibTorch. +Voice cloning requires an offline Python preparation step and a reference transcript. The Python interface supports sentence-chunk streaming; sample-by-sample streaming is not implemented. Waveform decoding currently depends on LibTorch, on the CPU or on CUDA. ## Credits diff --git a/packaging/README.md b/packaging/README.md index aae475d..efb1773 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -2,23 +2,47 @@ The `Python wheels` workflow runs on pushes to `py-interface` and supports manual dispatch. It builds and tests wheels as GitHub Actions artifacts; it does not publish to PyPI or create a release. +CI status below is from the [2026-10-22 run](https://github.com/haaziq-sl/kitten-tts-2-cpp/actions/runs/39751306869). + | Target | Python versions | Runner | Status | | --- | --- | --- | --- | -| Linux x86-64, glibc 2.34+ | 3.10-3.14 | ubuntu-24.04 + manylinux_2_34 | CI validation in progress; local x86-64 build tested | -| Linux ARM64, glibc 2.34+ | 3.10-3.14 | ubuntu-24.04-arm + manylinux_2_34 | CI validation in progress | -| macOS 14+, Apple Silicon | 3.10-3.14 | macos-14 | CI validation in progress | -| Windows x86-64 | 3.10-3.14 | windows-2022 | CI validation in progress | -| Windows ARM64 | 3.11-3.13 | windows-11-arm + ClangCL | Experimental; CI validation in progress | +| Linux x86-64, glibc 2.34+ | 3.10-3.14 | ubuntu-24.04 + manylinux_2_34 | CI smoke passed | +| Linux ARM64, glibc 2.34+ | 3.10-3.14 | ubuntu-24.04-arm + manylinux_2_34 | CI smoke passed | +| macOS 14+, Apple Silicon | 3.10-3.14 | macos-14 | CI smoke passed | +| Windows x86-64 | 3.10-3.14 | windows-2022 | CI smoke passed | +| Windows ARM64 | 3.11-3.13 | windows-11-arm + ClangCL | Experimental; CI smoke passed | These are 23 CPython builds. This is not universal architecture support: Intel macOS, 32-bit platforms, Alpine/musl, mobile, WebAssembly, PyPy, and free-threaded Python are outside this matrix. The pinned PyTorch 2.13.0 distribution does not provide compatible wheels for most of these targets. The CPU index has s390x builds, but this project does not yet have an s390x build/test runner. Sources for the matrix: [PyTorch CPU wheels](https://download.pytorch.org/whl/cpu/torch/), [cibuildwheel configuration](https://cibuildwheel.pypa.io/en/stable/options/), and [GitHub runner availability](https://docs.github.com/en/actions/reference/runners/github-hosted-runners). +### NVIDIA GPU (CUDA) wheels + +The same workflow also builds the package with NVIDIA GPU support: `kitten-tts-cpp-cu12` for Linux and `kitten-tts-cpp-cu13` for Windows. The import name stays `kitten_tts_cpp`. It runs the speech model on the CPU by default, and on an NVIDIA GPU when you ask for it (see [Using the GPU](#using-the-gpu)). + +| Target | Package | Python versions | Built with | Uses the CUDA libraries of | +| --- | --- | --- | --- | --- | +| Linux x86-64, glibc 2.34+ | `kitten-tts-cpp-cu12` | 3.10-3.14 | ubuntu-24.04 + manylinux_2_34, CUDA 12.8 compiler | PyTorch 2.13.0+cu129 | +| Windows x86-64 | `kitten-tts-cpp-cu13` | 3.10-3.14 | windows-2022, CUDA 13.0 compiler | PyTorch 2.13.0+cu130 | + +These are 10 CPython builds, one workflow job each. The Linux compiler is CUDA 12.8 and the PyTorch runtime is CUDA 12.9; both use CUDA 12 runtime and cuBLAS libraries. Windows uses CUDA 13.0 for both because PyTorch 2.13.0 has no CUDA 12.9 build for Windows. + +| | Linux (`kitten-tts-cpp-cu12`) | Windows (`kitten-tts-cpp-cu13`) | +| --- | --- | --- | +| Compiled GPU code | compute capability 6.1 (GTX 10xx), 7.5 (GTX 16xx, RTX 20xx, T4), 8.6 (RTX 30xx, A2000, A10), 8.9 (RTX 40xx, L4) and 12.0 (RTX 50xx, RTX PRO 6000) | the same without 6.1: CUDA 13 cannot compile for 6.1, so GTX 10xx cards need the Linux package or the CPU package | +| Portable GPU code | 8.0 PTX, compiled by the driver for GPUs without a matching binary and normally cached for reuse | the same | +| Waveform decoder on the GPU (`decoder_device="cuda"`) | 7.5 and newer; the PyTorch 2.13.0 CUDA 12.9 build has no code for 6.1 | 7.5 and newer | +| NVIDIA driver | one for CUDA 12; 570 or newer for 12.0 cards and for cards that use the portable code | 580 or newer (CUDA 13) | + +H100 uses the PTX fallback; its performance has not been benchmarked here. To build native H100 kernels, set `KITTEN_CUDA_ARCHITECTURES=90-real` (see [Building](#building)). + +The CUDA wheels do not bundle CUDA libraries. They use the CUDA runtime and cuBLAS that the matching PyTorch build installs, so install that PyTorch build first. Without an NVIDIA driver or GPU, the CUDA wheel still imports and runs on the CPU, and asking for the GPU gives a clear error. + ## Building Push to `py-interface` to start the build. Manual dispatch is also available once the workflow file exists on the default branch; select `py-interface` as the run branch. If the normalizer repository is private, provide `KITTEN_SUBMODULE_TOKEN` with read access to both repositories. Download the per-platform artifacts after the jobs pass. -Every wheel is installed into a separate test environment. The smoke test checks the Torch ABI version, packaged grammar data, native normalization, TorchScript loading/decoding, NumPy output, and an HTTPS config download without model credentials. Select the `model_test` workflow input for an additional real W4 generation on every wheel; private models require an `HF_TOKEN` Actions secret. A full release should pass that test on each platform first. +Every CI-built wheel is installed into a separate test environment. The smoke test checks the Torch ABI version, packaged grammar data, native normalization, TorchScript loading/decoding, NumPy output, and an HTTPS config download without model credentials. Select the `model_test` workflow input for an additional real W4 generation on every wheel; private models require an `HF_TOKEN` Actions secret. On a GPU machine, this also tests both published CUDA decoders and offline reuse. GitHub-hosted runners have no GPU; run those checks on GPU hardware before release. For local smoke tests, `KITTEN_WHEEL_MODEL_REVISION` pins the model commit and `KITTEN_WHEEL_MODEL_CACHE` selects the cache. To build a single target locally: @@ -29,6 +53,8 @@ python -m cibuildwheel --only cp311-manylinux_x86_64 --output-dir wheelhouse Linux cibuildwheel builds require Docker or Podman. macOS and Windows builds run on those operating systems with their compiler toolchains. Cross-platform wheel filenames cannot be obtained by renaming an existing wheel. +To build a single CUDA target locally, set the same options as the workflow's CUDA jobs (see `.github/workflows/python-wheels.yml`), for example `KITTEN_CUDA=1` and `PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu129` in `CIBW_ENVIRONMENT_LINUX`, and `CIBW_BEFORE_ALL_LINUX="sh {project}/packaging/install_cuda_linux.sh"`. + For a native portable build without containers: ```sh @@ -36,6 +62,17 @@ KITTEN_PORTABLE=1 CMAKE_ARGS='-DLLAMA_BUILD_LIBRESSL=ON' \ python setup.py build --build-base build-portable bdist_wheel ``` +For the CUDA package, install the tested CUDA toolkit and PyTorch combination (CUDA 12.8 and `cu129` on Linux; CUDA 13.0 and `cu130` on Windows), then add `KITTEN_CUDA=1`. On Linux: + +```sh +python -m pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cu129 +python -m pip install setuptools wheel "cmake>=3.28" ninja numpy soundfile certifi +KITTEN_CUDA=1 KITTEN_PORTABLE=1 CMAKE_ARGS='-DLLAMA_BUILD_LIBRESSL=ON' \ + python setup.py build --build-base build-cuda bdist_wheel +``` + +The package name follows the CUDA major version of the installed PyTorch (`kitten-tts-cpp-cu12` or `kitten-tts-cpp-cu13`). `KITTEN_CUDA_ARCHITECTURES` overrides the GPU architectures (default `61-real;75-real;80-virtual;86-real;89-real;120a-real` for CUDA 12, the same without `61-real` for CUDA 13). + This produces a wheel for the build host. On Linux, use `packaging/repair_wheel.py` with auditwheel/patchelf installed to repair and assign its actual glibc compatibility tag. A wheel built on a newer host cannot simply be labeled manylinux_2_28. The local CPython 3.11 test produced a manylinux_2_35_x86_64 wheel, not the CI matrix's manylinux_2_34 target. ## Installing an artifact @@ -44,9 +81,61 @@ Install the matching CPU PyTorch runtime, then the wheel for your platform and P ```sh python -m pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cpu -python -m pip install /path/to/kitten_tts_cpp-0.1.0---.whl +python -m pip install /path/to/kitten_tts_cpp----.whl +``` + +For the GPU package, install the CUDA build of PyTorch instead, then the GPU wheel. Use a virtual environment: `python3 -m venv .venv` and `. .venv/bin/activate` on Linux (recent distributions such as Ubuntu 24.04 do not let pip install into the system Python), or `py -m venv .venv` and `.venv\Scripts\activate` on Windows. On Linux: + +```sh +python -m pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cu129 +python -m pip install /path/to/kitten_tts_cpp_cu12----.whl ``` +On Windows, PyTorch needs the [Microsoft Visual C++ Redistributable (x64)](https://aka.ms/vs/17/release/vc_redist.x64.exe). A new Windows installation may not have it; PyTorch then fails to import with `WinError 126` for `c10.dll`. Then: + +```sh +python -m pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cu130 +python -m pip install /path/to/kitten_tts_cpp_cu13----win_amd64.whl +``` + +Install only one of the packages in an environment; all of them provide `kitten_tts_cpp`. The GPU package stops at import with an explanation if the installed PyTorch is a CPU-only build or was built for another CUDA major version. + No compiler, CMake, or exporter is needed to install the wheel. Model assets are still downloaded from the model repository on first use. The wheel includes the normalizer data and statically linked LibreSSL. It uses the separately installed PyTorch libraries rather than bundling another copy. Repair tools handle other native dependencies. HTTPS uses the certifi CA bundle by default; `SSL_CERT_FILE` or the constructor's `ca_file` parameter can select a custom trust bundle. This is why the wheel requires the exact PyTorch release it was compiled against and checks it before loading the native extension. Portable builds disable host-specific instruction flags, including AVX/AVX2/AVX-512 and AMX on x86, and use baseline GGML kernels. They prioritize compatibility and may be slower than local CPU-optimized builds. LibTorch retains its own CPU optimizations. Rebuild from source without `KITTEN_PORTABLE=1` for native CPU optimizations; quantized arithmetic can differ between kernel paths. + +## Using the GPU + +With the GPU package, pass `gpu_layers=99` to run the whole speech model on the GPU: + +```python +from kitten_tts_cpp import KittenTTS + +with KittenTTS("KittenML/kitten-tts-2", decoder="student_w4", gpu_layers=99) as model: + audio = model.generate("Hello from the GPU.", voice="Bruno", advanced={"seed": 1234}) + print(model.last_report["lm"]["placement"]) # "gpu" +``` + +The first CUDA call on Windows can take over a minute; keep the model open for faster repeated requests. + +The options match the command-line runner's flags, and invalid values give the same errors: + +| Option | Default | Meaning | +| --- | --- | --- | +| `gpu_layers` | `0` | `0` keeps the speech model on the CPU (same output as the CPU package). `99` or `"all"` puts the whole model on the GPU. Smaller values need `allow_hybrid=True`. | +| `main_gpu` | `0` | Which GPU to use. | +| `flash_attn` | `"auto"` | `"auto"`, `"on"` or `"off"`. | +| `allow_hybrid` | `False` | Accept a model that is only partly on the GPU. | +| `decoder_device` | `"cpu"` | `"cuda"` runs the waveform decoder on the GPU, with the decoder's CUDA export downloaded from the model repository (`default` and `student_w4`; `student_w8` has none). | + +For the recommended full GPU setup, add `decoder_device="cuda"`: `KittenTTS("KittenML/kitten-tts-2", decoder="student_w4", gpu_layers=99, decoder_device="cuda")`. The CUDA decoder downloads automatically; no export step is needed. + +`last_report["lm"]` lists where each part runs. On a machine without an NVIDIA GPU or driver, `gpu_layers` above 0 stops with "needs a visible CUDA GPU" and `gpu_layers=0` runs on the CPU. GPU results differ slightly from CPU results because the GPU computes with different precision; see the detailed guide. + +### Supported, but not the fastest setup + +These setups work and give correct audio, but they are slower than running both the speech model and the waveform decoder on the GPU. Real-time factor (RTF) is generation time divided by audio length, for one short sentence with the Linux GPU wheel on an RTX 6000 Ada (default decoder, 8 CPU threads); lower is faster. + +- **Waveform decoder on the CPU** (`decoder_device="cpu"`, the default): the decoder becomes the slowest step. RTF 0.48 against 0.12 with the decoder on the GPU. It is the only choice on compute capability 6.1 (GTX 10xx). +- **Speech model only partly on the GPU** (`gpu_layers` below 29 with `allow_hybrid=True`): the layers left on the CPU run at CPU speed. Use it only when the whole model does not fit in GPU memory. +- **CPU only** (`gpu_layers=0`): RTF 3.41 on the same machine. diff --git a/packaging/install_cuda_linux.sh b/packaging/install_cuda_linux.sh new file mode 100755 index 0000000..ff067f4 --- /dev/null +++ b/packaging/install_cuda_linux.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Install the CUDA 12.8 compiler, runtime, cuBLAS and NVRTC development files in a manylinux_2_34 (RHEL 9 family) +# build container. LibTorch's CMake package links NVRTC when torch is a CUDA build, so configuring fails without it. +set -eu +# EPEL is not needed and its mirrors can fail mid-sync, so skip it +dnf install -y --disablerepo='epel*' dnf-plugins-core +dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo +dnf install -y --disablerepo='epel*' cuda-nvcc-12-8 cuda-cudart-devel-12-8 libcublas-devel-12-8 cuda-nvrtc-devel-12-8 cuda-cccl-12-8 +/usr/local/cuda-12.8/bin/nvcc --version diff --git a/packaging/prepare_wheel.py b/packaging/prepare_wheel.py index a01a77e..56ed099 100644 --- a/packaging/prepare_wheel.py +++ b/packaging/prepare_wheel.py @@ -1,10 +1,13 @@ -"""Install the same CPU Torch version for the build and the resulting wheel.""" +"""Install the same Torch version for the build and the resulting wheel; with KITTEN_CUDA=1, its CUDA build.""" +import os import subprocess import sys +cuda = os.environ.get('KITTEN_CUDA') == '1' +index = ('cu130' if sys.platform == 'win32' else 'cu129') if cuda else 'cpu' subprocess.run([sys.executable, '-m', 'pip', 'install', 'torch==2.13.0', - '--index-url', 'https://download.pytorch.org/whl/cpu'], check=True) + '--index-url', 'https://download.pytorch.org/whl/' + index], check=True) subprocess.run([sys.executable, '-m', 'pip', 'install', 'setuptools>=68', 'wheel', - 'cmake>=3.21', 'numpy>=1.26', 'soundfile>=0.12', 'certifi>=2024.7.4'], check=True) + 'cmake>=3.21', 'numpy>=1.26', 'soundfile>=0.12', 'certifi>=2024.7.4'] + (['ninja'] if cuda else []), check=True) if sys.platform == 'win32': subprocess.run([sys.executable, '-m', 'pip', 'install', 'delvewheel>=1.9'], check=True) diff --git a/packaging/repair_wheel.py b/packaging/repair_wheel.py index 6db8eb1..0f6df3e 100644 --- a/packaging/repair_wheel.py +++ b/packaging/repair_wheel.py @@ -1,4 +1,8 @@ -"""Repair system dependencies, leaving LibTorch supplied by the pinned torch wheel.""" +"""Repair system dependencies, leaving LibTorch supplied by the pinned torch wheel. + +CUDA wheels (KITTEN_CUDA=1) also leave out the CUDA runtime and cuBLAS, which the CUDA build of torch installs, +and the driver library, which comes with the NVIDIA driver. +""" import importlib.util import os from pathlib import Path @@ -7,20 +11,31 @@ wheel, destination = sys.argv[1:] torch_lib = Path(importlib.util.find_spec('torch').origin).parent / 'lib' +cuda = os.environ.get('KITTEN_CUDA') == '1' env = os.environ.copy() if sys.platform == 'linux': - env['LD_LIBRARY_PATH'] = str(torch_lib) + os.pathsep + env.get('LD_LIBRARY_PATH', '') + paths = [torch_lib] + if cuda: + nvidia = torch_lib.parent.parent / 'nvidia' + paths += [nvidia / 'cuda_runtime/lib', nvidia / 'cublas/lib'] + env['LD_LIBRARY_PATH'] = os.pathsep.join(map(str, paths)) + os.pathsep + env.get('LD_LIBRARY_PATH', '') subprocess.run(['auditwheel', 'show', wheel], env=env, check=True) command = ['auditwheel', 'repair', '-w', destination] for library in sorted(torch_lib.glob('*.so*')): command += ['--exclude', library.name] + if cuda: + import torch + major = torch.version.cuda.split('.')[0] + for library in ('libcuda.so.1', f'libcudart.so.{major}', f'libcublas.so.{major}', f'libcublasLt.so.{major}'): + command += ['--exclude', library] command.append(wheel) elif sys.platform == 'darwin': env['DYLD_LIBRARY_PATH'] = str(torch_lib) + os.pathsep + env.get('DYLD_LIBRARY_PATH', '') command = [sys.executable, '-m', 'delocate.cmd.delocate_wheel', '--require-archs', 'arm64', '--exclude', '/torch/lib/', '-w', destination, wheel] elif sys.platform == 'win32': - excluded = ';'.join(p.name for p in sorted(torch_lib.glob('*.dll'))) + # A CUDA build of torch ships the CUDA runtime and cuBLAS DLLs in torch/lib. + excluded = ';'.join([p.name for p in sorted(torch_lib.glob('*.dll'))] + (['nvcuda.dll'] if cuda else [])) command = [sys.executable, '-m', 'delvewheel', 'repair', '--add-path', str(torch_lib), '--exclude', excluded, '-w', destination, wheel] else: diff --git a/packaging/smoke_wheel.py b/packaging/smoke_wheel.py index c8bf278..7ad82a6 100644 --- a/packaging/smoke_wheel.py +++ b/packaging/smoke_wheel.py @@ -1,4 +1,4 @@ -"""Exercise an installed wheel without downloading model weights or importing source code.""" +"""Exercise an installed wheel; opt into network and real-model checks with KITTEN_WHEEL_*_TEST.""" import json import os from pathlib import Path @@ -15,6 +15,10 @@ assert info['torch_version'] == torch.__version__.split('+')[0] assert (package / 'data/en/tagger.fst').is_file() assert (package / 'licenses/vendor_kitten-text-processing_LICENSE').is_file() +cuda_wheel = info.get('cuda', False) +if cuda_wheel: + assert torch.version.cuda is not None and info['cuda_runtime'].split('.')[0] == torch.version.cuda.split('.')[0], info +gpu_present = cuda_wheel and torch.cuda.is_available() class Decoder(torch.nn.Module): @@ -46,7 +50,30 @@ def forward(self, tokens, reference, features, embedding): np.testing.assert_allclose(audio, np.array([1, 2, 4299, 4299, 4299], dtype=np.float32) / 10000) assert json.loads(report)['samples'] == 5 del model -print('Installed wheel: import, ABI, grammar, TorchScript decoding, and NumPy output passed') + # The LM options are checked before the model loads, so a missing GGUF is enough here. + options = {'--assets': str(root), '--model': str(root / 'missing.gguf'), '--data': str(package / 'data'), '--threads': '1'} + for key, value, message in [('--gpu-layers', '-1', 'non-negative integer or all'), ('--gpu-layers', 'x', 'non-negative integer or all'), + ('--main-gpu', '-1', 'non-negative integer'), ('--flash-attn', 'maybe', 'auto, on or off'), + ('--decoder-device', 'tpu', 'cpu or cuda')]: + try: + _native.Engine({**options, key: value}) + except ValueError as error: + assert message in str(error), error + else: + raise AssertionError(f'{key} {value} was accepted') + if gpu_present: + print('CUDA device present: GPU generation is checked by the model test') + else: + # Without a CUDA device (or in the CPU package) GPU offload must fail with a clear message. + expected = 'needs a visible CUDA GPU' if cuda_wheel else 'needs a GPU build' + try: + _native.Engine({**options, '--gpu-layers': '99'}) + except RuntimeError as error: + assert expected in str(error), error + else: + raise AssertionError('--gpu-layers 99 was accepted without a CUDA device') + print(f'GPU checks skipped (no CUDA device); --gpu-layers 99 gives: {expected}') +print('Installed wheel: import, ABI, grammar, TorchScript decoding, NumPy output and option checks passed') if os.environ.get('KITTEN_WHEEL_NETWORK_TEST') == '1': with tempfile.TemporaryDirectory() as cache: @@ -60,7 +87,29 @@ def forward(self, tokens, reference, features, embedding): print('Native HTTPS config download passed') if os.environ.get('KITTEN_WHEEL_MODEL_TEST') == '1': - with kitten_tts_cpp.KittenTTS(decoder='student_w4', threads=2) as model: + options = {'threads': 2} + if os.environ.get('KITTEN_WHEEL_MODEL_REVISION'): + options['revision'] = os.environ['KITTEN_WHEEL_MODEL_REVISION'] + if os.environ.get('KITTEN_WHEEL_MODEL_CACHE'): + options['cache_dir'] = os.environ['KITTEN_WHEEL_MODEL_CACHE'] + with kitten_tts_cpp.KittenTTS(decoder='student_w4', **options) as model: audio = model.generate('Hello from Kitten T T S.', max_new_tokens=100, advanced={'seed': 1234}) assert audio.dtype == np.float32 and audio.size > 0 and np.isfinite(audio).all() + assert model.last_report['lm']['placement'] == 'cpu' print('Real W4 model synthesis passed') + if gpu_present: + with kitten_tts_cpp.KittenTTS(decoder='student_w4', gpu_layers=99, **options) as model: + audio = model.generate('Hello from Kitten T T S.', max_new_tokens=100, advanced={'seed': 1234}) + assert audio.dtype == np.float32 and audio.size > 0 and np.isfinite(audio).all() + assert model.last_report['lm']['placement'] == 'gpu', model.last_report['lm'] + print('Real W4 model synthesis with the LM on the GPU passed') + for decoder in ('default', 'student_w4'): + for offline in (False, True): + with kitten_tts_cpp.KittenTTS(decoder=decoder, gpu_layers=99, decoder_device='cuda', + offline=offline, **options) as model: + assert model.decoder_device == 'cuda' and model.device == 'cuda' + audio = model.generate('Hello from Kitten T T S.', max_new_tokens=100, advanced={'seed': 1234}) + assert model.sample_rate == 24000 and audio.dtype == np.float32 and audio.ndim == 1 + assert audio.size > 0 and np.isfinite(audio).all() and np.max(np.abs(audio)) > 1e-4 + assert model.last_report['lm']['placement'] == 'gpu', model.last_report['lm'] + print(f'Published {decoder} CUDA decoder synthesis passed (offline={offline})') diff --git a/python/kitten_tts_cpp/__init__.py b/python/kitten_tts_cpp/__init__.py index f795e76..9aabe37 100644 --- a/python/kitten_tts_cpp/__init__.py +++ b/python/kitten_tts_cpp/__init__.py @@ -1,4 +1,4 @@ -"""KittenTTS2's Python interface backed by the CPU C++ runtime.""" +"""KittenTTS2's Python interface backed by the C++ runtime (CPU, or NVIDIA GPU with the CUDA package).""" from .model import KittenTTS2, KittenTTS, get_model __version__ = '0.2.0' diff --git a/python/kitten_tts_cpp/model.py b/python/kitten_tts_cpp/model.py index ac0e753..ac6fb78 100644 --- a/python/kitten_tts_cpp/model.py +++ b/python/kitten_tts_cpp/model.py @@ -4,6 +4,7 @@ import os from pathlib import Path import queue +import sys import threading import numpy as np @@ -11,10 +12,27 @@ import torch # Load LibTorch's shared dependencies before the extension. _build_info = Path(__file__).with_name('_build_info.json') -if _build_info.is_file(): - _required_torch = json.loads(_build_info.read_text())['torch_version'] +_info = json.loads(_build_info.read_text()) if _build_info.is_file() else {} +if _info: + _required_torch = _info['torch_version'] if torch.__version__.split('+')[0] != _required_torch: raise ImportError(f'This kitten-tts-cpp wheel requires torch=={_required_torch}; install that version or rebuild from source') + if _info.get('cuda'): + # The extension uses the CUDA runtime and cuBLAS that a CUDA build of torch installs. + _major = _info.get('cuda_runtime', '12').split('.')[0] + _package = f'kitten-tts-cpp-cu{_major}' + _index = 'cu' + _info.get('cuda_runtime', f'{_major}.0').replace('.', '') + _install = (f'pip install torch=={_required_torch} --index-url https://download.pytorch.org/whl/{_index}, ' + 'or use the CPU package kitten-tts-cpp') + if torch.version.cuda is None: + raise ImportError(f'{_package} needs a CUDA {_major} build of torch=={_required_torch}, but the installed ' + f'torch {torch.__version__} is CPU-only. Install the CUDA build with {_install}') + if torch.version.cuda.split('.')[0] != _major: + raise ImportError(f'{_package} needs a CUDA {_major} build of torch=={_required_torch}, but the installed ' + f'torch {torch.__version__} uses CUDA {torch.version.cuda}. Install the CUDA {_major} build with {_install}') +if sys.platform == 'win32': + # LibTorch's DLLs, and for a CUDA torch the CUDA runtime and cuBLAS, are in torch/lib. + _dll_directory = os.add_dll_directory(str(Path(torch.__file__).parent / 'lib')) from . import _native @@ -43,18 +61,28 @@ def _finite(name, value): class KittenTTS2: - """Persistent CPU model. Uses the same native engine as the kitten-tts CLI. + """Persistent model. Uses the same native engine as the kitten-tts CLI. model_name is a Hugging Face repository or a local native asset directory. Raw recording cloning and Python checkpoint weight formats are not supported. + + The speech LM runs on the CPU by default. The CUDA package (kitten-tts-cpp-cu12 on Linux, + kitten-tts-cpp-cu13 on Windows) can run it on an NVIDIA GPU: gpu_layers=99 (or 'all') offloads + the whole LM, main_gpu selects the GPU, flash_attn is + 'auto', 'on' or 'off', and allow_hybrid=True accepts weights left on the CPU when gpu_layers is + below 29. decoder_device='cuda' runs the waveform decoder on the GPU with the decoder's CUDA export + (downloaded like the other assets; student_w8 has none). + These options are checked like the CLI's --gpu-layers, --main-gpu, --flash-attn, --allow-hybrid + and --decoder-device. last_report['lm']['placement'] is 'cpu', 'gpu' or 'hybrid'. """ def __init__(self, model_name='KittenML/kitten-tts-2', cache_dir=None, device=None, hf_token=None, decoder=None, weights='packed', *, threads=8, decoder_threads=None, revision=None, offline=False, - model_path=None, data_dir=None, repack=True, ca_file=None): + model_path=None, data_dir=None, repack=True, ca_file=None, + gpu_layers=0, main_gpu=0, decoder_device='cpu', flash_attn='auto', allow_hybrid=False): if device not in (None, 'cpu', torch.device('cpu')): - raise ValueError('kitten-tts-cpp supports device="cpu" only') + raise ValueError('device must be None or "cpu"; use gpu_layers and decoder_device="cuda" for an NVIDIA GPU') if weights != 'packed': raise ValueError('Only weights="packed" is supported; use model_path for a custom GGUF') _integer('threads', threads, 1) @@ -80,6 +108,13 @@ def __init__(self, model_name='KittenML/kitten-tts-2', cache_dir=None, args['--offline'] = '1' if not repack: args['--no-repack'] = '1' + if not isinstance(allow_hybrid, bool): + raise ValueError('allow_hybrid must be a boolean') + # Passed as text, so the native engine checks the values with the CLI's rules and messages. + args.update({'--gpu-layers': str(gpu_layers), '--main-gpu': str(main_gpu), + '--decoder-device': str(decoder_device), '--flash-attn': str(flash_attn)}) + if allow_hybrid: + args['--allow-hybrid'] = '1' self._streams_lock = threading.Lock() self._stops = set() self._lock = threading.RLock() @@ -87,7 +122,9 @@ def __init__(self, model_name='KittenML/kitten-tts-2', cache_dir=None, metadata = json.loads(self._engine.metadata()) self.config = metadata['config'] self._voices = metadata['voices'] - self.device = 'cpu' + self.placement = metadata.get('lm', {}).get('placement', 'cpu') + self.decoder_device = metadata.get('decoder_device', 'cpu') + self.device = 'cuda' if self.placement != 'cpu' or self.decoder_device == 'cuda' else 'cpu' self.decoder = decoder or self.config.get('cpp_decoder', self.config.get('default_decoder', 'default')) self.last_report = None diff --git a/setup.py b/setup.py index 3d0d6f7..fdd18e5 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,8 @@ -"""Build the KittenTTS2 CPU extension with CMake during package installation.""" +"""Build the KittenTTS2 extension with CMake during package installation. + +KITTEN_CUDA=1 builds the CUDA package, kitten-tts-cpp-cu, against the CUDA build of torch installed for the +build; N is that torch's CUDA major version. +""" import os import importlib.util import importlib.metadata @@ -14,6 +18,16 @@ ROOT = Path(__file__).resolve().parent TORCH_VERSION = importlib.metadata.version('torch').split('+')[0] +CUDA = os.environ.get('KITTEN_CUDA', '0') == '1' +if CUDA: + import torch + CUDA_MAJOR = (torch.version.cuda or '12').split('.')[0] # a CPU-only torch is rejected in build_extension +else: + CUDA_MAJOR = None +# Real cubins: 6.1 Pascal (CUDA 12 only; CUDA 13 cannot compile for it), 7.5 Turing, 8.6 Ampere, 8.9 Ada, 12.0 Blackwell. +# 8.0 PTX lets other cards from 8.0 up (for example A100, H100) compile the kernels at first start. +CUDA_ARCHITECTURES = os.environ.get('KITTEN_CUDA_ARCHITECTURES', + ('61-real;' if CUDA_MAJOR == '12' else '') + '75-real;80-virtual;86-real;89-real;120a-real') class CMakeBuild(build_ext): @@ -30,6 +44,9 @@ def run(self): def build_extension(self, ext): import torch + if CUDA and torch.version.cuda is None: + raise RuntimeError(f'KITTEN_CUDA=1 needs a CUDA build of torch=={TORCH_VERSION}; ' + 'packaging/prepare_wheel.py installs the one the release wheels use') if not (ROOT / 'vendor/kitten-text-processing/CMakeLists.txt').is_file(): raise RuntimeError('Initialize vendor/kitten-text-processing with git submodule update --init before installing') output = Path(self.get_ext_fullpath(ext.name)).resolve().parent @@ -49,12 +66,17 @@ def build_extension(self, ext): '-DLLAMA_BUILD_KITTEN_TTS=ON', '-DKITTEN_BUILD_PYTHON=ON', '-DLLAMA_BUILD_TESTS=OFF', '-DLLAMA_BUILD_EXAMPLES=OFF', '-DLLAMA_BUILD_SERVER=OFF', '-DLLAMA_BUILD_MTMD=OFF', '-DLLAMA_BUILD_APP=OFF', - '-DGGML_CUDA=OFF', '-DGGML_METAL=OFF', '-DGGML_BACKEND_DL=OFF', + f'-DGGML_CUDA={"ON" if CUDA else "OFF"}', '-DGGML_METAL=OFF', '-DGGML_BACKEND_DL=OFF', '-DGGML_NATIVE=ON', '-DLLAMA_OPENSSL=ON', f'-DPython3_EXECUTABLE={sys.executable}', f'-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}', f'-DKITTEN_PYTHON_OUTPUT_DIR={output}', ] + if CUDA: + # GGML_CUDA_NO_VMM: no link to the driver library, so the wheel also imports and runs on the CPU + # on machines without an NVIDIA driver. The CUDA runtime and cuBLAS come from torch's dependencies. + # NCCL is only for multi-GPU splits, which the engine does not use; a system NCCL would be bundled. + args += ['-DGGML_CUDA_NO_VMM=ON', '-DGGML_CUDA_NCCL=OFF', f'-DCMAKE_CUDA_ARCHITECTURES={CUDA_ARCHITECTURES}'] if (Path(sys.base_prefix) / 'include/openssl/ssl.h').is_file(): args.append(f'-DOPENSSL_ROOT_DIR={sys.base_prefix}') args += shlex.split(os.environ.get('CMAKE_ARGS', '')) @@ -72,9 +94,16 @@ def build_extension(self, ext): if not (data / 'en/tagger.fst').is_file(): raise RuntimeError('Native normalizer grammar data was not generated') shutil.copytree(data, output / 'data', dirs_exist_ok=True) - (output / '_build_info.json').write_text(json.dumps({ - 'torch_version': TORCH_VERSION, 'portable': portable, - }, indent=2) + '\n') + info = {'torch_version': TORCH_VERSION, 'portable': portable, 'cuda': CUDA} + if CUDA: + cache = (build / 'CMakeCache.txt').read_text(errors='replace') + nvcc = next(line.split('=', 1)[1] for line in cache.splitlines() if line.startswith('CMAKE_CUDA_COMPILER:')) + version = subprocess.run([nvcc, '--version'], check=True, capture_output=True, text=True).stdout + release = next(line for line in version.splitlines() if 'release' in line) # "..., release X.Y, VX.Y.Z" + toolkit = release.rsplit(',', 1)[-1].strip().lstrip('V') + info.update({'cuda_runtime': torch.version.cuda, 'cuda_toolkit': toolkit, + 'cuda_architectures': CUDA_ARCHITECTURES}) + (output / '_build_info.json').write_text(json.dumps(info, indent=2) + '\n') licenses = output / 'licenses' licenses.mkdir(exist_ok=True) for relative in ('LICENSE', 'tools/kitten-tts/LICENSE-KITTENTTS', @@ -90,8 +119,9 @@ def build_extension(self, ext): setup( - name='kitten-tts-cpp', version='0.2.0', - description='CPU KittenTTS2 inference with native C++ bindings', + name=f'kitten-tts-cpp-cu{CUDA_MAJOR}' if CUDA else 'kitten-tts-cpp', version='0.2.0', + description=(f'KittenTTS2 inference with native C++ bindings, NVIDIA GPU (CUDA {CUDA_MAJOR}) build' if CUDA + else 'CPU KittenTTS2 inference with native C++ bindings'), long_description=(ROOT / 'README.md').read_text(), long_description_content_type='text/markdown', author='KittenML', url='https://github.com/KittenML/kitten-tts-2-cpp', python_requires='>=3.10', diff --git a/tools/kitten-tts/CMakeLists.txt b/tools/kitten-tts/CMakeLists.txt index e1acd65..2a1b8b2 100644 --- a/tools/kitten-tts/CMakeLists.txt +++ b/tools/kitten-tts/CMakeLists.txt @@ -62,6 +62,11 @@ if(KITTEN_BUILD_PYTHON) set_target_properties(_native PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_RPATH "@loader_path/../torch/lib") elseif(UNIX) target_link_options(_native PRIVATE "-Wl,--exclude-libs,ALL") - set_target_properties(_native PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_RPATH "$ORIGIN/../torch/lib") + set(kitten_rpath "$ORIGIN/../torch/lib") + if(GGML_CUDA) + # the CUDA runtime and cuBLAS of the NVIDIA wheels that a CUDA build of torch installs + list(APPEND kitten_rpath "$ORIGIN/../nvidia/cuda_runtime/lib" "$ORIGIN/../nvidia/cublas/lib") + endif() + set_target_properties(_native PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_RPATH "${kitten_rpath}") endif() endif() diff --git a/tools/kitten-tts/README.md b/tools/kitten-tts/README.md index 6744b21..7b12504 100644 --- a/tools/kitten-tts/README.md +++ b/tools/kitten-tts/README.md @@ -1,18 +1,18 @@ -# KittenTTS2 CPU inference +# KittenTTS2 CPU and CUDA inference This is a local fork of DeepGrove's llama.cpp with a native C++ KittenTTS2 runner. -The speech LM runs in llama.cpp/GGML on CPU. Text normalization uses the pinned +The speech LM runs in llama.cpp/GGML on CPU, or on an NVIDIA GPU in a CUDA build +(see [GPU (CUDA) build](#gpu-cuda-build)). Text normalization uses the pinned `vendor/kitten-text-processing` submodule. The S3 waveform decoder runs through -CPU LibTorch, exported from the supplied KittenTTS Python package. No Python +LibTorch on the CPU, or on an NVIDIA GPU +(see [CUDA waveform decoder](#cuda-waveform-decoder)). It is +exported from the supplied KittenTTS Python package. No Python interpreter or subprocess is used during synthesis. Only KITTEN2 checkpoints are accepted. ## Build -Requires a C++17 compiler, CMake, and CPU LibTorch. A CPU PyTorch installation -can provide LibTorch's headers and libraries. Use matching torch/torchaudio -versions when preparing assets; export and runtime must use the same LibTorch -version. The normalizer decompresses its grammar data at build time using Python. +Requires a C++20 compiler, CMake, OpenSSL development headers, and LibTorch 2.13.0. Follow the root README's [environment setup](../../README.md#build) first. The normalizer decompresses its grammar data at build time using Python. Alternatively, set `KITTEN_DATA_DIR` to an already prepared grammar directory. ```sh @@ -32,11 +32,6 @@ cmake -S . -B build \ cmake --build build --target kitten-tts -j 8 ``` -The GitHub normalizer URL was unavailable during initial setup. This workspace's -submodule was populated from the clean sibling checkout, while `.gitmodules` -retains `https://github.com/KittenML/kitten-text-processing.git`. A fresh clone -needs access to that URL, or an equivalent checkout of the pinned commit. - ## Download for inference Users do not run the exporter. Run `build/bin/kitten-tts --text 'Hello.' --decoder student_w4 --output hello.wav`. The C++ runtime reads the model repository's `config.json` first and caches the selected native files. See the root README for repository, revision, cache, and offline options. @@ -49,15 +44,16 @@ This section is for model publishers. Export each supported decoder using the in python tools/kitten-tts/prepare_model_repo.py \ --repo ../kitten-tts-2 --assets models/kitten2 \ --student-w4 models/kitten2-student-w4 \ - --student-w8 models/kitten2-student-w8 + --cuda models/kitten2-cuda --student-w4-cuda models/kitten2-cuda-student-w4 ``` -This copies the shared GGUF into `cpp/model-tq2_1.gguf` and each decoder's TorchScript and matching prepared `voices.json` into `cpp//`. It preserves Python configuration fields and adds `cpp.version`, `cpp.gguf`, and `cpp.decoders`. Each file descriptor contains `file` (repository-relative path) and `size` (bytes). The runtime rejects unsupported manifest versions, unsafe paths, and size mismatches. Publish `cpp/` before the updated `config.json`, or upload both in one repository revision. Pin a commit with `--revision` for reproducible downloads. Existing Python decoder weights under `decoders/` remain separate from TorchScript exports. +The manifest lists only the supplied exports; include `--student-w8 PATH` to retain a W8 export. + +This copies the shared GGUF into `cpp/model-tq2_1.gguf` and each decoder's TorchScript and matching prepared `voices.json` into `cpp//`. A CUDA export (`--cuda`, `--student-w4-cuda`, `--student-w8-cuda`) is copied to `cpp//decoder-cuda.pt` and listed as `torchscript_cuda`; its `voices.json` must match the CPU export's. It preserves Python configuration fields and adds `cpp.version`, `cpp.gguf`, and `cpp.decoders`. Each file descriptor contains `file` (repository-relative path) and `size` (bytes). The runtime rejects unsupported manifest versions, unsafe paths, and size mismatches. Publish `cpp/` before the updated `config.json`, or upload both in one repository revision. Pin a commit with `--revision` for reproducible downloads. Existing Python decoder weights under `decoders/` remain separate from TorchScript exports. ## Export assets (publishers only) -Install `requirements-export.txt` in a conversion environment with a CPU -torch/torchaudio pair. The KittenTTS2 dependencies include +In a separate environment, install `torch==2.11.0` and `torchaudio==2.11.0` from the CPU index, then `tools/kitten-tts/requirements-export.txt` and the KittenTTS Python package. The KittenTTS2 dependencies include `s3tokenizer`, `diffusers`, `omegaconf`, `einops`, `transformers`, `safetensors`, `librosa`, and `soundfile`. The exporter uses the normalizer submodule directly. S3's weights must be cached or downloadable from @@ -70,15 +66,15 @@ python tools/kitten-tts/export.py \ ``` This writes `model-tq2_1.gguf`, `decoder.pt`, `voices.json`, `config.json`, -and a decoder parity report. All 38 built-in voices in the current checkpoint -are exported, including their speaker projections and S3 reference conditioning. +and a decoder parity report. All built-in voices in the checkpoint are exported, +including their speaker projections and S3 reference conditioning. `--stage lm` and `--stage decoder` can be run independently. The checkpoint contains folded ternary weights with 128-element scale groups. The default `--outtype tq2_1` preserves them with this fork's `TQ2_1` GGUF extension (tensor type 43, file type 42). Each 68-byte block stores 256 weights: 64 packed bytes followed by two FP16 scales. Bytes 0-31 encode the first 128 weights; bytes 32-63 encode the second 128. In each group, byte `m` stores values `m`, `m+32`, `m+64`, and `m+96` in consecutive two-bit fields. Codes 0, 1, and 2 mean -1, 0, and +1. Code 3 is invalid. The exporter verifies exact reconstruction and rejects non-ternary matrices or scales that cannot be stored exactly in FP16. Embeddings and the tied output head remain FP16; one-dimensional tensors remain FP32. The model is 1.03 GB, versus 1.45 GB for the previous Q4_0 export and 3.47 GB for FP16. Packed transformer storage is 53% smaller than Q4_0; total model savings are 29% because other tensors are unchanged. -This extension is CPU-only and requires this fork. Standard `TQ2_0` uses one scale per 256 weights and cannot preserve the checkpoint's two scales. `TQ2_1` has an AVX2 dot-product kernel with optional VNNI instructions and a portable scalar fallback. The compact kernels use Q8_K activations, so lossless weight storage does not imply identical logits or sampled audio. +This extension requires this fork, and only the CPU has TQ2_1 kernels; GPU builds repack it to Q2_0 (see [GPU (CUDA) build](#gpu-cuda-build)). Standard `TQ2_0` uses one scale per 256 weights and cannot preserve the checkpoint's two scales. `TQ2_1` has an AVX2 dot-product kernel with optional VNNI instructions and a portable scalar fallback. The compact kernels use Q8_K activations, so lossless weight storage does not imply identical logits or sampled audio. On supported Intel AMX builds, two-dimensional TQ2_1 matrices are expanded losslessly into the existing AMX Q4_0 layout at load time. This reuses the optimized prompt and single-token matrix kernels with Q8_0 activations. The original two FP16 scales are copied to their corresponding 32-weight blocks; no weight rounding is introduced. The file format and 1.03 GB GGUF remain unchanged. Runtime transformer-weight storage is about 756 MiB instead of 357 MiB, with temporary conversion buffers and mapped file pages also contributing to peak memory. @@ -104,10 +100,35 @@ python tools/kitten-tts/export.py \ Then pass that directory with `--assets` and reuse the main GGUF via `--model`. Student exports preserve Python's shorter reference conditioning and one-step -flow. `student_w8` is also accepted. The exported student uses LibTorch FP32 +flow. `student_w8` is also accepted when its Python weights are available. The exported student uses LibTorch FP32 operators over the reconstructed quantized weights; it is not an integer-only vocoder. +## CUDA waveform decoder + +Install the [Linux build prerequisites](../../README.md#build), NVIDIA driver, CMake 3.28 or newer, and CUDA 12.8 toolkit first. To run the S3 decoder on the GPU too, build against the CUDA build of LibTorch and add `--decoder-device cuda`. The runtime then downloads the decoder's CUDA export from the model repository; `default` and `student_w4` have one, `student_w8` does not: + +```sh +python3 -m venv venv-cuda +venv-cuda/bin/pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cu129 +git submodule update --init vendor/kitten-text-processing +cmake -S . -B build-cuda \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DGGML_CUDA=ON -DGGML_METAL=OFF \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12.8/bin/nvcc \ + -DCMAKE_CUDA_ARCHITECTURES='75-real;86-real;89-real;120a-real' \ + -DLLAMA_BUILD_KITTEN_TTS=ON \ + -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF \ + -DCMAKE_PREFIX_PATH="$(venv-cuda/bin/python -c 'import torch; print(torch.utils.cmake_prefix_path)')" +cmake --build build-cuda --target kitten-tts -j 8 +build-cuda/bin/kitten-tts --gpu-layers 99 --decoder-device cuda --text 'Hello.' --output hello.wav +``` + +- The CUDA export draws the decoder noise on the GPU, so a seed gives different, equally valid audio than the CPU decoder. +- The decoder runs on `cuda:0`; select the GPU with `CUDA_VISIBLE_DEVICES`. +- A CUDA decoder takes about 6 s to load. + ## Synthesize ```sh @@ -132,6 +153,47 @@ warmup affects the first two iterations. RTF is synthesis time divided by audio duration and excludes loading and initial text normalization. An RTF below 1 means synthesis is faster than playback. `--decoder-threads` tunes S3 separately. +## GPU (CUDA) build + +For both stages on the GPU, follow [CUDA waveform decoder](#cuda-waveform-decoder). For a GPU LM with CPU decoding, use the CPU PyTorch environment from [Build](#build) and CUDA 12.8 (`nvcc` on `PATH`): + +```sh +cmake -S . -B build-cuda \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DGGML_CUDA=ON -DGGML_METAL=OFF \ + -DCMAKE_CUDA_ARCHITECTURES='75-real;86-real;89-real;120a-real' \ + -DLLAMA_BUILD_KITTEN_TTS=ON \ + -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF \ + -DCMAKE_PREFIX_PATH="$(python -c 'import torch; print(torch.utils.cmake_prefix_path)')" +cmake --build build-cuda --target kitten-tts -j 8 +build-cuda/bin/kitten-tts --gpu-layers 99 --timings \ + --text 'Hello from the GPU.' --output gpu.wav +``` + +This build keeps the decoder on the CPU; to run it on the GPU as well, use the build in [CUDA waveform decoder](#cuda-waveform-decoder). `CMAKE_CUDA_ARCHITECTURES` lists the GPU generations compiled in: 75 (GTX 16xx, RTX 20xx, T4), 86 (RTX 30xx), 89 (RTX 40xx, L4) and 120a (RTX 50xx). Other cards stop with "no kernel image is available"; add theirs, for example `80-real` (A100), `90-real` (H100) or `61-real` (GTX 10xx, whose decoder stays on the CPU). + +- `--gpu-layers N|all` (default 0): `99` or `all` puts the whole LM on the GPU. 0 keeps the CPU path and its output unchanged. +- `--decoder-device cpu|cuda` (default `cpu`): `cuda` downloads the decoder's CUDA export (none for `student_w8`). +- `--main-gpu N` (default 0) and `--flash-attn auto|on|off` (default `auto`). +- `--allow-hybrid`: accept an LM that is only partly on the GPU (`--gpu-layers` below 29). Without it the runtime stops with an error. + +The ternary weights are repacked losslessly to the GPU's Q2_0 format at load time. Text processing, the token embedding lookup, sampling and WAV writing stay on the CPU. GPU and CPU builds compute with slightly different precision, so the same seed can give different audio. + +Speed on an RTX 4090 (real-time factor: synthesis time divided by audio length; lower is faster): + +| Setup | RTF | GPU memory | +|---|---:|---:| +| `--gpu-layers 99 --decoder-device cuda` (recommended) | 0.15 | 2.2-2.9 GB | +| `--gpu-layers 99`, decoder on the CPU | 0.41 | 1.7 GB | +| CPU only | 1.28 | none | + +The other setups are supported but not the fastest, because part of the work stays on the CPU: the CPU decoder becomes the slowest stage, and with `--allow-hybrid` the layers left on the CPU run at CPU speed (use it only when the LM does not fit in GPU memory). + +`--timings` prints per-stage times, tokens per second, RTF and peak GPU memory. The report's `lm` object shows where the LM ran (`"placement"`: `cpu`, `gpu` or `hybrid`), and the runtime warns if an operation falls back to the CPU. + +The CLI runs batch synthesis; the Python interface supports sentence-chunk streaming on both devices. Raw-audio voice cloning requires the offline preparation step below. + ## Prepare a cloned voice Voice extraction is an offline Python preparation step. The C++ runtime consumes @@ -150,6 +212,8 @@ build/bin/kitten-tts --assets models/kitten2 --voice MyVoice \ ## Verify parity +Use freshly exported assets and the same Python checkout used for export. + ```sh python tools/kitten-tts/check_parity.py \ --binary build/bin/kitten-tts --python-package ../KittenTTS \ @@ -172,7 +236,7 @@ so LM sampling differences cannot hide decoder errors. Use `--assets` pointing to a student bundle to check that decoder. Detailed measurements and source revisions are in [validation.json](validation.json). -Validated on Linux x86-64, Xeon Platinum 8462Y+, eight CPU threads: +Recorded in September 2026 on Linux x86-64, Xeon Platinum 8462Y+, eight CPU threads: - 228/228 frontend cases and 12/12 sampling cases passed. - FP16 LM versus FP32 Python: logit RMSE 0.002-0.003, all tested argmaxes equal. @@ -201,7 +265,18 @@ python tools/kitten-tts/test_tq2.py build/bin/test-backend-ops test -b CPU -o MUL_MAT,GET_ROWS -p tq2_1 ``` -Configure with `-DLLAMA_BUILD_TESTS=ON` first. The Python byte-layout check uses the Linux library in `build/bin`. The dedicated native test checks exact reconstruction and the dot product against dequantized Q8_K activations. Native VNNI, AVX2 without VNNI, and a build with AVX disabled passed locally. +Configure with `-DLLAMA_BUILD_TESTS=ON` first. The Python byte-layout check uses the Linux library in `build/bin`. The dedicated native test checks exact reconstruction and the dot product against dequantized Q8_K activations. It also checks the TQ2_1 to Q2_0 repack element by element. Native VNNI, AVX2 without VNNI, and a build with AVX disabled passed locally. + +For a CUDA build (`build-cuda`, configured with `-DLLAMA_BUILD_TESTS=ON`): + +```sh +cmake --build build-cuda --target test-tq2_1 test-tq2_1-load test-backend-ops -j 8 +ctest --test-dir build-cuda -R 'test-tq2_1$' --output-on-failure +build-cuda/bin/test-backend-ops test -b CUDA0 -o MUL_MAT,GET_ROWS -p q2_0 +build-cuda/bin/test-tq2_1-load ../kitten-tts-2/cpp/model-tq2_1.gguf +``` + +`test-tq2_1-load` needs the model file, so ctest does not run it. It checks that every TQ2_1 weight loads on the GPU as Q2_0 with the same values as the file. The optional CTest frontend check can be enabled with `-DKITTEN_REFERENCE_PACKAGE=/path/to/KittenTTS` and diff --git a/tools/kitten-tts/assets.cpp b/tools/kitten-tts/assets.cpp index d3c00bc..b365fe2 100644 --- a/tools/kitten-tts/assets.cpp +++ b/tools/kitten-tts/assets.cpp @@ -30,7 +30,7 @@ assets resolve_assets(const std::map & args) { // Encode slashes in revision names so cache paths cannot collide. std::string cache_revision = revision; for (char & c : cache_revision) if (c == '/') c = '@'; - fs::path root = local ? fs::path(args.at("--assets")) : fs::path(get("--cache-dir", env("XDG_CACHE_HOME", env("HOME", ".") + "/.cache") + "/kitten-tts")) / repo / cache_revision; + fs::path root = local ? fs::path(args.at("--assets")) : fs::path(get("--cache-dir", env("XDG_CACHE_HOME", env("HOME", env("USERPROFILE", ".")) + "/.cache") + "/kitten-tts")) / repo / cache_revision; common_download_opts opts; opts.offline = args.count("--offline"); opts.bearer_token = get("--hf-token", env("HF_TOKEN")); @@ -64,6 +64,10 @@ assets resolve_assets(const std::map & args) { std::string name = get("--decoder", result.config.value("default_decoder", "default")); if (!cpp.at("decoders").contains(name)) throw std::runtime_error("unknown decoder: " + name); const auto & decoder = cpp.at("decoders").at(name); + // A TorchScript trace keeps the device it was exported on, so a CUDA decoder is a separate file. + const bool cuda = get("--decoder-device", "cpu") == "cuda"; + if (need_decoder && cuda && !decoder.contains("torchscript_cuda")) + throw std::runtime_error("decoder " + name + " has no CUDA export in " + (local ? args.at("--assets") : repo) + "; use --decoder-device cpu, or a CUDA export via --assets"); auto asset = [&](const json & entry) { auto path = fetch(entry.at("file").get()); if (fs::file_size(path) != entry.at("size").get()) throw std::runtime_error("asset size mismatch; remove and download again: " + path); @@ -71,7 +75,7 @@ assets resolve_assets(const std::map & args) { }; result.voices = asset(decoder.at("voices")); if (need_model) result.model = args.count("--model") ? args.at("--model") : asset(cpp.at("gguf")); - if (need_decoder) result.decoder = asset(decoder.at("torchscript")); + if (need_decoder) result.decoder = asset(decoder.at(cuda ? "torchscript_cuda" : "torchscript")); return result; } } diff --git a/tools/kitten-tts/decoder.cpp b/tools/kitten-tts/decoder.cpp index 98d0dab..79a7782 100644 --- a/tools/kitten-tts/decoder.cpp +++ b/tools/kitten-tts/decoder.cpp @@ -1,28 +1,132 @@ #include "decoder.h" #include "json.hpp" #include +#include +#include +#include +#include #include +#include +#include +#include +#if TORCH_VERSION_MAJOR > 2 || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 9) +#include +#define KITTEN_DEVICE_ALLOCATOR_STATS 1 +#endif namespace kitten { -struct decoder::impl { int threads; torch::jit::Module module; explicit impl(const std::string & path, int n_threads): threads(n_threads), module(torch::jit::load(path,torch::kCPU)) { module.eval(); } }; -decoder::decoder(const std::string & path,int threads,int seed) { - at::set_num_threads(threads); at::manual_seed(seed); p=std::make_unique(path,threads); +namespace { +// export.py stores this next to the graph. Files without it were traced on the CPU. +constexpr const char * metadata_file = "kitten_decoder.json"; + +bool env_flag(const char * name, bool fallback) { + const char * v = std::getenv(name); + if (!v || !*v) return fallback; + return std::string(v) != "0"; +} + +void report_cuda_memory(const torch::Device & device, const char * when) { +#ifdef KITTEN_DEVICE_ALLOCATOR_STATS + using c10::CachingAllocator::StatType; + auto stats = c10::getDeviceAllocator(c10::DeviceType::CUDA)->getDeviceStats(device.has_index() ? device.index() : 0); + const auto & a = stats.allocated_bytes[size_t(StatType::AGGREGATE)]; + const auto & r = stats.reserved_bytes[size_t(StatType::AGGREGATE)]; + const double mib = 1024.0 * 1024.0; + std::fprintf(stderr, "kitten-tts: decoder %s %s: allocated %.1f MiB (peak %.1f), reserved %.1f MiB (peak %.1f)\n", + device.str().c_str(), when, a.current / mib, a.peak / mib, r.current / mib, r.peak / mib); +#else + (void) device; (void) when; +#endif +} +} + +struct decoder::voice { torch::Tensor prompt_token, prompt_feat, embedding; }; + +struct decoder::impl { + int threads; + torch::Device device; + torch::jit::Module module; + bool empty_cache = false; + std::string rng = "device"; + bool sinegen_on_device = false; + impl(const std::string & path, torch::Device dev, int n_threads): threads(n_threads), device(dev) { + torch::jit::ExtraFilesMap extra{{metadata_file, ""}}; + // A CUDA trace keeps some small constants on the CPU (s3gen draws its phase noise there), so load tensors where they were saved. + module = device.is_cuda() ? torch::jit::load(path, {}, extra) : torch::jit::load(path, device, extra); + module.eval(); + std::string traced = "cpu"; + if (!extra[metadata_file].empty()) { + auto m = nlohmann::json::parse(extra[metadata_file]); + traced = m.value("device", "cpu"); + rng = m.value("rng", "device"); + sinegen_on_device = m.value("sinegen", "cpu") == "device"; + } + if (traced != c10::DeviceTypeName(device.type(), true)) + throw std::runtime_error("decoder " + path + " was exported for " + traced + ", not " + device.str() + + "; export a matching decoder with export.py --device " + c10::DeviceTypeName(device.type(), true) + + " or change --decoder-device"); + } +}; + +decoder::decoder(const std::string & path, int threads, int seed, const std::string & device) { + if (device != "cpu" && device != "cuda") throw std::runtime_error("--decoder-device must be cpu or cuda"); + at::set_num_threads(threads); + if (device == "cuda") { + if (!torch::cuda::is_available()) + throw std::runtime_error("--decoder-device cuda needs a CUDA build of LibTorch and a visible GPU"); + // Keep GEMMs and convolutions in full FP32, as on the CPU. cuDNN allows TF32 convolutions on SM80+ by default. + at::globalContext().setAllowTF32CuBLAS(false); + at::globalContext().setAllowTF32CuDNN(false); + // No profiling runs: the profiling executor made the first two CUDA calls take 10 s and 18 s on an RTX 4090. + torch::jit::getProfilingMode() = false; + } + at::manual_seed(seed); + // Pin cuda:0, where the trace keeps its weights. A bare "cuda" means the thread's current GPU, which GGML moves to --main-gpu. + p = std::make_unique(path, device == "cuda" ? torch::Device(torch::kCUDA, 0) : torch::Device(torch::kCPU), threads); + if (p->device.is_cuda()) { + p->empty_cache = env_flag("KITTEN_DECODER_EMPTY_CACHE", false); + std::fprintf(stderr, "kitten-tts: decoder on %s (LibTorch %s, TF32 off, TorchScript graph passes off)\n", p->device.str().c_str(), TORCH_VERSION); + report_cuda_memory(p->device, "loaded"); + } } decoder::~decoder() = default; +// LibTorch's thread count is process-wide, so set this decoder's count again. at::manual_seed also seeds every CUDA generator when CUDA is available. void decoder::seed(int value) { at::set_num_threads(p->threads); at::manual_seed(value); } -std::vector decoder::decode(const std::vector & ids,const std::string & conditioning_json) { - if(ids.empty()) return {}; +std::shared_ptr decoder::prepare_voice(const nlohmann::ordered_json & j) const { c10::InferenceMode guard; - auto j=nlohmann::json::parse(conditioning_json); - std::vector tokens(ids.begin(),ids.end()); tokens.insert(tokens.end(),3,4299); auto ref=j.at("prompt_token").at(0).get>(); std::vector feat; for(auto & row:j.at("prompt_feat").at(0)) {auto v=row.get>();feat.insert(feat.end(),v.begin(),v.end());} auto emb=j.at("embedding").at(0).get>(); auto opts=torch::TensorOptions().dtype(torch::kFloat32); - auto output=p->module.forward({torch::from_blob(tokens.data(),{1,int64_t(tokens.size())},torch::kInt64), - torch::from_blob(ref.data(),{1,int64_t(ref.size())},torch::kInt64), - torch::from_blob(feat.data(),{1,int64_t(feat.size()/80),80},opts), - torch::from_blob(emb.data(),{1,int64_t(emb.size())},opts)}).toTensor().contiguous(); + // from_blob does not own the vectors: copy to the device, or clone on the CPU + auto keep=[&](const torch::Tensor & t){return p->device.is_cuda()?t.to(p->device):t.clone();}; + return std::make_shared(voice{keep(torch::from_blob(ref.data(),{1,int64_t(ref.size())},torch::kInt64)), + keep(torch::from_blob(feat.data(),{1,int64_t(feat.size()/80),80},opts)), + keep(torch::from_blob(emb.data(),{1,int64_t(emb.size())},opts))}); +} +std::vector decoder::cpu_stages() const { + if(!p->device.is_cuda()) return {"S3 decoder (CPU LibTorch)"}; + std::vector stages={"S3 decoder: SineGen phase draw (9 values per chunk, CPU generator, copied to the GPU)"}; + if(p->rng=="cpu") stages.push_back("S3 decoder: noise draws (CPU generator, copied to the GPU; --rng cpu export)"); + if(!p->sinegen_on_device) stages.push_back("S3 decoder: SineGen source buffer zero-fill (9 floats per sample, copied to the GPU; decoder.pt exported without the device-side SineGen)"); + stages.push_back("S3 decoder: token upload and waveform copy to host"); + return stages; +} +std::vector decoder::decode(const std::vector & ids, const voice & v) { + if(ids.empty()) return {}; + c10::InferenceMode guard; + // On CUDA, skip TorchScript graph passes. Constant pooling and CSE compare every pair of similar weight constants with a GPU equal() and a sync, which made the first call take 4 s instead of 0.8 s. Thread-local, so it is set here. + torch::jit::GraphOptimizerEnabledGuard optimize(!p->device.is_cuda() && torch::jit::getGraphExecutorOptimize()); + std::vector tokens(ids.begin(),ids.end()); tokens.insert(tokens.end(),3,4299); + const auto & dev=p->device; // .to() is a no-op for CPU tensors on a CPU decoder + auto output=p->module.forward({torch::from_blob(tokens.data(),{1,int64_t(tokens.size())},torch::kInt64).to(dev), + v.prompt_token,v.prompt_feat,v.embedding}).toTensor().to(torch::kCPU).contiguous(); + if(dev.is_cuda()) { + report_cuda_memory(dev, "after decode"); +#ifdef KITTEN_DEVICE_ALLOCATOR_STATS + if(p->empty_cache) c10::getDeviceAllocator(c10::DeviceType::CUDA)->emptyCache(); +#endif + } return std::vector(output.data_ptr(),output.data_ptr()+output.numel()); } } diff --git a/tools/kitten-tts/decoder.h b/tools/kitten-tts/decoder.h index 2e63f1e..4c5a11d 100644 --- a/tools/kitten-tts/decoder.h +++ b/tools/kitten-tts/decoder.h @@ -1,4 +1,5 @@ #pragma once +#include "json.hpp" #include #include #include @@ -7,9 +8,17 @@ class decoder { struct impl; std::unique_ptr p; public: - decoder(const std::string & path, int threads, int seed); + // One voice's S3 conditioning on the decoder device. The decoder keeps no voice, so one decoder can serve several voices. + struct voice; + // device is "cpu" (default) or "cuda". A CUDA decoder needs a TorchScript file + // exported with `export.py --device cuda` and a CUDA build of LibTorch. + decoder(const std::string & path, int threads, int seed, const std::string & device = "cpu"); ~decoder(); void seed(int value); - std::vector decode(const std::vector & ids, const std::string & conditioning_json); + // Parse one voice's S3 conditioning (voices.json entry) and copy it to the decoder device. + std::shared_ptr prepare_voice(const nlohmann::ordered_json & conditioning) const; + std::vector decode(const std::vector & ids, const voice & v); + // Decoder work that runs on the CPU, for reports. + std::vector cpu_stages() const; }; } diff --git a/tools/kitten-tts/engine.cpp b/tools/kitten-tts/engine.cpp index 6d9de2f..aee5e60 100644 --- a/tools/kitten-tts/engine.cpp +++ b/tools/kitten-tts/engine.cpp @@ -1,15 +1,21 @@ #include "engine.h" #include "llama.h" +#include "../../src/llama-ext.h" // staging API: model tensors, devices, graph splits and memory breakdown +#include "ggml-backend.h" #include "text.h" #include "sampling.h" #include "decoder.h" #include +#include +#include #include #include #include namespace kitten { using clock_type = std::chrono::steady_clock; +static double seconds_since(clock_type::time_point t) { return std::chrono::duration(clock_type::now()-t).count(); } +static double mib(size_t bytes) { return bytes/1048576.0; } static json read_json(const std::string & path) { std::ifstream f(path); if(!f) throw std::runtime_error("cannot open "+path); json j; f>>j;return j; } @@ -28,6 +34,76 @@ struct batch { ~batch(){llama_batch_free(b);} void position(int i,int pos,bool logits) {b.pos[i]=pos;b.n_seq_id[i]=1;b.seq_id[i][0]=0;b.logits[i]=logits;} }; +// A weight is on the GPU when a GPU device owns its buffer and the buffer is not that device's pinned host buffer. +static bool weight_on_gpu(const ggml_tensor * t) { + if(!t->buffer) return false; + auto buft=ggml_backend_buffer_get_type(t->buffer);auto dev=ggml_backend_buft_get_device(buft); + if(!dev) return false; + auto type=ggml_backend_dev_type(dev); + return (type==GGML_BACKEND_DEVICE_TYPE_GPU || type==GGML_BACKEND_DEVICE_TYPE_IGPU) && buft!=ggml_backend_dev_host_buffer_type(dev); +} +struct placement { + std::string mode="cpu"; // cpu, gpu or hybrid + int tq2_1_cpu=0; + std::string first_tq2_1_cpu; + std::vector cpu_weights; // CPU-resident weights except the input token embedding + json buffers=json::object(); +}; +static placement inspect_placement(const llama_model * model) { + struct usage { int tensors=0; size_t bytes=0; std::map types; }; + std::map by_buffer; placement p; bool any_gpu=false,input_seen=false; + for(int32_t i=0;ibuffer?ggml_backend_buffer_name(t->buffer):"none"]; + ++u.tensors;u.bytes+=t->buffer?ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(t->buffer),t):ggml_nbytes(t);++u.types[ggml_type_name(t->type)]; + if(t->type==GGML_TYPE_TQ2_1 && !gpu) {if(p.tq2_1_cpu==0) p.first_tq2_1_cpu=ggml_get_name(t); ++p.tq2_1_cpu;} + any_gpu|=gpu; + // the input layer (token_embd lookup) always stays on the CPU by design + if(!gpu) {if(!input_seen && std::strcmp(ggml_get_name(t),"token_embd.weight")==0) input_seen=true; else p.cpu_weights.push_back(ggml_get_name(t));} + } + for(auto & [name,u]:by_buffer) p.buffers[name]={{"tensors",u.tensors},{"mib",mib(u.bytes)},{"types",u.types}}; + p.mode=!any_gpu?"cpu":p.cpu_weights.empty()?"gpu":"hybrid"; + return p; +} +// Device-wide used memory (total - free), sampled at stage ends. GGML CUDA pools only grow while a context lives, +// so the sample taken after generation sees the LM high-water mark. +struct gpu_memory { + struct device { ggml_backend_dev_t dev; size_t total=0,baseline=0,peak=0,low=0; std::string peak_stage="baseline"; std::map stages; bool used=false; }; + std::vector devices; + void init() { + for(size_t i=0;id.peak) {d.peak=used;d.peak_stage=stage;} + } + } + json to_json() const { + json out=json::array(); + for(auto & d:devices) { + if(!d.used) continue; + json stages=json::object();for(auto & [k,v]:d.stages) stages[k]=mib(v); + out.push_back({{"device",ggml_backend_dev_name(d.dev)},{"description",ggml_backend_dev_description(d.dev)},{"total_mib",mib(d.total)}, + {"baseline_used_mib",mib(d.baseline)},{"peak_used_mib",mib(d.peak)},{"peak_over_baseline_mib",mib(d.peak-d.baseline)},{"peak_stage",d.peak_stage},{"used_mib_by_stage",stages}, + // usage below the baseline means another process freed memory during the run + {"other_process_changed_usage",d.low defaults; @@ -36,6 +112,25 @@ struct engine::impl { std::unique_ptr normalizer; std::unique_ptr model{nullptr, llama_model_free}; std::unique_ptr decoder; + // S3 conditioning per voice, already on the decoder device + std::map> conditioning; + int gpu_layers=0, main_gpu=0; + llama_flash_attn_type flash_attn=LLAMA_FLASH_ATTN_TYPE_AUTO; + std::string flash_attn_arg="auto", decoder_device="cpu"; + json lm; // LM placement, devices and precision; null when no LM is loaded + json startup=json::object(); // one-time load costs, reported with --timings + gpu_memory gpu_mem; + clock_type::time_point created=clock_type::now(); + int requests=0; + const kitten::decoder::voice & voice(const std::string & name, double & seconds) { + auto it=conditioning.find(name); + if(it==conditioning.end()) { + auto begin=clock_type::now(); + it=conditioning.emplace(name,decoder->prepare_voice(voices.at(name))).first; + seconds+=seconds_since(begin); + } + return *it->second; + } }; engine::engine(const std::map & args): p(std::make_unique()) { std::lock_guard lock(inference_mutex); @@ -44,25 +139,109 @@ engine::engine(const std::map & args): p(std::make_uni p->defaults.erase("--hf-token"); int threads=std::stoi(get("--threads","8")), decoder_threads=std::stoi(get("--decoder-threads",std::to_string(threads))); if(threads<1 || decoder_threads<1) throw std::invalid_argument("thread counts must be positive"); + auto count=[&](const std::string & key,const std::string & value){ + try {size_t end=0;int n=std::stoi(value,&end);if(end==value.size() && n>=0) return n;} catch(const std::exception &) {} + throw std::invalid_argument(key+" must be a non-negative integer"+(key=="--gpu-layers"?" or all":"")); + }; + std::string gpu_arg=get("--gpu-layers","0"); + p->gpu_layers=gpu_arg=="all"?-1:count("--gpu-layers",gpu_arg); + p->main_gpu=count("--main-gpu",get("--main-gpu","0")); + const std::map fa_types={{"auto",LLAMA_FLASH_ATTN_TYPE_AUTO},{"on",LLAMA_FLASH_ATTN_TYPE_ENABLED},{"off",LLAMA_FLASH_ATTN_TYPE_DISABLED}}; + p->flash_attn_arg=get("--flash-attn","auto"); + if(!fa_types.count(p->flash_attn_arg)) throw std::invalid_argument("--flash-attn must be auto, on or off"); + p->flash_attn=fa_types.at(p->flash_attn_arg); + p->decoder_device=get("--decoder-device","cpu"); + if(p->decoder_device!="cpu" && p->decoder_device!="cuda") throw std::invalid_argument("--decoder-device must be cpu or cuda"); + const bool timings=args.count("--timings"); + auto started=clock_type::now(); p->assets=resolve_assets(args); + p->startup["assets_seconds"]=seconds_since(started); p->config=p->assets.config; + started=clock_type::now(); p->voices=read_json(p->assets.voices); + p->startup["voices_load_seconds"]=seconds_since(started); + started=clock_type::now(); p->normalizer=std::make_unique(get("--data",KITTEN_DEFAULT_DATA)); + p->startup["normalizer_init_seconds"]=seconds_since(started); static std::once_flag initialized; std::call_once(initialized, [] { ggml_backend_load_all(); llama_backend_init(); }); + // the CUDA decoder (LibTorch cuda:0) shares device 0 with GGML's CUDA0 + const bool cuda_decoder=!p->assets.decoder.empty() && p->decoder_device=="cuda"; if(!args.count("--decode-tokens")) { - auto mp=llama_model_default_params();mp.n_gpu_layers=0;mp.use_extra_bufts=!args.count("--no-repack"); + if(p->gpu_layers!=0 && !llama_supports_gpu_offload()) { + if(ggml_backend_reg_by_name("CUDA")) throw std::runtime_error("--gpu-layers needs a visible CUDA GPU, but none was found (no NVIDIA driver, no device, or CUDA_VISIBLE_DEVICES hides it); use --gpu-layers 0 for CPU"); + throw std::runtime_error("--gpu-layers needs a GPU build (for example -DGGML_CUDA=ON) and a visible GPU; use --gpu-layers 0 for CPU"); + } + if(timings && (p->gpu_layers!=0 || cuda_decoder)) p->gpu_mem.init(); + if(cuda_decoder && !p->gpu_mem.devices.empty()) p->gpu_mem.devices[0].used=true; + auto mp=llama_model_default_params();mp.n_gpu_layers=p->gpu_layers;mp.use_extra_bufts=!args.count("--no-repack"); + // --gpu-layers 0 gives the LM no GPU device, so GPU builds do not offload prompt ops and match CPU builds. + // Otherwise one GPU holds the whole LM; splitting a 1.7B model across GPUs only adds transfers. + static ggml_backend_dev_t no_gpu[]={nullptr}; + if(p->gpu_layers==0) mp.devices=no_gpu; + else {mp.split_mode=LLAMA_SPLIT_MODE_NONE;mp.main_gpu=p->main_gpu;} + started=clock_type::now(); p->model.reset(llama_model_load_from_file(p->assets.model.c_str(),mp)); - if(!p->model) throw std::runtime_error("cannot load language model"); + if(!p->model) throw std::runtime_error(p->gpu_layers!=0?"cannot load language model (the llama log above gives the cause, for example a --main-gpu index that is not below the number of GPUs)":"cannot load language model"); + p->startup["model_load_seconds"]=seconds_since(started); token_map tm(p->config.at("token_map")); if(tm.base+tm.count>llama_vocab_n_tokens(llama_model_get_vocab(p->model.get()))) throw std::runtime_error("token map exceeds vocabulary"); + started=clock_type::now(); + auto pl=inspect_placement(p->model.get()); + p->startup["placement_seconds"]=seconds_since(started); + json devices=json::array(); + for(int i=0;imodel.get());++i) { + auto dev=llama_model_get_device(p->model.get(),i); + devices.push_back(std::string(ggml_backend_dev_name(dev))+" ("+ggml_backend_dev_description(dev)+")"); + for(auto & d:p->gpu_mem.devices) if(d.dev==dev) d.used=true; + } + if(p->gpu_layers!=0 || timings) { + std::string summary; + for(auto & buf:pl.buffers.items()) { + const auto & b=buf.value(); + summary+=" "+buf.key()+": "+std::to_string(b.at("tensors").get())+" weights, "+std::to_string(int(b.at("mib").get()+0.5))+" MiB ("; + bool first=true;for(auto & type:b.at("types").items()) {summary+=(first?"":", ")+type.key()+" "+std::to_string(type.value().get());first=false;} + summary+=");"; + } + std::fprintf(stderr,"kitten-tts: LM placement %s, gpu-layers %s:%s\n",pl.mode.c_str(),p->gpu_layers<0?"all":std::to_string(p->gpu_layers).c_str(),summary.c_str()); + } + if(p->gpu_layers!=0) { + if(pl.mode=="cpu") throw std::runtime_error("--gpu-layers was given, but no LM weight was placed on a GPU"); + if(pl.tq2_1_cpu>0 && !args.count("--allow-hybrid")) { + throw std::runtime_error(std::to_string(pl.tq2_1_cpu)+" TQ2_1 weight tensors stay on the CPU (for example "+pl.first_tq2_1_cpu+ + "): --gpu-layers is below 29 or the GPU backend cannot run TQ2_1. Use a build with the TQ2_1 GPU repack and --gpu-layers 99, or pass --allow-hybrid to run those matrix multiplies on the CPU"); + } + if(pl.mode=="hybrid") { + std::string names;for(size_t i=0;i4?", ...":""); + } + } + p->lm={{"gpu_layers",p->gpu_layers},{"placement",pl.mode},{"devices",devices},{"weights_by_buffer",pl.buffers},{"cpu_weights",pl.cpu_weights.size()}, + {"tq2_1_on_cpu",pl.tq2_1_cpu},{"flash_attn_requested",p->flash_attn_arg}}; + if(p->gpu_layers!=0) p->lm["main_gpu"]=p->main_gpu; + if(pl.mode!="cpu") p->lm["precision"]=precision_note; + if(timings) p->gpu_mem.sample("model_load"); + } + started=clock_type::now(); + if(!p->assets.decoder.empty()) p->decoder=std::make_unique(p->assets.decoder,decoder_threads,0,p->decoder_device); + p->startup["decoder_load_seconds"]=seconds_since(started); + if(timings) p->gpu_mem.sample("decoder_load"); + if(!p->lm.is_null()) { + const auto mode=p->lm.at("placement").get(); + json cpu_stages={"text normalization","tokenization","token_embd lookup (input layer)"}; + if(mode!="cpu") cpu_stages.push_back("logits copy to host (one row per step)"); + cpu_stages.push_back("sampling"); + if(p->decoder) {for(auto & s:p->decoder->cpu_stages()) cpu_stages.push_back(s); cpu_stages.push_back("chunk joins and audio output");} + if(mode!="gpu") cpu_stages.push_back(mode=="cpu"?"speech LM":"speech LM weights not offloaded"); + p->lm["cpu_stages"]=cpu_stages; } - if(!p->assets.decoder.empty()) p->decoder=std::make_unique(p->assets.decoder,decoder_threads,0); } engine::~engine() = default; json engine::metadata() const { json names=json::array(); for(auto it=p->voices.begin();it!=p->voices.end();++it) names.push_back(it.key()); - return {{"config",p->config},{"voices",names}}; + json result={{"config",p->config},{"voices",names},{"decoder_device",p->decoder_device}}; + if(!p->lm.is_null()) result["lm"]=p->lm; + return result; } std::string engine::normalize(const std::string & text) { std::lock_guard lock(inference_mutex); @@ -71,6 +250,7 @@ std::string engine::normalize(const std::string & text) { generation_result engine::generate(const std::map & options, const std::function &)> & on_chunk) { std::lock_guard lock(inference_mutex); + const auto request_begin=clock_type::now(); auto args=p->defaults; for(const auto & pair:options) args[pair.first]=pair.second; auto get=[&](const std::string & key,const std::string & fallback){auto it=args.find(key);return it==args.end()?fallback:it->second;}; const auto & config=p->config; @@ -78,32 +258,37 @@ generation_result engine::generate(const std::map & op auto & model=p->model; auto & decoder=p->decoder; auto & normalizer=*p->normalizer; + const bool timings=args.count("--timings"); token_map tm(config.at("token_map")); std::string voice=get("--voice",config.value("default_voice","Bruno")); if(!voices.contains(voice)) throw std::invalid_argument("unknown voice: "+voice); - const auto & ref=voices.at(voice); int threads=std::stoi(get("--threads","8")); int seed=std::stoi(get("--seed",std::to_string(std::random_device{}() % 2147483647))), max_tokens=std::stoi(get("--max-tokens","1000")); if(threads<1 || max_tokens<1) throw std::invalid_argument("thread and token counts must be positive"); + double voice_prepare_seconds=0; if(args.count("--decode-tokens")) { auto tokens=read_json(args.at("--decode-tokens")).get>(); tokens.erase(std::remove_if(tokens.begin(),tokens.end(),[&](int id){return id<0 || id>=tm.count;}),tokens.end()); if(tokens.empty()) throw std::runtime_error("no valid codec tokens"); if(!decoder) throw std::runtime_error("decoder is not loaded"); decoder->seed(seed); + const auto & conditioning=p->voice(voice,voice_prepare_seconds); auto begin=clock_type::now(); - auto audio=decoder->decode(tokens,ref.dump()); + auto audio=decoder->decode(tokens,conditioning); json report={{"samples",audio.size()},{"decoder_seconds",std::chrono::duration(clock_type::now()-begin).count()}}; return {std::move(audio),std::move(report)}; } if(!model) throw std::runtime_error("language model is not loaded"); + const auto & ref=voices.at(voice); auto gen=config.value("generation",json::object()); int chunk_chars=std::stoi(get("--chunk-chars",std::to_string(gen.value("chunk_chars",380)))); int chunk_min=std::stoi(get("--chunk-min-chars",std::to_string(gen.value("chunk_min_chars",130)))); float gap=std::stof(get("--chunk-gap","0.16")); if(!std::isfinite(gap) || gap<0 || gap>60) throw std::runtime_error("invalid chunk gap"); + auto started=clock_type::now(); auto spoken=kitten::normalize(args.at("--text"),normalizer,!args.count("--no-normalize")); auto chunks=kitten::split(spoken,chunk_chars,chunk_min); + const double normalize_seconds=seconds_since(started); if(chunks.empty()) throw std::runtime_error("no text to speak"); auto preset=config.at("decode_presets").at(get("--preset",config.value("default_preset","stable"))); kitten::sampling settings; @@ -122,22 +307,54 @@ generation_result engine::generate(const std::map & op std::vector emotion; if((args.count("--use-emotion") ? args.at("--use-emotion")=="1" : kitten::has_expression(args.at("--text"))) && gen.contains("emotion_control")) emotion=tokenize(vocab,gen.at("emotion_control")); std::mt19937 rng(seed); + const kitten::decoder::voice * conditioning=decoder?&p->voice(voice,voice_prepare_seconds):nullptr; std::vector> waves; json report={{"normalized",spoken},{"voice",voice},{"sample_rate",24000},{"chunks",json::array()}}; + if(!p->lm.is_null()) report["lm"]=p->lm; + if(decoder && args.count("--decoder-device")) report["decoder_device"]=p->decoder_device; std::ofstream logits_file; if(args.count("--logits")) {logits_file.open(args.at("--logits"),std::ios::binary);if(!logits_file) throw std::runtime_error("cannot open logits output");} std::vector forced; if(args.count("--force-tokens")) {forced=read_json(args.at("--force-tokens")).get>();if(chunks.size()!=1) throw std::runtime_error("teacher forcing requires one chunk");} report["iteration"]=1; report["seed"]=seed; if(decoder) decoder->seed(seed); + json llama_buffers=json::object(); + bool splits_warned=false; + // Splits beyond the input lookup mean an op runs on the CPU although every weight is on the GPU. + auto check_splits=[&](llama_context * c,const char * graph){ + if(!report.contains("lm")) return; + int n=llama_get_graph_splits(c);auto & gs=report["lm"]["graph_splits"]; + if(!gs.contains(graph) || gs[graph].get()2 && report["lm"]["placement"]=="gpu" && !splits_warned) { + std::fprintf(stderr,"kitten-tts: warning: the %s graph has %d backend splits, expected 2 with the whole LM on the GPU: some ops run on the CPU (GGML_SCHED_DEBUG=2 lists them)\n",graph,n); + splits_warned=true; + } + }; double lm_seconds=0,decoder_seconds=0;size_t total_tokens=0; + stage_times st; for(auto & chunk:chunks) { + auto t0=clock_type::now(); auto ids=kitten::prompt(tm,tokenize(vocab,chunk),ref_text,ref_audio,emotion,args.count("--use-reference") ? args.at("--use-reference")=="1" : (!args.count("--no-reference") && gen.value("use_reference_prompt",true))); + st.tokenize+=seconds_since(t0); int budget=chunks.size()==1?max_tokens:std::max(200,std::min(max_tokens,int(kitten::characters(chunk)/20.0*25*1.8))); if(args.count("--force-tokens")) budget=forced.size()+1; auto cp=llama_context_default_params();cp.n_ctx=ids.size()+budget;cp.n_batch=512;cp.n_ubatch=512;cp.n_threads=threads;cp.n_threads_batch=threads;cp.no_perf=false; + // Only the last position's logits are read, so reserve one output row instead of n_batch rows. + cp.n_outputs_max=1;cp.flash_attn_type=p->flash_attn; + t0=clock_type::now(); std::unique_ptr ctx(llama_init_from_model(model.get(),cp),llama_free); if(!ctx) throw std::runtime_error("cannot initialize context"); + st.context+=seconds_since(t0); + if(report.contains("lm")) report["lm"]["flash_attn"]=llama_get_flash_attn(ctx.get())?"on":"off"; + if(timings) { + p->gpu_mem.sample("context_init"); + for(auto & entry:llama_get_memory_breakdown(ctx.get())) { + auto & b=llama_buffers[ggml_backend_buft_name(entry.first)]; + const auto & m=entry.second; + auto keep_max=[&](const char * key,size_t bytes){double v=mib(bytes);if(!b.contains(key) || b[key].get() & op if(llama_decode(ctx.get(),b.b)) throw std::runtime_error("prompt prefill failed"); offset+=n; } + // GPU backends run asynchronously; wait so the prefill time includes the device work. + llama_synchronize(ctx.get()); + double prefill=seconds_since(begin),chunk_decode=0,chunk_sampling=0;size_t chunk_calls=0; + check_splits(ctx.get(),"prefill"); + st.prefill+=prefill;st.prefill_tokens+=ids.size(); auto history=ids;std::vector audio_ids,generated;bool ended=false; for(int step=0;step(raw),n_vocab*sizeof(float)); if(args.count("--force-tokens") && step==int(forced.size())) break; + t0=clock_type::now(); std::vector scores(raw,raw+n_vocab);kitten::process(scores,history,ids.size(),tm,settings); int token=args.count("--force-tokens")?forced.at(step):kitten::sample(scores,rng,args.count("--greedy")); + chunk_sampling+=seconds_since(t0); if(token<0 || token>=n_vocab) throw std::runtime_error("forced token out of range"); generated.push_back(token);history.push_back(token); if(!args.count("--force-tokens") && (token==tm.speech_end || token==tm.stop)) {ended=true;break;} if(token>=tm.base && token(clock_type::now()-begin).count();lm_seconds+=elapsed;total_tokens+=generated.size(); + st.decode+=chunk_decode;st.sampling+=chunk_sampling;st.decode_calls+=chunk_calls; report["chunks"].push_back({{"text",chunk},{"prompt",ids},{"generated",generated},{"audio_tokens",audio_ids},{"terminated",ended},{"lm_seconds",elapsed}}); + if(timings) { + auto & c=report["chunks"].back(); + c["prefill_seconds"]=prefill;c["prefill_tokens"]=ids.size();c["decode_seconds"]=chunk_decode;c["decode_calls"]=chunk_calls;c["sampling_seconds"]=chunk_sampling; + } llama_perf_context_print(ctx.get()); - if(decoder) {begin=clock_type::now();auto wave=decoder->decode(audio_ids,ref.dump());decoder_seconds+=std::chrono::duration(clock_type::now()-begin).count(); if(on_chunk && !wave.empty() && !on_chunk(wave)) throw std::runtime_error("generation cancelled"); waves.push_back(std::move(wave));} + if(timings) p->gpu_mem.sample("generation"); + // Free the KV cache and compute buffers before decoding audio, to lower peak memory. + t0=clock_type::now();ctx.reset();st.context+=seconds_since(t0); + if(decoder) { + begin=clock_type::now();auto wave=decoder->decode(audio_ids,*conditioning); + double d=std::chrono::duration(clock_type::now()-begin).count();decoder_seconds+=d; + if(timings) {report["chunks"].back()["decoder_seconds"]=d;p->gpu_mem.sample("decoder");} + if(on_chunk && !wave.empty() && !on_chunk(wave)) throw std::runtime_error("generation cancelled"); + waves.push_back(std::move(wave)); + } } report["lm_seconds"]=lm_seconds;report["decoder_seconds"]=decoder_seconds;report["generated_tokens"]=total_tokens; generation_result result; - if(decoder) {result.audio=kitten::join(waves,gap);report["audio_seconds"]=result.audio.size()/24000.0;report["rtf"]=(lm_seconds+decoder_seconds)/(result.audio.size()/24000.0);} + double audio_seconds=0; + if(decoder) { + auto t0=clock_type::now(); + result.audio=kitten::join(waves,gap); + st.output=seconds_since(t0);audio_seconds=result.audio.size()/24000.0; + report["audio_seconds"]=audio_seconds;report["rtf"]=(lm_seconds+decoder_seconds)/audio_seconds; + } + const int request=++p->requests; + if(timings) { + double total=seconds_since(request_begin),loop=st.decode+st.sampling; + auto rate=[](double n,double s){return s>0?n/s:0.0;}; + json t={{"startup",p->startup},{"normalize_seconds",normalize_seconds},{"voice_prepare_seconds",voice_prepare_seconds},{"tokenize_seconds",st.tokenize},{"context_seconds",st.context}, + {"prefill_seconds",st.prefill},{"prefill_tokens",st.prefill_tokens},{"prefill_tokens_per_second",rate(st.prefill_tokens,st.prefill)}, + {"decode_seconds",st.decode},{"decode_calls",st.decode_calls},{"decode_tokens_per_second",rate(st.decode_calls,st.decode)}, + {"sampling_seconds",st.sampling},{"sampling_share",loop>0?st.sampling/loop:0.0}, + {"generated_tokens",total_tokens},{"generation_tokens_per_second",rate(total_tokens,loop)}, + {"decoder_seconds",decoder_seconds},{"output_seconds",st.output},{"total_seconds",total}}; + if(decoder) {t["audio_seconds"]=audio_seconds;t["rtf"]=(lm_seconds+decoder_seconds)/audio_seconds;t["rtf_total"]=total/audio_seconds;} + // wall time since the engine was created (for the CLI, about the process start); the first request includes every startup cost + t["process_seconds"]=seconds_since(p->created); + t["llama_buffers_mib"]=llama_buffers; + auto gm=p->gpu_mem.to_json(); + if(!gm.empty()) {t["gpu_memory"]=gm;t["gpu_memory_method"]="device-wide used memory (total minus free) sampled at stage ends; the baseline is taken after this process creates its CUDA context and before the model load; valid only while no other process uses the device";} + report["timings"]=t; + std::fprintf(stderr,"kitten-tts timings, request %d:\n",request); + if(request==1) std::fprintf(stderr," startup model load %.3f s, placement check %.3f s, decoder load %.3f s, normalizer init %.3f s (once)\n", + p->startup.value("model_load_seconds",0.0),p->startup.value("placement_seconds",0.0),p->startup.value("decoder_load_seconds",0.0),p->startup.value("normalizer_init_seconds",0.0)); + std::fprintf(stderr," normalize %8.3f s\n voice prepare %8.3f s\n",normalize_seconds,voice_prepare_seconds); + std::fprintf(stderr," tokenize %8.3f s\n context %8.3f s (create and free, %zu chunks)\n",st.tokenize,st.context,chunks.size()); + std::fprintf(stderr," LM prefill %8.3f s (%zu positions, %.0f tok/s)\n",st.prefill,st.prefill_tokens,rate(st.prefill_tokens,st.prefill)); + std::fprintf(stderr," LM decode %8.3f s (%zu forward passes, %.1f tok/s)\n",st.decode,st.decode_calls,rate(st.decode_calls,st.decode)); + std::fprintf(stderr," sampling %8.3f s (%.1f%% of the generation loop; %zu tokens at %.1f tok/s overall)\n",st.sampling,loop>0?100*st.sampling/loop:0.0,total_tokens,rate(total_tokens,loop)); + std::fprintf(stderr," decoder %8.3f s\n output %8.3f s\n total %8.3f s\n",decoder_seconds,st.output,total); + if(decoder) std::fprintf(stderr," audio %8.3f s RTF %.3f (LM + decoder), %.3f (total)\n",audio_seconds,(lm_seconds+decoder_seconds)/audio_seconds,total/audio_seconds); + std::fprintf(stderr," since load %8.3f s (since the engine was created, startup included)\n",t["process_seconds"].get()); + for(auto & d:gm) std::fprintf(stderr," GPU memory %s peak %.0f MiB used of %.0f (baseline %.0f, +%.0f, at %s)%s\n",d["device"].get().c_str(), + d["peak_used_mib"].get(),d["total_mib"].get(),d["baseline_used_mib"].get(),d["peak_over_baseline_mib"].get(),d["peak_stage"].get().c_str(), + d["other_process_changed_usage"].get()?"; another process changed device memory during the run, so these numbers are not reliable":""); + } result.report=std::move(report); return result; } diff --git a/tools/kitten-tts/export.py b/tools/kitten-tts/export.py index 0d738bb..640056c 100644 --- a/tools/kitten-tts/export.py +++ b/tools/kitten-tts/export.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Export KittenTTS2 assets. Python is used only for conversion and validation.""" import argparse +import contextlib import json import pathlib import sys @@ -9,6 +10,16 @@ sys.path[:0] = [str(ROOT), str(ROOT / 'gguf-py'), str(ROOT / 'vendor/kitten-text-processing')] +def full_precision_tensors(directory): + from conversion.base import LazyTorchTensor, gguf + path = directory / 'model.safetensors' + if not path.is_file(): + return None + # Python's packed alternatives live beside the full checkpoint and are not GGUF inputs. + with gguf.utility.SafetensorsLocal(path) as part: + return {name: (lambda data=part[name]: LazyTorchTensor.from_local_tensor(data)) for name in part.keys()} + + def pack_ternary_q4(data): """Store exact {-gamma, 0, gamma} values in standard Q4_0 blocks.""" import numpy as np @@ -34,7 +45,18 @@ def main(): p.add_argument('--stage', choices=['lm', 'decoder', 'all'], default='all') p.add_argument('--threads', type=int, default=8) p.add_argument('--decoder', choices=['default','student_w4','student_w8'], default='default') + p.add_argument('--device', choices=['cpu', 'cuda'], default='cpu', + help='device to trace the decoder on; a CUDA decoder runs only with --decoder-device cuda') + p.add_argument('--rng', choices=['device', 'cpu'], default='device', + help='cpu: draw decoder noise from the CPU generator and copy it to the device, ' + 'so one seed gives the same noise as a CPU decoder (for parity checks)') + p.add_argument('--reuse-voices', type=pathlib.Path, + help='prepared voices.json to keep (for example the published cpp//voices.json) ' + 'instead of recomputing speaker projections and S3 conditioning') a = p.parse_args() + if a.rng == 'cpu' and a.decoder != 'default': + # student_w4 fails the trace self-check with CPU-drawn noise (max error 4.6e-4 > 2e-4); student_w8 is not checked + p.error('--rng cpu is supported only with --decoder default; use --rng device for student decoders') sys.path.insert(0, str(a.python_package)) import torch torch.set_num_threads(a.threads) @@ -52,6 +74,10 @@ def main(): import gguf class KittenModel(Qwen3Model): model_arch = Qwen3Model.model_arch + def index_tensors(self, remote_hf_model_id=None): + tensors = full_precision_tensors(self.dir_model) + return tensors if tensors is not None else super().index_tensors(remote_hf_model_id) + def modify_tensors(self, data_torch, name, bid): if name.startswith('spk_proj.'): return @@ -93,9 +119,7 @@ def quantize(data, qtype): def export_decoder(a, config): - import numpy as np import torch - from safetensors import safe_open from kittenml.kittentts2.vocoder import S3Codec from kittenml.kittentts2 import s3gen # The reference .item() freezes sequence length when tracing. This equivalent @@ -106,7 +130,11 @@ def make_pad_mask(lengths, max_len=0): s3gen.make_pad_mask = make_pad_mask s3gen.tqdm = lambda iterable, **kwargs: iterable student_weights = None if a.decoder == 'default' else a.repo / config['decoders'][a.decoder] - codec = S3Codec(device='cpu', student_weights=student_weights) + if a.device == 'cuda': + # Match the CPU decoder: FP32 GEMMs and convolutions without TF32. + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + codec = S3Codec(device=a.device, student_weights=student_weights) from kittenml.kittentts2 import student_flow student_flow.make_pad_mask = make_pad_mask class Decoder(torch.nn.Module): @@ -116,18 +144,89 @@ def __init__(self): self.hift = codec.s3gen.mel2wav self.register_buffer('fade', codec.s3gen.trim_fade) def forward(self, tokens, prompt_token, prompt_feat, embedding): - lengths = torch.ones(1, dtype=torch.long) * tokens.shape[1] - prompt_lengths = torch.ones(1, dtype=torch.long) * prompt_token.shape[1] + lengths = torch.ones(1, dtype=torch.long, device=tokens.device) * tokens.shape[1] + prompt_lengths = torch.ones(1, dtype=torch.long, device=tokens.device) * prompt_token.shape[1] if codec.student is None: - noise = torch.randn(1, 80, tokens.shape[1] * 2) + rng_device = 'cpu' if a.rng == 'cpu' else tokens.device + noise = torch.randn(1, 80, tokens.shape[1] * 2, device=rng_device).to(tokens.device) mel, _ = self.flow.inference(tokens, lengths, prompt_token, prompt_lengths, prompt_feat, None, embedding, True, n_timesteps=2, noised_mels=noise, meanflow=True) else: mel, _, _ = student_flow.meanflow_forward(self.flow, tokens, lengths, prompt_token, prompt_lengths, prompt_feat, embedding, n_steps=1) - wav, _ = self.hift.inference(mel, torch.zeros(1, 1, 0)) + wav, _ = self.hift.inference(mel, torch.zeros(1, 1, 0, device=tokens.device)) wav[:, :self.fade.shape[0]] *= self.fade return wav + if a.reuse_voices: + export_voices = json.loads(a.reuse_voices.read_text()) + voice = next(iter(export_voices.values())) + args = (torch.full((1, 53), 4299, dtype=torch.long), torch.tensor(voice['prompt_token'], dtype=torch.long), + torch.tensor(voice['prompt_feat'], dtype=torch.float32), torch.tensor(voice['embedding'], dtype=torch.float32)) + print('Reusing', len(export_voices), 'voices from', a.reuse_voices, flush=True) + else: + export_voices, args = export_conditioning(a, config, codec) + (a.output / 'voices.json').write_text(json.dumps(export_voices)) + args = tuple(x.to(a.device) for x in args) + decoder = Decoder().eval() + metadata = dict(device=a.device, rng=a.rng, decoder=a.decoder, torch=torch.__version__, sinegen='device') + with torch.inference_mode(), rng_on_cpu(a.rng == 'cpu'), sinegen_on_device(s3gen): + traced = torch.jit.trace(decoder, args, check_trace=False) + traced = torch.jit.freeze(traced) + traced.save(str(a.output / 'decoder.pending.pt'), _extra_files={'kitten_decoder.json': json.dumps(metadata)}) + results = [] + for n in [4, 17, 53, 101]: + inputs = (torch.full((1, n), 4299, dtype=torch.long, device=a.device), *args[1:]) + torch.manual_seed(123) + expected = decoder(*inputs) + torch.manual_seed(123) + actual = traced(*inputs) + torch.testing.assert_close(actual, expected, atol=2e-4, rtol=2e-4) + results.append(dict(tokens=n, samples=actual.numel(), max_error=float((actual-expected).abs().max()))) + (a.output / 'decoder.pending.pt').replace(a.output / 'decoder.pt') + (a.output / 'decoder-parity.json').write_text(json.dumps(dict(metadata, results=results), indent=2)) + print(results) + + +@contextlib.contextmanager +def rng_on_cpu(enabled): + """Trace randn_like as a CPU draw copied to the tensor's device, so a CUDA decoder uses the same noise as the CPU decoder for one seed. + + The CPU and CUDA generators give different numbers for one seed. The torch.rand calls in s3gen already use CPU tensors.""" + import torch + saved = torch.randn_like + if enabled: + torch.randn_like = lambda x, **kw: saved(x, **dict(kw, device='cpu')).to(x.device) + try: + yield + finally: + torch.randn_like = saved + + +@contextlib.contextmanager +def sinegen_on_device(s3gen): + """Create SineGen's harmonic buffer (9 floats per output sample) on f0's device. + + The reference creates it with torch.zeros on the CPU and copies it to f0's device on every call. The values are the same.""" + import torch + forward = s3gen.SineGen.forward + def patched(self, f0): + zeros = torch.zeros + torch.zeros = lambda *args, **kw: zeros(*args, **{'device': f0.device, **kw}) + try: + return forward(self, f0) + finally: + torch.zeros = zeros + s3gen.SineGen.forward = patched + try: + yield + finally: + s3gen.SineGen.forward = forward + + +def export_conditioning(a, config, codec): + import numpy as np + import torch + from safetensors import safe_open voices_path = a.repo / config['voices'] voices = json.loads(voices_path.read_text()) head = {} @@ -154,24 +253,7 @@ def forward(self, tokens, prompt_token, prompt_feat, embedding): args = (torch.full((1, 53), 4299, dtype=torch.long), cond['prompt_token'].long(), cond['prompt_feat'].float(), cond['embedding'].float()) print('Exported voice', name, flush=True) - (a.output / 'voices.json').write_text(json.dumps(export_voices)) - decoder = Decoder().eval() - with torch.inference_mode(): - traced = torch.jit.trace(decoder, args, check_trace=False) - traced = torch.jit.freeze(traced) - traced.save(str(a.output / 'decoder.pending.pt')) - results = [] - for n in [4, 17, 53, 101]: - inputs = (torch.full((1, n), 4299, dtype=torch.long), *args[1:]) - torch.manual_seed(123) - expected = decoder(*inputs) - torch.manual_seed(123) - actual = traced(*inputs) - torch.testing.assert_close(actual, expected, atol=2e-4, rtol=2e-4) - results.append(dict(tokens=n, samples=actual.numel(), max_error=float((actual-expected).abs().max()))) - (a.output / 'decoder.pending.pt').replace(a.output / 'decoder.pt') - (a.output / 'decoder-parity.json').write_text(json.dumps(results, indent=2)) - print(results) + return export_voices, args if __name__ == '__main__': diff --git a/tools/kitten-tts/main.cpp b/tools/kitten-tts/main.cpp index 74e0165..841076b 100644 --- a/tools/kitten-tts/main.cpp +++ b/tools/kitten-tts/main.cpp @@ -29,14 +29,14 @@ int main(int argc,char ** argv) try { std::setlocale(LC_ALL,"C.UTF-8"); try { std::locale::global(std::locale("C.UTF-8")); } catch (const std::runtime_error &) {} std::map args; - const std::vector flags={"--download-only","--offline","--no-normalize","--no-reference","--no-repack","--greedy","--tokens-only","--frontend","--help"}; + const std::vector flags={"--download-only","--offline","--no-normalize","--no-reference","--no-repack","--greedy","--tokens-only","--frontend","--allow-hybrid","--timings","--help"}; for(int i=1;i=argc) throw std::runtime_error("expected --option value");args[key]=argv[++i];} } if(args.count("--help") || argc==1) { - std::cout<<"KittenTTS2 CPU inference\n" + std::cout<<"KittenTTS2 inference (speech LM on CPU by default; CUDA builds can offload it with --gpu-layers)\n" " kitten-tts --text TEXT [--voice Bruno] [--output output.wav]\n" " --repo ID --revision REV --decoder default|student_w4|student_w8\n" " --cache-dir DIR --offline --download-only\n" @@ -44,7 +44,13 @@ int main(int argc,char ** argv) try { " --model FILE override the GGUF with a local file\n" " --data DIR prepared kitten-text-processing grammar directory\n" " --threads N --decoder-threads N --seed N --preset stable|expressive\n" - " --no-repack disable CPU weight repacking to reduce runtime memory\n" + " --decoder-device cpu|cuda (cuda downloads the decoder's CUDA export; needs a CUDA LibTorch)\n" + " --no-repack disable CPU (AMX) weight repacking to reduce runtime memory; the GPU TQ2_1 to Q2_0 repack still runs\n" + " --gpu-layers N|all LM layers to offload to the GPU (default 0 = CPU; 29, 99 or all = whole LM)\n" + " --main-gpu N GPU index for the LM (single-GPU mode, default 0)\n" + " --flash-attn auto|on|off (default auto)\n" + " --allow-hybrid accept TQ2_1 weights left on the CPU when --gpu-layers > 0\n" + " --timings print per-stage times, RTF and peak GPU memory; add them to the report\n" " --max-tokens N --temperature F --top-k N --top-p F --min-p F\n" " --repetition-penalty F --repetition-window N --run-penalty F --run-grace N\n" " --chunk-chars N --chunk-min-chars N --chunk-gap F\n" @@ -55,7 +61,7 @@ int main(int argc,char ** argv) try { return 0; } auto get=[&](const std::string & key,const std::string & fallback){auto it=args.find(key);return it==args.end()?fallback:it->second;}; - const std::vector known={"--repo","--revision","--decoder","--cache-dir","--assets","--model","--text","--voice","--output","--data","--threads","--decoder-threads","--seed","--preset","--max-tokens","--temperature","--top-k","--top-p","--min-p","--repetition-penalty","--repetition-window","--run-penalty","--run-grace","--chunk-chars","--chunk-min-chars","--chunk-gap","--report","--logits","--force-tokens","--decode-tokens","--repeat"}; + const std::vector known={"--repo","--revision","--decoder","--cache-dir","--assets","--model","--text","--voice","--output","--data","--threads","--decoder-threads","--seed","--preset","--max-tokens","--temperature","--top-k","--top-p","--min-p","--repetition-penalty","--repetition-window","--run-penalty","--run-grace","--chunk-chars","--chunk-min-chars","--chunk-gap","--report","--logits","--force-tokens","--decode-tokens","--repeat","--gpu-layers","--main-gpu","--flash-attn","--decoder-device"}; for(auto & a:args) if(std::find(flags.begin(),flags.end(),a.first)==flags.end() && std::find(known.begin(),known.end(),a.first)==known.end()) throw std::runtime_error("unknown option "+a.first); if(args.count("--frontend")) { kitten_text_processing::Normalizer normalizer(get("--data",KITTEN_DEFAULT_DATA)); diff --git a/tools/kitten-tts/prepare_model_repo.py b/tools/kitten-tts/prepare_model_repo.py index d57f4e1..ca25118 100644 --- a/tools/kitten-tts/prepare_model_repo.py +++ b/tools/kitten-tts/prepare_model_repo.py @@ -12,6 +12,9 @@ def main(): parser.add_argument('--assets', type=Path, required=True, help='Default decoder export including model-tq2_1.gguf') parser.add_argument('--student-w4', type=Path) parser.add_argument('--student-w8', type=Path) + parser.add_argument('--cuda', type=Path, help='Default decoder exported with export.py --device cuda') + parser.add_argument('--student-w4-cuda', type=Path) + parser.add_argument('--student-w8-cuda', type=Path) args = parser.parse_args() config_path = args.repo / 'config.json' config = json.loads(config_path.read_text()) @@ -41,12 +44,28 @@ def copy(source, relative): for file in ('decoder.pt', 'voices.json'): if not (source / file).is_file(): raise ValueError(f'missing {source / file}') + cuda_sources = {name: path for name, path in (('default', args.cuda), ('student_w4', args.student_w4_cuda), + ('student_w8', args.student_w8_cuda)) if path} + for name, source in cuda_sources.items(): + if name not in sources: + raise ValueError(f'{source}: a CUDA export of {name} needs its CPU export too') + exported = json.loads((source / 'config.json').read_text()) + if exported['type'] != 'KITTEN2' or exported.get('cpp_decoder', 'default') != name: + raise ValueError(f'{source}: wrong decoder export for {name}') + # Both devices share one voices.json: export the CUDA decoder with --reuse-voices. + if (source / 'voices.json').read_bytes() != (sources[name] / 'voices.json').read_bytes(): + raise ValueError(f'{source}: voices.json differs from the CPU export; export with --reuse-voices') + if not (source / 'decoder.pt').is_file(): + raise ValueError(f'missing {source / "decoder.pt"}') manifest = {'version': 1, 'gguf': copy(args.assets / 'model-tq2_1.gguf', 'cpp/model-tq2_1.gguf'), 'decoders': {}} for name, source in sources.items(): manifest['decoders'][name] = { 'torchscript': copy(source / 'decoder.pt', f'cpp/{name}/decoder.pt'), 'voices': copy(source / 'voices.json', f'cpp/{name}/voices.json'), } + if name in cuda_sources: + manifest['decoders'][name]['torchscript_cuda'] = copy(cuda_sources[name] / 'decoder.pt', + f'cpp/{name}/decoder-cuda.pt') config['cpp'] = manifest temporary = config_path.with_suffix('.json.tmp') temporary.write_text(json.dumps(config, indent=2) + '\n') diff --git a/tools/kitten-tts/test_assets.py b/tools/kitten-tts/test_assets.py index 361615f..8f5738e 100644 --- a/tools/kitten-tts/test_assets.py +++ b/tools/kitten-tts/test_assets.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Exercise native asset downloads against a local model repository.""" import functools +import hashlib import http.server import json import os @@ -22,6 +23,12 @@ class Handler(http.server.SimpleHTTPRequestHandler): def log_message(self, *args): pass + def end_headers(self): + path = Path(self.translate_path(self.path)) + if path.is_file(): + self.send_header('ETag', '"' + hashlib.sha256(path.read_bytes()).hexdigest() + '"') + super().end_headers() + def do_HEAD(self): heads.append(self.path) super().do_HEAD() @@ -39,6 +46,8 @@ def do_GET(self): 'torchscript': {'file': f'cpp/{name}/decoder.pt', 'size': 4}, 'voices': {'file': f'cpp/{name}/voices.json', 'size': 4}, } + if name != 'student_w8': + config['cpp']['decoders'][name]['torchscript_cuda'] = {'file': f'cpp/{name}/decoder-cuda.pt', 'size': 4} for entry in [config['cpp']['gguf']] + [entry for decoder in config['cpp']['decoders'].values() for entry in decoder.values()]: path = repo / entry['file'] path.parent.mkdir(parents=True, exist_ok=True) @@ -57,9 +66,34 @@ def run(*args, ok=True, cache='cache'): heads.clear() run('--decoder', 'student_w4', '--offline') assert not requests and not heads + profile = root / 'profile' + result = subprocess.run([binary, '--repo', 'org/model', '--download-only', '--decoder', 'student_w4'], + env={**env, 'HOME': '', 'XDG_CACHE_HOME': '', 'USERPROFILE': str(profile)}, + cwd=root, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert (profile / '.cache/kitten-tts/org/model/main/config.json').is_file() run('--decoder', 'student_w8', '--offline', ok=False) run('--decoder', 'missing', ok=False) requests.clear() + run('--decoder', 'student_w4', '--decoder-device', 'cuda', cache='cuda') + assert requests == ['/org/model/resolve/main/' + f for f in ('config.json', 'cpp/student_w4/voices.json', 'cpp/model.gguf', 'cpp/student_w4/decoder-cuda.pt')], requests + requests.clear() + heads.clear() + run('--decoder', 'student_w4', '--decoder-device', 'cuda', '--offline', cache='cuda') + assert not requests and not heads + cuda_entry = config['cpp']['decoders']['student_w4'].pop('torchscript_cuda') + run('--decoder', 'student_w4', cache='upgrade') + config['cpp']['decoders']['student_w4']['torchscript_cuda'] = cuda_entry + run('--decoder', 'student_w4', '--decoder-device', 'cuda', cache='upgrade') + assert (root / 'upgrade/org/model/main/cpp/student_w4/decoder-cuda.pt').read_bytes() == b'test' + cuda_entry['size'] = 5 + result = run('--decoder', 'student_w4', '--decoder-device', 'cuda', cache='cuda-size', ok=False) + assert 'asset size mismatch' in result.stderr, result.stderr + cuda_entry['size'] = 4 + requests.clear() + result = run('--decoder', 'student_w8', '--decoder-device', 'cuda', cache='cuda-missing', ok=False) + assert 'no CUDA export' in result.stderr and requests == ['/org/model/resolve/main/config.json'], (result.stderr, requests) + requests.clear() run('--tokens-only', cache='tokens') assert not any(path.endswith('decoder.pt') for path in requests) requests.clear() diff --git a/tools/kitten-tts/test_export.py b/tools/kitten-tts/test_export.py new file mode 100644 index 0000000..113e997 --- /dev/null +++ b/tools/kitten-tts/test_export.py @@ -0,0 +1,36 @@ +"""Run in the environment from requirements-export.txt.""" +import tempfile +import unittest +from pathlib import Path + +import torch +from safetensors.torch import save_file + +from export import full_precision_tensors +from conversion.base import LazyTorchTensor + + +class CheckpointTests(unittest.TestCase): + def test_full_checkpoint_ignores_packed_alternatives(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + weights = {'model.embed_tokens.weight': torch.arange(6).reshape(2, 3).float(), + 'model.layers.0.mlp.down_proj.weight': torch.ones(2, 3)} + save_file(weights, root / 'model.safetensors') + save_file({'model.layers.0.mlp.down_proj.weight::codes': torch.zeros(1, dtype=torch.uint8)}, + root / 'model-ternary.safetensors') + save_file({'model.embed_tokens.weight': torch.zeros(2, 3)}, root / 'model-tl2-emb4.safetensors') + tensors = full_precision_tensors(root) + self.assertEqual(set(tensors), set(weights)) + for name, expected in weights.items(): + torch.testing.assert_close(LazyTorchTensor.to_eager(tensors[name]()), expected) + + def test_defers_sharded_checkpoint(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + save_file({'weight': torch.ones(1)}, root / 'model-00001-of-00001.safetensors') + self.assertIsNone(full_precision_tensors(root)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/kitten-tts/test_python.py b/tools/kitten-tts/test_python.py index 75f4aae..164683e 100644 --- a/tools/kitten-tts/test_python.py +++ b/tools/kitten-tts/test_python.py @@ -48,6 +48,19 @@ def test_metadata_and_validation(self): self.assertEqual(self.model._options('Hello.', advanced={'repetition_window': None})['--repetition-window'], str(self.model.advanced_options['repetition_window'])) + def test_gpu_options(self): + for kwargs in [dict(gpu_layers=-1), dict(gpu_layers='x'), dict(gpu_layers=True), dict(main_gpu=-1), + dict(flash_attn='maybe'), dict(decoder_device='tpu'), dict(allow_hybrid='yes')]: + with self.subTest(kwargs=kwargs), self.assertRaises(ValueError): + KittenTTS(self.assets, model_path=self.model_path, decoder='student_w4', **kwargs) + self.model.generate('Hello.', advanced={'seed': 1234}) + self.assertEqual(self.model.last_report['lm']['placement'], 'cpu') + self.assertEqual((self.model.placement, self.model.device), ('cpu', 'cpu')) + import torch + if torch.version.cuda is None or not torch.cuda.is_available(): + with self.assertRaises(RuntimeError): + KittenTTS(self.assets, model_path=self.model_path, decoder='student_w4', gpu_layers=99) + def test_cli_parity_and_repeat(self): audio = self.model.generate('Hello again.', advanced={'seed': 1234}) ids = self.model.last_report['chunks'][0]['generated'] diff --git a/tools/kitten-tts/validation.json b/tools/kitten-tts/validation.json index 104283b..54b2319 100644 --- a/tools/kitten-tts/validation.json +++ b/tools/kitten-tts/validation.json @@ -358,5 +358,108 @@ ] } } + }, + "cuda_release_validation": { + "date": "2026-10-24", + "source_head": "8e0f03f41ae920a708c3dc119f442448a86b9915", + "source_state": "HEAD plus the accompanying uncommitted PR-readiness changes", + "model_repository": "KittenML/kitten-tts-2", + "model_revision": "c93cb7c271baa51d9ae8204d5277988906f235a2", + "python_reference_commit": "afd9dfda7ffdca35dd0151b149c221140c80ec3f", + "export_torch": "2.11.0+cu128", + "published_cuda_assets": [ + { + "decoder": "default", + "bytes": 534311680, + "sha256": "dc0d9588b8e36183f7cb7969353081533b03fe46bfaf8a6040a609ebf3cd444f" + }, + { + "decoder": "student_w4", + "bytes": 312339684, + "sha256": "7d815b39c27defc8f4d1007320436d7ccf36de9619ae17c1086f6845b42ac38e" + } + ], + "wheel_matrix": { + "python_versions": [ + "3.10", + "3.11", + "3.12", + "3.13", + "3.14" + ], + "linux": { + "platform": "manylinux_2_34_x86_64", + "build_container": "quay.io/pypa/manylinux_2_34_x86_64@sha256:fb501b3bde5ee751f2f4897678c5a82359448efb278b1718e8d375efe874b75f", + "compiler_cuda": "12.8.93", + "torch": "2.13.0+cu129", + "gpu": "NVIDIA GeForce RTX 3090", + "driver": "595.104.02", + "tested_wheels": 5, + "real_cpu_synthesis_without_driver": "5/5 passed; compiler library paths removed" + }, + "windows": { + "platform": "win_amd64, Windows Server 2022", + "compiler_cuda": "13.0.88", + "torch": "2.13.0+cu130", + "gpu": "NVIDIA L4", + "driver": "596.86", + "driver_model": "WDDM", + "tested_wheels": 5 + }, + "all_ten_wheels_passed": [ + "Clean environment install, native import, ABI check, grammar, synthetic TorchScript and anonymous HTTPS", + "Real W4 CPU generation and GPU LM with CPU decoder", + "Published default and W4 CUDA decoders, online and offline, finite nonzero mono float32 audio at 24 kHz", + "Voice selection, file output, sentence streaming, cancellation and reuse, local assets", + "Missing W8 CUDA export, partial offload guard, hybrid opt-in, empty offline cache", + "CPU generation and clear GPU errors with CUDA_VISIBLE_DEVICES=-1", + "Archive integrity, correct CPython ABI, current README metadata and no bundled model weights" + ] + }, + "source_and_publisher_checks": { + "readme_linux_cpu_and_cuda_builds": "passed with system OpenSSL; CUDA build also tested with CMake 3.28", + "cpu_sdist_install": "passed without Git or submodule network access", + "windows_native_cli": "default/W4 with CPU/CUDA decoder passed; cache resolution tests passed", + "incompatible_torch": "Linux and Windows reject Torch 2.12 and CPU-only Torch 2.13 with installation guidance", + "asset_cache": "cold/warm/offline, manifest upgrade, CUDA selection, size mismatch, USERPROFILE fallback passed", + "frontend_sampling_audio_join_cases": [ + 228, + 12, + 3 + ], + "tq2_1_repack_cases": 436, + "fp16_scale_patterns": 65536, + "cuda_q2_0_mul_mat_cases": 69, + "cuda_q2_0_get_rows_cases": 4, + "weight_parity": { + "tensors": 310, + "types": { + "F16": 1, + "F32": 113, + "TQ2_1": 196 + }, + "max_error": 0.0 + }, + "published_cuda_decoder_parity_cases": 6, + "published_cuda_decoder_max_error": 2.980232238769531e-07, + "cuda_export_eager_trace_token_lengths": [ + 4, + 17, + 53, + 101 + ], + "publisher_manifest_assembly": "passed for default/W4 and optional W8; voices and Python config preserved", + "reference_voice_preparation": "prepared voice synthesized successfully", + "all_w4_voices": "115/115 produced finite nonzero audio on the Linux GPU", + "cpu_regression": "6/6 WAV files byte-identical to base bfab0cbd31a5697e5cba4779f7eed78368eeede0, default/W4, seeds 1/42/1234", + "audio_review": "Representative generated clips checked with speech recognition; no human listening assessment" + }, + "observations": [ + "Portable scalar CPU LM exceeded the legacy reference RMSE threshold (0.15858495 vs 0.15); the documented native AVX2 build passed (0.07333/0.07642). No CPU kernel changes relative to the base branch.", + "Published CPU W4 matches Python TorchScript exactly but differs from the current eager decoder (max error 0.22980946 at 7 tokens). A fresh CPU W4 export passed all three eager/native cases exactly.", + "The static LibreSSL CLI test build required SSL_CERT_FILE; the documented source build with system OpenSSL passed without it.", + "Windows Python 3.12 wheel on L4/WDDM: first CUDA synthesis took about 71 seconds for W4 and 109 seconds for default; reusing the W4 model produced a longer second clip in about 3 seconds. The docs recommend keeping the model open. These are startup observations, not cross-platform performance guarantees.", + "This pass executed CUDA hardware tests on RTX 3090 and L4 only; other GPU architectures, ARM64 and macOS were not retested." + ] } }