vla.cpp performance review

.dockerignore

@@ -1,6 +1,7 @@
 # Keep the build context small: llama.cpp is fetched inside the image, models
 # are mounted at runtime, and build/output dirs are host artifacts.
 .git/
+.claude/
 models/
 build/
 build-*/

.github/workflows/build.yml

@@ -13,19 +13,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  cpp-unit:
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v7
-      # Compiled directly: pure, no llama.cpp or protobuf/zmq needed.
-      - name: pure unit tests
-        run: |
-          for t in test_vision_common test_rope_conventions; do
-            g++ -std=c++17 -Isrc -Wall -Wextra -fsanitize=address,undefined \
-                -fno-omit-frame-pointer "tests/$t.cpp" -o "/tmp/$t"
-            "/tmp/$t"
-          done
-
   py-tooling:
     runs-on: ubuntu-24.04
     steps:
@@ -56,21 +43,83 @@ jobs:
       - uses: actions/cache@v6
         with:
           path: build/_deps
-          key: llama-${{ steps.pin.outputs.tag }}-${{ runner.os }}
+          key: llama-${{ steps.pin.outputs.tag }}-${{ runner.os }}-asan
       # Everything, not a target list: ctest registers tests this job must build,
       # and a named list goes stale the next time one is added.
-      - name: build + ctest (CPU, -Wall -Wextra)
+      - name: build + ctest (CPU, ASan+UBSan, -Werror)
         run: |
-          cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=OFF -DVLA_BUILD_TESTS=ON
+          san="-fsanitize=address,undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer"
+          cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo -DGGML_CUDA=OFF -DVLA_BUILD_TESTS=ON \
+            -DVLA_WERROR=ON -DCMAKE_C_FLAGS="$san" -DCMAKE_CXX_FLAGS="$san"
           cmake --build build -j"$(nproc)"
           ctest --test-dir build --output-on-failure
-      # This job is the only one that fetches llama.cpp, and neither patch script
-      # runs on a CPU build, so their anchors would otherwise rot unnoticed until
-      # someone configures a CUDA or OpenVINO tree. Patch a copy: the real one is
+      # Neither patch script runs on a CPU build, and build-cuda patches only on a
+      # cache miss, so their anchors would otherwise rot unnoticed until someone
+      # configures a fresh CUDA or OpenVINO tree. Patch a copy: the real one is
       # cached.
       - name: patch anchors still apply
         run: |
           cp -r build/_deps/llama-src /tmp/llama-patchtest
           python3 scripts/patch_ggml_cuda_ext_hook.py /tmp/llama-patchtest
           python3 scripts/patch_ggml_openvino.py /tmp/llama-patchtest
           python3 scripts/patch_ggml_openvino.py /tmp/llama-patchtest   # idempotent
+
+  build-cuda:
+    runs-on: ubuntu-24.04
+    container: nvidia/cuda:12.9.1-devel-ubuntu24.04
+    steps:
+      - name: deps
+        run: |
+          apt-get update -qq
+          DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
+            build-essential cmake git ca-certificates pkg-config python3 \
+            libzmq3-dev cppzmq-dev libprotobuf-dev protobuf-compiler
+      - uses: actions/checkout@v7
+      - name: read llama.cpp pin
+        id: pin
+        run: echo "tag=$(bash scripts/llama_tag.sh)" >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@v6
+        with:
+          path: build/_deps
+          key: llama-${{ steps.pin.outputs.tag }}-${{ runner.os }}-cuda-${{ hashFiles('scripts/patch_ggml_cuda_ext_hook.py') }}
+      - name: build (compile only, the runner has no GPU)
+        run: |
+          cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=89-real \
+            -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined -DVLA_BUILD_TESTS=ON -DVLA_WERROR=ON
+          cmake --build build -j"$(nproc)"
+
+  build-backends:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { name: opencl, os: ubuntu-24.04, cmake: -DGGML_OPENCL=ON -DVLA_WERROR=ON }
+          - { name: metal,  os: macos-15,     cmake: -DGGML_METAL=ON, test: true }
+    steps:
+      - uses: actions/checkout@v7
+      - name: deps
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq --no-install-recommends \
+            build-essential cmake git ca-certificates pkg-config \
+            libzmq3-dev cppzmq-dev libprotobuf-dev protobuf-compiler \
+            ocl-icd-opencl-dev opencl-headers
+      - name: deps
+        if: runner.os == 'macOS'
+        run: brew install cmake zeromq cppzmq protobuf
+      - name: read llama.cpp pin
+        id: pin
+        run: echo "tag=$(bash scripts/llama_tag.sh)" >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@v6
+        with:
+          path: build/_deps
+          key: llama-${{ steps.pin.outputs.tag }}-${{ runner.os }}-${{ matrix.name }}
+      - name: build
+        run: |
+          cmake -B build -DCMAKE_BUILD_TYPE=Release ${{ matrix.cmake }} -DVLA_BUILD_TESTS=ON
+          cmake --build build -j"$(getconf _NPROCESSORS_ONLN)"
+      - name: ctest
+        if: matrix.test
+        run: ctest --test-dir build --output-on-failure

.github/workflows/release.yml

@@ -1,143 +1,206 @@
-# Tagged binaries and a container image. build.yml already compiles all of this
-# on every push; this is the same work with the artifacts kept.
+# Tagged binaries and a container image. A tag push publishes them; a manual
+# run builds the same artifacts on the chosen ref and publishes nothing.
 name: release
 
 on:
   push:
     tags: ['v*']
   workflow_dispatch:
-    inputs:
-      tag:
-        description: Tag to build (dry run, nothing is published)
-        required: true
 
 permissions:
-  contents: write
-  packages: write
+  contents: read
 
 env:
   BINARIES: vla-server vlm-server vla-cli vla-bench
 
 jobs:
   linux:
     runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.image }}
+    defaults:
+      run:
+        shell: bash
     strategy:
       fail-fast: false
       matrix:
         include:
           - name: linux-x86_64-cpu
+            runner: ubuntu-24.04
+            image: ubuntu:24.04
             cmake: -DGGML_CUDA=OFF
-            cuda: false
+          - name: linux-x86_64-cuda-12.8
             runner: ubuntu-24.04
-          - name: linux-x86_64-cuda
-            cmake: -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="75;86;89;120"
+            image: nvidia/cuda:12.8.1-devel-ubuntu24.04
+            cmake: -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90;120-real"
             cuda: true
+          - name: linux-x86_64-cuda-13.4
             runner: ubuntu-24.04
-          # Jetson and other aarch64 boards. Native arm64 runner, CPU only: the
-          # hosted images carry no CUDA for arm64, so a Jetson GPU build still
-          # has to happen on the device.
+            image: nvidia/cuda:13.4.1-devel-ubuntu24.04
+            cmake: -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90;120-real"
+            cuda: true
           - name: linux-aarch64-cpu
-            cmake: -DGGML_CUDA=OFF
-            cuda: false
             runner: ubuntu-24.04-arm
+            image: ubuntu:24.04
+            cmake: -DGGML_CUDA=OFF -DGGML_CPU_ARM_ARCH=armv8.2-a+dotprod+fp16
+          - name: linux-aarch64-cuda-13.4
+            runner: ubuntu-24.04-arm
+            image: nvidia/cuda:13.4.1-devel-ubuntu24.04
+            cmake: -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="87-real;110-real;121-real" -DGGML_CPU_ARM_ARCH=armv8.2-a+dotprod+fp16
+            cuda: true
     steps:
-      - uses: actions/checkout@v7
-
       - name: deps
         run: |
-          sudo apt-get update -qq
-          sudo apt-get install -y -qq --no-install-recommends \
-            build-essential cmake git ca-certificates pkg-config \
+          apt-get update -qq
+          DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
+            build-essential cmake git ca-certificates pkg-config python3 \
             libzmq3-dev cppzmq-dev libprotobuf-dev protobuf-compiler
 
-      - name: cuda toolkit
-        if: matrix.cuda
-        run: |
-          wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
-          sudo dpkg -i cuda-keyring_1.1-1_all.deb
-          sudo apt-get update -qq
-          sudo apt-get install -y -qq --no-install-recommends cuda-toolkit-12-8
-          echo "/usr/local/cuda/bin" >> "$GITHUB_PATH"
+      - uses: actions/checkout@v7
 
       - name: build
         run: |
-          cmake -B build -DCMAKE_BUILD_TYPE=Release ${{ matrix.cmake }}
+          cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF \
+            -DCMAKE_INSTALL_RPATH='$ORIGIN' -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+            ${{ matrix.cuda && '-DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined -DGGML_CUDA_NCCL=OFF' || '' }} ${{ matrix.cmake }}
           cmake --build build -j"$(nproc)" --target $BINARIES vla
 
       - name: package
         run: |
-          out="vla.cpp-${{ github.ref_name }}-${{ matrix.name }}"
-          mkdir -p "$out"
+          out="vla.cpp-${GITHUB_REF_NAME//\//-}-${{ matrix.name }}"
+          mkdir -p "$out/scripts"
           for b in $BINARIES; do cp "build/$b" "$out/"; done
-          cp build/libvla.so "$out/"
+          cp -P build/*.so* build/bin/*.so* "$out/"
+          ldd "$out/vla-server" | awk '/libgomp|libprotobuf/ {print $3}' | xargs cp -L -t "$out/"
+          cp /usr/share/doc/libprotobuf32t64/copyright "$out/LICENSE.protobuf"
+          cp /usr/share/doc/libgomp1/copyright "$out/LICENSE.libgomp"
+          cp build/_deps/llama-src/LICENSE "$out/LICENSE.llama.cpp"
+          cp build/_deps/llama-src/licenses/LICENSE-jsonhpp "$out/LICENSE.jsonhpp"
+          cp build/_deps/sentencepiece-src/LICENSE "$out/LICENSE.sentencepiece"
+          cp build/_deps/sentencepiece-src/third_party/darts_clone/LICENSE "$out/LICENSE.darts_clone"
           cp include/vla.h LICENSE.md README.md "$out/"
-          # vla-cli --text runs this; VLA_TOKENIZE_SCRIPT points at it.
-          mkdir -p "$out/scripts" && cp scripts/tokenize_prompt.py "$out/scripts/"
+          # vla-cli --text finds this in scripts/ next to itself.
+          cp scripts/tokenize_prompt.py "$out/scripts/"
           tar -czf "$out.tar.gz" "$out"
 
+      - name: smoke
+        run: |
+          out="vla.cpp-${GITHUB_REF_NAME//\//-}-${{ matrix.name }}"
+          mv build build.moved
+          ldd "$out"/vl*-* "$out"/*.so* > ldd.txt
+          if grep -v libcuda.so.1 ldd.txt | grep 'not found'; then exit 1; fi
+          LD_LIBRARY_PATH=/usr/local/cuda/compat "$out/vla-cli" --help
+
+      - name: cuda runtime
+        if: matrix.cuda
+        run: |
+          out="vla.cpp-${GITHUB_REF_NAME//\//-}-${{ matrix.name }}"
+          mkdir cudart
+          cp -L /usr/local/cuda/lib64/lib{cudart,cublas,cublasLt}.so."${CUDA_VERSION%%.*}" cudart/
+          tar -czf "cudart-$out.tar.gz" --transform "s,^cudart,$out," cudart
+
       - uses: actions/upload-artifact@v7
         with:
           name: ${{ matrix.name }}
           path: '*.tar.gz'
+          if-no-files-found: error
 
   macos:
-    runs-on: macos-14
+    runs-on: macos-15
+    env:
+      BINARIES: vla-cli vla-bench
     steps:
       - uses: actions/checkout@v7
 
       - name: deps
-        run: brew install cmake zeromq cppzmq protobuf
+        run: brew install cmake
 
       - name: build
         run: |
-          cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_METAL=ON
+          cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_METAL=ON -DGGML_NATIVE=OFF \
+            -DVLA_BUILD_SERVER=OFF -DVLA_SPM=OFF \
+            -DCMAKE_INSTALL_RPATH='@loader_path' -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
           cmake --build build -j"$(sysctl -n hw.ncpu)" --target $BINARIES vla
 
       - name: package
         run: |
-          out="vla.cpp-${{ github.ref_name }}-macos-arm64-metal"
-          mkdir -p "$out"
+          out="vla.cpp-${GITHUB_REF_NAME//\//-}-macos-arm64-metal"
+          mkdir -p "$out/scripts"
           for b in $BINARIES; do cp "build/$b" "$out/"; done
-          cp build/libvla.dylib "$out/"
+          cp -a build/*.dylib build/bin/*.dylib "$out/"
+          cp build/_deps/llama-src/LICENSE "$out/LICENSE.llama.cpp"
+          cp build/_deps/llama-src/licenses/LICENSE-jsonhpp "$out/LICENSE.jsonhpp"
           cp include/vla.h LICENSE.md README.md "$out/"
-          mkdir -p "$out/scripts" && cp scripts/tokenize_prompt.py "$out/scripts/"
+          cp scripts/tokenize_prompt.py "$out/scripts/"
           # Metal needs the shader library next to the binary.
           find build -name 'default.metallib' -exec cp {} "$out/" \;
           tar -czf "$out.tar.gz" "$out"
 
+      - name: smoke
+        run: |
+          out="vla.cpp-${GITHUB_REF_NAME//\//-}-macos-arm64-metal"
+          mv build build.moved
+          otool -L "$out"/vl*-* "$out"/*.dylib
+          if otool -L "$out"/vl*-* "$out"/*.dylib | grep -E '/opt/homebrew|/usr/local/'; then exit 1; fi
+          for lib in $(otool -L "$out"/vl*-* "$out"/*.dylib | awk '/@rpath\//{print $1}' | sort -u); do
+            test -e "$out/${lib#@rpath/}" || { echo "missing $lib"; exit 1; }
+          done
+          "$out/vla-cli" --help
+
       - uses: actions/upload-artifact@v7
         with:
           name: macos-arm64-metal
           path: '*.tar.gz'
+          if-no-files-found: error
 
   docker:
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      packages: write
     steps:
+      - name: free disk
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache
       - uses: actions/checkout@v7
       - uses: docker/setup-buildx-action@v4
       - uses: docker/login-action@v4
-        if: startsWith(github.ref, 'refs/tags/')
+        if: github.event_name == 'push'
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: image name
-        run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV"
+        run: |
+          echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV"
+          echo "REF=${GITHUB_REF_NAME//\//-}" >> "$GITHUB_ENV"
       - uses: docker/build-push-action@v7
         with:
           context: .
-          push: ${{ startsWith(github.ref, 'refs/tags/') }}
+          push: ${{ github.event_name == 'push' }}
+          build-args: |
+            CUDA_ARCH=75-real;80-real;86-real;89-real;90;120-real
+            GGML_NATIVE=OFF
           tags: |
-            ${{ env.IMAGE }}:${{ github.ref_name }}
+            ${{ env.IMAGE }}:${{ env.REF }}
             ${{ env.IMAGE }}:latest
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
   publish:
     needs: [linux, macos, docker]
-    if: startsWith(github.ref, 'refs/tags/')
+    if: github.event_name == 'push'
     runs-on: ubuntu-24.04
+    permissions:
+      contents: write
     steps:
+      - uses: actions/checkout@v7
+      # The release body is this tag's CHANGELOG.md section; a tag with none
+      # fails here rather than publishing an empty release.
+      - name: release notes
+        run: |
+          ver="${GITHUB_REF_NAME#v}"
+          awk -v h="## [$ver]" 'index($0, h) == 1 {on=1; next} on && /^## \[/ {exit} on' \
+            CHANGELOG.md > notes.md
+          test -s notes.md || { echo "no CHANGELOG.md section for $ver"; exit 1; }
       # Tarballs only. The docker job also leaves a .dockerbuild build record
       # artifact behind, and pulling that one fails the whole download.
       - uses: actions/download-artifact@v8
@@ -146,4 +209,5 @@ jobs:
         with:
           files: dist/*.tar.gz
           fail_on_unmatched_files: true
+          body_path: notes.md
           generate_release_notes: true