From 6696e4c713b9e700f6fd06998b3b5a94077fae81 Mon Sep 17 00:00:00 2001 From: sami jaghouar Date: Fri, 16 Oct 2026 12:04:16 +0000 Subject: [PATCH] feat(deep-gemm): build vLLM's DeepGEMM 2.8.0 (e1f418c) DeepGEMM 2.5.0 (891d57b) lacks the SM90 paged MQA logits path with 32-token KV blocks that DeepSeek-V4.1's indexer needs, and its SM90 k-grouped FP8 GEMM returns wrong weight gradients for some experts, nondeterministically. Build the commit vLLM 0.31.0 vendors instead, so vLLM and the trainer run the same kernels. Its DeepJIT needs the elfutils headers at build time. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build_kernels.yaml | 8 +++++--- scripts/install_deep_gemm.sh | 11 ++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_kernels.yaml b/.github/workflows/build_kernels.yaml index 100f4cc..5a24d62 100644 --- a/.github/workflows/build_kernels.yaml +++ b/.github/workflows/build_kernels.yaml @@ -2,7 +2,7 @@ name: Build and Release Kernels # Builds every wheel prime-rl (and other consumers) pin via `[tool.uv.sources]`: # prime-kernels (this repo's own kernels.toml-declared extensions), deep-ep and -# deep-gemm (from their pinned deepseek-ai revs), and our torchao (from the pytorch/ao +# deep-gemm (from their pinned upstream revs), and our torchao (from the pytorch/ao # rev the MXFP8 training path requires). This repo is the single build+release site for # all four — prime-rl only consumes the resulting release, it does not build anything # itself (see scripts/install_ep_kernels.sh, scripts/install_deep_gemm.sh, @@ -100,9 +100,11 @@ jobs: # git, curl, gcc/g++/make and cmake/ninja already ship in this image (via # /opt/conda and gcc-toolset-13). libibverbs-devel/librdmacm-devel: NVSHMEM's # IBGDA transport headers (infiniband/mlx5dv.h), compile-time only — deep-ep - # dlopens the real thing at runtime. `xz` unpacks the NVSHMEM archive. + # dlopens the real thing at runtime. elfutils-devel: DeepJIT's `elfutils/libdwfl.h` + # (deep-gemm's stack traces), also compile-time only and dlopened at runtime. `xz` + # unpacks the NVSHMEM archive. - name: Install build tooling - run: dnf install -y --setopt=install_weak_deps=False ca-certificates xz libibverbs-devel librdmacm-devel + run: dnf install -y --setopt=install_weak_deps=False ca-certificates xz libibverbs-devel librdmacm-devel elfutils-devel - name: Checkout uses: actions/checkout@v5 diff --git a/scripts/install_deep_gemm.sh b/scripts/install_deep_gemm.sh index 3783b91..cab73cb 100755 --- a/scripts/install_deep_gemm.sh +++ b/scripts/install_deep_gemm.sh @@ -10,20 +10,25 @@ # bash scripts/install_deep_gemm.sh --wheel-dir dist # # Options: -# --ref REF DeepGEMM commit hash (default: 891d57b) +# --repo URL DeepGEMM git repository (default: vllm-project/DeepGEMM) +# --ref REF DeepGEMM commit hash (default: e1f418c) # --wheel-dir DIR Output wheel to DIR (default: ./dist) +# +# The default is vLLM's DeepGEMM fork at the commit vLLM 0.31.0 vendors (2.8.0). vLLM imports +# an installed `deep_gemm` ahead of its vendored copy, so this keeps both on the same kernels. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(dirname "$SCRIPT_DIR")" -DEEPGEMM_GIT_REPO="https://github.com/deepseek-ai/DeepGEMM.git" -DEEPGEMM_GIT_REF="891d57b4db1071624b5c8fa0d1e51cb317fa709f" +DEEPGEMM_GIT_REPO="https://github.com/vllm-project/DeepGEMM.git" +DEEPGEMM_GIT_REF="e1f418c2a4f20818221f6b0e578b4c2f634d4c3f" WHEEL_DIR="$REPO_ROOT/dist" while [[ $# -gt 0 ]]; do case $1 in + --repo) DEEPGEMM_GIT_REPO="$2"; shift 2 ;; --ref) DEEPGEMM_GIT_REF="$2"; shift 2 ;; --wheel-dir) WHEEL_DIR="$2"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 1 ;;