From 8cf10ab2881cf4681fa5ee85ee7def57411aa51b Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 24 Aug 2026 12:56:49 +0200 Subject: [PATCH 01/46] Add DeepSeek-V4-Pro FP4 B200 llmd-vllm P/D disagg (agentX 1P-DEP8/1D-DEP8) Ports the agentX (llm-manifesto/agentx-mvp) DeepSeek-V4-Pro 1P-EP8/1D-EP8 K8s deployment to InferenceX's SLURM-based llmd-vllm framework on B200 (8 GPUs/node, so each DEP8 role fits on one node vs GB200's 4 GPUs/node needing two). New master-config key dsv4-fp4-b200-llmd-vllm-agentx and recipe carry the agentX EPP scheduling config (disagg-profile-handler's `deciders:` shape, prefix-cache-affinity-filter, token-load-scorer, etc., ported verbatim from a real agentx-mvp results snapshot) and per-role vLLM tuning, dropping the Kubernetes/manifesto-specific Mooncake KV chain in favor of server.sh's existing plain NixlConnector. Wires a self-contained FRAMEWORK==llmd-vllm branch into runners/launch_b200-dgxc.sh (ahead of the srtctl/srt-slurm path used by the other B200 multinode frameworks, which llm-d bypasses entirely), mirroring launch_gb200-nv.sh's pyxis/enroot + slurm_utils.sh helpers pattern. Pins a B200-only EPP/pd-sidecar v0.10.0 binary override (LLMD_BIN_DIR) for the newer EPP plugin shape, isolated from GB200's shared v0.9.0 binaries/path. Co-Authored-By: Cursor Agent Co-authored-by: Cursor --- .../dsv4_fp4_b200_llmd-vllm-disagg.sh | 60 ++++++ .../dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml | 181 ++++++++++++++++++ configs/nvidia-master.yaml | 48 +++++ perf-changelog.yaml | 8 + runners/launch_b200-dgxc.sh | 135 +++++++++++++ 5 files changed, 432 insertions(+) create mode 100755 benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh create mode 100644 benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml diff --git a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh new file mode 100755 index 0000000000..0589913d22 --- /dev/null +++ b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm P/D disagg benchmark +# (agentX-flavored 1P-DEP8/1D-DEP8). Sibling of +# dsv4_fp4_gb200_llmd-vllm-disagg.sh - same shape, different topology +# (B200 = 8 GPUs/node, so each DEP8 role fits on ONE node; GB200 = 4 +# GPUs/node, role spans 2 nodes). The runner resolves this script via +# SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" +# from launch_b200-dgxc-slurm.sh. + +set -euo pipefail + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + MODEL_PATH \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 + +# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity). +export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" + +export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +# Worker count per role (Option B multi-engine). Prefer an explicit +# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else +# fall back to the matrix num-worker fields (PREFILL_NUM_WORKERS/ +# DECODE_NUM_WORKERS); else 1 (single engine = unchanged 1P+1D). submit.sh +# reads these. +export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}" +export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}" + +JOB_ID=$(bash ./submit.sh \ + "$PREFILL_NODES" \ + "$DECODE_NODES" \ + "$ISL" "$OSL" "${CONC_LIST// /x}" inf \ + "$RANDOM_RANGE_RATIO") + +if [[ -z "$JOB_ID" ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml new file mode 100644 index 0000000000..e5b2ab97b5 --- /dev/null +++ b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml @@ -0,0 +1,181 @@ +# DeepSeek-V4-Pro (FP4) on B200, 1P-DEP8/1D-DEP8 P/D disagg via llmd-vllm. +# +# This is the agentX-flavored sibling of dsv4-fp4-gb200-mid-curve-megamoe.yaml: +# same topology shape (1 prefill DEP8 engine + 1 decode DEP8 engine), but the +# EPP scheduling config and per-role vLLM tuning are ported from the agentX +# (llm-manifesto/agentx-mvp) DeepSeek-V4-Pro K8s deployment instead of being +# InferenceX-native, per direct request. Selected via +# CONFIG_FILE=dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml in the +# dsv4-fp4-b200-llmd-vllm-agentx master-config key. +# +# Topology: B200 = 8 GPUs/node, so each DEP8 role fits on exactly ONE node +# (unlike GB200's 4 GPUs/node, where the same DEP8 role needs 2 nodes). +# PREFILL_NODES=1 DECODE_NODES=1 GPUS_PER_NODE=8 -> 2 nodes / 16 GPUs total. +# Because neither role spans >1 node, server.sh's LWS_GROUP_SIZE is always 1 +# here, so the cross-node --data-parallel-hybrid-lb / NVSHMEM-ibgda code paths +# never activate - the GB200 recipes' cross-node NIC/NVSHMEM tuning (e.g. +# NVSHMEM_HCA_PE_MAPPING) simply does not apply and is intentionally omitted. +# +# Source (agentX): llm-manifesto/models/deepseek-v4-ix/ix-disagg-base.yaml + +# 1P-EP8-1D-EP8.yaml. agentX's Mooncake/MultiConnector KV chain and CPU +# offload are Kubernetes/manifesto-specific and are NOT ported - server.sh +# hardcodes plain NixlConnector for every llmd-vllm disagg run (no per-recipe +# override point), matching what the existing GB200 llm-d recipes already do. +# agentX's kv_cache_metrics/kv_cache_metrics_sample (observability-only, not +# present in any InferenceX llm-d recipe or confirmed supported by this +# image's vLLM build) are also dropped as an unnecessary compatibility risk. +# max_model_len is set to 9280 (ISL 8192 + OSL 1024 + headroom, matching the +# existing InferenceX GB200 recipes) instead of agentX's general-purpose +# 1048576, since this recipe only ever serves the fixed 8k/1k benchmark. +# +# ---- EPP scheduling config ---- +# Ported EXACTLY (plugin list, parameters, weights) from the agentx-mvp +# results snapshot manifest.yaml +# (results/dsv4-pro-dspark/results_ilmarkov-ix-2p-ep8-1d-ep8-dspark-1308-long-v2), +# which is a real EPP config that ran a 1P-EP8-1D-EP8 DeepSeek-V4-Pro K8s +# deployment. Only the file-discovery plugin + dataLayer wiring is added on +# top (InferenceX's own no-Kubernetes addition - agentX's K8s deployment used +# InferencePool pod discovery instead, which doesn't apply here). +# +# This config uses disagg-profile-handler's `deciders: {prefill: ...}` +# parameter shape (not InferenceX's existing GB200 recipes' `deciderPluginName` +# shape) and additional plugins (inflight-load-producer, +# approx-prefix-cache-producer, prefix-cache-affinity-filter, +# token-load-scorer) not present in the current InferenceX EPP image. This +# requires the EPP/pd-sidecar binaries to be bumped to v0.10.0 for this +# recipe - see the LLMD_BIN_DIR override in runners/launch_b200-dgxc-slurm.sh +# (B200-only; GB200's shared v0.9.0 binaries/path are untouched). +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig + +plugins: + - name: file-disc + type: file-discovery + parameters: + path: /tmp/endpoints.yaml + watchFile: false + + - type: prefill-filter + - type: decode-filter + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 45272 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 4783 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 3000000 + - type: active-request-scorer + - type: queue-scorer + - type: always-disagg-pd-decider + - type: disagg-profile-handler + parameters: + deciders: + prefill: always-disagg-pd-decider + - type: max-score-picker + name: prefill-picker + - type: max-score-picker + name: decode-picker + +schedulingProfiles: + - name: prefill + plugins: + - pluginRef: prefill-filter + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: prefill-picker + - name: decode + plugins: + - pluginRef: decode-filter + - pluginRef: active-request-scorer + - pluginRef: decode-picker + +dataLayer: + discovery: + pluginRef: file-disc + +# ---- Per-role vLLM flags ---- +# Ported from agentX's ix-disagg-base.yaml (prefill/decode roles) + +# 1P-EP8-1D-EP8.yaml overrides. Both roles are DEP8 (TP=1, DP=8, EP on), +# FLASHINFER_MLA_SPARSE_DSV4 attention, deep_gemm_mega_moe MoE backend, +# EP weight filter (keeps only served experts resident - required for DEP8 +# to avoid OOM). See file header for what was intentionally dropped/adapted. +prefill: + tp: 1 + enable-expert-parallel: true + extra-args: >- + --kv-cache-dtype fp8 + --enforce-eager + --gpu-memory-utilization 0.97 + --max-model-len 9280 + --max-num-seqs 64 + --max-num-batched-tokens 8192 + --long-prefill-token-threshold 1024 + --enable-cumem-allocator + --block-size 256 + --tokenizer-mode deepseek_v4 + --moe-backend deep_gemm_mega_moe + --enable-ep-weight-filter + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --numa-bind + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + PYTHONHASHSEED: "0" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + +decode: + tp: 1 + enable-expert-parallel: true + extra-args: >- + --kv-cache-dtype fp8 + --max-num-seqs 64 + --max-num-batched-tokens 256 + --max-cudagraph-capture-size 64 + --gpu-memory-utilization 0.95 + --max-model-len 9280 + --enable-cumem-allocator + --block-size 256 + --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --tokenizer-mode deepseek_v4 + --moe-backend deep_gemm_mega_moe + --enable-ep-weight-filter + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --numa-bind + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + PYTHONHASHSEED: "0" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + +# ---- SLURM resource directives ---- +slurm: + time_limit: "08:00:00" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f0842fe558..6cf8b32b9e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4522,6 +4522,54 @@ dsv4-fp4-gb200-llmd-vllm: - "DECODE_NODES=2" - "GPUS_PER_NODE=4" +# DeepSeek-V4-Pro FP4 B200 llmd-vllm, agentX-flavored: EPP scheduling config + +# per-role vLLM tuning ported from the agentX (llm-manifesto/agentx-mvp) K8s +# deployment (see benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx- +# 1p1d-dep8-dep8.yaml for the exact port/adaptation notes), rather than +# InferenceX-native tuning like dsv4-fp4-gb200-llmd-vllm above. Requires the +# B200-only EPP/pd-sidecar v0.10.0 binary override wired in +# runners/launch_b200-dgxc-slurm.sh. +# +# B200 = 8 GPUs/node, so each DEP8 role fits on ONE node (GB200's 4 GPUs/node +# needs 2 nodes per role for the same DEP8 shape) - see the recipe file for +# the full node-math writeup. +dsv4-fp4-b200-llmd-vllm-agentx: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b200-multinode + precision: fp4 + framework: llmd-vllm + router: { name: llm-d-router, version: "0.10.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1P DEP8 + 1D DEP8 (2 nodes / 16 GPUs). + - spec-decoding: "none" + conc-list: [64, 128, 192, 256] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + - "GPUS_PER_NODE=8" + - "CONFIG_FILE=dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "GPUS_PER_NODE=8" + # MTP2 variant of dsv4-fp4-gb200-dynamo-vllm. Uses the vLLM 0.20.1 image # and hand-picked 8k/1k Pareto points mirrored from NVIDIA/srt-slurm. dsv4-fp4-gb200-dynamo-vllm-mtp2: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a9d2492a08..71bbd725f5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6387,3 +6387,11 @@ - "Replace the internal TileRT 0.1.5.post2+inferencex.1 queueing backport with the official 0.1.5.post3 PyPI wheel now that upstream has published it; remove the internal wheel builder and router patch" - "Bump the shared TileRT dependency and router metadata to 0.1.5.post3 for both fixed-seq-length and AgentX configurations while retaining the AgentX 1800-second queue timeout" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2650 + +- config-keys: + - dsv4-fp4-b200-llmd-vllm-agentx + description: + - "Add DeepSeek-V4-Pro FP4 B200 llmd-vllm P/D disagg benchmark (1P DEP8 + 1D DEP8, 2 nodes / 16 GPUs), porting the agentX (llm-manifesto/agentx-mvp) EPP scheduling config and per-role vLLM tuning" + - "Wire a FRAMEWORK==llmd-vllm dispatch into runners/launch_b200-dgxc-slurm.sh (self-contained, bypasses the srtctl/srt-slurm path used by the other B200 multinode frameworks)" + - "Pin a B200-only EPP/pd-sidecar v0.10.0 binary override (LLMD_BIN_DIR) for the disagg-profile-handler deciders: EPP shape; GB200's shared v0.9.0 binaries/path are untouched" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 498ad7d1c1..e044b5f5cf 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -129,6 +129,141 @@ fi export AIPERF_MMAP_CACHE_HOST_PATH="/lustre/fsw/gharunners/aiperf-cache" if [[ "$IS_MULTINODE" == "true" ]]; then + if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then + # Self-contained: bypasses the srtctl/srt-slurm path entirely (the + # "Validate framework" guard and the dsv4-only-dynamo-vllm guard + # further below are srtctl-specific and are never reached from here). + # MODEL_PATH is already resolved above (the dsv4/fp4 branch); llm-d + # additionally needs MODEL_NAME (served-model-name), which this file + # doesn't otherwise set outside the srtctl block - reuse $MODEL (the + # master-config `model:` field), matching this file's own + # `export SERVED_MODEL_NAME=$MODEL` convention below. + if [[ ! ( "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ) ]]; then + echo "Unsupported MODEL_PREFIX/PRECISION for llmd-vllm on B200: $MODEL_PREFIX/$PRECISION" >&2 + exit 1 + fi + export MODEL_NAME="$MODEL" + + source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" + + # Enroot 3.x does not parse Docker's tag@digest syntax (the llm-d + # image is digest-pinned); build the explicit registry#repo:digest + # URI enroot expects. + llmd_enroot_uri_for_image() { + local image="$1" + local image_without_digest="$image" + local digest="" + local first_component registry repository + + if [[ "$image" == *@sha256:* ]]; then + image_without_digest="${image%@*}" + digest="${image##*@}" + fi + + first_component="${image_without_digest%%/*}" + if [[ "$image_without_digest" == */* && ( "$first_component" == *.* || "$first_component" == *:* || "$first_component" == "localhost" ) ]]; then + registry="$first_component" + repository="${image_without_digest#*/}" + else + registry="registry-1.docker.io" + repository="$image_without_digest" + fi + + if [[ -z "$digest" ]]; then + if [[ "$registry" == "registry-1.docker.io" ]]; then + printf 'docker://%s\n' "$image" + else + printf 'docker://%s#%s\n' "$registry" "$repository" + fi + return + fi + if [[ "$registry" == "registry-1.docker.io" && "$repository" != */* ]]; then + repository="library/$repository" + fi + printf 'docker://%s#%s:%s\n' "$registry" "$repository" "$digest" + } + + # Separate name from this file's existing (later) `import_squash` used + # by the srtctl path, to avoid redefining that function. + llmd_import_squash() { + local squash="$1" image="$2" + local lock="${squash}.lock" + local enroot_uri + enroot_uri=$(llmd_enroot_uri_for_image "$image") || exit 1 + ( + exec 9>"$lock" + flock -w 1800 9 || { echo "Failed to acquire lock for $squash" >&2; exit 1; } + if unsquashfs -l "$squash" > /dev/null 2>&1; then + echo "Squash file already exists and is valid, skipping import: $squash" + else + rm -f "$squash" + if ! enroot import -o "$squash" "$enroot_uri"; then + echo "Error: enroot import failed for $enroot_uri" >&2 + exit 1 + fi + fi + ) || exit 1 + } + + LLMD_SQUASH_DIR="${LLMD_SQUASH_DIR:-/home/sa-shared/containers}" + mkdir -p "$LLMD_SQUASH_DIR" 2>/dev/null || true + LLMD_SQUASH_FILE="${LLMD_SQUASH_DIR}/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + llmd_import_squash "$LLMD_SQUASH_FILE" "$IMAGE" + + export LLMD_CONTAINER_ENGINE=pyxis + export LLMD_SQUASH_FILE + + # B200-only EPP/pd-sidecar binary pin (v0.10.0, needed for the + # agentX-ported disagg-profile-handler `deciders:` EPP shape). Points + # at a dedicated path so it never collides with GB200's shared + # binaries.env / v0.9.0 path - see benchmarks/llm-d/binaries.env for + # the mechanism. NOTE: this path must be populated once (out of band, + # on a host with docker + registry access) via: + # LLMD_BIN_DIR=/home/sa-shared/llm-d-bins-v0.10.0 \ + # EPP_FROM_IMAGE=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0 \ + # ROUTING_SIDECAR_IMAGE=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0 \ + # benchmarks/llm-d/extract-binaries.sh + # before this recipe can actually run; job.slurm's pyxis mount loop is + # a no-op (falls back to the image's baked-in v0.9.0) until then. + export LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}" + + export DOCKER_IMAGE_NAME=$IMAGE + export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" + mkdir -p "$BENCHMARK_LOGS_DIR" + + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" + BENCH_SCRIPT="benchmarks/multi_node/${SCRIPT_NAME}" + if [[ ! -f "$BENCH_SCRIPT" ]]; then + echo "Error: llm-d wrapper not found: $BENCH_SCRIPT" >&2 + exit 1 + fi + + JOB_ID=$(bash "$BENCH_SCRIPT") + if [[ -z "$JOB_ID" ]]; then + echo "Error: failed to submit llm-d job" >&2 + exit 1 + fi + echo "Submitted llm-d job: $JOB_ID" + + trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP + + LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + + while IFS= read -r -d '' result_file; do + copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 + done < <(find "$BENCHMARK_LOGS_DIR" -name "${RESULT_FILENAME}*.json" -print0 2>/dev/null) + + if [[ "${RUN_EVAL:-false}" == "true" ]]; then + EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR" -type d -name eval_results -print -quit 2>/dev/null) + [[ -z "$EVAL_DIR" ]] && EVAL_DIR="$BENCHMARK_LOGS_DIR/eval_results" + copy_eval_artifacts "$EVAL_DIR" "$GITHUB_WORKSPACE" || exit 1 + fi + + scancel "$JOB_ID" 2>/dev/null || true + exit 0 + fi + if [[ "$FRAMEWORK" == "tilert" ]]; then export SLURM_PARTITION SLURM_ACCOUNT export TILERT_WEIGHTS_DIR="${TILERT_WEIGHTS_DIR:-/lustre/fsw/gharunners/models/${MODEL_PREFIX}-${PRECISION}-tilert-8shard}" From 83c30609d7f88ca521413d27cedbd4c0e05a8943 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Wed, 26 Aug 2026 16:34:30 +0200 Subject: [PATCH 02/46] Trim disagg conc-list to 192 and script the v0.10.0 EPP binary pull - dsv4-fp4-b200-llmd-vllm-agentx: drop c256 from the 1P-DEP8/1D-DEP8 conc-list per request. - extract-binaries.sh: support BINARIES_ENV_FILE to source an alternate pins file, so a cluster-scoped router-version bump doesn't require editing the shared binaries.env (whose EPP_FROM_IMAGE/ ROUTING_SIDECAR_IMAGE are plain assignments and can't be overridden via pre-exported env vars). - Add binaries-b200-v0.10.0.env: B200-only v0.10.0 EPP/pd-sidecar pins, linux/amd64 platform (B200 is x86_64, unlike GB200's arm64 default), dedicated LLMD_BIN_DIR. - launch_b200-dgxc.sh: fix the extraction command documented in the LLMD_BIN_DIR comment, which previously relied on the non-overridable env vars above and would have silently extracted v0.9.0 binaries. Co-authored-by: Cursor --- benchmarks/llm-d/binaries-b200-v0.10.0.env | 33 ++++++++++++++++++++++ benchmarks/llm-d/extract-binaries.sh | 9 +++++- configs/nvidia-master.yaml | 2 +- runners/launch_b200-dgxc.sh | 14 ++++----- 4 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 benchmarks/llm-d/binaries-b200-v0.10.0.env diff --git a/benchmarks/llm-d/binaries-b200-v0.10.0.env b/benchmarks/llm-d/binaries-b200-v0.10.0.env new file mode 100644 index 0000000000..c485112f61 --- /dev/null +++ b/benchmarks/llm-d/binaries-b200-v0.10.0.env @@ -0,0 +1,33 @@ +# B200-only binary pins for the agentX-ported EPP config used by +# dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml (disagg-profile-handler +# `deciders:` shape + inflight-load-producer / approx-prefix-cache-producer / +# prefix-cache-affinity-filter / token-load-scorer plugins), which require +# the v0.10.0 EPP/pd-sidecar router build. Kept as a sibling file (not a +# change to binaries.env) so GB200's shared v0.9.0 pins and bin path are +# completely untouched. +# +# Pull with extract-binaries.sh instead of binaries.env's defaults: +# BINARIES_ENV_FILE=binaries-b200-v0.10.0.env \ +# benchmarks/llm-d/extract-binaries.sh +# Run once (or whenever these pins change) on a host with docker + registry +# access and the /home/sa-shared filesystem mounted. + +# --- Endpoint Picker (EPP / inference scheduler) --- +EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0" +EPP_BIN_PATH="/app/epp" + +# --- P/D routing sidecar (pd-sidecar) --- +ROUTING_SIDECAR_IMAGE="ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0" +ROUTING_SIDECAR_BIN_PATH="/app/pd-sidecar" + +# --- Envoy front proxy --- +# Unchanged from binaries.env; re-declared here so this file is +# self-contained (extract-binaries.sh always extracts all three). +ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2" +ENVOY_BIN_PATH="/usr/local/bin/envoy" + +# B200 is x86_64 (unlike GB200's Grace/arm64, binaries.env's default). +LLMD_BIN_PLATFORM="${LLMD_BIN_PLATFORM:-linux/amd64}" + +# Dedicated path so this never collides with GB200's shared bin dir. +LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}" diff --git a/benchmarks/llm-d/extract-binaries.sh b/benchmarks/llm-d/extract-binaries.sh index ed9f00e577..bd17fc2734 100755 --- a/benchmarks/llm-d/extract-binaries.sh +++ b/benchmarks/llm-d/extract-binaries.sh @@ -13,12 +13,19 @@ # ./extract-binaries.sh # uses binaries.env defaults # LLMD_BIN_DIR=/some/dir ./extract-binaries.sh # LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh # for an x86 test +# BINARIES_ENV_FILE=binaries-b200-v0.10.0.env ./extract-binaries.sh +# # pull a different set of image pins (e.g. a router-version bump +# # scoped to one cluster) without touching the shared binaries.env set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BINARIES_ENV_FILE="${BINARIES_ENV_FILE:-$HERE/binaries.env}" +# Resolve a bare filename against this script's directory, so +# BINARIES_ENV_FILE=foo.env works regardless of the caller's cwd. +[[ "$BINARIES_ENV_FILE" != /* ]] && BINARIES_ENV_FILE="$HERE/$BINARIES_ENV_FILE" # shellcheck source=/dev/null -source "$HERE/binaries.env" +source "$BINARIES_ENV_FILE" echo "Extracting llm-d binaries -> $LLMD_BIN_DIR (platform $LLMD_BIN_PLATFORM)" mkdir -p "$LLMD_BIN_DIR" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6cf8b32b9e..041c8cbfa0 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4551,7 +4551,7 @@ dsv4-fp4-b200-llmd-vllm-agentx: search-space: # 1P DEP8 + 1D DEP8 (2 nodes / 16 GPUs). - spec-decoding: "none" - conc-list: [64, 128, 192, 256] + conc-list: [64, 128, 192] prefill: num-worker: 1 tp: 1 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index e044b5f5cf..f9d4da045b 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -216,13 +216,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # B200-only EPP/pd-sidecar binary pin (v0.10.0, needed for the # agentX-ported disagg-profile-handler `deciders:` EPP shape). Points # at a dedicated path so it never collides with GB200's shared - # binaries.env / v0.9.0 path - see benchmarks/llm-d/binaries.env for - # the mechanism. NOTE: this path must be populated once (out of band, - # on a host with docker + registry access) via: - # LLMD_BIN_DIR=/home/sa-shared/llm-d-bins-v0.10.0 \ - # EPP_FROM_IMAGE=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0 \ - # ROUTING_SIDECAR_IMAGE=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0 \ - # benchmarks/llm-d/extract-binaries.sh + # binaries.env / v0.9.0 path - see + # benchmarks/llm-d/binaries-b200-v0.10.0.env for the pins and + # benchmarks/llm-d/binaries.env for the general mechanism. NOTE: this + # path must be populated once (out of band, on a host with docker + + # registry access and the /home/sa-shared mount) via: + # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env \ + # benchmarks/llm-d/extract-binaries.sh # before this recipe can actually run; job.slurm's pyxis mount loop is # a no-op (falls back to the image's baked-in v0.9.0) until then. export LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}" From fe7e680a8894d41483e63d9ab3d0a5376f730faa Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Wed, 26 Aug 2026 17:37:25 +0200 Subject: [PATCH 03/46] Add DeepSeek-V4-Pro FP4 B200 llmd-vllm aggregated (TP8/DEP8) benchmark Sibling of the P/D disagg benchmark: one engine per topology does both prefill and decode (1 node / 8 GPUs each), no P/D KV handoff. - server.sh/job.slurm: add an aggregated mode (DECODE_NODES=0) - no decode role, no pd-sidecar, no --kv_transfer_config (there is nothing to hand KV off to), and the sole engine's rank 0 takes over the EPP/Envoy/bench coordinator duties normally owned by the decode leader. Also fixes the underlying DECODE_NODES=0 bugs: job.slurm's out-of-bounds IPS[] read for DECODE_LEADER_IP, and server.sh emitting a bogus empty-address decode-0 endpoint in endpoints.yaml. - New recipes dsv4-fp4-b200-agentx-agg-{tp8,dep8}.yaml: per-role vLLM tuning ported from agentX's ix-agg-base/agg-tp8 and ix-agg-dep-base/agg-dep8, plus a minimal single-schedulingProfile EPP config (no disagg-profile-handler needed - llm-d auto-selects single-profile-handler for one profile). Needs no EPP router bump since every plugin used is already in the default v0.9.0 binaries. - New dsv4_fp4_b200_llmd-vllm-agg.sh wrapper + launch_b200-dgxc.sh now branches its llmd-vllm wrapper-script choice on $DISAGG instead of hardcoding the disagg wrapper. - New dsv4-fp4-b200-llmd-vllm-agentx-agg master-config key (disagg: false, prefill num-worker=1 / decode num-worker=0 per the process_result.py / dynamo-vllm aggregated convention). conc-lists mirror the agentX dspark historical runs (results_ilmarkov-2-ix-agg- tp8-dspark-v3-long, results_ilmarkov-ix-agg-dep8-dspark-1308-long). - README.md + perf-changelog.yaml updated. Co-authored-by: Cursor --- .../multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh | 64 +++++++++ .../dsv4-fp4-b200-agentx-agg-dep8.yaml | 113 +++++++++++++++ .../dsv4-fp4-b200-agentx-agg-tp8.yaml | 132 ++++++++++++++++++ benchmarks/multi_node/llm-d/README.md | 22 +++ benchmarks/multi_node/llm-d/job.slurm | 11 +- benchmarks/multi_node/llm-d/server.sh | 55 ++++++-- configs/nvidia-master.yaml | 66 +++++++++ perf-changelog.yaml | 9 ++ runners/launch_b200-dgxc.sh | 10 +- 9 files changed, 466 insertions(+), 16 deletions(-) create mode 100755 benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh create mode 100644 benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-dep8.yaml create mode 100644 benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-tp8.yaml diff --git a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh new file mode 100755 index 0000000000..730824bc2d --- /dev/null +++ b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm aggregated benchmark +# (agentX-flavored TP8 / DEP8, one engine does both prefill and decode). +# Sibling of dsv4_fp4_b200_llmd-vllm-disagg.sh - same shape, but always +# submits with DECODE_NODES=0 (no decode role at all; see the aggregated +# recipes' header comments and server.sh's IS_AGGREGATED handling). The +# runner resolves this script via +# SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" +# from launch_b200-dgxc-slurm.sh when DISAGG=false. + +set -euo pipefail + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + MODEL_PATH \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$DECODE_NODES" != "0" ]]; then + echo "Error: dsv4_fp4_b200_llmd-vllm-agg.sh requires DECODE_NODES=0 (got $DECODE_NODES); aggregated mode has no decode role" >&2 + exit 1 +fi + +set -x + +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 + +# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity). +export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" + +export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +# Aggregated is always a single engine (no multi-engine high-tpt split), so +# PREFILL_WORKERS is always 1. DECODE_WORKERS is unused (DECODE_NODES=0) but +# still exported since submit.sh/server.sh read it unconditionally. +export PREFILL_WORKERS="${PREFILL_WORKERS:-1}" +export DECODE_WORKERS="${DECODE_WORKERS:-1}" + +JOB_ID=$(bash ./submit.sh \ + "$PREFILL_NODES" \ + "$DECODE_NODES" \ + "$ISL" "$OSL" "${CONC_LIST// /x}" inf \ + "$RANDOM_RANGE_RATIO") + +if [[ -z "$JOB_ID" ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-dep8.yaml b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-dep8.yaml new file mode 100644 index 0000000000..996b5e3e5c --- /dev/null +++ b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-dep8.yaml @@ -0,0 +1,113 @@ +# DeepSeek-V4-Pro (FP4) on B200, aggregated DEP8 (one engine does both +# prefill and decode) via llmd-vllm. agentX-flavored sibling of +# dsv4-fp4-b200-agentx-agg-tp8.yaml (same aggregated wiring, DP+EP instead +# of pure TP). +# +# Topology: B200 = 8 GPUs/node, so DEP8 (DP=8, EP on) fits on exactly ONE +# node (unlike GB200's 4 GPUs/node, where agentX's ix-agg-dep-base.yaml/ +# agg-dep8.yaml needs `lws: {size: 2, replicas: 1}` spanning 2 nodes via +# hybrid-lb). PREFILL_NODES=1 DECODE_NODES=0 GPUS_PER_NODE=8 -> 1 node / 8 +# GPUs total. Since the role never spans >1 node, server.sh's +# LWS_GROUP_SIZE is always 1 here, so the cross-node --data-parallel-hybrid-lb +# / NVSHMEM-ibgda code paths never activate - vLLM runs its DP=8 ranks +# in-process on the one node. +# +# Aggregated-mode wiring: see the header comment in +# dsv4-fp4-b200-agentx-agg-tp8.yaml (no decode role / no pd-sidecar / no +# kv_transfer_config, engine mapped to the "prefill" role, rank 0 doubles +# as EPP/Envoy/bench coordinator). The EPP config below is identical in +# shape to that recipe's (same rationale: agentX's aggregated K8s config has +# no llm-d EPP schema to port, so this is a purpose-built minimal +# single-profile config, no v0.10.0 router bump needed). +# +# Source (agentX): llm-manifesto/models/deepseek-v4-ix/ix-agg-dep-base.yaml +# + agg-dep8.yaml. Dropped entirely (no InferenceX equivalent needed): +# the `mooncake:` sidecar block and the MultiConnector +# NixlConnector+SimpleCPUOffloadConnector+MooncakeStoreConnector KV chain +# (aggregated has no P/D KV transfer at all, see above), and the +# Mooncake/UCX-specific env (VLLM_MOONCAKE_LOAD_RECV_THREADS, +# MC_ENABLE_DEST_DEVICE_AFFINITY, UCX_TLS=rc,cuda_copy) that existed only to +# support that KV chain. max_model_len trimmed to 9280 (ISL 8192 + OSL 1024 +# + headroom) instead of agentX's general-purpose 1048576, matching every +# other recipe in this directory. +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig + +plugins: + - name: file-disc + type: file-discovery + parameters: + path: /tmp/endpoints.yaml + watchFile: false + + - type: prefill-filter + - type: queue-scorer + - type: kv-cache-utilization-scorer + - type: active-request-scorer + - type: prefix-cache-scorer + +schedulingProfiles: + - name: default + plugins: + - pluginRef: prefill-filter + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + - pluginRef: active-request-scorer + weight: 1 + - pluginRef: prefix-cache-scorer + weight: 3 + +dataLayer: + discovery: + pluginRef: file-disc + +# ---- Per-role vLLM flags ---- +# Ported from agentX's ix-agg-dep-base.yaml (roles[0], extended by +# agg-dep8.yaml). DEP8 (TP=1, DP=8, EP on), FLASHINFER_MLA_SPARSE_DSV4 +# attention, deep_gemm_mega_moe MoE backend, EP weight filter (keeps only +# served experts resident - required for DEP8 to avoid OOM), matching the +# disagg recipe's DEP8 roles. tp/enable-expert-parallel are explicit here +# even though they match server.sh's defaults, for self-documentation. +# +# No `decode:` block: server.sh only ever reads the section named after the +# node's own ROLE, which is always "prefill" for this recipe (DECODE_NODES=0). +prefill: + tp: 1 + enable-expert-parallel: true + extra-args: >- + --kv-cache-dtype fp8 + --gpu-memory-utilization 0.92 + --max-model-len 9280 + --max-num-seqs 64 + --max-num-batched-tokens 8192 + --max-cudagraph-capture-size 64 + --block-size 256 + --tokenizer-mode deepseek_v4 + --moe-backend deep_gemm_mega_moe + --enable-ep-weight-filter + --enable-cumem-allocator + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --numa-bind + --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + # Differs from agg-tp8.yaml's "0" - ported as-is from + # ix-agg-dep-base.yaml (DEP8's DP-ranks-as-processes path apparently + # wants NCCL symmetric memory enabled, unlike pure TP8's allreduce path). + VLLM_USE_NCCL_SYMM_MEM: "1" + +# ---- SLURM resource directives ---- +slurm: + time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-tp8.yaml b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-tp8.yaml new file mode 100644 index 0000000000..31bdf64617 --- /dev/null +++ b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-tp8.yaml @@ -0,0 +1,132 @@ +# DeepSeek-V4-Pro (FP4) on B200, aggregated TP8 (one engine does both +# prefill and decode) via llmd-vllm. agentX-flavored sibling of the +# disagg dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml recipe. +# +# Topology: B200 = 8 GPUs/node, so TP8 fits on exactly ONE node (unlike +# GB200's 4 GPUs/node, where agentX's ix-agg-base.yaml/agg-tp8.yaml needs +# `lws: {size: 2, replicas: 1}` spanning 2 nodes). PREFILL_NODES=1 +# DECODE_NODES=0 GPUS_PER_NODE=8 -> 1 node / 8 GPUs total. Since the role +# never spans >1 node, server.sh's LWS_GROUP_SIZE is always 1 here, so the +# cross-node --headless multi-node-TP / NVSHMEM-ibgda code paths never +# activate; this is a plain single-node `vllm serve --tensor-parallel-size 8`. +# +# Aggregated-mode wiring (InferenceX-side, applies to every agg recipe): +# - No decode role exists at all (DECODE_NODES=0), so there is no +# pd-sidecar and no P/D KV handoff - server.sh skips +# --kv_transfer_config entirely for this recipe. +# - Per the process_result.py / dynamo-vllm precedent (e.g. +# dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg in nvidia-master.yaml), +# the single engine is mapped to the "prefill" role (PREFILL_NODES=all, +# DECODE_NODES=0) so multinode result aggregation counts its GPUs once +# and process_result.py's `prefill_gpus > 0` invariant holds. This is +# the opposite of agentX's own K8s naming (roles: [{name: decode}]), +# which is a gateway/InferencePool convention that doesn't carry over. +# - Rank 0 (the only rank here) takes over EPP/Envoy/bench coordinator +# duties normally owned by the decode leader (server.sh's aggregated +# coordinator gate). +# +# Source (agentX): llm-manifesto/models/deepseek-v4-ix/ix-agg-base.yaml + +# agg-tp8.yaml. Dropped: the commented-out SimpleCPUOffloadConnector +# kv_transfer_config (aggregated needs no KV transfer at all here), +# VLLM_USE_V2_MODEL_RUNNER (no precedent in any InferenceX llm-d recipe), +# DG_JIT_CACHE_DIR (a K8s-pod-local /tmp path, not meaningful across SLURM +# job containers). max_model_len is set to 9280 (ISL 8192 + OSL 1024 + +# headroom, matching the existing InferenceX recipes) instead of agentX's +# general-purpose 1048576, since this recipe only ever serves the fixed +# 8k/1k benchmark. +# +# ---- EPP scheduling config ---- +# Unlike the disagg recipe (which ports an exact agentX EPP snapshot), +# agentX's aggregated K8s deployments route via a Gateway API InferencePool +# (`routing: {kind: load_aware, target_role: decode}`), not llm-d's EPP +# YAML schema - there is nothing to port directly. This is a purpose-built, +# minimal EPP config instead: exactly one schedulingProfile, so llm-d +# auto-selects the single-profile-handler (no disagg-profile-handler/decider +# needed - there is only one pool to route to). `prefill-filter` matches +# the endpoints.yaml labels server.sh always emits for the sole role +# (llm-d.ai/role: prefill, see above). Every plugin here already exists in +# the default v0.9.0 image-baked binaries, so this recipe does NOT need the +# v0.10.0 router bump the disagg recipe requires. +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig + +plugins: + - name: file-disc + type: file-discovery + parameters: + path: /tmp/endpoints.yaml + watchFile: false + + - type: prefill-filter + - type: queue-scorer + - type: kv-cache-utilization-scorer + - type: active-request-scorer + - type: prefix-cache-scorer + +schedulingProfiles: + - name: default + plugins: + - pluginRef: prefill-filter + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + - pluginRef: active-request-scorer + weight: 1 + - pluginRef: prefix-cache-scorer + weight: 3 + +dataLayer: + discovery: + pluginRef: file-disc + +# ---- Per-role vLLM flags ---- +# Ported from agentX's ix-agg-base.yaml (roles[0], extended by agg-tp8.yaml). +# TP=8, no DP/EP (pure tensor-parallel). FLASHINFER_MLA_SPARSE_DSV4 +# attention, matching the disagg recipe's DEP8 roles. enable-expert-parallel +# must be explicit false here: server.sh defaults ROLE_ENABLE_EP to true +# when a recipe omits the key (the H200 1P+1D DEP8 shape), which would be +# wrong for pure TP8. +# +# No `decode:` block: server.sh only ever reads the section named after the +# node's own ROLE, which is always "prefill" for this recipe (DECODE_NODES=0). +prefill: + tp: 8 + enable-expert-parallel: false + extra-args: >- + --kv-cache-dtype fp8 + --gpu-memory-utilization 0.92 + --max-model-len 9280 + --max-num-seqs 32 + --max-num-batched-tokens 8192 + --max-cudagraph-capture-size 32 + --block-size 256 + --tokenizer-mode deepseek_v4 + --disable-custom-all-reduce + --enable-cumem-allocator + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + TILELANG_CLEANUP_TEMP_FILES: "1" + # Intra-node (NVLink) TP8 allreduce tuning - real cross-GPU allreduce + # traffic here (unlike the DEP8 recipes, which have no TP>1 role), so + # ported as-is from ix-agg-base.yaml rather than dropped. + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + NCCL_P2P_LEVEL: "NVL" + +# ---- SLURM resource directives ---- +slurm: + time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d/README.md b/benchmarks/multi_node/llm-d/README.md index 81dbd51995..8f914eddd4 100644 --- a/benchmarks/multi_node/llm-d/README.md +++ b/benchmarks/multi_node/llm-d/README.md @@ -25,6 +25,28 @@ the coordinator (EPP + Envoy + bench), exactly like the AMD path's | `xP` | decode leader + pd-sidecar + EPP + Envoy + benchmark client | | `xP+1 .. xP+yD-1` | decode workers | +### Aggregated mode (`yD = 0`) + +Setting `DECODE_NODES=0` runs a single engine that does both prefill and +decode in-process - there is no decode role, no pd-sidecar, and no +`--kv_transfer_config` (server.sh's `IS_AGGREGATED` flag, derived from +`DECODE_NODES=0`, gates all three). Rank `0` (the sole engine's leader) +takes over the coordinator duties (endpoints.yaml + EPP + Envoy + bench) +that the decode leader normally owns. + +Per the `process_result.py` / `dynamo-vllm` precedent, the aggregated +engine is mapped to the **prefill** role in the master config +(`prefill: {num-worker: 1, ...}`, `decode: {num-worker: 0, ...}`) so +multinode result aggregation counts its GPUs exactly once and +`process_result.py`'s `prefill_gpus > 0` invariant holds. `endpoints.yaml` +still only ever emits `llm-d.ai/role: prefill` entries in this mode (no +`decode-*` entries at all), so an aggregated recipe's EPP config needs a +filter matching that role, not a `decode-filter` / disagg split - see +`benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-{tp8,dep8}.yaml` +for a minimal single-`schedulingProfile` example (llm-d auto-selects +`single-profile-handler` when there is exactly one profile, so no +`disagg-profile-handler`/decider is needed). + Each instance (prefill or decode) is one vLLM engine spanning multiple nodes via `--data-parallel-hybrid-lb`. With `xP=2, yD=2, GPUS_PER_NODE=8` you get DP=16 prefill + DP=16 decode (the wide-EP diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index f260c1a6fb..816fbe4b17 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -48,8 +48,17 @@ fi # prefill workers = ranks 1 .. PREFILL_NODES-1 # decode leader = rank PREFILL_NODES (also coordinator: EPP + Envoy + bench) # decode workers = ranks PREFILL_NODES+1 .. NUM_NODES-1 +# +# Aggregated mode (DECODE_NODES=0): there is no decode role at all, so +# IPS[$PREFILL_NODES] would be an out-of-bounds read (IPS only has +# PREFILL_NODES entries). Leave the decode addresses empty; server.sh's +# aggregated coordinator gate (rank 0, ROLE=prefill) never reads them. PREFILL_LEADER_IP="${IPS[0]}" -DECODE_LEADER_IP="${IPS[$PREFILL_NODES]}" +if [[ "$DECODE_NODES" -gt 0 ]]; then + DECODE_LEADER_IP="${IPS[$PREFILL_NODES]}" +else + DECODE_LEADER_IP="" +fi # DP leader addresses for vLLM --data-parallel-address (rank 0 of each instance). PREFILL_DP_ADDR="$PREFILL_LEADER_IP" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 7e189767bc..90a43c917b 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -25,6 +25,15 @@ NODE_RANK="${NODE_RANK:-${SLURM_PROCID:-0}}" PREFILL_NODES="${PREFILL_NODES:-1}" DECODE_NODES="${DECODE_NODES:-1}" GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +# Aggregated mode: no decode role at all, so no P/D KV handoff (no +# pd-sidecar, no NixlConnector) - the single engine does prefill+decode +# in-process. DECODE_NODES=0 is the only signal for this; everything else +# (coordinator gating, kv_transfer_config) derives from it below. +if [[ "$DECODE_NODES" -eq 0 ]]; then + IS_AGGREGATED=1 +else + IS_AGGREGATED=0 +fi VLLM_PORT=8200 SIDECAR_PORT=8000 ENVOY_PORT=8080 @@ -210,24 +219,28 @@ fi # ---------------------------------------------------------------- # Bring up vLLM engine (every node) # ---------------------------------------------------------------- -# KV role: prefill=producer, decode=consumer (override via KV_ROLE_OVERRIDE). -if [[ -n "${KV_ROLE_OVERRIDE:-}" ]]; then - KV_ROLE="$KV_ROLE_OVERRIDE" -elif [[ "$ROLE" == "prefill" ]]; then - KV_ROLE="kv_producer" -else - KV_ROLE="kv_consumer" -fi -KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" - COMMON_ARGS=( --port "$VLLM_PORT" --served-model-name "$MODEL_NAME" --trust-remote-code --disable-access-log-for-endpoints=/health,/metrics --tensor-parallel-size "$TP_SIZE" - --kv_transfer_config "$KV_TRANSFER_CONFIG" ) +# KV role: prefill=producer, decode=consumer (override via KV_ROLE_OVERRIDE). +# Aggregated mode has no second engine to hand KV off to (no pd-sidecar +# either), so skip --kv_transfer_config entirely rather than standing up a +# NixlConnector producer with no consumer ever pulling from it. +if [[ "$IS_AGGREGATED" -eq 0 ]]; then + if [[ -n "${KV_ROLE_OVERRIDE:-}" ]]; then + KV_ROLE="$KV_ROLE_OVERRIDE" + elif [[ "$ROLE" == "prefill" ]]; then + KV_ROLE="kv_producer" + else + KV_ROLE="kv_consumer" + fi + KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" + COMMON_ARGS+=(--kv_transfer_config "$KV_TRANSFER_CONFIG") +fi # A single frontend (HTTP + tokenize + DP load-balance) is CPU-bound and caps # throughput, so run several. Incompatible with --headless, so it is the one # flag the headless-worker branch below drops. Overridable via LLMD_API_SERVER_COUNT. @@ -311,6 +324,10 @@ fi # endpoint per node. Pure-TP: only the TP-group leader has an api-server # (followers are --headless), so only the leader runs a sidecar and only leaders # are listed as endpoints. +# +# Aggregated mode (IS_AGGREGATED=1): ROLE is never "decode" (no decode nodes +# exist at all), so this is already a no-op there - correct, since the single +# engine needs no P/D handoff. if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX" -eq 0 ) ]]; then SIDECAR_CONNECTOR="nixlv2" SIDECAR_FLAGS=(--port="$SIDECAR_PORT" --vllm-port="$VLLM_PORT" @@ -325,9 +342,14 @@ if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX fi # ================================================================ -# Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval +# Coordinator: endpoints, EPP, Envoy, bench, eval # ================================================================ -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then +# Normally the decode leader. In aggregated mode (IS_AGGREGATED=1) there is +# no decode role at all, so the sole engine's leader (rank 0, which is +# always ROLE=prefill there since PREFILL_NODES=NUM_NODES) takes over the +# coordinator duties instead. +if [[ ( "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ) || \ + ( "$IS_AGGREGATED" -eq 1 && "$ROLE" == "prefill" && "$NODE_RANK" -eq 0 ) ]]; then # Release the allocation whenever the coordinator exits. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" @@ -351,7 +373,10 @@ VLLM_PORT = int('$VLLM_PORT') SIDECAR_PORT = int('$SIDECAR_PORT') # ALL_IPS is rank-ordered: ranks [0:pn] are prefill nodes, [pn:pn+dn] decode. prefill_ips = all_ips[:pn] or [os.environ['PREFILL_LEADER_IP']] -decode_ips = all_ips[pn:pn + dn] or [os.environ['DECODE_LEADER_IP']] +# dn == 0 (aggregated mode): no decode role at all. DECODE_LEADER_IP is '' +# in that case (see job.slurm), so decode_ips must NOT fall back to it - +# that would emit a bogus decode-0 endpoint with an empty address. +decode_ips = (all_ips[pn:pn + dn] or [os.environ['DECODE_LEADER_IP']]) if dn > 0 else [] endpoints = [] def add_role(role, ips, base_port, group_size=1): @@ -369,6 +394,8 @@ def add_role(role, ips, base_port, group_size=1): # endpoint per node for DEP8, or one per TP-group leader for pure-TP. add_role('prefill', prefill_ips, VLLM_PORT) decode_group = 1 if decode_ep else max(1, dn // decode_workers) +# dn == 0 -> decode_ips == [] -> add_role emits zero decode endpoints +# (aggregated mode: everything routes through the 'prefill'-labeled pool). add_role('decode', decode_ips, SIDECAR_PORT, group_size=decode_group) yaml.safe_dump({'endpoints': endpoints}, open('/tmp/endpoints.yaml', 'w')) print(f'endpoints.yaml ({len(endpoints)} endpoints):') diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 041c8cbfa0..4c0e024ebf 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4570,6 +4570,72 @@ dsv4-fp4-b200-llmd-vllm-agentx: - "DECODE_NODES=1" - "GPUS_PER_NODE=8" +# Aggregated (TP8 / DEP8) sibling of dsv4-fp4-b200-llmd-vllm-agentx: one +# engine per topology does both prefill and decode (no P/D split, no +# pd-sidecar, no kv_transfer_config - see the recipe files' header +# comments). Separate top-level key because `disagg:` is a whole-key +# property. Uses the default v0.9.0 EPP/pd-sidecar binaries (no router +# bump needed - the aggregated EPP config only uses plugins already +# present there). conc-lists mirror the agentX dspark historical runs: +# results/dsv4-pro-dspark/results_ilmarkov-2-ix-agg-tp8-dspark-v3-long +# (tp8) and results_ilmarkov-ix-agg-dep8-dspark-1308-long (dep8). +dsv4-fp4-b200-llmd-vllm-agentx-agg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b200-multinode + precision: fp4 + framework: llmd-vllm + router: { name: llm-d-router, version: "0.9.0" } + multinode: true + disagg: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # Aggregated TP8 (1 node / 8 GPUs; pure tensor-parallel, no EP). + - spec-decoding: "none" + conc-list: [1, 2, 4, 8, 12] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "GPUS_PER_NODE=8" + - "CONFIG_FILE=dsv4-fp4-b200-agentx-agg-tp8.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once + # (matches dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg's convention). + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=0" + # Aggregated DEP8 (1 node / 8 GPUs; DP=8 + EP). + - spec-decoding: "none" + conc-list: [12, 16, 32, 52, 72] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + - "GPUS_PER_NODE=8" + - "CONFIG_FILE=dsv4-fp4-b200-agentx-agg-dep8.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=0" + # MTP2 variant of dsv4-fp4-gb200-dynamo-vllm. Uses the vLLM 0.20.1 image # and hand-picked 8k/1k Pareto points mirrored from NVIDIA/srt-slurm. dsv4-fp4-gb200-dynamo-vllm-mtp2: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 71bbd725f5..59d17ef9c9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6395,3 +6395,12 @@ - "Wire a FRAMEWORK==llmd-vllm dispatch into runners/launch_b200-dgxc-slurm.sh (self-contained, bypasses the srtctl/srt-slurm path used by the other B200 multinode frameworks)" - "Pin a B200-only EPP/pd-sidecar v0.10.0 binary override (LLMD_BIN_DIR) for the disagg-profile-handler deciders: EPP shape; GB200's shared v0.9.0 binaries/path are untouched" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER + +- config-keys: + - dsv4-fp4-b200-llmd-vllm-agentx-agg + description: + - "Add DeepSeek-V4-Pro FP4 B200 llmd-vllm aggregated benchmark (TP8 and DEP8, 1 node / 8 GPUs each, one engine does both prefill and decode), porting the agentX (llm-manifesto) ix-agg-base/agg-tp8 and ix-agg-dep-base/agg-dep8 per-role vLLM tuning" + - "Extend server.sh/job.slurm with an aggregated (DECODE_NODES=0) mode: no decode role, no pd-sidecar, no --kv_transfer_config, and the sole engine's rank 0 takes over the EPP/Envoy/bench coordinator duties normally owned by the decode leader" + - "Add a minimal single-schedulingProfile EPP config for aggregated recipes (no disagg-profile-handler needed); uses the existing default v0.9.0 EPP/pd-sidecar binaries, no router bump required" + - "Branch runners/launch_b200-dgxc-slurm.sh's llmd-vllm wrapper-script selection on DISAGG (true -> disagg wrapper, false -> new aggregated wrapper)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index f9d4da045b..e0c3049ca0 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -231,7 +231,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" mkdir -p "$BENCHMARK_LOGS_DIR" - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" + # DISAGG is exported job-wide from the master config's `disagg:` key + # (see benchmark-multinode-tmpl.yml). true -> the P/D disagg wrapper + # (1P-DEP8/1D-DEP8); false -> the aggregated wrapper (TP8/DEP8, one + # engine does both prefill and decode, DECODE_NODES=0). + if [[ "${DISAGG:-true}" == "true" ]]; then + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" + else + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" + fi BENCH_SCRIPT="benchmarks/multi_node/${SCRIPT_NAME}" if [[ ! -f "$BENCH_SCRIPT" ]]; then echo "Error: llm-d wrapper not found: $BENCH_SCRIPT" >&2 From 79f3e12ed7231726ecb774afcb063a5a1b6a7dd5 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 31 Aug 2026 12:05:06 +0200 Subject: [PATCH 04/46] Update configs --- .../agg-gb200-dep8-dspark-agentic.yaml} | 1 + ...gg-gb200-dep8-dspark-mooncake-agentic.yaml | 96 +++++++++++++++++++ .../agg-gb200-tp8-dspark-agentic.yaml} | 1 + ...-gb200-1p1d-dep8-dep8-dspark-agentic.yaml} | 34 ++++++- benchmarks/multi_node/llm-d/server.sh | 62 +++++++++++- configs/nvidia-master.yaml | 39 ++++++-- perf-changelog.yaml | 8 ++ 7 files changed, 224 insertions(+), 17 deletions(-) rename benchmarks/multi_node/llm-d-recipes/{dsv4-fp4-b200-agentx-agg-dep8.yaml => agentic/agg-gb200-dep8-dspark-agentic.yaml} (95%) create mode 100644 benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml rename benchmarks/multi_node/llm-d-recipes/{dsv4-fp4-b200-agentx-agg-tp8.yaml => agentic/agg-gb200-tp8-dspark-agentic.yaml} (96%) rename benchmarks/multi_node/llm-d-recipes/{dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml => agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml} (82%) diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-dep8.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml similarity index 95% rename from benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-dep8.yaml rename to benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index 996b5e3e5c..058ba9ae0f 100644 --- a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-dep8.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -92,6 +92,7 @@ prefill: --no-enable-flashinfer-autotune --numa-bind --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: VLLM_USE_RUST_FRONTEND: "1" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml new file mode 100644 index 0000000000..80e151df16 --- /dev/null +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -0,0 +1,96 @@ +# DeepSeek-V4-Pro (FP4) on B200, aggregated DEP8 with Mooncake prefix-cache +# KV store enabled. Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds a +# `mooncake:` section so server.sh wires a MultiConnector +# (NixlConnector + MooncakeStoreConnector) for cross-run prefix-cache sharing +# via RDMA between DP ranks. See server.sh's aggregated+Mooncake branch for +# the kv_both role assignment. +# +# EPP config and per-role vLLM flags are identical to the base dep8 recipe. +# Mooncake store config (P2PHANDSHAKE embedded mode, 140 GB segment) matches +# the disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml recipe. +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig + +plugins: + - name: file-disc + type: file-discovery + parameters: + path: /tmp/endpoints.yaml + watchFile: false + + - type: prefill-filter + - type: queue-scorer + - type: kv-cache-utilization-scorer + - type: active-request-scorer + - type: prefix-cache-scorer + +schedulingProfiles: + - name: default + plugins: + - pluginRef: prefill-filter + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + - pluginRef: active-request-scorer + weight: 1 + - pluginRef: prefix-cache-scorer + weight: 3 + +dataLayer: + discovery: + pluginRef: file-disc + +# ---- Per-role vLLM flags ---- +prefill: + tp: 1 + enable-expert-parallel: true + extra-args: >- + --kv-cache-dtype fp8 + --gpu-memory-utilization 0.92 + --max-model-len 9280 + --max-num-seqs 64 + --max-num-batched-tokens 8192 + --max-cudagraph-capture-size 64 + --block-size 256 + --tokenizer-mode deepseek_v4 + --moe-backend deep_gemm_mega_moe + --enable-ep-weight-filter + --enable-cumem-allocator + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --numa-bind + --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +# ---- Mooncake KV store config ---- +mooncake: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "140GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + +# ---- SLURM resource directives ---- +slurm: + time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-tp8.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml similarity index 96% rename from benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-tp8.yaml rename to benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml index 31bdf64617..f180c54b72 100644 --- a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-tp8.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -107,6 +107,7 @@ prefill: --no-disable-hybrid-kv-cache-manager --no-enable-flashinfer-autotune --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: VLLM_USE_RUST_FRONTEND: "1" diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml similarity index 82% rename from benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml rename to benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index e5b2ab97b5..0d86e3fd88 100644 --- a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -121,9 +121,10 @@ prefill: --enforce-eager --gpu-memory-utilization 0.97 --max-model-len 9280 - --max-num-seqs 64 + --max-num-seqs 16 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 + --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --enable-cumem-allocator --block-size 256 --tokenizer-mode deepseek_v4 @@ -144,15 +145,21 @@ prefill: VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" decode: tp: 1 enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --max-num-seqs 64 + --max-num-seqs 32 --max-num-batched-tokens 256 - --max-cudagraph-capture-size 64 + --max-cudagraph-capture-size 256 + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --gpu-memory-utilization 0.95 --max-model-len 9280 --enable-cumem-allocator @@ -175,6 +182,27 @@ decode: VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +# ---- Mooncake KV store config ---- +# P2PHANDSHAKE embedded mode — no K8s mooncake-master pod needed; nodes +# negotiate RDMA directly. global_segment_size 140 GB matches the dynamo +# agentic recipe (srt-slurm-recipes/vllm/deepseek-v4/agentic/). Prefill +# uses kv_both (store new KV + load cache hits via RDMA); decode uses +# kv_consumer with lookup disabled (NIXL-only receive, no Mooncake lookup). +mooncake: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "140GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false # ---- SLURM resource directives ---- slurm: diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 90a43c917b..c0576e2f26 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -166,6 +166,38 @@ PY fi echo "Resolved $ROLE TP_SIZE=$TP_SIZE ROLE_ENABLE_EP=$ROLE_ENABLE_EP" +# ---------------------------------------------------------------- +# Mooncake KV store (optional, from recipe top-level `mooncake:` key) +# ---------------------------------------------------------------- +# When a recipe sets `mooncake.store_config`, write the JSON config to +# /tmp/mooncake_config.json and set MOONCAKE_CONFIG_PATH. The KV transfer +# config section below will then build a MultiConnector chain +# (NixlConnector + MooncakeStoreConnector) instead of plain NixlConnector. +# Mooncake uses P2PHANDSHAKE embedded mode (no external metadata server) +# so no K8s sidecar is required - nodes negotiate directly via RDMA. +MOONCAKE_CONFIG_PATH="" +if [[ -n "${CONFIG_FILE:-}" && -f "/etc/llmd-recipes/${CONFIG_FILE}" ]]; then + _MC_JSON=$(python3 - < /tmp/mooncake_config.json + MOONCAKE_CONFIG_PATH=/tmp/mooncake_config.json + export MOONCAKE_CONFIG_PATH + echo "Mooncake enabled: config at $MOONCAKE_CONFIG_PATH" + # Install mooncake if the image does not bundle it. + python3 -c "import mooncake_transfer_engine" 2>/dev/null || \ + pip install --quiet mooncake-transfer-engine-cuda13==0.3.12.post1 + fi +fi + # ---------------------------------------------------------------- # Transport env (NCCL / UCX / NIXL), recipe-overridable # ---------------------------------------------------------------- @@ -227,9 +259,11 @@ COMMON_ARGS=( --tensor-parallel-size "$TP_SIZE" ) # KV role: prefill=producer, decode=consumer (override via KV_ROLE_OVERRIDE). -# Aggregated mode has no second engine to hand KV off to (no pd-sidecar -# either), so skip --kv_transfer_config entirely rather than standing up a -# NixlConnector producer with no consumer ever pulling from it. +# Aggregated mode normally has no second engine to hand KV off to (no +# pd-sidecar either), so --kv_transfer_config is skipped. Exception: when +# Mooncake is enabled on an aggregated recipe, a MultiConnector +# (NixlConnector + MooncakeStoreConnector) is wired with kv_both so DP ranks +# can share prefix-cache blocks across runs via the Mooncake RDMA store. if [[ "$IS_AGGREGATED" -eq 0 ]]; then if [[ -n "${KV_ROLE_OVERRIDE:-}" ]]; then KV_ROLE="$KV_ROLE_OVERRIDE" @@ -238,7 +272,27 @@ if [[ "$IS_AGGREGATED" -eq 0 ]]; then else KV_ROLE="kv_consumer" fi - KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" + if [[ -n "${MOONCAKE_CONFIG_PATH:-}" ]]; then + # MultiConnector: NixlConnector handles direct P/D KV transfer; + # MooncakeStoreConnector enables cross-node prefix-cache lookup via RDMA. + # Prefill uses kv_both so it can both store new KV and load cache hits. + # Decode uses kv_consumer with lookup disabled (it only receives from NIXL). + if [[ "$ROLE" == "prefill" ]]; then + _KV_OUTER="kv_both" + _MC_EXTRA='"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false' + else + _KV_OUTER="kv_consumer" + _MC_EXTRA='"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false' + fi + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + else + KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" + fi + COMMON_ARGS+=(--kv_transfer_config "$KV_TRANSFER_CONFIG") +elif [[ -n "${MOONCAKE_CONFIG_PATH:-}" ]]; then + # Aggregated + Mooncake: single role acts as kv_both (stores new KV and + # loads cache hits from the Mooncake RDMA store for prefix-cache sharing). + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"load_async\":true,\"lookup_async\":true,\"enable_cross_layers_blocks\":false,\"enable_offload\":false}}]}}" COMMON_ARGS+=(--kv_transfer_config "$KV_TRANSFER_CONFIG") fi # A single frontend (HTTP + tokenize + DP load-balance) is CPU-bound and caps diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4c0e024ebf..4b26b5d65f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4534,8 +4534,8 @@ dsv4-fp4-gb200-llmd-vllm: # needs 2 nodes per role for the same DEP8 shape) - see the recipe file for # the full node-math writeup. dsv4-fp4-b200-llmd-vllm-agentx: - image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4 - model: deepseek-ai/DeepSeek-V4-Pro + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: b200-multinode precision: fp4 @@ -4549,7 +4549,7 @@ dsv4-fp4-b200-llmd-vllm-agentx: - isl: 8192 osl: 1024 search-space: - # 1P DEP8 + 1D DEP8 (2 nodes / 16 GPUs). + # 1P DEP8 + 1D DEP8 (2 nodes / 16 GPUs). Always Mooncake. - spec-decoding: "none" conc-list: [64, 128, 192] prefill: @@ -4560,7 +4560,7 @@ dsv4-fp4-b200-llmd-vllm-agentx: additional-settings: - "PREFILL_NODES=1" - "GPUS_PER_NODE=8" - - "CONFIG_FILE=dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml" + - "CONFIG_FILE=agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml" decode: num-worker: 1 tp: 1 @@ -4580,8 +4580,8 @@ dsv4-fp4-b200-llmd-vllm-agentx: # results/dsv4-pro-dspark/results_ilmarkov-2-ix-agg-tp8-dspark-v3-long # (tp8) and results_ilmarkov-ix-agg-dep8-dspark-1308-long (dep8). dsv4-fp4-b200-llmd-vllm-agentx-agg: - image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4 - model: deepseek-ai/DeepSeek-V4-Pro + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: b200-multinode precision: fp4 @@ -4605,7 +4605,7 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: additional-settings: - "PREFILL_NODES=1" - "GPUS_PER_NODE=8" - - "CONFIG_FILE=dsv4-fp4-b200-agentx-agg-tp8.yaml" + - "CONFIG_FILE=agentic/agg-gb200-tp8-dspark-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker # count at zero so result aggregation counts eight GPUs only once # (matches dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg's convention). @@ -4616,9 +4616,28 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: dp-attn: false additional-settings: - "DECODE_NODES=0" - # Aggregated DEP8 (1 node / 8 GPUs; DP=8 + EP). + # Aggregated DEP8 (1 node / 8 GPUs; DP=8 + EP), no Mooncake. + - spec-decoding: "none" + conc-list: [12, 16, 32] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + - "GPUS_PER_NODE=8" + - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=0" + # Aggregated DEP8 with Mooncake prefix-cache KV store. - spec-decoding: "none" - conc-list: [12, 16, 32, 52, 72] + conc-list: [52, 72] prefill: num-worker: 1 tp: 1 @@ -4627,7 +4646,7 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: additional-settings: - "PREFILL_NODES=1" - "GPUS_PER_NODE=8" - - "CONFIG_FILE=dsv4-fp4-b200-agentx-agg-dep8.yaml" + - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml" decode: num-worker: 0 tp: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 59d17ef9c9..fa98caac85 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6404,3 +6404,11 @@ - "Add a minimal single-schedulingProfile EPP config for aggregated recipes (no disagg-profile-handler needed); uses the existing default v0.9.0 EPP/pd-sidecar binaries, no router bump required" - "Branch runners/launch_b200-dgxc-slurm.sh's llmd-vllm wrapper-script selection on DISAGG (true -> disagg wrapper, false -> new aggregated wrapper)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER + +- config-keys: + - dsv4-fp4-b200-llmd-vllm-agentx + description: + - "Add Mooncake prefix-cache KV store for the disagg DEP8 arm at concurrencies [52, 72]" + - "Extend server.sh to parse a `mooncake.store_config` key from llm-d recipe YAMLs: writes /tmp/mooncake_config.json, sets MOONCAKE_CONFIG_PATH, and wires MultiConnector (NixlConnector + MooncakeStoreConnector) instead of plain NixlConnector; prefill uses kv_both, decode kv_consumer with lookup disabled" + - "Add dsv4-fp4-b200-agentx-1p1d-dep8-dep8-mooncake.yaml recipe with Mooncake store_config (P2PHANDSHAKE embedded, 140 GB segment, mlx5_0..3 RDMA) and MC_* env vars, mirroring the dynamo-vllm agentic GB200 disagg recipe" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 81368237305794c36536078fed1d943cd317b43d Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 31 Aug 2026 14:11:06 +0200 Subject: [PATCH 05/46] Cleanup comments --- .../agg-gb200-dep8-dspark-agentic.yaml | 48 +------------ ...gg-gb200-dep8-dspark-mooncake-agentic.yaml | 13 +--- .../agentic/agg-gb200-tp8-dspark-agentic.yaml | 68 ++----------------- ...g-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 64 ++--------------- 4 files changed, 13 insertions(+), 180 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index 058ba9ae0f..7b90a8d645 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -1,35 +1,5 @@ -# DeepSeek-V4-Pro (FP4) on B200, aggregated DEP8 (one engine does both -# prefill and decode) via llmd-vllm. agentX-flavored sibling of -# dsv4-fp4-b200-agentx-agg-tp8.yaml (same aggregated wiring, DP+EP instead -# of pure TP). -# -# Topology: B200 = 8 GPUs/node, so DEP8 (DP=8, EP on) fits on exactly ONE -# node (unlike GB200's 4 GPUs/node, where agentX's ix-agg-dep-base.yaml/ -# agg-dep8.yaml needs `lws: {size: 2, replicas: 1}` spanning 2 nodes via -# hybrid-lb). PREFILL_NODES=1 DECODE_NODES=0 GPUS_PER_NODE=8 -> 1 node / 8 -# GPUs total. Since the role never spans >1 node, server.sh's -# LWS_GROUP_SIZE is always 1 here, so the cross-node --data-parallel-hybrid-lb -# / NVSHMEM-ibgda code paths never activate - vLLM runs its DP=8 ranks -# in-process on the one node. -# -# Aggregated-mode wiring: see the header comment in -# dsv4-fp4-b200-agentx-agg-tp8.yaml (no decode role / no pd-sidecar / no -# kv_transfer_config, engine mapped to the "prefill" role, rank 0 doubles -# as EPP/Envoy/bench coordinator). The EPP config below is identical in -# shape to that recipe's (same rationale: agentX's aggregated K8s config has -# no llm-d EPP schema to port, so this is a purpose-built minimal -# single-profile config, no v0.10.0 router bump needed). -# -# Source (agentX): llm-manifesto/models/deepseek-v4-ix/ix-agg-dep-base.yaml -# + agg-dep8.yaml. Dropped entirely (no InferenceX equivalent needed): -# the `mooncake:` sidecar block and the MultiConnector -# NixlConnector+SimpleCPUOffloadConnector+MooncakeStoreConnector KV chain -# (aggregated has no P/D KV transfer at all, see above), and the -# Mooncake/UCX-specific env (VLLM_MOONCAKE_LOAD_RECV_THREADS, -# MC_ENABLE_DEST_DEVICE_AFFINITY, UCX_TLS=rc,cuda_copy) that existed only to -# support that KV chain. max_model_len trimmed to 9280 (ISL 8192 + OSL 1024 -# + headroom) instead of agentX's general-purpose 1048576, matching every -# other recipe in this directory. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated DEP8, 1 node. +# DSpark speculative decoding: 5 tokens. No Mooncake. apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -63,16 +33,6 @@ dataLayer: discovery: pluginRef: file-disc -# ---- Per-role vLLM flags ---- -# Ported from agentX's ix-agg-dep-base.yaml (roles[0], extended by -# agg-dep8.yaml). DEP8 (TP=1, DP=8, EP on), FLASHINFER_MLA_SPARSE_DSV4 -# attention, deep_gemm_mega_moe MoE backend, EP weight filter (keeps only -# served experts resident - required for DEP8 to avoid OOM), matching the -# disagg recipe's DEP8 roles. tp/enable-expert-parallel are explicit here -# even though they match server.sh's defaults, for self-documentation. -# -# No `decode:` block: server.sh only ever reads the section named after the -# node's own ROLE, which is always "prefill" for this recipe (DECODE_NODES=0). prefill: tp: 1 enable-expert-parallel: true @@ -104,11 +64,7 @@ prefill: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" TILELANG_CLEANUP_TEMP_FILES: "1" - # Differs from agg-tp8.yaml's "0" - ported as-is from - # ix-agg-dep-base.yaml (DEP8's DP-ranks-as-processes path apparently - # wants NCCL symmetric memory enabled, unlike pure TP8's allreduce path). VLLM_USE_NCCL_SYMM_MEM: "1" -# ---- SLURM resource directives ---- slurm: time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 80e151df16..05e8322a23 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -1,13 +1,6 @@ -# DeepSeek-V4-Pro (FP4) on B200, aggregated DEP8 with Mooncake prefix-cache -# KV store enabled. Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds a -# `mooncake:` section so server.sh wires a MultiConnector -# (NixlConnector + MooncakeStoreConnector) for cross-run prefix-cache sharing -# via RDMA between DP ranks. See server.sh's aggregated+Mooncake branch for -# the kv_both role assignment. -# -# EPP config and per-role vLLM flags are identical to the base dep8 recipe. -# Mooncake store config (P2PHANDSHAKE embedded mode, 140 GB segment) matches -# the disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml recipe. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated DEP8 with Mooncake prefix-cache. +# Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds Mooncake (P2PHANDSHAKE embedded RDMA) +# so server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role). apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml index f180c54b72..6245c9485d 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -1,52 +1,5 @@ -# DeepSeek-V4-Pro (FP4) on B200, aggregated TP8 (one engine does both -# prefill and decode) via llmd-vllm. agentX-flavored sibling of the -# disagg dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml recipe. -# -# Topology: B200 = 8 GPUs/node, so TP8 fits on exactly ONE node (unlike -# GB200's 4 GPUs/node, where agentX's ix-agg-base.yaml/agg-tp8.yaml needs -# `lws: {size: 2, replicas: 1}` spanning 2 nodes). PREFILL_NODES=1 -# DECODE_NODES=0 GPUS_PER_NODE=8 -> 1 node / 8 GPUs total. Since the role -# never spans >1 node, server.sh's LWS_GROUP_SIZE is always 1 here, so the -# cross-node --headless multi-node-TP / NVSHMEM-ibgda code paths never -# activate; this is a plain single-node `vllm serve --tensor-parallel-size 8`. -# -# Aggregated-mode wiring (InferenceX-side, applies to every agg recipe): -# - No decode role exists at all (DECODE_NODES=0), so there is no -# pd-sidecar and no P/D KV handoff - server.sh skips -# --kv_transfer_config entirely for this recipe. -# - Per the process_result.py / dynamo-vllm precedent (e.g. -# dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg in nvidia-master.yaml), -# the single engine is mapped to the "prefill" role (PREFILL_NODES=all, -# DECODE_NODES=0) so multinode result aggregation counts its GPUs once -# and process_result.py's `prefill_gpus > 0` invariant holds. This is -# the opposite of agentX's own K8s naming (roles: [{name: decode}]), -# which is a gateway/InferencePool convention that doesn't carry over. -# - Rank 0 (the only rank here) takes over EPP/Envoy/bench coordinator -# duties normally owned by the decode leader (server.sh's aggregated -# coordinator gate). -# -# Source (agentX): llm-manifesto/models/deepseek-v4-ix/ix-agg-base.yaml + -# agg-tp8.yaml. Dropped: the commented-out SimpleCPUOffloadConnector -# kv_transfer_config (aggregated needs no KV transfer at all here), -# VLLM_USE_V2_MODEL_RUNNER (no precedent in any InferenceX llm-d recipe), -# DG_JIT_CACHE_DIR (a K8s-pod-local /tmp path, not meaningful across SLURM -# job containers). max_model_len is set to 9280 (ISL 8192 + OSL 1024 + -# headroom, matching the existing InferenceX recipes) instead of agentX's -# general-purpose 1048576, since this recipe only ever serves the fixed -# 8k/1k benchmark. -# -# ---- EPP scheduling config ---- -# Unlike the disagg recipe (which ports an exact agentX EPP snapshot), -# agentX's aggregated K8s deployments route via a Gateway API InferencePool -# (`routing: {kind: load_aware, target_role: decode}`), not llm-d's EPP -# YAML schema - there is nothing to port directly. This is a purpose-built, -# minimal EPP config instead: exactly one schedulingProfile, so llm-d -# auto-selects the single-profile-handler (no disagg-profile-handler/decider -# needed - there is only one pool to route to). `prefill-filter` matches -# the endpoints.yaml labels server.sh always emits for the sole role -# (llm-d.ai/role: prefill, see above). Every plugin here already exists in -# the default v0.9.0 image-baked binaries, so this recipe does NOT need the -# v0.10.0 router bump the disagg recipe requires. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated TP8, 1 node. +# DSpark speculative decoding: 5 tokens. No Mooncake (single-node TP8). apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -80,18 +33,9 @@ dataLayer: discovery: pluginRef: file-disc -# ---- Per-role vLLM flags ---- -# Ported from agentX's ix-agg-base.yaml (roles[0], extended by agg-tp8.yaml). -# TP=8, no DP/EP (pure tensor-parallel). FLASHINFER_MLA_SPARSE_DSV4 -# attention, matching the disagg recipe's DEP8 roles. enable-expert-parallel -# must be explicit false here: server.sh defaults ROLE_ENABLE_EP to true -# when a recipe omits the key (the H200 1P+1D DEP8 shape), which would be -# wrong for pure TP8. -# -# No `decode:` block: server.sh only ever reads the section named after the -# node's own ROLE, which is always "prefill" for this recipe (DECODE_NODES=0). prefill: tp: 8 + # Must be explicit false: server.sh defaults enable-expert-parallel to true. enable-expert-parallel: false extra-args: >- --kv-cache-dtype fp8 @@ -119,15 +63,11 @@ prefill: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" TILELANG_CLEANUP_TEMP_FILES: "1" - # Intra-node (NVLink) TP8 allreduce tuning - real cross-GPU allreduce - # traffic here (unlike the DEP8 recipes, which have no TP>1 role), so - # ported as-is from ix-agg-base.yaml rather than dropped. - VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_USE_NCCL_SYMM_MEM: "0" # TP8 intra-node allreduce; DEP8 recipe uses "1" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" NCCL_P2P_LEVEL: "NVL" -# ---- SLURM resource directives ---- slurm: time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 0d86e3fd88..4bd13b0572 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -1,50 +1,7 @@ -# DeepSeek-V4-Pro (FP4) on B200, 1P-DEP8/1D-DEP8 P/D disagg via llmd-vllm. -# -# This is the agentX-flavored sibling of dsv4-fp4-gb200-mid-curve-megamoe.yaml: -# same topology shape (1 prefill DEP8 engine + 1 decode DEP8 engine), but the -# EPP scheduling config and per-role vLLM tuning are ported from the agentX -# (llm-manifesto/agentx-mvp) DeepSeek-V4-Pro K8s deployment instead of being -# InferenceX-native, per direct request. Selected via -# CONFIG_FILE=dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml in the -# dsv4-fp4-b200-llmd-vllm-agentx master-config key. -# -# Topology: B200 = 8 GPUs/node, so each DEP8 role fits on exactly ONE node -# (unlike GB200's 4 GPUs/node, where the same DEP8 role needs 2 nodes). -# PREFILL_NODES=1 DECODE_NODES=1 GPUS_PER_NODE=8 -> 2 nodes / 16 GPUs total. -# Because neither role spans >1 node, server.sh's LWS_GROUP_SIZE is always 1 -# here, so the cross-node --data-parallel-hybrid-lb / NVSHMEM-ibgda code paths -# never activate - the GB200 recipes' cross-node NIC/NVSHMEM tuning (e.g. -# NVSHMEM_HCA_PE_MAPPING) simply does not apply and is intentionally omitted. -# -# Source (agentX): llm-manifesto/models/deepseek-v4-ix/ix-disagg-base.yaml + -# 1P-EP8-1D-EP8.yaml. agentX's Mooncake/MultiConnector KV chain and CPU -# offload are Kubernetes/manifesto-specific and are NOT ported - server.sh -# hardcodes plain NixlConnector for every llmd-vllm disagg run (no per-recipe -# override point), matching what the existing GB200 llm-d recipes already do. -# agentX's kv_cache_metrics/kv_cache_metrics_sample (observability-only, not -# present in any InferenceX llm-d recipe or confirmed supported by this -# image's vLLM build) are also dropped as an unnecessary compatibility risk. -# max_model_len is set to 9280 (ISL 8192 + OSL 1024 + headroom, matching the -# existing InferenceX GB200 recipes) instead of agentX's general-purpose -# 1048576, since this recipe only ever serves the fixed 8k/1k benchmark. -# -# ---- EPP scheduling config ---- -# Ported EXACTLY (plugin list, parameters, weights) from the agentx-mvp -# results snapshot manifest.yaml -# (results/dsv4-pro-dspark/results_ilmarkov-ix-2p-ep8-1d-ep8-dspark-1308-long-v2), -# which is a real EPP config that ran a 1P-EP8-1D-EP8 DeepSeek-V4-Pro K8s -# deployment. Only the file-discovery plugin + dataLayer wiring is added on -# top (InferenceX's own no-Kubernetes addition - agentX's K8s deployment used -# InferencePool pod discovery instead, which doesn't apply here). -# -# This config uses disagg-profile-handler's `deciders: {prefill: ...}` -# parameter shape (not InferenceX's existing GB200 recipes' `deciderPluginName` -# shape) and additional plugins (inflight-load-producer, -# approx-prefix-cache-producer, prefix-cache-affinity-filter, -# token-load-scorer) not present in the current InferenceX EPP image. This -# requires the EPP/pd-sidecar binaries to be bumped to v0.10.0 for this -# recipe - see the LLMD_BIN_DIR override in runners/launch_b200-dgxc-slurm.sh -# (B200-only; GB200's shared v0.9.0 binaries/path are untouched). +# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, P/D disagg 1P-DEP8/1D-DEP8. +# Always uses Mooncake (P2PHANDSHAKE embedded RDMA prefix-cache). +# DSpark speculative decoding: prefill 1 token, decode 5 tokens. +# Requires EPP/pd-sidecar v0.10.0 (disagg-profile-handler deciders shape). apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -107,12 +64,6 @@ dataLayer: discovery: pluginRef: file-disc -# ---- Per-role vLLM flags ---- -# Ported from agentX's ix-disagg-base.yaml (prefill/decode roles) + -# 1P-EP8-1D-EP8.yaml overrides. Both roles are DEP8 (TP=1, DP=8, EP on), -# FLASHINFER_MLA_SPARSE_DSV4 attention, deep_gemm_mega_moe MoE backend, -# EP weight filter (keeps only served experts resident - required for DEP8 -# to avoid OOM). See file header for what was intentionally dropped/adapted. prefill: tp: 1 enable-expert-parallel: true @@ -188,12 +139,6 @@ decode: MC_STORE_CLIENT_METRIC_INTERVAL: "5" MC_TE_METRIC: "0" -# ---- Mooncake KV store config ---- -# P2PHANDSHAKE embedded mode — no K8s mooncake-master pod needed; nodes -# negotiate RDMA directly. global_segment_size 140 GB matches the dynamo -# agentic recipe (srt-slurm-recipes/vllm/deepseek-v4/agentic/). Prefill -# uses kv_both (store new KV + load cache hits via RDMA); decode uses -# kv_consumer with lookup disabled (NIXL-only receive, no Mooncake lookup). mooncake: store_config: metadata_server: "P2PHANDSHAKE" @@ -204,6 +149,5 @@ mooncake: mode: "embedded" enable_offload: false -# ---- SLURM resource directives ---- slurm: time_limit: "08:00:00" From af3818dcb3ccd744735d2bbe5da23a890532b30b Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 31 Aug 2026 15:23:34 +0200 Subject: [PATCH 06/46] Address review comments --- configs/nvidia-master.yaml | 54 ++++++++-------------------- perf-changelog.yaml | 42 +++++++++------------- runners/launch_b200-nscale-compat.sh | 27 +++++++------- 3 files changed, 43 insertions(+), 80 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 37e4c9badc..85e6af68bd 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4502,22 +4502,14 @@ dsv4-fp4-gb200-llmd-vllm: - "DECODE_NODES=2" - "GPUS_PER_NODE=4" -# DeepSeek-V4-Pro FP4 B200 llmd-vllm, agentX-flavored: EPP scheduling config + -# per-role vLLM tuning ported from the agentX (llm-manifesto/agentx-mvp) K8s -# deployment (see benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx- -# 1p1d-dep8-dep8.yaml for the exact port/adaptation notes), rather than -# InferenceX-native tuning like dsv4-fp4-gb200-llmd-vllm above. Requires the -# B200-only EPP/pd-sidecar v0.10.0 binary override wired in -# runners/launch_b200-dgxc-slurm.sh. -# -# B200 = 8 GPUs/node, so each DEP8 role fits on ONE node (GB200's 4 GPUs/node -# needs 2 nodes per role for the same DEP8 shape) - see the recipe file for -# the full node-math writeup. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, P/D disagg via llmd-vllm. +# Always uses Mooncake; DSpark speculative decoding. Requires EPP v0.10.0 +# (LLMD_BIN_DIR override in runners/launch_b200-nscale-compat.sh). dsv4-fp4-b200-llmd-vllm-agentx: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 - runner: b200-multinode + runner: cluster:b200-nscale precision: fp4 framework: llmd-vllm router: { name: llm-d-router, version: "0.10.0" } @@ -4563,7 +4555,7 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 - runner: b200-multinode + runner: cluster:b200-nscale precision: fp4 framework: llmd-vllm router: { name: llm-d-router, version: "0.9.0" } @@ -4577,63 +4569,45 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: # Aggregated TP8 (1 node / 8 GPUs; pure tensor-parallel, no EP). - spec-decoding: "none" conc-list: [1, 2, 4, 8, 12] - prefill: + num-nodes: 1 + worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - "PREFILL_NODES=1" + - "DECODE_NODES=0" - "GPUS_PER_NODE=8" - "CONFIG_FILE=agentic/agg-gb200-tp8-dspark-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts eight GPUs only once - # (matches dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg's convention). - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=0" # Aggregated DEP8 (1 node / 8 GPUs; DP=8 + EP), no Mooncake. - spec-decoding: "none" conc-list: [12, 16, 32] - prefill: + num-nodes: 1 + worker: num-worker: 1 tp: 1 ep: 8 dp-attn: true additional-settings: - "PREFILL_NODES=1" + - "DECODE_NODES=0" - "GPUS_PER_NODE=8" - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-agentic.yaml" - decode: - num-worker: 0 - tp: 1 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=0" # Aggregated DEP8 with Mooncake prefix-cache KV store. - spec-decoding: "none" conc-list: [52, 72] - prefill: + num-nodes: 1 + worker: num-worker: 1 tp: 1 ep: 8 dp-attn: true additional-settings: - "PREFILL_NODES=1" + - "DECODE_NODES=0" - "GPUS_PER_NODE=8" - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml" - decode: - num-worker: 0 - tp: 1 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=0" # MTP2 variant of dsv4-fp4-gb200-dynamo-vllm. Uses the vLLM 0.20.1 image # and hand-picked 8k/1k Pareto points mirrored from NVIDIA/srt-slurm. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c371582e24..d8830b1555 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6388,31 +6388,6 @@ - "Bump the shared TileRT dependency and router metadata to 0.1.5.post3 for both fixed-seq-length and AgentX configurations while retaining the AgentX 1800-second queue timeout" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2650 -- config-keys: - - dsv4-fp4-b200-llmd-vllm-agentx - description: - - "Add DeepSeek-V4-Pro FP4 B200 llmd-vllm P/D disagg benchmark (1P DEP8 + 1D DEP8, 2 nodes / 16 GPUs), porting the agentX (llm-manifesto/agentx-mvp) EPP scheduling config and per-role vLLM tuning" - - "Wire a FRAMEWORK==llmd-vllm dispatch into runners/launch_b200-dgxc-slurm.sh (self-contained, bypasses the srtctl/srt-slurm path used by the other B200 multinode frameworks)" - - "Pin a B200-only EPP/pd-sidecar v0.10.0 binary override (LLMD_BIN_DIR) for the disagg-profile-handler deciders: EPP shape; GB200's shared v0.9.0 binaries/path are untouched" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER - -- config-keys: - - dsv4-fp4-b200-llmd-vllm-agentx-agg - description: - - "Add DeepSeek-V4-Pro FP4 B200 llmd-vllm aggregated benchmark (TP8 and DEP8, 1 node / 8 GPUs each, one engine does both prefill and decode), porting the agentX (llm-manifesto) ix-agg-base/agg-tp8 and ix-agg-dep-base/agg-dep8 per-role vLLM tuning" - - "Extend server.sh/job.slurm with an aggregated (DECODE_NODES=0) mode: no decode role, no pd-sidecar, no --kv_transfer_config, and the sole engine's rank 0 takes over the EPP/Envoy/bench coordinator duties normally owned by the decode leader" - - "Add a minimal single-schedulingProfile EPP config for aggregated recipes (no disagg-profile-handler needed); uses the existing default v0.9.0 EPP/pd-sidecar binaries, no router bump required" - - "Branch runners/launch_b200-dgxc-slurm.sh's llmd-vllm wrapper-script selection on DISAGG (true -> disagg wrapper, false -> new aggregated wrapper)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER - -- config-keys: - - dsv4-fp4-b200-llmd-vllm-agentx - description: - - "Add Mooncake prefix-cache KV store for the disagg DEP8 arm at concurrencies [52, 72]" - - "Extend server.sh to parse a `mooncake.store_config` key from llm-d recipe YAMLs: writes /tmp/mooncake_config.json, sets MOONCAKE_CONFIG_PATH, and wires MultiConnector (NixlConnector + MooncakeStoreConnector) instead of plain NixlConnector; prefill uses kv_both, decode kv_consumer with lookup disabled" - - "Add dsv4-fp4-b200-agentx-1p1d-dep8-dep8-mooncake.yaml recipe with Mooncake store_config (P2PHANDSHAKE embedded, 140 GB segment, mlx5_0..3 RDMA) and MC_* env vars, mirroring the dynamo-vllm agentic GB200 disagg recipe" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 - - config-keys: - kimik3-fp4-mi355x-atom-agentic-mtp scenario-type: @@ -6631,3 +6606,20 @@ - "Make speculative decoding concurrency-adaptive: concurrency 1 uses 6 draft tokens (synthetic acceptance 3.75), concurrency <=10 uses 2 tokens (2.51), and higher concurrencies disable MTP; MAX_NUM_SEQS is now 2*CONC with CUDA-graph capture sizes scaled to (1+SPEC_NUM_TOKENS)." - "Raise dram-utilization from 0.50 to 0.60, extend the none-offload conc-list from [1, 4, 8] to [1, 4, 8, 10, 12], and set the lmcache arm conc-list to [4, 8, 10, 12]." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2787 + +- config-keys: + - dsv4-fp4-b200-llmd-vllm-agentx + description: + - "Add DeepSeek-V4-Pro-0813 (DSpark) FP4 B200 llmd-vllm P/D disagg benchmark (1P DEP8 + 1D DEP8, 2 nodes / 16 GPUs total); always uses Mooncake RDMA prefix-cache; DSpark speculative decoding (prefill 1 token, decode 5 tokens)" + - "Wire a FRAMEWORK==llmd-vllm dispatch into runners/launch_b200-nscale-compat.sh (self-contained, bypasses the srtctl/srt-slurm path)" + - "Pin a disagg-only EPP/pd-sidecar v0.10.0 binary override (LLMD_BIN_DIR=/home/sa-shared/llm-d-bins-v0.10.0) required for the disagg-profile-handler deciders: EPP shape; aggregated runs use the image-baked v0.9.0 binaries" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-b200-llmd-vllm-agentx-agg + description: + - "Add DeepSeek-V4-Pro-0813 (DSpark) FP4 B200 llmd-vllm aggregated benchmark: TP8 at conc-list [1,2,4,8,12], DEP8 at [12,16,32], DEP8+Mooncake at [52,72] (all 1 node / 8 GPUs, one engine does both prefill and decode)" + - "Extend server.sh/job.slurm with aggregated (DECODE_NODES=0) mode: no decode role, no pd-sidecar; rank 0 takes over EPP/Envoy/bench coordinator duties" + - "Add Mooncake (P2PHANDSHAKE embedded RDMA) for the DEP8+Mooncake arm via agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml; server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role) when MOONCAKE_CONFIG_PATH is set in aggregated mode" + - "Branch runners/launch_b200-nscale-compat.sh llmd-vllm wrapper selection on DISAGG (true -> disagg, false -> aggregated)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 60bc6d465b..a837765e90 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -20,12 +20,15 @@ if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/scratch/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && "$MODEL" == *-0813 ]]; then + export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro-0813}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then SELECTED_MODEL_PATH="" if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then SELECTED_MODEL_PATH="$MODEL_PATH" else - for candidate in /scratch/models/DeepSeek-V4-Pro /scratch/models/DeepSeek-V4-Pro-NVFP4 /scratch/models/DeepSeek-V4-Pro-0813; do + for candidate in /scratch/models/DeepSeek-V4-Pro /scratch/models/DeepSeek-V4-Pro-NVFP4; do if [[ -d "$candidate" ]]; then SELECTED_MODEL_PATH="$candidate" break @@ -153,6 +156,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$registry" == "registry-1.docker.io" && "$repository" != */* ]]; then repository="library/$repository" fi + # Strip any :tag from repository before appending the digest. + repository="${repository%%:*}" printf 'docker://%s#%s:%s\n' "$registry" "$repository" "$digest" } @@ -186,20 +191,6 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export LLMD_CONTAINER_ENGINE=pyxis export LLMD_SQUASH_FILE - # B200-only EPP/pd-sidecar binary pin (v0.10.0, needed for the - # agentX-ported disagg-profile-handler `deciders:` EPP shape). Points - # at a dedicated path so it never collides with GB200's shared - # binaries.env / v0.9.0 path - see - # benchmarks/llm-d/binaries-b200-v0.10.0.env for the pins and - # benchmarks/llm-d/binaries.env for the general mechanism. NOTE: this - # path must be populated once (out of band, on a host with docker + - # registry access and the /home/sa-shared mount) via: - # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env \ - # benchmarks/llm-d/extract-binaries.sh - # before this recipe can actually run; job.slurm's pyxis mount loop is - # a no-op (falls back to the image's baked-in v0.9.0) until then. - export LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}" - export DOCKER_IMAGE_NAME=$IMAGE export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" mkdir -p "$BENCHMARK_LOGS_DIR" @@ -210,6 +201,11 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # engine does both prefill and decode, DECODE_NODES=0). if [[ "${DISAGG:-true}" == "true" ]]; then SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" + # EPP/pd-sidecar binaries needed for the disagg-profile-handler `deciders:` + # shape. Populate once via: + # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env benchmarks/llm-d/extract-binaries.sh + # job.slurm falls back to the image-baked v0.9.0 until this path exists. + export LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}" else SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" fi @@ -219,6 +215,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then exit 1 fi + export SLURM_PARTITION SLURM_ACCOUNT JOB_ID=$(bash "$BENCH_SCRIPT") if [[ -z "$JOB_ID" ]]; then echo "Error: failed to submit llm-d job" >&2 From e648883a391c8302b0999aba13f70983ce2be133 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 31 Aug 2026 15:47:02 +0200 Subject: [PATCH 07/46] Update DSV4-Pro model to 0813 version --- perf-changelog.yaml | 2 ++ runners/launch_b200-nscale-compat.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d8830b1555..31bcb9cc5c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6613,6 +6613,7 @@ - "Add DeepSeek-V4-Pro-0813 (DSpark) FP4 B200 llmd-vllm P/D disagg benchmark (1P DEP8 + 1D DEP8, 2 nodes / 16 GPUs total); always uses Mooncake RDMA prefix-cache; DSpark speculative decoding (prefill 1 token, decode 5 tokens)" - "Wire a FRAMEWORK==llmd-vllm dispatch into runners/launch_b200-nscale-compat.sh (self-contained, bypasses the srtctl/srt-slurm path)" - "Pin a disagg-only EPP/pd-sidecar v0.10.0 binary override (LLMD_BIN_DIR=/home/sa-shared/llm-d-bins-v0.10.0) required for the disagg-profile-handler deciders: EPP shape; aggregated runs use the image-baked v0.9.0 binaries" + - "Fix MODEL_PATH for -0813 checkpoint: unconditionally set /scratch/models/DeepSeek-V4-Pro-0813 in the launcher's -0813 branch so the outer script's pre-exported non-0813 path is always overridden" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 - config-keys: @@ -6622,4 +6623,5 @@ - "Extend server.sh/job.slurm with aggregated (DECODE_NODES=0) mode: no decode role, no pd-sidecar; rank 0 takes over EPP/Envoy/bench coordinator duties" - "Add Mooncake (P2PHANDSHAKE embedded RDMA) for the DEP8+Mooncake arm via agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml; server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role) when MOONCAKE_CONFIG_PATH is set in aggregated mode" - "Branch runners/launch_b200-nscale-compat.sh llmd-vllm wrapper selection on DISAGG (true -> disagg, false -> aggregated)" + - "Fix MODEL_PATH for -0813 checkpoint: unconditionally set /scratch/models/DeepSeek-V4-Pro-0813 in the launcher's -0813 branch so the outer script's pre-exported non-0813 path is always overridden" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index a837765e90..a30d648eaa 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -21,7 +21,7 @@ elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/scratch/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && "$MODEL" == *-0813 ]]; then - export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro-0813}" + export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-0813" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then SELECTED_MODEL_PATH="" From a4edd452fe80e32bc7471435a0c19a80523a13a9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 09:39:50 -0500 Subject: [PATCH 08/46] fix: run llm-d DSpark with AgentX golden acceptance and explicit metadata --- benchmarks/llm-d/binaries-b200-v0.10.0.env | 4 +- benchmarks/llm-d/extract-binaries.sh | 10 +- .../multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh | 25 +- .../dsv4_fp4_b200_llmd-vllm-disagg.sh | 31 +- .../agg-gb200-dep8-dspark-agentic.yaml | 10 +- ...gg-gb200-dep8-dspark-mooncake-agentic.yaml | 14 +- .../agentic/agg-gb200-tp8-dspark-agentic.yaml | 10 +- ...g-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 24 +- benchmarks/multi_node/llm-d/agentic.sh | 27 ++ benchmarks/multi_node/llm-d/job.slurm | 33 ++- benchmarks/multi_node/llm-d/recipe.py | 99 +++++++ benchmarks/multi_node/llm-d/server.sh | 268 ++++++++---------- benchmarks/multi_node/llm-d/submit.sh | 47 +-- benchmarks/multi_node/llm-d/test_recipe.py | 72 +++++ configs/nvidia-master.yaml | 35 +-- docs/configuration-procedures.md | 2 + docs/configuration-procedures_zh.md | 2 + perf-changelog.yaml | 12 + runners/launch_b200-nscale-compat.sh | 74 +++-- runners/test_slurm_utils.py | 27 ++ .../test_generate_sweep_configs.py | 30 ++ 21 files changed, 580 insertions(+), 276 deletions(-) create mode 100644 benchmarks/multi_node/llm-d/agentic.sh create mode 100644 benchmarks/multi_node/llm-d/recipe.py create mode 100644 benchmarks/multi_node/llm-d/test_recipe.py diff --git a/benchmarks/llm-d/binaries-b200-v0.10.0.env b/benchmarks/llm-d/binaries-b200-v0.10.0.env index c485112f61..b04199da96 100644 --- a/benchmarks/llm-d/binaries-b200-v0.10.0.env +++ b/benchmarks/llm-d/binaries-b200-v0.10.0.env @@ -27,7 +27,7 @@ ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2" ENVOY_BIN_PATH="/usr/local/bin/envoy" # B200 is x86_64 (unlike GB200's Grace/arm64, binaries.env's default). -LLMD_BIN_PLATFORM="${LLMD_BIN_PLATFORM:-linux/amd64}" +LLMD_BIN_PLATFORM="linux/amd64" # Dedicated path so this never collides with GB200's shared bin dir. -LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}" +LLMD_BIN_DIR="/home/sa-shared/llm-d-bins-v0.10.0" diff --git a/benchmarks/llm-d/extract-binaries.sh b/benchmarks/llm-d/extract-binaries.sh index bd17fc2734..a97a42a375 100755 --- a/benchmarks/llm-d/extract-binaries.sh +++ b/benchmarks/llm-d/extract-binaries.sh @@ -10,17 +10,17 @@ # benchmarks/llm-d/binaries.env. Idempotent: overwrites in place. # # Usage: -# ./extract-binaries.sh # uses binaries.env defaults -# LLMD_BIN_DIR=/some/dir ./extract-binaries.sh -# LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh # for an x86 test +# BINARIES_ENV_FILE=binaries.env ./extract-binaries.sh +# BINARIES_ENV_FILE=binaries.env LLMD_BIN_DIR=/some/dir ./extract-binaries.sh +# BINARIES_ENV_FILE=binaries.env LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env ./extract-binaries.sh # # pull a different set of image pins (e.g. a router-version bump # # scoped to one cluster) without touching the shared binaries.env -set -euo pipefail +set -eo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BINARIES_ENV_FILE="${BINARIES_ENV_FILE:-$HERE/binaries.env}" +: "${BINARIES_ENV_FILE:?Set binaries.env or a cluster-specific pins file}" # Resolve a bare filename against this script's directory, so # BINARIES_ENV_FILE=foo.env works regardless of the caller's cwd. [[ "$BINARIES_ENV_FILE" != /* ]] && BINARIES_ENV_FILE="$HERE/$BINARIES_ENV_FILE" diff --git a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh index 730824bc2d..c326b1058c 100755 --- a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh +++ b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh @@ -9,21 +9,28 @@ # SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" # from launch_b200-dgxc-slurm.sh when DISAGG=false. -set -euo pipefail +set -eo pipefail source "$(dirname "$0")/../benchmark_lib.sh" check_env_vars \ CONC_LIST \ - ISL \ - OSL \ IMAGE \ MODEL_PATH \ PREFILL_NODES \ DECODE_NODES \ RANDOM_RANGE_RATIO -if [[ -n "${SLURM_JOB_ID:-}" ]]; then +if [[ "${IS_AGENTIC}" == "1" ]]; then + check_env_vars DURATION KV_OFFLOADING + # Positional submit.sh placeholders; AgentX never uses fixed token lengths. + ISL=0 + OSL=0 +else + check_env_vars ISL OSL +fi + +if [[ -n "${SLURM_JOB_ID}" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi @@ -36,10 +43,10 @@ set -x cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 -# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity). -export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +# B200 DGX = 8 GPUs per node. +export GPUS_PER_NODE="8" -export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export TIME_LIMIT="08:00:00" export MODEL_PATH=$MODEL_PATH export MODEL_NAME=$MODEL_NAME export CONTAINER_IMAGE=$IMAGE @@ -47,8 +54,8 @@ export CONTAINER_IMAGE=$IMAGE # Aggregated is always a single engine (no multi-engine high-tpt split), so # PREFILL_WORKERS is always 1. DECODE_WORKERS is unused (DECODE_NODES=0) but # still exported since submit.sh/server.sh read it unconditionally. -export PREFILL_WORKERS="${PREFILL_WORKERS:-1}" -export DECODE_WORKERS="${DECODE_WORKERS:-1}" +export PREFILL_WORKERS="1" +export DECODE_WORKERS="1" JOB_ID=$(bash ./submit.sh \ "$PREFILL_NODES" \ diff --git a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh index 0589913d22..93cbf89238 100755 --- a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh +++ b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh @@ -8,21 +8,28 @@ # SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" # from launch_b200-dgxc-slurm.sh. -set -euo pipefail +set -eo pipefail source "$(dirname "$0")/../benchmark_lib.sh" check_env_vars \ CONC_LIST \ - ISL \ - OSL \ IMAGE \ MODEL_PATH \ PREFILL_NODES \ DECODE_NODES \ RANDOM_RANGE_RATIO -if [[ -n "${SLURM_JOB_ID:-}" ]]; then +if [[ "${IS_AGENTIC}" == "1" ]]; then + check_env_vars DURATION KV_OFFLOADING + # Positional submit.sh placeholders; AgentX never uses fixed token lengths. + ISL=0 + OSL=0 +else + check_env_vars ISL OSL +fi + +if [[ -n "${SLURM_JOB_ID}" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi @@ -30,21 +37,17 @@ set -x cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 -# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity). -export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +# B200 DGX = 8 GPUs per node. +export GPUS_PER_NODE="8" -export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export TIME_LIMIT="08:00:00" export MODEL_PATH=$MODEL_PATH export MODEL_NAME=$MODEL_NAME export CONTAINER_IMAGE=$IMAGE -# Worker count per role (Option B multi-engine). Prefer an explicit -# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else -# fall back to the matrix num-worker fields (PREFILL_NUM_WORKERS/ -# DECODE_NUM_WORKERS); else 1 (single engine = unchanged 1P+1D). submit.sh -# reads these. -export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}" -export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}" +# Worker counts come from the generated matrix. +export PREFILL_WORKERS="$PREFILL_NUM_WORKERS" +export DECODE_WORKERS="$DECODE_NUM_WORKERS" JOB_ID=$(bash ./submit.sh \ "$PREFILL_NODES" \ diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index 7b90a8d645..f372e73c4c 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -36,15 +36,17 @@ dataLayer: prefill: tp: 1 enable-expert-parallel: true + # No workload-length or sequence cap: use native context and vLLM scheduling defaults. + # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --gpu-memory-utilization 0.92 - --max-model-len 9280 - --max-num-seqs 64 --max-num-batched-tokens 8192 - --max-cudagraph-capture-size 64 --block-size 256 --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 --moe-backend deep_gemm_mega_moe --enable-ep-weight-filter --enable-cumem-allocator @@ -52,7 +54,7 @@ prefill: --no-enable-flashinfer-autotune --numa-bind --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: VLLM_USE_RUST_FRONTEND: "1" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 05e8322a23..28f9b58ee9 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -38,15 +38,17 @@ dataLayer: prefill: tp: 1 enable-expert-parallel: true + # No workload-length or sequence cap: use native context and vLLM scheduling defaults. + # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --gpu-memory-utilization 0.92 - --max-model-len 9280 - --max-num-seqs 64 --max-num-batched-tokens 8192 - --max-cudagraph-capture-size 64 --block-size 256 --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 --moe-backend deep_gemm_mega_moe --enable-ep-weight-filter --enable-cumem-allocator @@ -54,9 +56,10 @@ prefill: --no-enable-flashinfer-autotune --numa-bind --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: + PYTHONHASHSEED: "0" VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" @@ -77,12 +80,13 @@ prefill: mooncake: store_config: metadata_server: "P2PHANDSHAKE" + # Standalone fallback; AgentX uses the master YAML per-node DRAM budget / 8 GPUs. global_segment_size: "140GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" - enable_offload: false + enable_offload: false # SSD only; the embedded DRAM pool is still enabled. # ---- SLURM resource directives ---- slurm: diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml index 6245c9485d..282af9b7b1 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -37,21 +37,23 @@ prefill: tp: 8 # Must be explicit false: server.sh defaults enable-expert-parallel to true. enable-expert-parallel: false + # No workload-length or sequence cap: use native context and vLLM scheduling defaults. + # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --gpu-memory-utilization 0.92 - --max-model-len 9280 - --max-num-seqs 32 --max-num-batched-tokens 8192 - --max-cudagraph-capture-size 32 --block-size 256 --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 --disable-custom-all-reduce --enable-cumem-allocator --no-disable-hybrid-kv-cache-manager --no-enable-flashinfer-autotune --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: VLLM_USE_RUST_FRONTEND: "1" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 4bd13b0572..653e151abb 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -67,18 +67,21 @@ dataLayer: prefill: tp: 1 enable-expert-parallel: true + # No workload-length or sequence cap: use native context and vLLM scheduling defaults. + # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --enforce-eager --gpu-memory-utilization 0.97 - --max-model-len 9280 - --max-num-seqs 16 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 - --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --enable-cumem-allocator --block-size 256 --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 --moe-backend deep_gemm_mega_moe --enable-ep-weight-filter --no-disable-hybrid-kv-cache-manager @@ -105,18 +108,20 @@ prefill: decode: tp: 1 enable-expert-parallel: true + # No workload-length or sequence cap: use native context and vLLM scheduling defaults. + # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 - --max-num-seqs 32 - --max-num-batched-tokens 256 - --max-cudagraph-capture-size 256 - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --max-num-batched-tokens 8192 + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --gpu-memory-utilization 0.95 - --max-model-len 9280 --enable-cumem-allocator --block-size 256 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 --moe-backend deep_gemm_mega_moe --enable-ep-weight-filter --no-disable-hybrid-kv-cache-manager @@ -142,12 +147,13 @@ decode: mooncake: store_config: metadata_server: "P2PHANDSHAKE" + # Standalone fallback; AgentX uses the master YAML per-node DRAM budget / 8 GPUs. global_segment_size: "140GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" - enable_offload: false + enable_offload: false # SSD only; the embedded DRAM pool is still enabled. slurm: time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d/agentic.sh b/benchmarks/multi_node/llm-d/agentic.sh new file mode 100644 index 0000000000..550ed78a29 --- /dev/null +++ b/benchmarks/multi_node/llm-d/agentic.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Client-only AgentX adapter for an already-ready llm-d Envoy frontend. +set -eo pipefail + +: "${INFMAX_CONTAINER_WORKSPACE:?Set the repository mount path}" +export MODEL="$MODEL_NAME" +export SERVED_MODEL_NAME="$MODEL_NAME" +export PORT="$VLLM_PORT" +export AIPERF_SERVER_URL="http://localhost:$ENVOY_PORT" +export RESULT_DIR="$BENCHMARK_LOGS_DIR/agentic" +export AGENTIC_OUTPUT_DIR="$BENCHMARK_LOGS_DIR" +export CONC_LIST="${BENCH_MAX_CONCURRENCY//x/ }" +export CONC="${CONC_LIST%% *}" + +# Envoy does not expose vLLM /metrics. Scrape each engine leader directly; +# headless members of multi-node engines do not run HTTP servers. +IFS=',' read -r -a ips <<< "$ALL_IPS" +metrics_urls=() +for ((i=0; i/dev/null || true + [[ -n "${WATCHER_PID}" ]] && kill "$WATCHER_PID" 2>/dev/null || true } trap cleanup INT TERM HUP EXIT @@ -97,9 +97,29 @@ WATCHER_PID=$! # for clusters that require enroot+pyxis srun (e.g. gb200-nv, where the # SLURM user is not in the docker group). Same env contract on both # paths; only the per-node container launch differs. -LLMD_CONTAINER_ENGINE="${LLMD_CONTAINER_ENGINE:-docker}" +: "${LLMD_CONTAINER_ENGINE:?Set docker or pyxis in the invoking launcher}" echo "LLMD_CONTAINER_ENGINE=$LLMD_CONTAINER_ENGINE" +# Preserve the workflow's AgentX protocol, provenance, offload and physical GPU +# metadata on both engines. Forward names, not interpolated values: JSON and +# HF_TOKEN must never be embedded in the nested shell command or printed. +AGENTIC_ENV_NAMES=( + IS_AGENTIC SCENARIO_TYPE CONC CONC_LIST DURATION AIPERF_EXPERIMENTAL_FAST + IMAGE RECIPE_FINGERPRINT DISAGG HF_TOKEN + KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA + ROUTER_METADATA KV_P2P_TRANSFER TOTAL_CPU_DRAM_GB + PREFILL_NUM_WORKERS PREFILL_TP PREFILL_PP_SIZE PREFILL_PCP_SIZE + PREFILL_DCP_SIZE PREFILL_EP PREFILL_DP_ATTN + DECODE_NUM_WORKERS DECODE_TP DECODE_PP_SIZE DECODE_PCP_SIZE + DECODE_DCP_SIZE DECODE_EP DECODE_DP_ATTN +) +AGENTIC_DOCKER_ENV="" +for env_name in "${AGENTIC_ENV_NAMES[@]}"; do + export "$env_name" + AGENTIC_DOCKER_ENV+=" -e $env_name" +done +AGENTIC_PYXIS_ENV=$(IFS=,; echo "${AGENTIC_ENV_NAMES[*]}") + if [[ "$LLMD_CONTAINER_ENGINE" == "docker" ]]; then # One docker run per node, one task per node. server.sh dispatches by NODE_RANK. srun \ @@ -107,7 +127,7 @@ if [[ "$LLMD_CONTAINER_ENGINE" == "docker" ]]; then --signal=TERM@30 \ --unbuffered \ bash -lc " -set -euo pipefail +set -eo pipefail echo \"Rank \$SLURM_PROCID on \$(hostname)\" docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f || true @@ -167,6 +187,7 @@ exec docker run --rm \ -e SPEC_DECODING=$SPEC_DECODING \ -e IS_MULTINODE=$IS_MULTINODE \ -e CONFIG_FILE=$CONFIG_FILE \ + $AGENTIC_DOCKER_ENV \ --name \"${DOCKER_CONT_NAME}_\$SLURM_PROCID\" \ \"\$DOCKER_IMAGE_NAME\" -lc ' set -o pipefail @@ -207,6 +228,8 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then PYXIS_ENV_LIST+=",RUN_EVAL,EVAL_ONLY,EVAL_CONC,FRAMEWORK,PRECISION,MODEL_PREFIX" PYXIS_ENV_LIST+=",RUNNER_TYPE,RESULT_FILENAME,SPEC_DECODING,IS_MULTINODE,CONFIG_FILE" + PYXIS_ENV_LIST+=",$AGENTIC_PYXIS_ENV" + PYXIS_MOUNTS="${MODEL_DIR}:/models:ro" PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs" PYXIS_MOUNTS+=",${DI_REPO_DIR}:${DOCKER_MOUNT_PATH}" @@ -226,7 +249,7 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then [[ -f "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" ]] && \ source "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" for _bin in epp pd-sidecar envoy; do - if [[ -n "${LLMD_BIN_DIR:-}" && -x "${LLMD_BIN_DIR}/${_bin}" ]]; then + if [[ -n "${LLMD_BIN_DIR}" && -x "${LLMD_BIN_DIR}/${_bin}" ]]; then PYXIS_MOUNTS+=",${LLMD_BIN_DIR}/${_bin}:/usr/local/bin/${_bin}:ro" echo "Mounting ${LLMD_BIN_DIR}/${_bin} -> /usr/local/bin/${_bin}" fi diff --git a/benchmarks/multi_node/llm-d/recipe.py b/benchmarks/multi_node/llm-d/recipe.py new file mode 100644 index 0000000000..59f1fda5f9 --- /dev/null +++ b/benchmarks/multi_node/llm-d/recipe.py @@ -0,0 +1,99 @@ +"""Render llm-d role arguments and enforce AgentX benchmark metadata.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shlex +import sys + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def validate_agentic_offload(recipe: dict, env: dict) -> None: + """An embedded Mooncake store is DRAM offload even with SSD disabled.""" + if env.get("IS_AGENTIC") != "1": + return + store = recipe.get("mooncake", {}).get("store_config") + expected = "dram" if store else "none" + if env.get("KV_OFFLOADING") != expected: + raise ValueError(f"Recipe requires KV_OFFLOADING={expected}; fix the master YAML") + if store and env.get("KV_OFFLOAD_BACKEND") != "mooncake": + raise ValueError("Mooncake recipe requires KV_OFFLOAD_BACKEND=mooncake") + + +def role_assignments(recipe: dict, role: str, env: dict) -> str: + validate_agentic_offload(recipe, env) + section = recipe.get(role) or {} + extra = (section.get("extra-args") or "").strip() + if env.get("IS_AGENTIC") == "1": + match = re.search(r"--speculative-config\s+", extra) + if match: + config, length = json.JSONDecoder().raw_decode(extra[match.end():]) + if config.get("method") == "dspark": + if env.get("SPEC_DECODING") != "mtp": + raise ValueError("DSpark requires SPEC_DECODING=mtp in the master YAML") + if env.get("EVAL_ONLY", "false") == "true": + config.pop("synthetic_acceptance_length", None) + config.pop("rejection_sample_method", None) + else: + if env.get("RUN_EVAL", "false") == "true": + raise ValueError("Run accuracy evals separately with EVAL_ONLY=true, not synthetic AL") + if env.get("MODEL_NAME") != "deepseek-ai/DeepSeek-V4-Pro-0813": + raise ValueError("No registered DSpark golden AL for this model") + golden_path = REPO_ROOT / "golden_al_distribution/dsv4-pro-0813-dspark.yaml" + golden = yaml.safe_load(golden_path.read_text()) + k = config["num_speculative_tokens"] + al = golden["deepseek-v4-pro-0813"]["thinking_on"][k] + config.update( + rejection_sample_method="synthetic", + synthetic_acceptance_length=al, + enable_adaptive_verification=False, + ) + print(f"DSpark {role}: K={k}, golden AL={al} ({golden_path.name})", file=sys.stderr) + extra = extra[:match.end()] + json.dumps(config, separators=(",", ":")) + extra[match.end() + length:] + assignments = [f"ROLE_EXTRA_ARGS={shlex.quote(extra)}"] + if section.get("tp") is not None: + assignments.append(f"TP_SIZE={int(section['tp'])}") + if section.get("enable-expert-parallel") is not None: + assignments.append(f"ROLE_ENABLE_EP={str(section['enable-expert-parallel']).lower()}") + for key, value in (section.get("env") or {}).items(): + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise ValueError(f"Invalid recipe environment variable: {key}") + assignments.append(f"export {key}={shlex.quote(str(value))}") + return "\n".join(assignments) + + +def mooncake_config(recipe: dict, env: dict) -> str: + validate_agentic_offload(recipe, env) + config = dict(recipe.get("mooncake", {}).get("store_config") or {}) + if not config: + return "" + if env.get("IS_AGENTIC") == "1": + budget_gb = int(env["TOTAL_CPU_DRAM_GB"]) + gpus_per_node = int(env["GPUS_PER_NODE"]) + if budget_gb <= 0 or gpus_per_node <= 0: + raise ValueError("Mooncake requires a positive per-node DRAM budget and GPU count") + # The master budget is per node. Each embedded per-GPU store owns a + # share; transfer buffers are separate from the reusable KV pool. + config["global_segment_size"] = budget_gb * 10**9 // gpus_per_node + return json.dumps(config) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("recipe", type=Path) + output = parser.add_mutually_exclusive_group(required=True) + output.add_argument("--role", choices=("prefill", "decode")) + output.add_argument("--mooncake", action="store_true") + args = parser.parse_args() + recipe = yaml.safe_load(args.recipe.read_text()) + print(mooncake_config(recipe, os.environ) if args.mooncake else role_assignments(recipe, args.role, os.environ)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index c0576e2f26..2976a172f9 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -14,17 +14,14 @@ # via --data-parallel-hybrid-lb; the leader accepts traffic, workers serve their # local DP ranks. -set -euo pipefail +set -eo pipefail source /workspace/benchmarks/benchmark_lib.sh # ---------------------------------------------------------------- # Config + service ports # ---------------------------------------------------------------- -NODE_RANK="${NODE_RANK:-${SLURM_PROCID:-0}}" -PREFILL_NODES="${PREFILL_NODES:-1}" -DECODE_NODES="${DECODE_NODES:-1}" -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +check_env_vars NODE_RANK PREFILL_NODES DECODE_NODES GPUS_PER_NODE PREFILL_WORKERS DECODE_WORKERS ALL_IPS # Aggregated mode: no decode role at all, so no P/D KV handoff (no # pd-sidecar, no NixlConnector) - the single engine does prefill+decode # in-process. DECODE_NODES=0 is the only signal for this; everything else @@ -72,7 +69,7 @@ print(ip, iface) ' 2>/dev/null) || true HOST_IP=$(echo "$_HOST_INFO" | awk '{print $1}') DEFAULT_IFACE=$(echo "$_HOST_INFO" | awk '{print $2}') -DEFAULT_IFACE="${DEFAULT_IFACE:-eth0}" +check_env_vars HOST_IP DEFAULT_IFACE VLLM_LOG="/benchmark_logs/vllm_rank${NODE_RANK}.log" SIDECAR_LOG="/benchmark_logs/sidecar_rank${NODE_RANK}.log" @@ -88,9 +85,7 @@ echo "=== rank=$NODE_RANK host=$HOST_IP model=$MODEL ===" # engines, each spanning (role_nodes / role_workers) nodes with its own DP # coordinator (leader IP) and rank range. workers=1 => one engine over all role # nodes (1P+1D / mid-curve); >1 => high-tpt (e.g. 2 prefill : 1 decode, DEP8 each). -PREFILL_WORKERS="${PREFILL_WORKERS:-1}" -DECODE_WORKERS="${DECODE_WORKERS:-1}" -IFS=',' read -r -a _ALL_IPS <<< "${ALL_IPS:-}" +IFS=',' read -r -a _ALL_IPS <<< "${ALL_IPS}" if [[ "$NODE_RANK" -lt "$PREFILL_NODES" ]]; then ROLE="prefill" @@ -113,15 +108,9 @@ else exit 1 fi -# Each engine's DP coordinator = its leader node's IP (ALL_IPS[leader rank]); -# fall back to the role leader env when ALL_IPS is unset. -if [[ -n "${_ALL_IPS[${_group_leader_rank}]:-}" ]]; then - DP_ADDR="${_ALL_IPS[${_group_leader_rank}]}" -elif [[ "$ROLE" == "prefill" ]]; then - DP_ADDR="$PREFILL_DP_ADDR" -else - DP_ADDR="$DECODE_DP_ADDR" -fi +# job.slurm supplies the complete rank-ordered address list. +DP_ADDR="${_ALL_IPS[$_group_leader_rank]}" +check_env_vars DP_ADDR DP_SIZE_LOCAL="$GPUS_PER_NODE" START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) @@ -132,6 +121,45 @@ ROLE_ENABLE_EP=true echo "ROLE=$ROLE DP_SIZE=$DP_SIZE DP_ADDR=$DP_ADDR LWS_WORKER_INDEX=$LWS_WORKER_INDEX START_RANK=$START_RANK" +# Explicit transport baseline; role recipe env is applied afterwards. +export GLOO_SOCKET_IFNAME=$DEFAULT_IFACE +export NCCL_SOCKET_IFNAME=$DEFAULT_IFACE +export VLLM_SKIP_P2P_CHECK=1 +# Randomized DP dummy inputs make idle DP ranks fan their lockstep dummy passes +# across all experts (full MoE all-to-all), wasting prefill bandwidth; a recipe +# may set this to 0. +export VLLM_RANDOMIZE_DP_DUMMY_INPUTS=1 +export VLLM_USE_DEEP_GEMM=1 +# Cold-start budget for engine-core readiness. DSV4-Pro on GB200 cold-starts in +# ~9-11 min (weight load + DeepGEMM JIT warmup + cudagraph capture + NIXL/UCX +# handshake); the 600s vLLM default is too tight, so allow 30 min. +export VLLM_ENGINE_READY_TIMEOUT_S=1800 +# DeepGEMM JIT links -l:libcuda.so.1 at warmup; the compat dir is on +# LD_LIBRARY_PATH (runtime) but not LIBRARY_PATH (link time). Prepend it, plus +# the arch-specific toolkit lib dir resolved from `uname -m`. +case "$(uname -m)" in + aarch64|arm64) _NCT_LIB=/usr/lib/aarch64-linux-gnu ;; + *) _NCT_LIB=/usr/lib/x86_64-linux-gnu ;; +esac +export LIBRARY_PATH=/usr/local/cuda/compat:${_NCT_LIB}:${LIBRARY_PATH} +export VLLM_NIXL_SIDE_CHANNEL_HOST="$HOST_IP" +export VLLM_LOGGING_LEVEL=INFO + +# Pin NIXL/UCX to IB verbs (rc) so cross-node KV rides the IB HCAs (job.slurm +# exposes /dev/infiniband + IPC_LOCK); cuda_copy/cuda_ipc cover intra-node. +export UCX_TLS=cuda_copy,cuda_ipc,rc + + +if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then + export NVIDIA_GDRCOPY=enabled + # ibgda default kept for future DeepEP/wide-EP recipes; a recipe may override + # NVSHMEM_REMOTE_TRANSPORT to none. + export NVSHMEM_REMOTE_TRANSPORT=ibgda + export NVSHMEM_IB_ENABLE_IBGDA=true + export NVSHMEM_SYMMETRIC_SIZE=16G + export NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME=$DEFAULT_IFACE +fi + # ---------------------------------------------------------------- # Recipe: per-role serve args + env (/etc/llmd-recipes/$CONFIG_FILE) # ---------------------------------------------------------------- @@ -140,27 +168,20 @@ echo "ROLE=$ROLE DP_SIZE=$DP_SIZE DP_ADDR=$DP_ADDR LWS_WORKER_INDEX=$LWS_WORKER_ # verbatim), env (map, exported before vllm serve). Absent keys keep the # defaults above, so a recipe with neither tp nor EP is a plain TP=1 DP+EP run. ROLE_EXTRA_ARGS="" -if [[ -n "${CONFIG_FILE:-}" ]]; then +if [[ -n "${CONFIG_FILE}" ]]; then RECIPE_PATH="/etc/llmd-recipes/${CONFIG_FILE}" if [[ -f "$RECIPE_PATH" ]]; then echo "Loading $ROLE recipe from $RECIPE_PATH" - eval "$(python3 - <&2 + exit 1 + fi echo "WARNING: CONFIG_FILE=$CONFIG_FILE but $RECIPE_PATH not found; using defaults" >&2 fi fi @@ -176,17 +197,9 @@ echo "Resolved $ROLE TP_SIZE=$TP_SIZE ROLE_ENABLE_EP=$ROLE_ENABLE_EP" # Mooncake uses P2PHANDSHAKE embedded mode (no external metadata server) # so no K8s sidecar is required - nodes negotiate directly via RDMA. MOONCAKE_CONFIG_PATH="" -if [[ -n "${CONFIG_FILE:-}" && -f "/etc/llmd-recipes/${CONFIG_FILE}" ]]; then - _MC_JSON=$(python3 - < /tmp/mooncake_config.json MOONCAKE_CONFIG_PATH=/tmp/mooncake_config.json @@ -198,54 +211,13 @@ PY fi fi -# ---------------------------------------------------------------- -# Transport env (NCCL / UCX / NIXL), recipe-overridable -# ---------------------------------------------------------------- -export GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-$DEFAULT_IFACE} -export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-$DEFAULT_IFACE} -export VLLM_SKIP_P2P_CHECK=1 -# Randomized DP dummy inputs make idle DP ranks fan their lockstep dummy passes -# across all experts (full MoE all-to-all), wasting prefill bandwidth; a recipe -# may set this to 0. -export VLLM_RANDOMIZE_DP_DUMMY_INPUTS=${VLLM_RANDOMIZE_DP_DUMMY_INPUTS:-1} -export VLLM_USE_DEEP_GEMM=1 -# Cold-start budget for engine-core readiness. DSV4-Pro on GB200 cold-starts in -# ~9-11 min (weight load + DeepGEMM JIT warmup + cudagraph capture + NIXL/UCX -# handshake); the 600s vLLM default is too tight, so allow 30 min. -export VLLM_ENGINE_READY_TIMEOUT_S=${VLLM_ENGINE_READY_TIMEOUT_S:-1800} -# DeepGEMM JIT links -l:libcuda.so.1 at warmup; the compat dir is on -# LD_LIBRARY_PATH (runtime) but not LIBRARY_PATH (link time). Prepend it, plus -# the arch-specific toolkit lib dir resolved from `uname -m`. -case "$(uname -m)" in - aarch64|arm64) _NCT_LIB=/usr/lib/aarch64-linux-gnu ;; - *) _NCT_LIB=/usr/lib/x86_64-linux-gnu ;; -esac -export LIBRARY_PATH=/usr/local/cuda/compat:${_NCT_LIB}:${LIBRARY_PATH:-} -export VLLM_NIXL_SIDE_CHANNEL_HOST="$HOST_IP" -export VLLM_LOGGING_LEVEL=${VLLM_LOGGING_LEVEL:-INFO} - -# Pin NIXL/UCX to IB verbs (rc) so cross-node KV rides the IB HCAs (job.slurm -# exposes /dev/infiniband + IPC_LOCK); cuda_copy/cuda_ipc cover intra-node. -export UCX_TLS=${UCX_TLS:-cuda_copy,cuda_ipc,rc} - # ---------------------------------------------------------------- # Wide-EP NVSHMEM / ibgda env (only when an engine spans >1 node) # ---------------------------------------------------------------- # Single-node-per-role recipes avoid DeepEP / NVSHMEM ibgda, so leave these off # there to avoid triggering ibgda code paths that are not needed. -if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - export NVIDIA_GDRCOPY=enabled - # ibgda default kept for future DeepEP/wide-EP recipes; a recipe may override - # NVSHMEM_REMOTE_TRANSPORT to none. - export NVSHMEM_REMOTE_TRANSPORT=${NVSHMEM_REMOTE_TRANSPORT:-ibgda} - export NVSHMEM_IB_ENABLE_IBGDA=${NVSHMEM_IB_ENABLE_IBGDA:-true} - export NVSHMEM_SYMMETRIC_SIZE=${NVSHMEM_SYMMETRIC_SIZE:-16G} - export NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME=${NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME:-$DEFAULT_IFACE} - # NVSHMEM ignores NVSHMEM_HCA_PE_MAPPING when NVSHMEM_HCA_LIST is set, so - # clear the latter when the recipe provides an explicit PE mapping. - if [[ -n "${NVSHMEM_HCA_PE_MAPPING:-}" ]]; then - unset NVSHMEM_HCA_LIST 2>/dev/null || true - fi +if [[ "$LWS_GROUP_SIZE" -gt 1 && -n "$NVSHMEM_HCA_PE_MAPPING" ]]; then + unset NVSHMEM_HCA_LIST fi # ---------------------------------------------------------------- @@ -265,14 +237,14 @@ COMMON_ARGS=( # (NixlConnector + MooncakeStoreConnector) is wired with kv_both so DP ranks # can share prefix-cache blocks across runs via the Mooncake RDMA store. if [[ "$IS_AGGREGATED" -eq 0 ]]; then - if [[ -n "${KV_ROLE_OVERRIDE:-}" ]]; then + if [[ -n "${KV_ROLE_OVERRIDE}" ]]; then KV_ROLE="$KV_ROLE_OVERRIDE" elif [[ "$ROLE" == "prefill" ]]; then KV_ROLE="kv_producer" else KV_ROLE="kv_consumer" fi - if [[ -n "${MOONCAKE_CONFIG_PATH:-}" ]]; then + if [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then # MultiConnector: NixlConnector handles direct P/D KV transfer; # MooncakeStoreConnector enables cross-node prefix-cache lookup via RDMA. # Prefill uses kv_both so it can both store new KV and load cache hits. @@ -289,7 +261,7 @@ if [[ "$IS_AGGREGATED" -eq 0 ]]; then KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" fi COMMON_ARGS+=(--kv_transfer_config "$KV_TRANSFER_CONFIG") -elif [[ -n "${MOONCAKE_CONFIG_PATH:-}" ]]; then +elif [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then # Aggregated + Mooncake: single role acts as kv_both (stores new KV and # loads cache hits from the Mooncake RDMA store for prefix-cache sharing). KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"load_async\":true,\"lookup_async\":true,\"enable_cross_layers_blocks\":false,\"enable_offload\":false}}]}}" @@ -302,7 +274,7 @@ fi # load-balances its local DP ranks -> ONE serving port (VLLM_PORT) per node, so # the local rank-0 health port is always VLLM_PORT. HEALTH_PORT="$VLLM_PORT" -API_SERVER_COUNT="${LLMD_API_SERVER_COUNT:-4}" +API_SERVER_COUNT="4" # Multiple frontends only help the DP (wide-EP) path, where they load-balance # across the node's local DP ranks. A pure-TP engine has a single core with one # frontend, so it keeps the default count (also avoids --api-server-count @@ -548,7 +520,7 @@ PY { echo "=== NET DIAG: decode -> prefill ${ip}:${port} ===" echo "[diag] decode node: $(hostname -f 2>/dev/null || hostname) local-ips: $(hostname -I 2>/dev/null)" - echo "[diag] ifaces: DEFAULT_IFACE=${DEFAULT_IFACE:-} NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-} GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-}" + echo "[diag] ifaces: DEFAULT_IFACE=${DEFAULT_IFACE} NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME} GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME}" # Local source address the kernel would pick to reach ip: reveals which # subnet/interface the route uses, without needing iproute2. python3 - "$ip" <<'PY' 2>&1 || true @@ -632,59 +604,65 @@ PY done echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready" - # ---- Benchmark sweep (one run per concurrency level) ---- - # BENCH_MAX_CONCURRENCY is an 'x'-delimited list from submit.sh (e.g. "1024x512"). - IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" - # GPU counts embedded in the result filename as _gpus_/_ctx_/_gen_ tokens so the - # CI "Process result" step (benchmark-multinode-tmpl.yml) can parse them and run - # process_result.py for llm-d -- same filename convention as amd_utils/bench.sh. - # ctx = prefill GPUs, gen = decode GPUs; nodes*GPUS_PER_NODE is correct for any - # PREFILL_WORKERS/DECODE_WORKERS split (e.g. high-tpt 2P -> 16 prefill GPUs). - _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) - _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) - _bench_total_gpus=$(( _bench_prefill_gpus + _bench_decode_gpus )) - for max_concurrency in "${CONCURRENCIES[@]}"; do - num_prompts=$(( max_concurrency * BENCH_NUM_PROMPTS_MULTIPLIER )) - [[ "$num_prompts" -lt 16 ]] && num_prompts=16 - # Bench against Envoy (EPP routes to decode; the sidecar pulls from - # prefill via NIXL). --bench-serving-dir = the /workspace repo bind-mount; - # --tokenizer = /models (served-model-name is not a valid HF repo id). - # DSV4-Pro needs trust-remote-code + tokenizer-mode deepseek_v4 (the older - # transformers wheel does not register it) + chat template / --dsv4 to - # match the dynamo-vllm bench prompt formatting. - bench_extra_args=() - if [[ "${MODEL_NAME,,}" == *"deepseek-v4"* ]]; then - bench_extra_args+=( - --trust-remote-code - --tokenizer-mode deepseek_v4 - --use-chat-template - --dsv4 - ) - fi + if [[ "${IS_AGENTIC}" == "1" && "${EVAL_ONLY}" != "true" ]]; then + export ENVOY_PORT VLLM_PORT INFMAX_CONTAINER_WORKSPACE + bash /workspace/benchmarks/multi_node/llm-d/agentic.sh + elif [[ "${EVAL_ONLY}" != "true" ]]; then + # ---- Benchmark sweep (one run per concurrency level) ---- + # BENCH_MAX_CONCURRENCY is an 'x'-delimited list from submit.sh (e.g. "1024x512"). + IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" + # GPU counts embedded in the result filename as _gpus_/_ctx_/_gen_ tokens so the + # CI "Process result" step (benchmark-multinode-tmpl.yml) can parse them and run + # process_result.py for llm-d -- same filename convention as amd_utils/bench.sh. + # ctx = prefill GPUs, gen = decode GPUs; nodes*GPUS_PER_NODE is correct for any + # PREFILL_WORKERS/DECODE_WORKERS split (e.g. high-tpt 2P -> 16 prefill GPUs). + _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) + _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) + _bench_total_gpus=$(( _bench_prefill_gpus + _bench_decode_gpus )) + for max_concurrency in "${CONCURRENCIES[@]}"; do + num_prompts=$(( max_concurrency * BENCH_NUM_PROMPTS_MULTIPLIER )) + [[ "$num_prompts" -lt 16 ]] && num_prompts=16 + # Bench against Envoy (EPP routes to decode; the sidecar pulls from + # prefill via NIXL). --bench-serving-dir = the /workspace repo bind-mount; + # --tokenizer = /models (served-model-name is not a valid HF repo id). + # DSV4-Pro needs trust-remote-code + tokenizer-mode deepseek_v4 (the older + # transformers wheel does not register it) + chat template / --dsv4 to + # match the dynamo-vllm bench prompt formatting. + bench_extra_args=() + if [[ "${MODEL_NAME,,}" == *"deepseek-v4"* ]]; then + bench_extra_args+=( + --trust-remote-code + --tokenizer-mode deepseek_v4 + --use-chat-template + --dsv4 + ) + fi - # Non-fatal: a failed or timed-out conc point must not abort the sweep - # or (under set -e) skip the allocation release below. The EXIT trap - # releases the allocation regardless, but continuing here lets a - # multi-conc sweep record every point it can. - run_benchmark_serving \ - --bench-serving-dir /workspace \ - --tokenizer /models \ - --model "$MODEL_NAME" \ - --port "$ENVOY_PORT" \ - --backend openai \ - --input-len "$BENCH_INPUT_LEN" \ - --output-len "$BENCH_OUTPUT_LEN" \ - --random-range-ratio "$BENCH_RANDOM_RANGE_RATIO" \ - --num-prompts "$num_prompts" \ - --max-concurrency "$max_concurrency" \ - --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ - --result-dir "$BENCHMARK_LOGS_DIR/" \ - "${bench_extra_args[@]}" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" - done + # Non-fatal: a failed or timed-out conc point must not abort the sweep + # or (under set -e) skip the allocation release below. The EXIT trap + # releases the allocation regardless, but continuing here lets a + # multi-conc sweep record every point it can. + run_benchmark_serving \ + --bench-serving-dir /workspace \ + --tokenizer /models \ + --model "$MODEL_NAME" \ + --port "$ENVOY_PORT" \ + --backend openai \ + --input-len "$BENCH_INPUT_LEN" \ + --output-len "$BENCH_OUTPUT_LEN" \ + --random-range-ratio "$BENCH_RANDOM_RANGE_RATIO" \ + --num-prompts "$num_prompts" \ + --max-concurrency "$max_concurrency" \ + --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ + --result-dir "$BENCHMARK_LOGS_DIR/" \ + "${bench_extra_args[@]}" \ + || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" + done + + fi # ---- Eval (optional) ---- - if [[ "${RUN_EVAL:-false}" == "true" ]]; then + if [[ "${RUN_EVAL}" == "true" ]]; then # Concurrency for the eval and, crucially, for the concurrency stamped # into meta_env.json. run_eval/append_lm_eval_summary read # EVAL_CONCURRENT_REQUESTS and CONC (not EVAL_CONC), so mirror the AMD @@ -695,7 +673,7 @@ PY # CONC is empty, the metadata records conc=1, and score verification # fails ("eval metadata concurrency does not match workflow request") # even when accuracy passes. - if [[ -n "${EVAL_CONC:-}" ]]; then + if [[ -n "${EVAL_CONC}" ]]; then export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" else export EVAL_CONCURRENT_REQUESTS=$(printf '%s' "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh index 11c34736f8..15060a46a0 100755 --- a/benchmarks/multi_node/llm-d/submit.sh +++ b/benchmarks/multi_node/llm-d/submit.sh @@ -10,7 +10,7 @@ # each instance spans PREFILL_NODES / DECODE_NODES nodes via vLLM # --data-parallel-hybrid-lb. Total nodes = PREFILL_NODES + DECODE_NODES. -set -euo pipefail +set -eo pipefail # Repo root resolved from this script's location, so paths below are # independent of the caller's $PWD (the wrapper cd's into llm-d/ before @@ -19,7 +19,7 @@ REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" check_env() { local name="$1" - if [[ -z "${!name:-}" ]]; then + if [[ -z "${!name}" ]]; then echo "Error: ${name} not set" >&2 exit 1 fi @@ -39,11 +39,11 @@ DECODE_NODES=$2 ISL=$3 OSL=$4 CONCURRENCIES=$5 -REQUEST_RATE=${6:-inf} -RANDOM_RANGE_RATIO=${7:-0.8} +REQUEST_RATE=$6 +RANDOM_RANGE_RATIO=$7 NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +check_env GPUS_PER_NODE export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE export MODEL_DIR=$MODEL_PATH @@ -56,8 +56,10 @@ export GPUS_PER_NODE=$GPUS_PER_NODE # INDEPENDENT DP/EP engines (default 1 = one engine over all role nodes). Each # engine spans role_nodes/workers nodes, so DP_SIZE is PER-ENGINE. Matches how # dynamo/AMD and upstream oci-high-tpt run 2P high-tpt (2 prefill : 1 decode). -export PREFILL_WORKERS="${PREFILL_WORKERS:-1}" -export DECODE_WORKERS="${DECODE_WORKERS:-1}" +check_env PREFILL_WORKERS +export PREFILL_WORKERS +check_env DECODE_WORKERS +export DECODE_WORKERS if (( PREFILL_NODES % PREFILL_WORKERS != 0 )); then echo "Error: PREFILL_NODES ($PREFILL_NODES) not divisible by PREFILL_WORKERS ($PREFILL_WORKERS)" >&2 exit 1 @@ -74,19 +76,26 @@ export BENCH_MAX_CONCURRENCY=$CONCURRENCIES export BENCH_REQUEST_RATE=$REQUEST_RATE export BENCH_RANDOM_RANGE_RATIO=$RANDOM_RANGE_RATIO # Match the AMD multinode default. -export BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" +export BENCH_NUM_PROMPTS_MULTIPLIER="10" -export RUN_EVAL="${RUN_EVAL:-false}" -export EVAL_ONLY="${EVAL_ONLY:-false}" -export EVAL_CONC="${EVAL_CONC:-}" -export FRAMEWORK="${FRAMEWORK:-llmd-vllm}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-none}" -export IS_MULTINODE="${IS_MULTINODE:-true}" -export CONFIG_FILE="${CONFIG_FILE:-}" +export RUN_EVAL="${RUN_EVAL}" +export EVAL_ONLY="${EVAL_ONLY}" +export EVAL_CONC="${EVAL_CONC}" +export FRAMEWORK="${FRAMEWORK}" +export PRECISION="${PRECISION}" +export MODEL_PREFIX="${MODEL_PREFIX}" +export RUNNER_TYPE="${RUNNER_TYPE}" +export RESULT_FILENAME="${RESULT_FILENAME}" +export SPEC_DECODING="${SPEC_DECODING}" +export IS_MULTINODE="${IS_MULTINODE}" +export CONFIG_FILE="${CONFIG_FILE}" + +# Explicit fleet/container mapping shared by every llm-d wrapper. +case "$RUNNER_TYPE" in + b200|gb200) export LLMD_CONTAINER_ENGINE=pyxis ;; + h200) export LLMD_CONTAINER_ENGINE=docker ;; + *) echo "Unsupported llm-d runner hardware: $RUNNER_TYPE" >&2; exit 1 ;; +esac # Recipe may override SLURM time limit (longer topologies need more wall time). if [[ -n "$CONFIG_FILE" ]]; then diff --git a/benchmarks/multi_node/llm-d/test_recipe.py b/benchmarks/multi_node/llm-d/test_recipe.py new file mode 100644 index 0000000000..0eb0cc8274 --- /dev/null +++ b/benchmarks/multi_node/llm-d/test_recipe.py @@ -0,0 +1,72 @@ +"""CPU-only checks for the llm-d recipe-to-vLLM contract.""" + +import json +from pathlib import Path +import shlex + +import pytest +import yaml + +from recipe import mooncake_config, role_assignments + + +RECIPES = Path(__file__).resolve().parents[1] / "llm-d-recipes/agentic" + + +@pytest.mark.parametrize("path", sorted(RECIPES.glob("*.yaml")), ids=lambda path: path.stem) +@pytest.mark.parametrize("eval_only", ["false", "true"]) +def test_dspark_uses_role_golden_al_and_native_context(path, eval_only): + recipe = yaml.safe_load(path.read_text()) + env = { + "IS_AGENTIC": "1", "EVAL_ONLY": eval_only, "SPEC_DECODING": "mtp", + "MODEL_NAME": "deepseek-ai/DeepSeek-V4-Pro-0813", + "KV_OFFLOADING": "dram" if "mooncake" in recipe else "none", + "KV_OFFLOAD_BACKEND": "mooncake" if "mooncake" in recipe else "", + } + for role in ("prefill", "decode"): + if role not in recipe: + continue + assignment = role_assignments(recipe, role, env).splitlines()[0] + args = shlex.split(assignment)[0].removeprefix("ROLE_EXTRA_ARGS=") + start = args.index("--speculative-config ") + len("--speculative-config ") + config, _ = json.JSONDecoder().raw_decode(args[start:]) + if eval_only == "true": + assert "synthetic_acceptance_length" not in config + assert "rejection_sample_method" not in config + else: + assert config["rejection_sample_method"] == "synthetic" + assert config["synthetic_acceptance_length"] == {1: 1.84, 5: 3.61}[config["num_speculative_tokens"]] + assert config["enable_adaptive_verification"] is False + assert "--max-model-len" not in args + assert "--max-num-seqs" not in args + assert "--tool-call-parser deepseek_v4" in args + assert "--reasoning-parser deepseek_v4" in args + + +def test_offload_metadata_and_runtime_budget_agree(): + recipe = yaml.safe_load((RECIPES / "agg-gb200-dep8-dspark-mooncake-agentic.yaml").read_text()) + env = {"IS_AGENTIC": "1", "KV_OFFLOADING": "none"} + with pytest.raises(ValueError, match="KV_OFFLOADING=dram"): + mooncake_config(recipe, env) + env.update(KV_OFFLOADING="dram", KV_OFFLOAD_BACKEND="mooncake", TOTAL_CPU_DRAM_GB="1298", GPUS_PER_NODE="8") + config = json.loads(mooncake_config(recipe, env)) + assert config["enable_offload"] is False + assert config["global_segment_size"] * 8 == 1298 * 10**9 + + +def test_no_offload_recipe_rejects_dram_metadata(): + with pytest.raises(ValueError, match="KV_OFFLOADING=none"): + role_assignments({}, "prefill", {"IS_AGENTIC": "1", "KV_OFFLOADING": "dram"}) + + +def test_fixed_length_recipe_is_not_rewritten(): + recipe = {"prefill": {"extra-args": '--speculative-config {"method":"dspark","num_speculative_tokens":5}'}} + assignment = role_assignments(recipe, "prefill", {"IS_AGENTIC": "0"}) + assert "synthetic" not in assignment + + +def test_unregistered_golden_al_fails_closed(): + recipe = yaml.safe_load((RECIPES / "agg-gb200-tp8-dspark-agentic.yaml").read_text()) + env = {"IS_AGENTIC": "1", "KV_OFFLOADING": "none", "SPEC_DECODING": "mtp", "MODEL_NAME": "other"} + with pytest.raises(ValueError, match="No registered DSpark golden AL"): + role_assignments(recipe, "prefill", env) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 85e6af68bd..db2df09894 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4517,16 +4517,17 @@ dsv4-fp4-b200-llmd-vllm-agentx: multinode: true disagg: true scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 + agentic-coding: + - dram-utilization: 0.60 search-space: # 1P DEP8 + 1D DEP8 (2 nodes / 16 GPUs). Always Mooncake. - - spec-decoding: "none" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } conc-list: [64, 128, 192] prefill: num-worker: 1 - tp: 1 + tp: 8 ep: 8 dp-attn: true additional-settings: @@ -4535,7 +4536,7 @@ dsv4-fp4-b200-llmd-vllm-agentx: - "CONFIG_FILE=agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml" decode: num-worker: 1 - tp: 1 + tp: 8 ep: 8 dp-attn: true additional-settings: @@ -4544,8 +4545,7 @@ dsv4-fp4-b200-llmd-vllm-agentx: # Aggregated (TP8 / DEP8) sibling of dsv4-fp4-b200-llmd-vllm-agentx: one # engine per topology does both prefill and decode (no P/D split, no -# pd-sidecar, no kv_transfer_config - see the recipe files' header -# comments). Separate top-level key because `disagg:` is a whole-key +# pd-sidecar; only the Mooncake arm uses a KV store connector). Separate top-level key because `disagg:` is a whole-key # property. Uses the default v0.9.0 EPP/pd-sidecar binaries (no router # bump needed - the aggregated EPP config only uses plugins already # present there). conc-lists mirror the agentX dspark historical runs: @@ -4562,12 +4562,12 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: multinode: true disagg: false scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 + agentic-coding: + - dram-utilization: 0.60 search-space: # Aggregated TP8 (1 node / 8 GPUs; pure tensor-parallel, no EP). - - spec-decoding: "none" + - spec-decoding: mtp + kv-offloading: none conc-list: [1, 2, 4, 8, 12] num-nodes: 1 worker: @@ -4581,12 +4581,13 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: - "GPUS_PER_NODE=8" - "CONFIG_FILE=agentic/agg-gb200-tp8-dspark-agentic.yaml" # Aggregated DEP8 (1 node / 8 GPUs; DP=8 + EP), no Mooncake. - - spec-decoding: "none" + - spec-decoding: mtp + kv-offloading: none conc-list: [12, 16, 32] num-nodes: 1 worker: num-worker: 1 - tp: 1 + tp: 8 ep: 8 dp-attn: true additional-settings: @@ -4595,12 +4596,14 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: - "GPUS_PER_NODE=8" - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-agentic.yaml" # Aggregated DEP8 with Mooncake prefix-cache KV store. - - spec-decoding: "none" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } conc-list: [52, 72] num-nodes: 1 worker: num-worker: 1 - tp: 1 + tp: 8 ep: 8 dp-attn: true additional-settings: diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index e42027abb5..12b7b38df3 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -142,6 +142,8 @@ llm-d is not the srt-slurm path: InferenceX owns the Slurm allocation and starts A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml` fallback and removes recipe-specific vLLM flags. Treat that as a validation failure unless fallback is explicitly intended. +For the B200 DSpark AgentX keys, select `agentic-coding`, `spec-decoding: mtp`, and explicit `kv-offloading`. `server.sh` uses the shared AgentX client via `llm-d/agentic.sh`, with no fixed context/sequence cap. `recipe.py` loads the committed DSpark thinking-on golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Mooncake's embedded store is **DRAM offloading even with `enable_offload: false`** (that flag controls SSD), so declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides the generated per-node budget among GPU ranks. The plain TP8/DEP8 arms declare `none`. Bash inputs are explicit constants or caller-supplied values, not fallback expressions. Binary extraction requires an explicit `BINARIES_ENV_FILE`. + ## Update an image Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants), the matching master configs, runtime scripts, and checked-in recipes. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 0ebd17fc68..164d9d94df 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -142,6 +142,8 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 `CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 +B200 DSpark AgentX 配置使用 `agentic-coding`、`spec-decoding: mtp`，并显式声明 `kv-offloading`。`server.sh` 通过 `llm-d/agentic.sh` 运行共享 AgentX 客户端，不设置固定上下文或序列数上限。`recipe.py` 为吞吐测试读取已提交的 DSpark thinking-on golden AL；`EVAL_ONLY=true` 保留真实验证。Mooncake 的嵌入式存储即使设置 `enable_offload: false` 也属于 **DRAM 卸载**（该开关控制 SSD），因此必须声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将生成的每节点内存预算均分给 GPU rank。普通 TP8/DEP8 配置声明 `none`。Bash 输入采用显式常量或调用方提供的值，不使用回退表达式。提取二进制文件时必须显式设置 `BINARIES_ENV_FILE`。 + ## 更新镜像 来源：[`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants)、对应主配置、运行时脚本与检入的 Recipe。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 31bcb9cc5c..1efe376a65 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6625,3 +6625,15 @@ - "Branch runners/launch_b200-nscale-compat.sh llmd-vllm wrapper selection on DISAGG (true -> disagg, false -> aggregated)" - "Fix MODEL_PATH for -0813 checkpoint: unconditionally set /scratch/models/DeepSeek-V4-Pro-0813 in the launcher's -0813 branch so the outer script's pre-exported non-0813 path is always overridden" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-b200-llmd-vllm-agentx + - dsv4-fp4-b200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Run the shared AgentX trace replay instead of fixed 8k/1k requests; remove fixed context/sequence/cudagraph limits and enable DeepSeek-V4 chat tool/reasoning parsers." + - "Report DSpark as spec-decoding mtp and read the committed DeepSeek-V4-Pro-0813 thinking-on golden AL (K1=1.84, K5=3.61) for synthetic throughput verification; accuracy evals retain real verification." + - "Declare Mooncake arms as DRAM offloading, apply the master per-node 0.60 DRAM budget to the embedded per-GPU pools, and report DEP8 as eight physical GPUs. Non-Mooncake arms explicitly disable offloading." + - "Forward AgentX provenance, topology and offload metadata through both container engines and stage aggregate/raw artifacts. Remove nounset and environment fallback expressions from the changed Bash launch paths." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index a30d648eaa..2cf5b2b7f4 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -2,8 +2,8 @@ # Compatibility launcher for B200 Nscale configurations that have not yet # moved to the native srt-slurm path in launch_b200-nscale-slurm.sh. -SLURM_PARTITION="${SLURM_PARTITION:-batch_1}" -SLURM_ACCOUNT="${SLURM_ACCOUNT:-benchmark}" +SLURM_PARTITION="batch_1" +SLURM_ACCOUNT="benchmark" POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" @@ -24,18 +24,7 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && "$MODEL" == *-0813 ]]; export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-0813" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - SELECTED_MODEL_PATH="" - if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then - SELECTED_MODEL_PATH="$MODEL_PATH" - else - for candidate in /scratch/models/DeepSeek-V4-Pro /scratch/models/DeepSeek-V4-Pro-NVFP4; do - if [[ -d "$candidate" ]]; then - SELECTED_MODEL_PATH="$candidate" - break - fi - done - fi - export MODEL_PATH="${SELECTED_MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" + export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "bf16" ]]; then export MODEL_PATH="/scratch/models/Qwen3.5-397B-A17B" @@ -66,7 +55,7 @@ elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/models/GLM-5-NVFP4" export SRT_SLURM_MODEL_PREFIX="glm5-fp4" elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="${MODEL_PATH:-/scratch/models/GLM-5.2-NVFP4}" + export MODEL_PATH="/scratch/models/GLM-5.2-NVFP4" export SRT_SLURM_MODEL_PREFIX="glm5.2-fp4" elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then export MODEL_PATH="/scratch/models/Kimi-K2.5" @@ -75,7 +64,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="${MODEL_PATH:-/scratch/models/Kimi-K2.6-NVFP4}" + export MODEL_PATH="/scratch/models/Kimi-K2.6-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/scratch/models/MiniMax-M2.5" @@ -183,7 +172,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then ) || exit 1 } - LLMD_SQUASH_DIR="${LLMD_SQUASH_DIR:-/home/sa-shared/containers}" + LLMD_SQUASH_DIR="/home/sa-shared/containers" mkdir -p "$LLMD_SQUASH_DIR" 2>/dev/null || true LLMD_SQUASH_FILE="${LLMD_SQUASH_DIR}/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" llmd_import_squash "$LLMD_SQUASH_FILE" "$IMAGE" @@ -199,13 +188,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # (see benchmark-multinode-tmpl.yml). true -> the P/D disagg wrapper # (1P-DEP8/1D-DEP8); false -> the aggregated wrapper (TP8/DEP8, one # engine does both prefill and decode, DECODE_NODES=0). - if [[ "${DISAGG:-true}" == "true" ]]; then + if [[ "${DISAGG}" == "true" ]]; then SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" # EPP/pd-sidecar binaries needed for the disagg-profile-handler `deciders:` # shape. Populate once via: # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env benchmarks/llm-d/extract-binaries.sh # job.slurm falls back to the image-baked v0.9.0 until this path exists. - export LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}" + export LLMD_BIN_DIR="/home/sa-shared/llm-d-bins-v0.10.0" else SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" fi @@ -228,11 +217,18 @@ if [[ "$IS_MULTINODE" == "true" ]]; then LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 - while IFS= read -r -d '' result_file; do - copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 - done < <(find "$BENCHMARK_LOGS_DIR" -name "${RESULT_FILENAME}*.json" -print0 2>/dev/null) + if [[ "${IS_AGENTIC}" == "1" && "${EVAL_ONLY}" != "true" ]]; then + copy_agentic_results "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" || exit 1 + # Host-owned artifact staging; containers create no workspace dirs. + mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" + cp -R "$BENCHMARK_LOGS_DIR/agentic/." "$GITHUB_WORKSPACE/LOGS/agentic/" + else + while IFS= read -r -d '' result_file; do + copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 + done < <(find "$BENCHMARK_LOGS_DIR" -name "${RESULT_FILENAME}*.json" -print0 2>/dev/null) + fi - if [[ "${RUN_EVAL:-false}" == "true" ]]; then + if [[ "${RUN_EVAL}" == "true" ]]; then EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR" -type d -name eval_results -print -quit 2>/dev/null) [[ -z "$EVAL_DIR" ]] && EVAL_DIR="$BENCHMARK_LOGS_DIR/eval_results" copy_eval_artifacts "$EVAL_DIR" "$GITHUB_WORKSPACE" || exit 1 @@ -244,11 +240,11 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$FRAMEWORK" == "tilert" ]]; then export SLURM_PARTITION SLURM_ACCOUNT - export TILERT_WEIGHTS_DIR="${TILERT_WEIGHTS_DIR:-/scratch/models/${MODEL_PREFIX}-${PRECISION}-tilert-8shard}" + export TILERT_WEIGHTS_DIR="/scratch/models/${MODEL_PREFIX}-${PRECISION}-tilert-8shard" # Nscale exposes eight RoCE HCAs, mlx5_0..mlx5_7. - export UCX_NET_DEVICES="${UCX_NET_DEVICES:-mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1}" - export UCX_MEMTYPE_CACHE="${UCX_MEMTYPE_CACHE:-n}" - export UCX_MEMTYPE_REG_WHOLE="${UCX_MEMTYPE_REG_WHOLE:-n}" + export UCX_NET_DEVICES="mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1" + export UCX_MEMTYPE_CACHE="n" + export UCX_MEMTYPE_REG_WHOLE="n" TILERT_SUBDIR="multi_node" [[ "${SCENARIO_SUBDIR}" == "agentic/" ]] && TILERT_SUBDIR="multi_node/agentic" TILERT_DISAGG="$GITHUB_WORKSPACE/benchmarks/${TILERT_SUBDIR}/${EXP_NAME%%_*}_${PRECISION}_b200_${FRAMEWORK}-disagg.sh" @@ -270,8 +266,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi USES_DCGM_POWER=0 - _POWER_CONFIG_FILE="${CONFIG_FILE:-}" - if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + _POWER_CONFIG_FILE="${CONFIG_FILE}" + if [[ "${EVAL_ONLY}" == "true" && -n "${EVAL_CONFIG_FILE}" ]]; then _POWER_CONFIG_FILE="$EVAL_CONFIG_FILE" fi _RECIPE_REL="${_POWER_CONFIG_FILE%%:*}" @@ -286,7 +282,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then USES_DCGM_POWER=1 fi if [[ "$USES_DCGM_POWER" == "1" && ( - "${IS_AGENTIC:-0}" == "1" || + "${IS_AGENTIC}" == "1" || "$MODEL_PREFIX" != "dsv4" || "$PRECISION" != "fp4" || "$FRAMEWORK" != "dynamo-vllm" @@ -388,9 +384,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # Map container images to local squash files NGINX_IMAGE="nginx:1.27.4" - SQUASH_DIR="${B200_SQUASH_DIR:-/data/home/sa-shared/containers}" + SQUASH_DIR="/data/home/sa-shared/containers" if [[ $MODEL_PREFIX == "minimaxm2.5" && $FRAMEWORK == "dynamo-vllm" ]]; then - SQUASH_DIR="${B200_SQUASH_DIR:-/home/slurm-shared/gharunners/squash}" + SQUASH_DIR="/home/slurm-shared/gharunners/squash" fi if ! mkdir -p "$SQUASH_DIR" 2>/dev/null || [[ ! -w "$SQUASH_DIR" ]]; then echo "Warning: $SQUASH_DIR is not writable; using workspace-local squash cache" >&2 @@ -413,7 +409,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then local lock_file="${lock_dir}/${image_key}.lock" ( - flock -w "${B200_SQUASH_LOCK_TIMEOUT:-600}" 9 || { echo "Failed to acquire lock for $squash_file" >&2; exit 1; } + flock -w "600" 9 || { echo "Failed to acquire lock for $squash_file" >&2; exit 1; } if unsquashfs -l "$squash_file" > /dev/null 2>&1; then echo "Squash file already exists and is valid, skipping import: $squash_file" else @@ -444,7 +440,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export ISL="$ISL" export OSL="$OSL" - export EVAL_ONLY="${EVAL_ONLY:-false}" + export EVAL_ONLY="${EVAL_ONLY}" # Agentic runs bind-mount two persistent caches into every worker # container (Lustre, shared across nodes): aiperf's content-addressed @@ -514,7 +510,7 @@ EOF # throughput row keeps synthetic golden acceptance. Only configs that set # EVAL_CONFIG_FILE opt into this selection; all other configs keep using # CONFIG_FILE unchanged. - if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + if [[ "${EVAL_ONLY}" == "true" && -n "${EVAL_CONFIG_FILE}" ]]; then CONFIG_FILE="$EVAL_CONFIG_FILE" echo "EVAL_ONLY=true: selecting real-verification recipe $CONFIG_FILE" fi @@ -605,7 +601,7 @@ EOF cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . - if [[ "${EVAL_ONLY:-false}" != "true" ]]; then + if [[ "${EVAL_ONLY}" != "true" ]]; then # Find all result subdirectories RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null) @@ -653,7 +649,7 @@ EOF fi # Collect eval results if eval was requested - if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then + if [[ "${RUN_EVAL}" == "true" || "${EVAL_ONLY}" == "true" ]]; then EVAL_DIR="$LOGS_DIR/eval_results" if [ -d "$EVAL_DIR" ]; then echo "Extracting eval results from $EVAL_DIR" @@ -707,9 +703,9 @@ else # The runner lease reserves the Slurm nodes before this single-node job is # submitted to the Nscale batch_1 partition. - export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" + export GPU_COUNT="${GPU_COUNT:?GPU_COUNT must be set by the invoking launcher}" - SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}" + SALLOC_TIME_LIMIT="480" salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index c027cc9b2f..4e5fe7eb26 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -1,3 +1,5 @@ +import json +import os import subprocess from pathlib import Path @@ -55,3 +57,28 @@ def test_copy_agentic_results_fails_when_aggregate_is_missing( assert result.returncode != 0 assert "no run_conc*.json results found" in result.stderr + + +def test_llmd_agentic_adapter_uses_envoy_and_all_engine_metrics(tmp_path: Path) -> None: + """Execute the real adapter with a recording client, without any GPUs.""" + client = tmp_path / "benchmarks/multi_node/agentic_srt.sh" + client.parent.mkdir(parents=True) + client.write_text("python3 -c 'import json, os; print(json.dumps(dict(os.environ)))'\n") + env = dict(os.environ, INFMAX_CONTAINER_WORKSPACE=str(tmp_path), + MODEL_NAME="deepseek-ai/DeepSeek-V4-Pro-0813", + ENVOY_PORT="8080", VLLM_PORT="8200", BENCHMARK_LOGS_DIR=str(tmp_path / "logs"), + BENCH_MAX_CONCURRENCY="64", ALL_IPS="10.0.0.1,10.0.0.2", + PREFILL_NODES="1", DECODE_NODES="1", PREFILL_WORKERS="1", DECODE_WORKERS="1", + SPEC_DECODING="mtp", KV_OFFLOADING="dram", KV_OFFLOAD_BACKEND="mooncake") + result = subprocess.run(["bash", str(REPO_ROOT / "benchmarks/multi_node/llm-d/agentic.sh")], + env=env, text=True, capture_output=True) + assert result.returncode == 0, result.stderr + recorded = json.loads(result.stdout) + assert recorded["MODEL"] == env["MODEL_NAME"] + assert recorded["AIPERF_SERVER_URL"] == "http://localhost:8080" + assert recorded["PORT"] == "8200" + assert recorded["AIPERF_SERVER_METRICS_URLS"] == "http://10.0.0.1:8200/metrics,http://10.0.0.2:8200/metrics" + assert recorded["AGENTIC_OUTPUT_DIR"] == env["BENCHMARK_LOGS_DIR"] + assert recorded["CONC"] == "64" + assert recorded["SPEC_DECODING"] == "mtp" + assert recorded["KV_OFFLOADING"] == "dram" diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index cbde1afc4a..753fdd2446 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -23,6 +23,36 @@ from validation import load_config_files, load_runner_file +def test_llmd_dspark_agentx_registered_matrix(): + root = Path(__file__).resolve().parents[2] + config = load_config_files([str(root / "configs/nvidia-master.yaml")]) + runners = load_runner_file(str(root / "configs/runners.yaml")) + args = argparse.Namespace( + config_keys=["dsv4-fp4-b200-llmd-vllm-agentx", "dsv4-fp4-b200-llmd-vllm-agentx-agg"], + seq_lens=None, conc=None, scenario_type=["agentic-coding"], runner_node_filter=None, + ) + rows = generate_test_config_sweep(args, config, runners) + assert len(rows) == 13 + recipes = set() + for row in rows: + assert row["scenario-type"] == "agentic-coding" + assert row["spec-decoding"] == "mtp" + assert row["prefill"]["tp"] == 8 + assert row["node-count"] == (2 if row["disagg"] else 1) + assert row["duration"] == 3600 + settings = row["prefill"]["additional-settings"] + recipe = next(setting.split("=", 1)[1] for setting in settings if setting.startswith("CONFIG_FILE=")) + recipes.add(recipe) + offloads = row["disagg"] or "mooncake" in recipe + assert row["kv-offloading"] == ("dram" if offloads else "none") + if offloads: + assert row["kv-offload-backend"]["name"] == "mooncake" + assert row["total-cpu-dram-gb"] > 0 + else: + assert row["total-cpu-dram-gb"] == 0 + assert len(recipes) == 4 + + def test_aggregated_multinode_node_count_uses_explicit_num_nodes(): entry = { "runner": "unknown", From e3d6d0cf1c2f23b71cd61a7f1125f0cf94a70b4e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 09:42:19 -0500 Subject: [PATCH 09/46] fix: accept cluster runner labels in llm-d container selection --- benchmarks/multi_node/llm-d/submit.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh index 84bdca6c1e..1c08b7a97f 100755 --- a/benchmarks/multi_node/llm-d/submit.sh +++ b/benchmarks/multi_node/llm-d/submit.sh @@ -94,9 +94,9 @@ export IS_MULTINODE="${IS_MULTINODE}" export CONFIG_FILE="${CONFIG_FILE}" # Explicit fleet/container mapping shared by every llm-d wrapper. -case "$RUNNER_TYPE" in - b200|gb200) export LLMD_CONTAINER_ENGINE=pyxis ;; - h200) export LLMD_CONTAINER_ENGINE=docker ;; +case "${RUNNER_TYPE#cluster:}" in + b200|b200-*|gb200|gb200-*) export LLMD_CONTAINER_ENGINE=pyxis ;; + h200|h200-*) export LLMD_CONTAINER_ENGINE=docker ;; *) echo "Unsupported llm-d runner hardware: $RUNNER_TYPE" >&2; exit 1 ;; esac From 8d00f590ae0f9a0f74a3a7287b1fc5720a681ec4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 09:48:50 -0500 Subject: [PATCH 10/46] chore: remove standalone llm-d recipe and redundant config tests --- benchmarks/multi_node/llm-d/test_recipe.py | 72 ------------------- runners/test_slurm_utils.py | 2 - .../test_generate_sweep_configs.py | 31 -------- 3 files changed, 105 deletions(-) delete mode 100644 benchmarks/multi_node/llm-d/test_recipe.py diff --git a/benchmarks/multi_node/llm-d/test_recipe.py b/benchmarks/multi_node/llm-d/test_recipe.py deleted file mode 100644 index 0eb0cc8274..0000000000 --- a/benchmarks/multi_node/llm-d/test_recipe.py +++ /dev/null @@ -1,72 +0,0 @@ -"""CPU-only checks for the llm-d recipe-to-vLLM contract.""" - -import json -from pathlib import Path -import shlex - -import pytest -import yaml - -from recipe import mooncake_config, role_assignments - - -RECIPES = Path(__file__).resolve().parents[1] / "llm-d-recipes/agentic" - - -@pytest.mark.parametrize("path", sorted(RECIPES.glob("*.yaml")), ids=lambda path: path.stem) -@pytest.mark.parametrize("eval_only", ["false", "true"]) -def test_dspark_uses_role_golden_al_and_native_context(path, eval_only): - recipe = yaml.safe_load(path.read_text()) - env = { - "IS_AGENTIC": "1", "EVAL_ONLY": eval_only, "SPEC_DECODING": "mtp", - "MODEL_NAME": "deepseek-ai/DeepSeek-V4-Pro-0813", - "KV_OFFLOADING": "dram" if "mooncake" in recipe else "none", - "KV_OFFLOAD_BACKEND": "mooncake" if "mooncake" in recipe else "", - } - for role in ("prefill", "decode"): - if role not in recipe: - continue - assignment = role_assignments(recipe, role, env).splitlines()[0] - args = shlex.split(assignment)[0].removeprefix("ROLE_EXTRA_ARGS=") - start = args.index("--speculative-config ") + len("--speculative-config ") - config, _ = json.JSONDecoder().raw_decode(args[start:]) - if eval_only == "true": - assert "synthetic_acceptance_length" not in config - assert "rejection_sample_method" not in config - else: - assert config["rejection_sample_method"] == "synthetic" - assert config["synthetic_acceptance_length"] == {1: 1.84, 5: 3.61}[config["num_speculative_tokens"]] - assert config["enable_adaptive_verification"] is False - assert "--max-model-len" not in args - assert "--max-num-seqs" not in args - assert "--tool-call-parser deepseek_v4" in args - assert "--reasoning-parser deepseek_v4" in args - - -def test_offload_metadata_and_runtime_budget_agree(): - recipe = yaml.safe_load((RECIPES / "agg-gb200-dep8-dspark-mooncake-agentic.yaml").read_text()) - env = {"IS_AGENTIC": "1", "KV_OFFLOADING": "none"} - with pytest.raises(ValueError, match="KV_OFFLOADING=dram"): - mooncake_config(recipe, env) - env.update(KV_OFFLOADING="dram", KV_OFFLOAD_BACKEND="mooncake", TOTAL_CPU_DRAM_GB="1298", GPUS_PER_NODE="8") - config = json.loads(mooncake_config(recipe, env)) - assert config["enable_offload"] is False - assert config["global_segment_size"] * 8 == 1298 * 10**9 - - -def test_no_offload_recipe_rejects_dram_metadata(): - with pytest.raises(ValueError, match="KV_OFFLOADING=none"): - role_assignments({}, "prefill", {"IS_AGENTIC": "1", "KV_OFFLOADING": "dram"}) - - -def test_fixed_length_recipe_is_not_rewritten(): - recipe = {"prefill": {"extra-args": '--speculative-config {"method":"dspark","num_speculative_tokens":5}'}} - assignment = role_assignments(recipe, "prefill", {"IS_AGENTIC": "0"}) - assert "synthetic" not in assignment - - -def test_unregistered_golden_al_fails_closed(): - recipe = yaml.safe_load((RECIPES / "agg-gb200-tp8-dspark-agentic.yaml").read_text()) - env = {"IS_AGENTIC": "1", "KV_OFFLOADING": "none", "SPEC_DECODING": "mtp", "MODEL_NAME": "other"} - with pytest.raises(ValueError, match="No registered DSpark golden AL"): - role_assignments(recipe, "prefill", env) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 04707b0cad..44f279a9b6 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -1,7 +1,5 @@ import json import os - - import runpy import subprocess from pathlib import Path diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 6e8dec7a6a..ef305ab23a 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -12,7 +12,6 @@ import yaml import generate_sweep_configs -from validation import load_config_files, load_runner_file from generate_sweep_configs import ( MIN_EVAL_CONC, add_multinode_node_count, @@ -30,36 +29,6 @@ ) -def test_llmd_dspark_agentx_registered_matrix(): - root = Path(__file__).resolve().parents[2] - config = load_config_files([str(root / "configs/nvidia-master.yaml")]) - runners = load_runner_file(str(root / "configs/runners.yaml")) - args = argparse.Namespace( - config_keys=["dsv4-fp4-b200-llmd-vllm-agentx", "dsv4-fp4-b200-llmd-vllm-agentx-agg"], - seq_lens=None, conc=None, scenario_type=["agentic-coding"], runner_node_filter=None, - ) - rows = generate_test_config_sweep(args, config, runners) - assert len(rows) == 13 - recipes = set() - for row in rows: - assert row["scenario-type"] == "agentic-coding" - assert row["spec-decoding"] == "mtp" - assert row["prefill"]["tp"] == 8 - assert row["node-count"] == (2 if row["disagg"] else 1) - assert row["duration"] == 3600 - settings = row["prefill"]["additional-settings"] - recipe = next(setting.split("=", 1)[1] for setting in settings if setting.startswith("CONFIG_FILE=")) - recipes.add(recipe) - offloads = row["disagg"] or "mooncake" in recipe - assert row["kv-offloading"] == ("dram" if offloads else "none") - if offloads: - assert row["kv-offload-backend"]["name"] == "mooncake" - assert row["total-cpu-dram-gb"] > 0 - else: - assert row["total-cpu-dram-gb"] == 0 - assert len(recipes) == 4 - - def test_aggregated_multinode_node_count_uses_explicit_num_nodes(): entry = { "runner": "unknown", From 039147729857b4bfb0679b1b521799601552ab4b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 09:49:24 -0500 Subject: [PATCH 11/46] docs: consolidate DSpark performance changelog --- perf-changelog.yaml | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 96aa9db71f..7bf6516461 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6822,33 +6822,13 @@ - "Refresh to collect TensorRT-LLM server metrics." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 -- config-keys: - - dsv4-fp4-b200-llmd-vllm-agentx - description: - - "Add DeepSeek-V4-Pro-0813 (DSpark) FP4 B200 llmd-vllm P/D disagg benchmark (1P DEP8 + 1D DEP8, 2 nodes / 16 GPUs total); always uses Mooncake RDMA prefix-cache; DSpark speculative decoding (prefill 1 token, decode 5 tokens)" - - "Wire a FRAMEWORK==llmd-vllm dispatch into runners/launch_b200-nscale-compat.sh (self-contained, bypasses the srtctl/srt-slurm path)" - - "Pin a disagg-only EPP/pd-sidecar v0.10.0 binary override (LLMD_BIN_DIR=/home/sa-shared/llm-d-bins-v0.10.0) required for the disagg-profile-handler deciders: EPP shape; aggregated runs use the image-baked v0.9.0 binaries" - - "Fix MODEL_PATH for -0813 checkpoint: unconditionally set /scratch/models/DeepSeek-V4-Pro-0813 in the launcher's -0813 branch so the outer script's pre-exported non-0813 path is always overridden" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 - -- config-keys: - - dsv4-fp4-b200-llmd-vllm-agentx-agg - description: - - "Add DeepSeek-V4-Pro-0813 (DSpark) FP4 B200 llmd-vllm aggregated benchmark: TP8 at conc-list [1,2,4,8,12], DEP8 at [12,16,32], DEP8+Mooncake at [52,72] (all 1 node / 8 GPUs, one engine does both prefill and decode)" - - "Extend server.sh/job.slurm with aggregated (DECODE_NODES=0) mode: no decode role, no pd-sidecar; rank 0 takes over EPP/Envoy/bench coordinator duties" - - "Add Mooncake (P2PHANDSHAKE embedded RDMA) for the DEP8+Mooncake arm via agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml; server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role) when MOONCAKE_CONFIG_PATH is set in aggregated mode" - - "Branch runners/launch_b200-nscale-compat.sh llmd-vllm wrapper selection on DISAGG (true -> disagg, false -> aggregated)" - - "Fix MODEL_PATH for -0813 checkpoint: unconditionally set /scratch/models/DeepSeek-V4-Pro-0813 in the launcher's -0813 branch so the outer script's pre-exported non-0813 path is always overridden" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 - - config-keys: - dsv4-fp4-b200-llmd-vllm-agentx - dsv4-fp4-b200-llmd-vllm-agentx-agg scenario-type: - agentic-coding description: - - "Run the shared AgentX trace replay instead of fixed 8k/1k requests; remove fixed context/sequence/cudagraph limits and enable DeepSeek-V4 chat tool/reasoning parsers." - - "Report DSpark as spec-decoding mtp and read the committed DeepSeek-V4-Pro-0813 thinking-on golden AL (K1=1.84, K5=3.61) for synthetic throughput verification; accuracy evals retain real verification." - - "Declare Mooncake arms as DRAM offloading, apply the master per-node 0.60 DRAM budget to the embedded per-GPU pools, and report DEP8 as eight physical GPUs. Non-Mooncake arms explicitly disable offloading." - - "Forward AgentX provenance, topology and offload metadata through both container engines and stage aggregate/raw artifacts. Remove nounset and environment fallback expressions from the changed Bash launch paths." + - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache for B200 AgentX trace replay at native context: aggregated TP8/DEP8 (8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (16 GPUs)." + - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification." + - "Enable Mooncake DRAM offloading for disaggregated and aggregated DEP8+Mooncake variants with dram-utilization=0.60; no offloading for plain TP8/DEP8. SSD offloading remains disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From a65c1e3885e2de3e4b4aa17e183fd566a400a0b9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 09:53:02 -0500 Subject: [PATCH 12/46] fix: expose and verify llm-d worker metrics for AIPerf --- benchmarks/multi_node/llm-d/agentic.sh | 42 +++++++++++++---- benchmarks/multi_node/llm-d/server.sh | 6 ++- docs/configuration-procedures.md | 2 + docs/configuration-procedures_zh.md | 2 + runners/test_slurm_utils.py | 65 +++++++++++++++++++------- 5 files changed, 89 insertions(+), 28 deletions(-) diff --git a/benchmarks/multi_node/llm-d/agentic.sh b/benchmarks/multi_node/llm-d/agentic.sh index 550ed78a29..d8c793c9b0 100644 --- a/benchmarks/multi_node/llm-d/agentic.sh +++ b/benchmarks/multi_node/llm-d/agentic.sh @@ -12,16 +12,38 @@ export AGENTIC_OUTPUT_DIR="$BENCHMARK_LOGS_DIR" export CONC_LIST="${BENCH_MAX_CONCURRENCY//x/ }" export CONC="${CONC_LIST%% *}" -# Envoy does not expose vLLM /metrics. Scrape each engine leader directly; -# headless members of multi-node engines do not run HTTP servers. -IFS=',' read -r -a ips <<< "$ALL_IPS" -metrics_urls=() -for ((i=0; i&2 + exit 1 + fi + echo "vLLM metrics ready: $metrics_url" done -export AIPERF_SERVER_METRICS_URLS="$(IFS=,; echo "${metrics_urls[*]}")" +rm -f "$metrics_probe" +trap - EXIT exec bash "$INFMAX_CONTAINER_WORKSPACE/benchmarks/multi_node/agentic_srt.sh" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 97acdcfd4d..0d8ef795e5 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -224,6 +224,7 @@ fi # Bring up vLLM engine (every node) # ---------------------------------------------------------------- COMMON_ARGS=( + --host 0.0.0.0 --port "$VLLM_PORT" --served-model-name "$MODEL_NAME" --trust-remote-code @@ -384,6 +385,7 @@ if [[ ( "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ) || \ # ---- Write endpoints.yaml (file-discovery) ---- # namespace must match EPP's --pool-namespace (file-discovery filters by it; # the schema default 'default' would drop every entry). See README.md. + export LLMD_ENDPOINTS_FILE=/tmp/endpoints.yaml python3 - < decode_ips == [] -> add_role emits zero decode endpoints # (aggregated mode: everything routes through the 'prefill'-labeled pool). add_role('decode', decode_ips, SIDECAR_PORT, group_size=decode_group) -yaml.safe_dump({'endpoints': endpoints}, open('/tmp/endpoints.yaml', 'w')) +yaml.safe_dump({'endpoints': endpoints}, open(os.environ['LLMD_ENDPOINTS_FILE'], 'w')) print(f'endpoints.yaml ({len(endpoints)} endpoints):') -print(open('/tmp/endpoints.yaml').read()) +print(open(os.environ['LLMD_ENDPOINTS_FILE']).read()) PY # ---- Bring up EPP ---- diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 12b7b38df3..7ca83eef1c 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -144,6 +144,8 @@ A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml For the B200 DSpark AgentX keys, select `agentic-coding`, `spec-decoding: mtp`, and explicit `kv-offloading`. `server.sh` uses the shared AgentX client via `llm-d/agentic.sh`, with no fixed context/sequence cap. `recipe.py` loads the committed DSpark thinking-on golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Mooncake's embedded store is **DRAM offloading even with `enable_offload: false`** (that flag controls SSD), so declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides the generated per-node budget among GPU ranks. The plain TP8/DEP8 arms declare `none`. Bash inputs are explicit constants or caller-supplied values, not fallback expressions. Binary extraction requires an explicit `BINARIES_ENV_FILE`. +llm-d exports the discovered vLLM `/metrics` endpoints as `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, checks that each returns vLLM metrics, and forwards them to AIPerf through `--server-metrics`. Scrapes use each serving node’s vLLM port, not Envoy or the P/D sidecar. + ## Update an image Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants), the matching master configs, runtime scripts, and checked-in recipes. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 164d9d94df..4cdbfc11d2 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -144,6 +144,8 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 B200 DSpark AgentX 配置使用 `agentic-coding`、`spec-decoding: mtp`，并显式声明 `kv-offloading`。`server.sh` 通过 `llm-d/agentic.sh` 运行共享 AgentX 客户端，不设置固定上下文或序列数上限。`recipe.py` 为吞吐测试读取已提交的 DSpark thinking-on golden AL；`EVAL_ONLY=true` 保留真实验证。Mooncake 的嵌入式存储即使设置 `enable_offload: false` 也属于 **DRAM 卸载**（该开关控制 SSD），因此必须声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将生成的每节点内存预算均分给 GPU rank。普通 TP8/DEP8 配置声明 `none`。Bash 输入采用显式常量或调用方提供的值，不使用回退表达式。提取二进制文件时必须显式设置 `BINARIES_ENV_FILE`。 +llm-d 将发现的 vLLM `/metrics` 端点导出为 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，确认每个端点返回 vLLM 指标后，通过 `--server-metrics` 传给 AIPerf。抓取使用各服务节点的 vLLM 端口，而非 Envoy 或 P/D sidecar。 + ## 更新镜像 来源：[`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants)、对应主配置、运行时脚本与检入的 Recipe。 diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 44f279a9b6..078775d082 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -68,29 +68,62 @@ def test_copy_agentic_results_fails_when_aggregate_is_missing( assert "no run_conc*.json results found" in result.stderr -def test_llmd_agentic_adapter_uses_envoy_and_all_engine_metrics(tmp_path: Path) -> None: - """Execute the real adapter with a recording client, without any GPUs.""" +@pytest.mark.parametrize("metrics_body", ["vllm:num_requests_running 0\n", "envoy_http_requests_total 0\n"]) +def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, metrics_body: str) -> None: + """Check endpoint selection, preflight failure, and the real AIPerf CLI builder.""" client = tmp_path / "benchmarks/multi_node/agentic_srt.sh" client.parent.mkdir(parents=True) - client.write_text("python3 -c 'import json, os; print(json.dumps(dict(os.environ)))'\n") + client.write_text('''source "$REAL_BENCHMARK_LIB" +build_replay_cmd "$RESULT_DIR" +export REPLAY_CMD +python3 - <<'PY' +import json, os +keys = ["AIPERF_METRIC_URLS", "AIPERF_SERVER_METRICS_URLS", "REPLAY_CMD"] +print(json.dumps({key: os.environ[key] for key in keys})) +PY +''') + # Mock only the HTTP collaborator; the real adapter parses discovery and + # selects each serving node, including non-leader DP nodes. + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + curl = bin_dir / "curl" + curl.write_text( + '#!/usr/bin/env python3\n' + 'import os, sys\nfrom pathlib import Path\n' + 'Path(sys.argv[sys.argv.index("--output") + 1]).write_text(os.environ["METRICS_BODY"])\n' + 'with open(os.environ["METRICS_REQUESTS"], "a") as out:\n' + ' out.write(next(arg for arg in sys.argv if arg.startswith("http://")) + "\\n")\n' + ) + curl.chmod(0o755) + endpoints = tmp_path / "endpoints.yaml" + endpoints.write_text(yaml.safe_dump({"endpoints": [ + {"address": "10.0.0.1", "port": "8200"}, + {"address": "10.0.0.2", "port": "8200"}, + {"address": "10.0.0.3", "port": "8000"}, + ]})) + requests = tmp_path / "metrics-requests.txt" env = dict(os.environ, INFMAX_CONTAINER_WORKSPACE=str(tmp_path), - MODEL_NAME="deepseek-ai/DeepSeek-V4-Pro-0813", + REAL_BENCHMARK_LIB=str(REPO_ROOT / "benchmarks/benchmark_lib.sh"), + PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], + METRICS_BODY=metrics_body, METRICS_REQUESTS=str(requests), + LLMD_ENDPOINTS_FILE=str(endpoints), MODEL_NAME="test-model", MODEL_PREFIX="dsv4", + FRAMEWORK="llmd-vllm", DURATION="3600", IS_AGENTIC="1", KV_OFFLOADING="none", ENVOY_PORT="8080", VLLM_PORT="8200", BENCHMARK_LOGS_DIR=str(tmp_path / "logs"), - BENCH_MAX_CONCURRENCY="64", ALL_IPS="10.0.0.1,10.0.0.2", - PREFILL_NODES="1", DECODE_NODES="1", PREFILL_WORKERS="1", DECODE_WORKERS="1", - SPEC_DECODING="mtp", KV_OFFLOADING="dram", KV_OFFLOAD_BACKEND="mooncake") + BENCH_MAX_CONCURRENCY="64") result = subprocess.run(["bash", str(REPO_ROOT / "benchmarks/multi_node/llm-d/agentic.sh")], env=env, text=True, capture_output=True) + if metrics_body.startswith("envoy_"): + assert result.returncode != 0 + assert "no vLLM metrics exposed" in result.stderr + return assert result.returncode == 0, result.stderr - recorded = json.loads(result.stdout) - assert recorded["MODEL"] == env["MODEL_NAME"] - assert recorded["AIPERF_SERVER_URL"] == "http://localhost:8080" - assert recorded["PORT"] == "8200" - assert recorded["AIPERF_SERVER_METRICS_URLS"] == "http://10.0.0.1:8200/metrics,http://10.0.0.2:8200/metrics" - assert recorded["AGENTIC_OUTPUT_DIR"] == env["BENCHMARK_LOGS_DIR"] - assert recorded["CONC"] == "64" - assert recorded["SPEC_DECODING"] == "mtp" - assert recorded["KV_OFFLOADING"] == "dram" + recorded = json.loads(result.stdout.splitlines()[-1]) + expected_urls = ["http://10.0.0.1:8200/metrics", "http://10.0.0.2:8200/metrics", "http://10.0.0.3:8200/metrics"] + assert requests.read_text().splitlines() == expected_urls + assert recorded["AIPERF_METRIC_URLS"].split(",") == expected_urls + assert recorded["AIPERF_SERVER_METRICS_URLS"].split(",") == expected_urls + assert "--url http://localhost:8080 " in recorded["REPLAY_CMD"] + assert "--server-metrics " + " ".join(expected_urls) + " " in recorded["REPLAY_CMD"] def test_patch_srt_eval_dispatch_forwards_selection_and_is_idempotent( From f6cfad925b27d932f037f10bb5143af435d42085 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 10:31:22 -0500 Subject: [PATCH 13/46] fix(llm-d): use the B200 Nscale shared home for cached artifacts --- benchmarks/llm-d/binaries-b200-v0.10.0.env | 4 ++-- runners/launch_b200-nscale-compat.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/llm-d/binaries-b200-v0.10.0.env b/benchmarks/llm-d/binaries-b200-v0.10.0.env index b04199da96..dec7615293 100644 --- a/benchmarks/llm-d/binaries-b200-v0.10.0.env +++ b/benchmarks/llm-d/binaries-b200-v0.10.0.env @@ -10,7 +10,7 @@ # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env \ # benchmarks/llm-d/extract-binaries.sh # Run once (or whenever these pins change) on a host with docker + registry -# access and the /home/sa-shared filesystem mounted. +# access and the /data/home/sa-shared filesystem mounted. # --- Endpoint Picker (EPP / inference scheduler) --- EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0" @@ -30,4 +30,4 @@ ENVOY_BIN_PATH="/usr/local/bin/envoy" LLMD_BIN_PLATFORM="linux/amd64" # Dedicated path so this never collides with GB200's shared bin dir. -LLMD_BIN_DIR="/home/sa-shared/llm-d-bins-v0.10.0" +LLMD_BIN_DIR="/data/home/sa-shared/llm-d-bins-v0.10.0" diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index b689c88690..f71dceba66 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -158,7 +158,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then local enroot_uri enroot_uri=$(llmd_enroot_uri_for_image "$image") || exit 1 ( - exec 9>"$lock" + exec 9>"$lock" || exit 1 flock -w 1800 9 || { echo "Failed to acquire lock for $squash" >&2; exit 1; } if unsquashfs -l "$squash" > /dev/null 2>&1; then echo "Squash file already exists and is valid, skipping import: $squash" @@ -172,8 +172,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then ) || exit 1 } - LLMD_SQUASH_DIR="/home/sa-shared/containers" - mkdir -p "$LLMD_SQUASH_DIR" 2>/dev/null || true + LLMD_SQUASH_DIR="/data/home/sa-shared/containers" + mkdir -p "$LLMD_SQUASH_DIR" || exit 1 LLMD_SQUASH_FILE="${LLMD_SQUASH_DIR}/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" llmd_import_squash "$LLMD_SQUASH_FILE" "$IMAGE" @@ -194,7 +194,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # shape. Populate once via: # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env benchmarks/llm-d/extract-binaries.sh # job.slurm falls back to the image-baked v0.9.0 until this path exists. - export LLMD_BIN_DIR="/home/sa-shared/llm-d-bins-v0.10.0" + export LLMD_BIN_DIR="/data/home/sa-shared/llm-d-bins-v0.10.0" else SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" fi From 1758eaca8bf1bab281631a6a1331bb13983517f9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 10:37:06 -0500 Subject: [PATCH 14/46] fix(llm-d): match AgentX router metadata to the bundled version --- configs/nvidia-master.yaml | 11 +++++------ docs/configuration-procedures.md | 2 ++ docs/configuration-procedures_zh.md | 2 ++ perf-changelog.yaml | 2 +- runners/launch_b200-nscale-compat.sh | 3 ++- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 92703f95f0..f3dd98ebd9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4503,8 +4503,8 @@ dsv4-fp4-gb200-llmd-vllm: - "GPUS_PER_NODE=4" # DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, P/D disagg via llmd-vllm. -# Always uses Mooncake; DSpark speculative decoding. Requires EPP v0.10.0 -# (LLMD_BIN_DIR override in runners/launch_b200-nscale-compat.sh). +# Always uses Mooncake; DSpark speculative decoding. The DSpark image +# bundles EPP/pd-sidecar v0.10.0. dsv4-fp4-b200-llmd-vllm-agentx: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -4546,9 +4546,8 @@ dsv4-fp4-b200-llmd-vllm-agentx: # Aggregated (TP8 / DEP8) sibling of dsv4-fp4-b200-llmd-vllm-agentx: one # engine per topology does both prefill and decode (no P/D split, no # pd-sidecar; only the Mooncake arm uses a KV store connector). Separate top-level key because `disagg:` is a whole-key -# property. Uses the default v0.9.0 EPP/pd-sidecar binaries (no router -# bump needed - the aggregated EPP config only uses plugins already -# present there). conc-lists mirror the agentX dspark historical runs: +# property. Uses the same image-bundled v0.10.0 router as disaggregated runs. +# conc-lists mirror the agentX dspark historical runs: # results/dsv4-pro-dspark/results_ilmarkov-2-ix-agg-tp8-dspark-v3-long # (tp8) and results_ilmarkov-ix-agg-dep8-dspark-1308-long (dep8). dsv4-fp4-b200-llmd-vllm-agentx-agg: @@ -4558,7 +4557,7 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: runner: cluster:b200-nscale precision: fp4 framework: llmd-vllm - router: { name: llm-d-router, version: "0.9.0" } + router: { name: llm-d-router, version: "0.10.0" } multinode: true disagg: false scenarios: diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 7ca83eef1c..ee0ee0bbed 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -146,6 +146,8 @@ For the B200 DSpark AgentX keys, select `agentic-coding`, `spec-decoding: mtp`, llm-d exports the discovered vLLM `/metrics` endpoints as `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, checks that each returns vLLM metrics, and forwards them to AIPerf through `--server-metrics`. Scrapes use each serving node’s vLLM port, not Envoy or the P/D sidecar. +Both B200 DSpark keys declare router v0.10.0, matching the DSpark image's bundled EPP/pd-sidecar. Verify that the serving image publishes `linux/amd64` before running on B200 Nscale; an ARM64-only GB200 build is not compatible. + ## Update an image Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants), the matching master configs, runtime scripts, and checked-in recipes. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 4cdbfc11d2..ee85389937 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -146,6 +146,8 @@ B200 DSpark AgentX 配置使用 `agentic-coding`、`spec-decoding: mtp`，并显 llm-d 将发现的 vLLM `/metrics` 端点导出为 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，确认每个端点返回 vLLM 指标后，通过 `--server-metrics` 传给 AIPerf。抓取使用各服务节点的 vLLM 端口，而非 Envoy 或 P/D sidecar。 +两个 B200 DSpark 配置均声明 router v0.10.0，与 DSpark 镜像内置的 EPP/pd-sidecar 一致。在 B200 Nscale 上运行前，确认服务镜像提供 `linux/amd64` 版本；仅支持 ARM64 的 GB200 镜像不兼容。 + ## 更新镜像 来源：[`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants)、对应主配置、运行时脚本与检入的 Recipe。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7bf6516461..f5438506a8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6828,7 +6828,7 @@ scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache for B200 AgentX trace replay at native context: aggregated TP8/DEP8 (8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (16 GPUs)." + - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache and llm-d router 0.10.0 for B200 AgentX trace replay at native context: aggregated TP8/DEP8 (8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (16 GPUs)." - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification." - "Enable Mooncake DRAM offloading for disaggregated and aggregated DEP8+Mooncake variants with dram-utilization=0.60; no offloading for plain TP8/DEP8. SSD offloading remains disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index f71dceba66..9fe40d072e 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -193,7 +193,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # EPP/pd-sidecar binaries needed for the disagg-profile-handler `deciders:` # shape. Populate once via: # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env benchmarks/llm-d/extract-binaries.sh - # job.slurm falls back to the image-baked v0.9.0 until this path exists. + # The DSpark image already bundles v0.10.0; any mounted copies + # must use the same version as the master config. export LLMD_BIN_DIR="/data/home/sa-shared/llm-d-bins-v0.10.0" else SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" From 4e5f6c69d6173dc668afd557c8d6093b38644cf5 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 10:51:16 -0500 Subject: [PATCH 15/46] fix(llm-d): move AgentX to GB200 and simplify launch paths --- benchmarks/llm-d/binaries-b200-v0.10.0.env | 33 --- benchmarks/llm-d/extract-binaries.sh | 17 +- .../multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh | 71 ------ .../dsv4_fp4_b200_llmd-vllm-disagg.sh | 63 ------ .../dsv4_fp4_gb200_llmd-vllm-agg.sh | 9 + .../dsv4_fp4_gb200_llmd-vllm-disagg.sh | 63 +----- .../agg-gb200-dep8-dspark-agentic.yaml | 4 +- ...gg-gb200-dep8-dspark-mooncake-agentic.yaml | 6 +- .../agentic/agg-gb200-tp8-dspark-agentic.yaml | 6 +- ...g-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 8 +- benchmarks/multi_node/llm-d/README.md | 27 +-- benchmarks/multi_node/llm-d/job.slurm | 30 +-- benchmarks/multi_node/llm-d/recipe.py | 7 +- benchmarks/multi_node/llm-d/server.sh | 97 +++------ benchmarks/multi_node/llm-d/submit.sh | 14 +- configs/nvidia-master.yaml | 56 +++-- docs/configuration-procedures.md | 6 +- docs/configuration-procedures_zh.md | 6 +- perf-changelog.yaml | 6 +- runners/launch_b200-nscale-compat.sh | 205 +++--------------- runners/launch_gb200-nv.sh | 30 ++- 21 files changed, 167 insertions(+), 597 deletions(-) delete mode 100644 benchmarks/llm-d/binaries-b200-v0.10.0.env delete mode 100755 benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh delete mode 100755 benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh create mode 100644 benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-agg.sh diff --git a/benchmarks/llm-d/binaries-b200-v0.10.0.env b/benchmarks/llm-d/binaries-b200-v0.10.0.env deleted file mode 100644 index dec7615293..0000000000 --- a/benchmarks/llm-d/binaries-b200-v0.10.0.env +++ /dev/null @@ -1,33 +0,0 @@ -# B200-only binary pins for the agentX-ported EPP config used by -# dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml (disagg-profile-handler -# `deciders:` shape + inflight-load-producer / approx-prefix-cache-producer / -# prefix-cache-affinity-filter / token-load-scorer plugins), which require -# the v0.10.0 EPP/pd-sidecar router build. Kept as a sibling file (not a -# change to binaries.env) so GB200's shared v0.9.0 pins and bin path are -# completely untouched. -# -# Pull with extract-binaries.sh instead of binaries.env's defaults: -# BINARIES_ENV_FILE=binaries-b200-v0.10.0.env \ -# benchmarks/llm-d/extract-binaries.sh -# Run once (or whenever these pins change) on a host with docker + registry -# access and the /data/home/sa-shared filesystem mounted. - -# --- Endpoint Picker (EPP / inference scheduler) --- -EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0" -EPP_BIN_PATH="/app/epp" - -# --- P/D routing sidecar (pd-sidecar) --- -ROUTING_SIDECAR_IMAGE="ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0" -ROUTING_SIDECAR_BIN_PATH="/app/pd-sidecar" - -# --- Envoy front proxy --- -# Unchanged from binaries.env; re-declared here so this file is -# self-contained (extract-binaries.sh always extracts all three). -ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2" -ENVOY_BIN_PATH="/usr/local/bin/envoy" - -# B200 is x86_64 (unlike GB200's Grace/arm64, binaries.env's default). -LLMD_BIN_PLATFORM="linux/amd64" - -# Dedicated path so this never collides with GB200's shared bin dir. -LLMD_BIN_DIR="/data/home/sa-shared/llm-d-bins-v0.10.0" diff --git a/benchmarks/llm-d/extract-binaries.sh b/benchmarks/llm-d/extract-binaries.sh index a97a42a375..ed9f00e577 100755 --- a/benchmarks/llm-d/extract-binaries.sh +++ b/benchmarks/llm-d/extract-binaries.sh @@ -10,22 +10,15 @@ # benchmarks/llm-d/binaries.env. Idempotent: overwrites in place. # # Usage: -# BINARIES_ENV_FILE=binaries.env ./extract-binaries.sh -# BINARIES_ENV_FILE=binaries.env LLMD_BIN_DIR=/some/dir ./extract-binaries.sh -# BINARIES_ENV_FILE=binaries.env LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh -# BINARIES_ENV_FILE=binaries-b200-v0.10.0.env ./extract-binaries.sh -# # pull a different set of image pins (e.g. a router-version bump -# # scoped to one cluster) without touching the shared binaries.env +# ./extract-binaries.sh # uses binaries.env defaults +# LLMD_BIN_DIR=/some/dir ./extract-binaries.sh +# LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh # for an x86 test -set -eo pipefail +set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -: "${BINARIES_ENV_FILE:?Set binaries.env or a cluster-specific pins file}" -# Resolve a bare filename against this script's directory, so -# BINARIES_ENV_FILE=foo.env works regardless of the caller's cwd. -[[ "$BINARIES_ENV_FILE" != /* ]] && BINARIES_ENV_FILE="$HERE/$BINARIES_ENV_FILE" # shellcheck source=/dev/null -source "$BINARIES_ENV_FILE" +source "$HERE/binaries.env" echo "Extracting llm-d binaries -> $LLMD_BIN_DIR (platform $LLMD_BIN_PLATFORM)" mkdir -p "$LLMD_BIN_DIR" diff --git a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh deleted file mode 100755 index c326b1058c..0000000000 --- a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# -# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm aggregated benchmark -# (agentX-flavored TP8 / DEP8, one engine does both prefill and decode). -# Sibling of dsv4_fp4_b200_llmd-vllm-disagg.sh - same shape, but always -# submits with DECODE_NODES=0 (no decode role at all; see the aggregated -# recipes' header comments and server.sh's IS_AGGREGATED handling). The -# runner resolves this script via -# SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" -# from launch_b200-dgxc-slurm.sh when DISAGG=false. - -set -eo pipefail - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - IMAGE \ - MODEL_PATH \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO - -if [[ "${IS_AGENTIC}" == "1" ]]; then - check_env_vars DURATION KV_OFFLOADING - # Positional submit.sh placeholders; AgentX never uses fixed token lengths. - ISL=0 - OSL=0 -else - check_env_vars ISL OSL -fi - -if [[ -n "${SLURM_JOB_ID}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$DECODE_NODES" != "0" ]]; then - echo "Error: dsv4_fp4_b200_llmd-vllm-agg.sh requires DECODE_NODES=0 (got $DECODE_NODES); aggregated mode has no decode role" >&2 - exit 1 -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 - -# B200 DGX = 8 GPUs per node. -export GPUS_PER_NODE="8" - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# Aggregated is always a single engine (no multi-engine high-tpt split), so -# PREFILL_WORKERS is always 1. DECODE_WORKERS is unused (DECODE_NODES=0) but -# still exported since submit.sh/server.sh read it unconditionally. -export PREFILL_WORKERS="1" -export DECODE_WORKERS="1" - -JOB_ID=$(bash ./submit.sh \ - "$PREFILL_NODES" \ - "$DECODE_NODES" \ - "$ISL" "$OSL" "${CONC_LIST// /x}" inf \ - "$RANDOM_RANGE_RATIO") - -if [[ -z "$JOB_ID" ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh b/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh deleted file mode 100755 index 93cbf89238..0000000000 --- a/benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# -# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm P/D disagg benchmark -# (agentX-flavored 1P-DEP8/1D-DEP8). Sibling of -# dsv4_fp4_gb200_llmd-vllm-disagg.sh - same shape, different topology -# (B200 = 8 GPUs/node, so each DEP8 role fits on ONE node; GB200 = 4 -# GPUs/node, role spans 2 nodes). The runner resolves this script via -# SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" -# from launch_b200-dgxc-slurm.sh. - -set -eo pipefail - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - IMAGE \ - MODEL_PATH \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO - -if [[ "${IS_AGENTIC}" == "1" ]]; then - check_env_vars DURATION KV_OFFLOADING - # Positional submit.sh placeholders; AgentX never uses fixed token lengths. - ISL=0 - OSL=0 -else - check_env_vars ISL OSL -fi - -if [[ -n "${SLURM_JOB_ID}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 - -# B200 DGX = 8 GPUs per node. -export GPUS_PER_NODE="8" - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# Worker counts come from the generated matrix. -export PREFILL_WORKERS="$PREFILL_NUM_WORKERS" -export DECODE_WORKERS="$DECODE_NUM_WORKERS" - -JOB_ID=$(bash ./submit.sh \ - "$PREFILL_NODES" \ - "$DECODE_NODES" \ - "$ISL" "$OSL" "${CONC_LIST// /x}" inf \ - "$RANDOM_RANGE_RATIO") - -if [[ -z "$JOB_ID" ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-agg.sh b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-agg.sh new file mode 100644 index 0000000000..772ce13802 --- /dev/null +++ b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-agg.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eo pipefail + +export GPUS_PER_NODE=4 TIME_LIMIT=08:00:00 CONTAINER_IMAGE="$IMAGE" +export PREFILL_WORKERS=1 DECODE_WORKERS=1 + +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" +exec bash ./submit.sh "$PREFILL_NODES" "$DECODE_NODES" \ + "$ISL" "$OSL" "${CONC_LIST// /x}" inf "$RANDOM_RANGE_RATIO" diff --git a/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh index f3eca3f27b..58ce0262a5 100755 --- a/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh +++ b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh @@ -1,60 +1,9 @@ #!/usr/bin/env bash -# -# Wrapper for the DeepSeek-V4-Pro GB200 llmd-vllm P/D disagg benchmark -# (mid-curve 1P1D and high-tpt 2P1D). Sibling of gptoss_fp4_h200_llmd-vllm.sh - -# same shape, different topology (GB200 = 4 GPUs/node, role spans 2 nodes; -# H200 = 8 GPUs/node, role on a single node). The runner resolves this script via -# SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh" -# from launch_gb200-nv.sh. +set -eo pipefail -set -euo pipefail +export GPUS_PER_NODE=4 TIME_LIMIT=08:00:00 CONTAINER_IMAGE="$IMAGE" +export PREFILL_WORKERS="$PREFILL_NUM_WORKERS" DECODE_WORKERS="$DECODE_NUM_WORKERS" -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 - -# GB200 = 4 GPUs per node (Grace+Blackwell). The shared submit.sh -# defaults GPUS_PER_NODE to 8, which is wrong for this SKU and would -# overshoot DP_SIZE = nodes * 8. -export GPUS_PER_NODE="${GPUS_PER_NODE:-4}" - -export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# Worker count per role (Option B multi-engine). Prefer an explicit -# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else fall -# back to the matrix num-worker fields (PREFILL_NUM_WORKERS/DECODE_NUM_WORKERS); -# else 1 (single engine = unchanged 1P+1D / mid-curve). submit.sh reads these. -export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}" -export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}" - -JOB_ID=$(bash ./submit.sh \ - "$PREFILL_NODES" \ - "$DECODE_NODES" \ - "$ISL" "$OSL" "${CONC_LIST// /x}" inf \ - "$RANDOM_RANGE_RATIO") - -if [[ -z "$JOB_ID" ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" +exec bash ./submit.sh "$PREFILL_NODES" "$DECODE_NODES" \ + "$ISL" "$OSL" "${CONC_LIST// /x}" inf "$RANDOM_RANGE_RATIO" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index f372e73c4c..85963725ec 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated DEP8, 1 node. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated DEP8, 2 nodes. # DSpark speculative decoding: 5 tokens. No Mooncake. apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -36,8 +36,6 @@ dataLayer: prefill: tp: 1 enable-expert-parallel: true - # No workload-length or sequence cap: use native context and vLLM scheduling defaults. - # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --gpu-memory-utilization 0.92 diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 28f9b58ee9..8607e861f1 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated DEP8 with Mooncake prefix-cache. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated DEP8 with Mooncake prefix-cache. # Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds Mooncake (P2PHANDSHAKE embedded RDMA) # so server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role). apiVersion: llm-d.ai/v1alpha1 @@ -38,8 +38,6 @@ dataLayer: prefill: tp: 1 enable-expert-parallel: true - # No workload-length or sequence cap: use native context and vLLM scheduling defaults. - # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --gpu-memory-utilization 0.92 @@ -80,8 +78,6 @@ prefill: mooncake: store_config: metadata_server: "P2PHANDSHAKE" - # Standalone fallback; AgentX uses the master YAML per-node DRAM budget / 8 GPUs. - global_segment_size: "140GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml index 282af9b7b1..32fae3f735 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -1,5 +1,5 @@ -# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated TP8, 1 node. -# DSpark speculative decoding: 5 tokens. No Mooncake (single-node TP8). +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated TP8, 2 nodes. +# DSpark speculative decoding: 5 tokens. No Mooncake (two-node TP8). apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -37,8 +37,6 @@ prefill: tp: 8 # Must be explicit false: server.sh defaults enable-expert-parallel to true. enable-expert-parallel: false - # No workload-length or sequence cap: use native context and vLLM scheduling defaults. - # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --gpu-memory-utilization 0.92 diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 653e151abb..ba109bca54 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, P/D disagg 1P-DEP8/1D-DEP8. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, P/D disagg 1P-DEP8/1D-DEP8 (4 nodes). # Always uses Mooncake (P2PHANDSHAKE embedded RDMA prefix-cache). # DSpark speculative decoding: prefill 1 token, decode 5 tokens. # Requires EPP/pd-sidecar v0.10.0 (disagg-profile-handler deciders shape). @@ -67,8 +67,6 @@ dataLayer: prefill: tp: 1 enable-expert-parallel: true - # No workload-length or sequence cap: use native context and vLLM scheduling defaults. - # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --enforce-eager @@ -108,8 +106,6 @@ prefill: decode: tp: 1 enable-expert-parallel: true - # No workload-length or sequence cap: use native context and vLLM scheduling defaults. - # recipe.py injects DSpark golden AL for AgentX throughput (never accuracy evals). extra-args: >- --kv-cache-dtype fp8 --max-num-batched-tokens 8192 @@ -147,8 +143,6 @@ decode: mooncake: store_config: metadata_server: "P2PHANDSHAKE" - # Standalone fallback; AgentX uses the master YAML per-node DRAM budget / 8 GPUs. - global_segment_size: "140GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" diff --git a/benchmarks/multi_node/llm-d/README.md b/benchmarks/multi_node/llm-d/README.md index 8f914eddd4..37180d4bf6 100644 --- a/benchmarks/multi_node/llm-d/README.md +++ b/benchmarks/multi_node/llm-d/README.md @@ -27,25 +27,14 @@ the coordinator (EPP + Envoy + bench), exactly like the AMD path's ### Aggregated mode (`yD = 0`) -Setting `DECODE_NODES=0` runs a single engine that does both prefill and -decode in-process - there is no decode role, no pd-sidecar, and no -`--kv_transfer_config` (server.sh's `IS_AGGREGATED` flag, derived from -`DECODE_NODES=0`, gates all three). Rank `0` (the sole engine's leader) -takes over the coordinator duties (endpoints.yaml + EPP + Envoy + bench) -that the decode leader normally owns. - -Per the `process_result.py` / `dynamo-vllm` precedent, the aggregated -engine is mapped to the **prefill** role in the master config -(`prefill: {num-worker: 1, ...}`, `decode: {num-worker: 0, ...}`) so -multinode result aggregation counts its GPUs exactly once and -`process_result.py`'s `prefill_gpus > 0` invariant holds. `endpoints.yaml` -still only ever emits `llm-d.ai/role: prefill` entries in this mode (no -`decode-*` entries at all), so an aggregated recipe's EPP config needs a -filter matching that role, not a `decode-filter` / disagg split - see -`benchmarks/multi_node/llm-d-recipes/dsv4-fp4-b200-agentx-agg-{tp8,dep8}.yaml` -for a minimal single-`schedulingProfile` example (llm-d auto-selects -`single-profile-handler` when there is exactly one profile, so no -`disagg-profile-handler`/decider is needed). +`DECODE_NODES=0` runs one engine for prefill and decode. Rank 0 runs EPP, +Envoy, and the client; no P/D sidecar is started. Only the Mooncake variant +uses a KV connector. The master config uses `disagg: false`, `worker`, and +`num-nodes`; discovery labels its serving endpoints `prefill`. + +The GB200 AgentX recipes live under `llm-d-recipes/agentic/`. TP8/DEP8 uses +2 nodes (8 GPUs); 1P-DEP8/1D-DEP8 uses 4 nodes (16 GPUs). Pure TP publishes +only its leader's API endpoint; DEP publishes an endpoint on every node. Each instance (prefill or decode) is one vLLM engine spanning multiple nodes via `--data-parallel-hybrid-lb`. With `xP=2, yD=2, diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index cb81b5b72d..526a69a839 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -48,11 +48,7 @@ fi # prefill workers = ranks 1 .. PREFILL_NODES-1 # decode leader = rank PREFILL_NODES (also coordinator: EPP + Envoy + bench) # decode workers = ranks PREFILL_NODES+1 .. NUM_NODES-1 -# -# Aggregated mode (DECODE_NODES=0): there is no decode role at all, so -# IPS[$PREFILL_NODES] would be an out-of-bounds read (IPS only has -# PREFILL_NODES entries). Leave the decode addresses empty; server.sh's -# aggregated coordinator gate (rank 0, ROLE=prefill) never reads them. +# Aggregated mode has no decode address. PREFILL_LEADER_IP="${IPS[0]}" if [[ "$DECODE_NODES" -gt 0 ]]; then DECODE_LEADER_IP="${IPS[$PREFILL_NODES]}" @@ -250,23 +246,15 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro" PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro" - # Optional: mount the epp / pd-sidecar / envoy binaries from a shared - # filesystem instead of relying on them being baked into the image. - # This lets a STOCK vllm/vllm-openai image be used directly (no - # combined-image rebuild per vLLM version bump) - see - # benchmarks/llm-d/binaries.env + extract-binaries.sh. Each mount is - # gated on the file existing, so this is a no-op when the binaries - # have not been extracted (the baked-image path keeps working), and - # harmless when they have (mounting a binary over the identical one). - # shellcheck source=/dev/null - [[ -f "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" ]] && \ + # AgentX uses the image-bundled router; legacy runs can mount extracted binaries. + if [[ "$IS_AGENTIC" != "1" ]]; then source "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" - for _bin in epp pd-sidecar envoy; do - if [[ -n "${LLMD_BIN_DIR}" && -x "${LLMD_BIN_DIR}/${_bin}" ]]; then - PYXIS_MOUNTS+=",${LLMD_BIN_DIR}/${_bin}:/usr/local/bin/${_bin}:ro" - echo "Mounting ${LLMD_BIN_DIR}/${_bin} -> /usr/local/bin/${_bin}" - fi - done + for _bin in epp pd-sidecar envoy; do + if [[ -x "${LLMD_BIN_DIR}/${_bin}" ]]; then + PYXIS_MOUNTS+=",${LLMD_BIN_DIR}/${_bin}:/usr/local/bin/${_bin}:ro" + fi + done + fi # MODEL_DIR / BENCHMARK_LOGS_DIR / NODE_RANK are translated to their # in-container values inside bash -lc (host MODEL_DIR is the source diff --git a/benchmarks/multi_node/llm-d/recipe.py b/benchmarks/multi_node/llm-d/recipe.py index 59f1fda5f9..da0d0ffc56 100644 --- a/benchmarks/multi_node/llm-d/recipe.py +++ b/benchmarks/multi_node/llm-d/recipe.py @@ -37,11 +37,11 @@ def role_assignments(recipe: dict, role: str, env: dict) -> str: if config.get("method") == "dspark": if env.get("SPEC_DECODING") != "mtp": raise ValueError("DSpark requires SPEC_DECODING=mtp in the master YAML") - if env.get("EVAL_ONLY", "false") == "true": + if env.get("EVAL_ONLY") == "true": config.pop("synthetic_acceptance_length", None) config.pop("rejection_sample_method", None) else: - if env.get("RUN_EVAL", "false") == "true": + if env.get("RUN_EVAL") == "true": raise ValueError("Run accuracy evals separately with EVAL_ONLY=true, not synthetic AL") if env.get("MODEL_NAME") != "deepseek-ai/DeepSeek-V4-Pro-0813": raise ValueError("No registered DSpark golden AL for this model") @@ -56,7 +56,8 @@ def role_assignments(recipe: dict, role: str, env: dict) -> str: ) print(f"DSpark {role}: K={k}, golden AL={al} ({golden_path.name})", file=sys.stderr) extra = extra[:match.end()] + json.dumps(config, separators=(",", ":")) + extra[match.end() + length:] - assignments = [f"ROLE_EXTRA_ARGS={shlex.quote(extra)}"] + assignments = [f"ROLE_EXTRA_ARGS={shlex.quote(extra)}", + f"PREFILL_ENABLE_EP={str(recipe.get('prefill', {}).get('enable-expert-parallel', True)).lower()}"] if section.get("tp") is not None: assignments.append(f"TP_SIZE={int(section['tp'])}") if section.get("enable-expert-parallel") is not None: diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 0d8ef795e5..b873b62e2f 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -22,15 +22,7 @@ source /workspace/benchmarks/benchmark_lib.sh # Config + service ports # ---------------------------------------------------------------- check_env_vars NODE_RANK PREFILL_NODES DECODE_NODES GPUS_PER_NODE PREFILL_WORKERS DECODE_WORKERS ALL_IPS -# Aggregated mode: no decode role at all, so no P/D KV handoff (no -# pd-sidecar, no NixlConnector) - the single engine does prefill+decode -# in-process. DECODE_NODES=0 is the only signal for this; everything else -# (coordinator gating, kv_transfer_config) derives from it below. -if [[ "$DECODE_NODES" -eq 0 ]]; then - IS_AGGREGATED=1 -else - IS_AGGREGATED=0 -fi +IS_AGGREGATED=$(( DECODE_NODES == 0 )) VLLM_PORT=8200 SIDECAR_PORT=8000 ENVOY_PORT=8080 @@ -118,6 +110,7 @@ START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) # Defaults: TP=1, DP=role_total, EP on (the H200 1P+1D shape). Recipe overrides below. TP_SIZE=1 ROLE_ENABLE_EP=true +PREFILL_ENABLE_EP=true echo "ROLE=$ROLE DP_SIZE=$DP_SIZE DP_ADDR=$DP_ADDR LWS_WORKER_INDEX=$LWS_WORKER_INDEX START_RANK=$START_RANK" @@ -205,9 +198,6 @@ if [[ -n "${CONFIG_FILE}" && -f "/etc/llmd-recipes/${CONFIG_FILE}" ]]; then MOONCAKE_CONFIG_PATH=/tmp/mooncake_config.json export MOONCAKE_CONFIG_PATH echo "Mooncake enabled: config at $MOONCAKE_CONFIG_PATH" - # Install mooncake if the image does not bundle it. - python3 -c "import mooncake_transfer_engine" 2>/dev/null || \ - pip install --quiet mooncake-transfer-engine-cuda13==0.3.12.post1 fi fi @@ -231,12 +221,7 @@ COMMON_ARGS=( --disable-access-log-for-endpoints=/health,/metrics --tensor-parallel-size "$TP_SIZE" ) -# KV role: prefill=producer, decode=consumer (override via KV_ROLE_OVERRIDE). -# Aggregated mode normally has no second engine to hand KV off to (no -# pd-sidecar either), so --kv_transfer_config is skipped. Exception: when -# Mooncake is enabled on an aggregated recipe, a MultiConnector -# (NixlConnector + MooncakeStoreConnector) is wired with kv_both so DP ranks -# can share prefix-cache blocks across runs via the Mooncake RDMA store. +# Aggregated engines need KV transfer only when Mooncake is enabled. if [[ "$IS_AGGREGATED" -eq 0 ]]; then if [[ -n "${KV_ROLE_OVERRIDE}" ]]; then KV_ROLE="$KV_ROLE_OVERRIDE" @@ -352,9 +337,6 @@ fi # (followers are --headless), so only the leader runs a sidecar and only leaders # are listed as endpoints. # -# Aggregated mode (IS_AGGREGATED=1): ROLE is never "decode" (no decode nodes -# exist at all), so this is already a no-op there - correct, since the single -# engine needs no P/D handoff. if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX" -eq 0 ) ]]; then SIDECAR_CONNECTOR="nixlv2" SIDECAR_FLAGS=(--port="$SIDECAR_PORT" --vllm-port="$VLLM_PORT" @@ -371,10 +353,7 @@ fi # ================================================================ # Coordinator: endpoints, EPP, Envoy, bench, eval # ================================================================ -# Normally the decode leader. In aggregated mode (IS_AGGREGATED=1) there is -# no decode role at all, so the sole engine's leader (rank 0, which is -# always ROLE=prefill there since PREFILL_NODES=NUM_NODES) takes over the -# coordinator duties instead. +# Rank 0 coordinates aggregated runs; the decode leader coordinates P/D runs. if [[ ( "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ) || \ ( "$IS_AGGREGATED" -eq 1 && "$ROLE" == "prefill" && "$NODE_RANK" -eq 0 ) ]]; then @@ -382,52 +361,29 @@ if [[ ( "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ) || \ BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT - # ---- Write endpoints.yaml (file-discovery) ---- - # namespace must match EPP's --pool-namespace (file-discovery filters by it; - # the schema default 'default' would drop every entry). See README.md. + # DEP registers every node; pure TP registers only each engine's API leader. export LLMD_ENDPOINTS_FILE=/tmp/endpoints.yaml python3 - < DEP8 hybrid-LB (an api-server per node); -# EP off => pure-TP (only each TP-group leader has an api-server). -decode_ep = ('$ROLE_ENABLE_EP' == 'true') -VLLM_PORT = int('$VLLM_PORT') -SIDECAR_PORT = int('$SIDECAR_PORT') -# ALL_IPS is rank-ordered: ranks [0:pn] are prefill nodes, [pn:pn+dn] decode. -prefill_ips = all_ips[:pn] or [os.environ['PREFILL_LEADER_IP']] -# dn == 0 (aggregated mode): no decode role at all. DECODE_LEADER_IP is '' -# in that case (see job.slurm), so decode_ips must NOT fall back to it - -# that would emit a bogus decode-0 endpoint with an empty address. -decode_ips = (all_ips[pn:pn + dn] or [os.environ['DECODE_LEADER_IP']]) if dn > 0 else [] +ips = os.environ['ALL_IPS'].split(',') +pn = int(os.environ['PREFILL_NODES']) +dn = int(os.environ['DECODE_NODES']) endpoints = [] -def add_role(role, ips, base_port, group_size=1): - # group_size == 1: one endpoint per node (DEP8 hybrid-LB: each node's - # api-server / sidecar load-balances its local DP ranks). - # group_size > 1: one endpoint per TP-group leader (pure-TP: followers are - # --headless with no api-server), i.e. every group_size-th node IP. - serving_ips = ips[::group_size] if group_size > 1 else ips - for i, ip in enumerate(serving_ips): - endpoints.append({'name': f'{role}-{i}', 'namespace': NS, 'address': ip, - 'port': str(base_port), 'labels': {'llm-d.ai/role': role}}) - -# Prefill (DEP8 in every current recipe): one endpoint per node, EPP hits vLLM -# directly (VLLM_PORT). Decode: EPP hits the pd-sidecar (SIDECAR_PORT); one -# endpoint per node for DEP8, or one per TP-group leader for pure-TP. -add_role('prefill', prefill_ips, VLLM_PORT) -decode_group = 1 if decode_ep else max(1, dn // decode_workers) -# dn == 0 -> decode_ips == [] -> add_role emits zero decode endpoints -# (aggregated mode: everything routes through the 'prefill'-labeled pool). -add_role('decode', decode_ips, SIDECAR_PORT, group_size=decode_group) -yaml.safe_dump({'endpoints': endpoints}, open(os.environ['LLMD_ENDPOINTS_FILE'], 'w')) -print(f'endpoints.yaml ({len(endpoints)} endpoints):') -print(open(os.environ['LLMD_ENDPOINTS_FILE']).read()) +def add_role(role, addresses, port, group_size): + for i, address in enumerate(addresses[::group_size]): + endpoints.append({'name': f'{role}-{i}', 'namespace': 'inferencex', + 'address': address, 'port': str(port), + 'labels': {'llm-d.ai/role': role}}) + +prefill_group = 1 if '$PREFILL_ENABLE_EP' == 'true' else pn // int('$PREFILL_WORKERS') +add_role('prefill', ips[:pn], int('$VLLM_PORT'), prefill_group) +if dn: + decode_group = 1 if '$ROLE_ENABLE_EP' == 'true' else dn // int('$DECODE_WORKERS') + add_role('decode', ips[pn:pn + dn], int('$SIDECAR_PORT'), decode_group) +with open(os.environ['LLMD_ENDPOINTS_FILE'], 'w') as output: + yaml.safe_dump({'endpoints': endpoints}, output) +print(yaml.safe_dump({'endpoints': endpoints})) PY # ---- Bring up EPP ---- @@ -510,8 +466,13 @@ PY # IPS[0]. curl gets an explicit connect/max timeout so a blackholed endpoint # trips the deadline instead of hanging the whole run (a single timeout-less # curl once wedged a 2P run for 7h before it was cancelled). - _prefill_ips=( "${_ALL_IPS[@]:0:${PREFILL_NODES}}" ) - [[ ${#_prefill_ips[@]} -gt 0 ]] || _prefill_ips=( "$PREFILL_LEADER_IP" ) + mapfile -t _prefill_ips < <(python3 - "$LLMD_ENDPOINTS_FILE" <<'PY' +import sys, yaml +for endpoint in yaml.safe_load(open(sys.argv[1]))['endpoints']: + if endpoint['labels']['llm-d.ai/role'] == 'prefill': + print(endpoint['address']) +PY + ) # On failure, dump enough to tell a server-not-ready problem (TCP connects but # /health is slow) apart from a network/subnet problem (TCP connect refused or diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh index 1c08b7a97f..7bea96ccff 100755 --- a/benchmarks/multi_node/llm-d/submit.sh +++ b/benchmarks/multi_node/llm-d/submit.sh @@ -78,20 +78,12 @@ export BENCH_RANDOM_RANGE_RATIO=$RANDOM_RANGE_RATIO # Match the AMD multinode default. export BENCH_NUM_PROMPTS_MULTIPLIER="10" -export RUN_EVAL="${RUN_EVAL}" -export EVAL_ONLY="${EVAL_ONLY}" -export EVAL_CONC="${EVAL_CONC}" +export RUN_EVAL EVAL_ONLY EVAL_CONC export EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE SWEBENCH_GEN_MODE export SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET export IS_AGENTIC SCENARIO_TYPE -export FRAMEWORK="${FRAMEWORK}" -export PRECISION="${PRECISION}" -export MODEL_PREFIX="${MODEL_PREFIX}" -export RUNNER_TYPE="${RUNNER_TYPE}" -export RESULT_FILENAME="${RESULT_FILENAME}" -export SPEC_DECODING="${SPEC_DECODING}" -export IS_MULTINODE="${IS_MULTINODE}" -export CONFIG_FILE="${CONFIG_FILE}" +export FRAMEWORK PRECISION MODEL_PREFIX RUNNER_TYPE RESULT_FILENAME +export SPEC_DECODING IS_MULTINODE CONFIG_FILE # Explicit fleet/container mapping shared by every llm-d wrapper. case "${RUNNER_TYPE#cluster:}" in diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f3dd98ebd9..a3495c4ba5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4502,14 +4502,14 @@ dsv4-fp4-gb200-llmd-vllm: - "DECODE_NODES=2" - "GPUS_PER_NODE=4" -# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, P/D disagg via llmd-vllm. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, P/D disagg via llmd-vllm. # Always uses Mooncake; DSpark speculative decoding. The DSpark image # bundles EPP/pd-sidecar v0.10.0. -dsv4-fp4-b200-llmd-vllm-agentx: - image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly +dsv4-fp4-gb200-llmd-vllm-agentx: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 - runner: cluster:b200-nscale + runner: cluster:gb200-nv precision: fp4 framework: llmd-vllm router: { name: llm-d-router, version: "0.10.0" } @@ -4520,7 +4520,7 @@ dsv4-fp4-b200-llmd-vllm-agentx: agentic-coding: - dram-utilization: 0.60 search-space: - # 1P DEP8 + 1D DEP8 (2 nodes / 16 GPUs). Always Mooncake. + # 1P DEP8 + 1D DEP8 (4 nodes / 16 GPUs). Always Mooncake. - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake } @@ -4531,8 +4531,8 @@ dsv4-fp4-b200-llmd-vllm-agentx: ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=1" - - "GPUS_PER_NODE=8" + - "PREFILL_NODES=2" + - "GPUS_PER_NODE=4" - "CONFIG_FILE=agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml" decode: num-worker: 1 @@ -4540,21 +4540,15 @@ dsv4-fp4-b200-llmd-vllm-agentx: ep: 8 dp-attn: true additional-settings: - - "DECODE_NODES=1" - - "GPUS_PER_NODE=8" - -# Aggregated (TP8 / DEP8) sibling of dsv4-fp4-b200-llmd-vllm-agentx: one -# engine per topology does both prefill and decode (no P/D split, no -# pd-sidecar; only the Mooncake arm uses a KV store connector). Separate top-level key because `disagg:` is a whole-key -# property. Uses the same image-bundled v0.10.0 router as disaggregated runs. -# conc-lists mirror the agentX dspark historical runs: -# results/dsv4-pro-dspark/results_ilmarkov-2-ix-agg-tp8-dspark-v3-long -# (tp8) and results_ilmarkov-ix-agg-dep8-dspark-1308-long (dep8). -dsv4-fp4-b200-llmd-vllm-agentx-agg: - image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly + - "DECODE_NODES=2" + - "GPUS_PER_NODE=4" + +# Aggregated TP8/DEP8; only the Mooncake arm enables DRAM offloading. +dsv4-fp4-gb200-llmd-vllm-agentx-agg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 - runner: cluster:b200-nscale + runner: cluster:gb200-nv precision: fp4 framework: llmd-vllm router: { name: llm-d-router, version: "0.10.0" } @@ -4564,51 +4558,51 @@ dsv4-fp4-b200-llmd-vllm-agentx-agg: agentic-coding: - dram-utilization: 0.60 search-space: - # Aggregated TP8 (1 node / 8 GPUs; pure tensor-parallel, no EP). + # Aggregated TP8 (2 nodes / 8 GPUs; pure tensor-parallel, no EP). - spec-decoding: mtp kv-offloading: none conc-list: [1, 2, 4, 8, 12] - num-nodes: 1 + num-nodes: 2 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" + - "PREFILL_NODES=2" - "DECODE_NODES=0" - - "GPUS_PER_NODE=8" + - "GPUS_PER_NODE=4" - "CONFIG_FILE=agentic/agg-gb200-tp8-dspark-agentic.yaml" - # Aggregated DEP8 (1 node / 8 GPUs; DP=8 + EP), no Mooncake. + # Aggregated DEP8 (2 nodes / 8 GPUs; DP=8 + EP), no Mooncake. - spec-decoding: mtp kv-offloading: none conc-list: [12, 16, 32] - num-nodes: 1 + num-nodes: 2 worker: num-worker: 1 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=1" + - "PREFILL_NODES=2" - "DECODE_NODES=0" - - "GPUS_PER_NODE=8" + - "GPUS_PER_NODE=4" - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-agentic.yaml" # Aggregated DEP8 with Mooncake prefix-cache KV store. - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake } conc-list: [52, 72] - num-nodes: 1 + num-nodes: 2 worker: num-worker: 1 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=1" + - "PREFILL_NODES=2" - "DECODE_NODES=0" - - "GPUS_PER_NODE=8" + - "GPUS_PER_NODE=4" - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml" # MTP2 variant of dsv4-fp4-gb200-dynamo-vllm. Uses the vLLM 0.20.1 image diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index ee0ee0bbed..59755e319f 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -142,11 +142,11 @@ llm-d is not the srt-slurm path: InferenceX owns the Slurm allocation and starts A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml` fallback and removes recipe-specific vLLM flags. Treat that as a validation failure unless fallback is explicitly intended. -For the B200 DSpark AgentX keys, select `agentic-coding`, `spec-decoding: mtp`, and explicit `kv-offloading`. `server.sh` uses the shared AgentX client via `llm-d/agentic.sh`, with no fixed context/sequence cap. `recipe.py` loads the committed DSpark thinking-on golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Mooncake's embedded store is **DRAM offloading even with `enable_offload: false`** (that flag controls SSD), so declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides the generated per-node budget among GPU ranks. The plain TP8/DEP8 arms declare `none`. Bash inputs are explicit constants or caller-supplied values, not fallback expressions. Binary extraction requires an explicit `BINARIES_ENV_FILE`. +The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. -llm-d exports the discovered vLLM `/metrics` endpoints as `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, checks that each returns vLLM metrics, and forwards them to AIPerf through `--server-metrics`. Scrapes use each serving node’s vLLM port, not Envoy or the P/D sidecar. +`recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Plain TP8/DEP8 declares `none`. -Both B200 DSpark keys declare router v0.10.0, matching the DSpark image's bundled EPP/pd-sidecar. Verify that the serving image publishes `linux/amd64` before running on B200 Nscale; an ARM64-only GB200 build is not compatible. +Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. ## Update an image diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index ee85389937..0eec3d2a67 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -142,11 +142,11 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 `CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 -B200 DSpark AgentX 配置使用 `agentic-coding`、`spec-decoding: mtp`，并显式声明 `kv-offloading`。`server.sh` 通过 `llm-d/agentic.sh` 运行共享 AgentX 客户端，不设置固定上下文或序列数上限。`recipe.py` 为吞吐测试读取已提交的 DSpark thinking-on golden AL；`EVAL_ONLY=true` 保留真实验证。Mooncake 的嵌入式存储即使设置 `enable_offload: false` 也属于 **DRAM 卸载**（该开关控制 SSD），因此必须声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将生成的每节点内存预算均分给 GPU rank。普通 TP8/DEP8 配置声明 `none`。Bash 输入采用显式常量或调用方提供的值，不使用回退表达式。提取二进制文件时必须显式设置 `BINARIES_ENV_FILE`。 +GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。 -llm-d 将发现的 vLLM `/metrics` 端点导出为 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，确认每个端点返回 vLLM 指标后，通过 `--server-metrics` 传给 AIPerf。抓取使用各服务节点的 vLLM 端口，而非 Envoy 或 P/D sidecar。 +`recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。普通 TP8/DEP8 声明 `none`。 -两个 B200 DSpark 配置均声明 router v0.10.0，与 DSpark 镜像内置的 EPP/pd-sidecar 一致。在 B200 Nscale 上运行前，确认服务镜像提供 `linux/amd64` 版本；仅支持 ARM64 的 GB200 镜像不兼容。 +服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 ## 更新镜像 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f5438506a8..df11b307dd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6823,12 +6823,12 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 - config-keys: - - dsv4-fp4-b200-llmd-vllm-agentx - - dsv4-fp4-b200-llmd-vllm-agentx-agg + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache and llm-d router 0.10.0 for B200 AgentX trace replay at native context: aggregated TP8/DEP8 (8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (16 GPUs)." + - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache and llm-d router 0.10.0 for GB200 AgentX trace replay at native context: aggregated TP8/DEP8 (2 nodes, 8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (4 nodes, 16 GPUs)." - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification." - "Enable Mooncake DRAM offloading for disaggregated and aggregated DEP8+Mooncake variants with dram-utilization=0.60; no offloading for plain TP8/DEP8. SSD offloading remains disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 9fe40d072e..a93f0e5ccb 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -2,8 +2,8 @@ # Compatibility launcher for B200 Nscale configurations that have not yet # moved to the native srt-slurm path in launch_b200-nscale-slurm.sh. -SLURM_PARTITION="batch_1" -SLURM_ACCOUNT="benchmark" +SLURM_PARTITION="${SLURM_PARTITION:-batch_1}" +SLURM_ACCOUNT="${SLURM_ACCOUNT:-benchmark}" POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" @@ -20,11 +20,19 @@ if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/scratch/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" -elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && "$MODEL" == *-0813 ]]; then - export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-0813" - export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro" + SELECTED_MODEL_PATH="" + if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then + SELECTED_MODEL_PATH="$MODEL_PATH" + else + for candidate in /scratch/models/DeepSeek-V4-Pro /scratch/models/DeepSeek-V4-Pro-NVFP4 /scratch/models/DeepSeek-V4-Pro-0813; do + if [[ -d "$candidate" ]]; then + SELECTED_MODEL_PATH="$candidate" + break + fi + done + fi + export MODEL_PATH="${SELECTED_MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "bf16" ]]; then export MODEL_PATH="/scratch/models/Qwen3.5-397B-A17B" @@ -55,7 +63,7 @@ elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/models/GLM-5-NVFP4" export SRT_SLURM_MODEL_PREFIX="glm5-fp4" elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/scratch/models/GLM-5.2-NVFP4" + export MODEL_PATH="${MODEL_PATH:-/scratch/models/GLM-5.2-NVFP4}" export SRT_SLURM_MODEL_PREFIX="glm5.2-fp4" elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then export MODEL_PATH="/scratch/models/Kimi-K2.5" @@ -64,7 +72,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/scratch/models/Kimi-K2.6-NVFP4" + export MODEL_PATH="${MODEL_PATH:-/scratch/models/Kimi-K2.6-NVFP4}" export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/scratch/models/MiniMax-M2.5" @@ -94,158 +102,13 @@ fi export AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/aiperf-cache" if [[ "$IS_MULTINODE" == "true" ]]; then - if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then - # Self-contained: bypasses the srtctl/srt-slurm path entirely (the - # "Validate framework" guard and the dsv4-only-dynamo-vllm guard - # further below are srtctl-specific and are never reached from here). - # MODEL_PATH is already resolved above (the dsv4/fp4 branch); llm-d - # additionally needs MODEL_NAME (served-model-name), which this file - # doesn't otherwise set outside the srtctl block - reuse $MODEL (the - # master-config `model:` field), matching this file's own - # `export SERVED_MODEL_NAME=$MODEL` convention below. - if [[ ! ( "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ) ]]; then - echo "Unsupported MODEL_PREFIX/PRECISION for llmd-vllm on B200: $MODEL_PREFIX/$PRECISION" >&2 - exit 1 - fi - export MODEL_NAME="$MODEL" - - source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" - - # Enroot 3.x does not parse Docker's tag@digest syntax (the llm-d - # image is digest-pinned); build the explicit registry#repo:digest - # URI enroot expects. - llmd_enroot_uri_for_image() { - local image="$1" - local image_without_digest="$image" - local digest="" - local first_component registry repository - - if [[ "$image" == *@sha256:* ]]; then - image_without_digest="${image%@*}" - digest="${image##*@}" - fi - - first_component="${image_without_digest%%/*}" - if [[ "$image_without_digest" == */* && ( "$first_component" == *.* || "$first_component" == *:* || "$first_component" == "localhost" ) ]]; then - registry="$first_component" - repository="${image_without_digest#*/}" - else - registry="registry-1.docker.io" - repository="$image_without_digest" - fi - - if [[ -z "$digest" ]]; then - if [[ "$registry" == "registry-1.docker.io" ]]; then - printf 'docker://%s\n' "$image" - else - printf 'docker://%s#%s\n' "$registry" "$repository" - fi - return - fi - if [[ "$registry" == "registry-1.docker.io" && "$repository" != */* ]]; then - repository="library/$repository" - fi - # Strip any :tag from repository before appending the digest. - repository="${repository%%:*}" - printf 'docker://%s#%s:%s\n' "$registry" "$repository" "$digest" - } - - # Separate name from this file's existing (later) `import_squash` used - # by the srtctl path, to avoid redefining that function. - llmd_import_squash() { - local squash="$1" image="$2" - local lock="${squash}.lock" - local enroot_uri - enroot_uri=$(llmd_enroot_uri_for_image "$image") || exit 1 - ( - exec 9>"$lock" || exit 1 - flock -w 1800 9 || { echo "Failed to acquire lock for $squash" >&2; exit 1; } - if unsquashfs -l "$squash" > /dev/null 2>&1; then - echo "Squash file already exists and is valid, skipping import: $squash" - else - rm -f "$squash" - if ! enroot import -o "$squash" "$enroot_uri"; then - echo "Error: enroot import failed for $enroot_uri" >&2 - exit 1 - fi - fi - ) || exit 1 - } - - LLMD_SQUASH_DIR="/data/home/sa-shared/containers" - mkdir -p "$LLMD_SQUASH_DIR" || exit 1 - LLMD_SQUASH_FILE="${LLMD_SQUASH_DIR}/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - llmd_import_squash "$LLMD_SQUASH_FILE" "$IMAGE" - - export LLMD_CONTAINER_ENGINE=pyxis - export LLMD_SQUASH_FILE - - export DOCKER_IMAGE_NAME=$IMAGE - export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" - mkdir -p "$BENCHMARK_LOGS_DIR" - - # DISAGG is exported job-wide from the master config's `disagg:` key - # (see benchmark-multinode-tmpl.yml). true -> the P/D disagg wrapper - # (1P-DEP8/1D-DEP8); false -> the aggregated wrapper (TP8/DEP8, one - # engine does both prefill and decode, DECODE_NODES=0). - if [[ "${DISAGG}" == "true" ]]; then - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh" - # EPP/pd-sidecar binaries needed for the disagg-profile-handler `deciders:` - # shape. Populate once via: - # BINARIES_ENV_FILE=binaries-b200-v0.10.0.env benchmarks/llm-d/extract-binaries.sh - # The DSpark image already bundles v0.10.0; any mounted copies - # must use the same version as the master config. - export LLMD_BIN_DIR="/data/home/sa-shared/llm-d-bins-v0.10.0" - else - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh" - fi - BENCH_SCRIPT="benchmarks/multi_node/${SCRIPT_NAME}" - if [[ ! -f "$BENCH_SCRIPT" ]]; then - echo "Error: llm-d wrapper not found: $BENCH_SCRIPT" >&2 - exit 1 - fi - - export SLURM_PARTITION SLURM_ACCOUNT - JOB_ID=$(bash "$BENCH_SCRIPT") - if [[ -z "$JOB_ID" ]]; then - echo "Error: failed to submit llm-d job" >&2 - exit 1 - fi - echo "Submitted llm-d job: $JOB_ID" - - trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP - - LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 - - if [[ "${IS_AGENTIC}" == "1" && "${EVAL_ONLY}" != "true" ]]; then - copy_agentic_results "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" || exit 1 - # Host-owned artifact staging; containers create no workspace dirs. - mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" - cp -R "$BENCHMARK_LOGS_DIR/agentic/." "$GITHUB_WORKSPACE/LOGS/agentic/" - else - while IFS= read -r -d '' result_file; do - copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 - done < <(find "$BENCHMARK_LOGS_DIR" -name "${RESULT_FILENAME}*.json" -print0 2>/dev/null) - fi - - if [[ "${RUN_EVAL}" == "true" ]]; then - EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR" -type d -name eval_results -print -quit 2>/dev/null) - [[ -z "$EVAL_DIR" ]] && EVAL_DIR="$BENCHMARK_LOGS_DIR/eval_results" - copy_eval_artifacts "$EVAL_DIR" "$GITHUB_WORKSPACE" || exit 1 - fi - - scancel "$JOB_ID" 2>/dev/null || true - exit 0 - fi - if [[ "$FRAMEWORK" == "tilert" ]]; then export SLURM_PARTITION SLURM_ACCOUNT - export TILERT_WEIGHTS_DIR="/scratch/models/${MODEL_PREFIX}-${PRECISION}-tilert-8shard" + export TILERT_WEIGHTS_DIR="${TILERT_WEIGHTS_DIR:-/scratch/models/${MODEL_PREFIX}-${PRECISION}-tilert-8shard}" # Nscale exposes eight RoCE HCAs, mlx5_0..mlx5_7. - export UCX_NET_DEVICES="mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1" - export UCX_MEMTYPE_CACHE="n" - export UCX_MEMTYPE_REG_WHOLE="n" + export UCX_NET_DEVICES="${UCX_NET_DEVICES:-mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1}" + export UCX_MEMTYPE_CACHE="${UCX_MEMTYPE_CACHE:-n}" + export UCX_MEMTYPE_REG_WHOLE="${UCX_MEMTYPE_REG_WHOLE:-n}" TILERT_SUBDIR="multi_node" [[ "${SCENARIO_SUBDIR}" == "agentic/" ]] && TILERT_SUBDIR="multi_node/agentic" TILERT_DISAGG="$GITHUB_WORKSPACE/benchmarks/${TILERT_SUBDIR}/${EXP_NAME%%_*}_${PRECISION}_b200_${FRAMEWORK}-disagg.sh" @@ -267,8 +130,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi USES_DCGM_POWER=0 - _POWER_CONFIG_FILE="${CONFIG_FILE}" - if [[ "${EVAL_ONLY}" == "true" && -n "${EVAL_CONFIG_FILE}" ]]; then + _POWER_CONFIG_FILE="${CONFIG_FILE:-}" + if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then _POWER_CONFIG_FILE="$EVAL_CONFIG_FILE" fi _RECIPE_REL="${_POWER_CONFIG_FILE%%:*}" @@ -283,7 +146,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then USES_DCGM_POWER=1 fi if [[ "$USES_DCGM_POWER" == "1" && ( - "${IS_AGENTIC}" == "1" || + "${IS_AGENTIC:-0}" == "1" || "$MODEL_PREFIX" != "dsv4" || "$PRECISION" != "fp4" || "$FRAMEWORK" != "dynamo-vllm" @@ -364,7 +227,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then cd "$SRT_REPO_DIR" || exit 1 git checkout sa-submission-q2-2026 fi - if [[ "${EVAL_FRAMEWORK}" != "lm-eval" ]]; then + if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 fi @@ -388,9 +251,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # Map container images to local squash files NGINX_IMAGE="nginx:1.27.4" - SQUASH_DIR="/data/home/sa-shared/containers" + SQUASH_DIR="${B200_SQUASH_DIR:-/data/home/sa-shared/containers}" if [[ $MODEL_PREFIX == "minimaxm2.5" && $FRAMEWORK == "dynamo-vllm" ]]; then - SQUASH_DIR="/home/slurm-shared/gharunners/squash" + SQUASH_DIR="${B200_SQUASH_DIR:-/home/slurm-shared/gharunners/squash}" fi if ! mkdir -p "$SQUASH_DIR" 2>/dev/null || [[ ! -w "$SQUASH_DIR" ]]; then echo "Warning: $SQUASH_DIR is not writable; using workspace-local squash cache" >&2 @@ -413,7 +276,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then local lock_file="${lock_dir}/${image_key}.lock" ( - flock -w "600" 9 || { echo "Failed to acquire lock for $squash_file" >&2; exit 1; } + flock -w "${B200_SQUASH_LOCK_TIMEOUT:-600}" 9 || { echo "Failed to acquire lock for $squash_file" >&2; exit 1; } if unsquashfs -l "$squash_file" > /dev/null 2>&1; then echo "Squash file already exists and is valid, skipping import: $squash_file" else @@ -444,7 +307,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export ISL="$ISL" export OSL="$OSL" - export EVAL_ONLY="${EVAL_ONLY}" + export EVAL_ONLY="${EVAL_ONLY:-false}" # Agentic runs bind-mount two persistent caches into every worker # container (Lustre, shared across nodes): aiperf's content-addressed @@ -514,7 +377,7 @@ EOF # throughput row keeps synthetic golden acceptance. Only configs that set # EVAL_CONFIG_FILE opt into this selection; all other configs keep using # CONFIG_FILE unchanged. - if [[ "${EVAL_ONLY}" == "true" && -n "${EVAL_CONFIG_FILE}" ]]; then + if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then CONFIG_FILE="$EVAL_CONFIG_FILE" echo "EVAL_ONLY=true: selecting real-verification recipe $CONFIG_FILE" fi @@ -531,7 +394,7 @@ EOF # so large-model loads (e.g. DSR1-FP8 ~680GB off shared FS) finish in time. # Uses ${CONFIG_FILE%%:*} because CONFIG_FILE may carry an :override[N] suffix. sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" - if [[ "${EVAL_ONLY}" == "true" ]]; then + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ "${CONFIG_FILE%%:*}" "$FRAMEWORK" || exit 1 fi @@ -609,7 +472,7 @@ EOF cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . - if [[ "${EVAL_ONLY}" != "true" ]]; then + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then # Find all result subdirectories RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null) @@ -657,7 +520,7 @@ EOF fi # Collect eval results if eval was requested - if [[ "${RUN_EVAL}" == "true" || "${EVAL_ONLY}" == "true" ]]; then + if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then EVAL_DIR="$LOGS_DIR/eval_results" if [ -d "$EVAL_DIR" ]; then echo "Extracting eval results from $EVAL_DIR" @@ -711,9 +574,9 @@ else # The runner lease reserves the Slurm nodes before this single-node job is # submitted to the Nscale batch_1 partition. - export GPU_COUNT="${GPU_COUNT:?GPU_COUNT must be set by the invoking launcher}" + export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" - SALLOC_TIME_LIMIT="480" + SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}" salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 9799aa3b71..48bcd6f3d7 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -79,7 +79,7 @@ import_squash() { echo "Squash file already exists and is valid, skipping import: $squash" else local enroot_runtime - enroot_runtime=$(mktemp -d "${TMPDIR:-/tmp}/enroot-import.XXXXXX") || exit 1 + enroot_runtime=$(mktemp -d /tmp/enroot-import.XXXXXX) || exit 1 trap 'rm -rf -- "$enroot_runtime"' EXIT export ENROOT_RUNTIME_PATH="$enroot_runtime" @@ -100,7 +100,10 @@ import_squash() { } if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then - if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then + if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$MODEL" == *-0813 ]]; then + export MODEL_PATH="/mnt/lustre01/users-public/sa-shared/models/DeepSeek-V4-Pro-0813" + export MODEL_NAME="$MODEL" + elif [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" export MODEL_NAME="deepseek-ai/DeepSeek-V4-Pro" else @@ -117,14 +120,18 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" mkdir -p "$BENCHMARK_LOGS_DIR" - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh" + if [[ "$DISAGG" == "true" ]]; then + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh" + else + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-agg.sh" + fi BENCH_SCRIPT="benchmarks/multi_node/${SCRIPT_NAME}" if [[ ! -f "$BENCH_SCRIPT" ]]; then echo "Error: llm-d wrapper not found: $BENCH_SCRIPT" >&2 exit 1 fi - JOB_ID=$(bash "$BENCH_SCRIPT") + JOB_ID=$(bash "$BENCH_SCRIPT") || exit 1 if [[ -z "$JOB_ID" ]]; then echo "Error: failed to submit llm-d job" >&2 exit 1 @@ -140,7 +147,12 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 done < <(find "$BENCHMARK_LOGS_DIR" -name "${RESULT_FILENAME}*.json" -print0 2>/dev/null) - if [[ "${RUN_EVAL:-false}" == "true" ]]; then + if [[ "$IS_AGENTIC" == "1" && "$EVAL_ONLY" != "true" ]]; then + mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" + cp -R "$BENCHMARK_LOGS_DIR/agentic/." "$GITHUB_WORKSPACE/LOGS/agentic/" || exit 1 + fi + + if [[ "${RUN_EVAL}" == "true" ]]; then EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR" -type d -name eval_results -print -quit 2>/dev/null) if [[ -z "$EVAL_DIR" ]]; then EVAL_DIR="$BENCHMARK_LOGS_DIR/eval_results" @@ -320,7 +332,7 @@ if [[ "$USES_DCGM_POWER" == "1" ]]; then sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" fi -export EVAL_ONLY="${EVAL_ONLY:-false}" +export EVAL_ONLY export ISL="$ISL" export OSL="$OSL" @@ -554,7 +566,7 @@ else git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" fi -if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then +if [[ "${EVAL_FRAMEWORK}" != "lm-eval" ]]; then python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 fi @@ -802,7 +814,7 @@ else echo "Warning: Logs directory not found at $LOGS_DIR" fi -if [[ "${EVAL_ONLY:-false}" != "true" ]]; then +if [[ "${EVAL_ONLY}" != "true" ]]; then if [ ! -d "$LOGS_DIR" ]; then exit 1 fi @@ -863,6 +875,6 @@ else fi # Collect eval results if eval was requested -if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then +if [[ "${RUN_EVAL}" == "true" || "${EVAL_ONLY}" == "true" ]]; then copy_eval_artifacts "$LOGS_DIR/eval_results" "$GITHUB_WORKSPACE" || exit 1 fi From 775e5e936f95a59ad1ba23664a8517bc1dcb91de Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Wed, 2 Sep 2026 18:24:33 +0200 Subject: [PATCH 16/46] Adapt optimized configs and map to agentic gb200 --- .../agg-gb200-dep8-dspark-agentic.yaml | 56 +++++++++++++------ ...gg-gb200-dep8-dspark-mooncake-agentic.yaml | 55 +++++++++++++----- .../agentic/agg-gb200-tp8-dspark-agentic.yaml | 32 ++++++----- ...g-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 39 +++++++++++-- configs/nvidia-master.yaml | 24 ++++---- perf-changelog.yaml | 8 ++- 6 files changed, 149 insertions(+), 65 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index 7b90a8d645..1c294ba4fd 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -1,5 +1,5 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated DEP8, 1 node. -# DSpark speculative decoding: 5 tokens. No Mooncake. +# DSpark speculative decoding: 5 tokens. No Mooncake. conc-list [12,16,32]. apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -10,24 +10,38 @@ plugins: path: /tmp/endpoints.yaml watchFile: false - - type: prefill-filter - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: active-request-scorer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 8967 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 4783 + maxTTFTPenaltyMs: 30000 - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 2295620 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker schedulingProfiles: - name: default plugins: - - pluginRef: prefill-filter - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: active-request-scorer - weight: 1 + - pluginRef: prefix-cache-affinity-filter - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer weight: 3 + - pluginRef: max-score-picker dataLayer: discovery: @@ -39,10 +53,10 @@ prefill: extra-args: >- --kv-cache-dtype fp8 --gpu-memory-utilization 0.92 - --max-model-len 9280 + --max-model-len 1048576 --max-num-seqs 64 --max-num-batched-tokens 8192 - --max-cudagraph-capture-size 64 + --max-cudagraph-capture-size 192 --block-size 256 --tokenizer-mode deepseek_v4 --moe-backend deep_gemm_mega_moe @@ -52,19 +66,29 @@ prefill: --no-enable-flashinfer-autotune --numa-bind --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":true,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: + PYTHONHASHSEED: "0" VLLM_USE_RUST_FRONTEND: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" slurm: time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 05e8322a23..a9b6d35c1e 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -1,6 +1,7 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated DEP8 with Mooncake prefix-cache. # Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds Mooncake (P2PHANDSHAKE embedded RDMA) # so server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role). +# DSpark 5 speculative tokens. conc-list [52,72]. apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -11,24 +12,38 @@ plugins: path: /tmp/endpoints.yaml watchFile: false - - type: prefill-filter - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: active-request-scorer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 8967 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 4783 + maxTTFTPenaltyMs: 30000 - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 3518020 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker schedulingProfiles: - name: default plugins: - - pluginRef: prefill-filter - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: active-request-scorer - weight: 1 + - pluginRef: prefix-cache-affinity-filter - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer weight: 3 + - pluginRef: max-score-picker dataLayer: discovery: @@ -41,10 +56,10 @@ prefill: extra-args: >- --kv-cache-dtype fp8 --gpu-memory-utilization 0.92 - --max-model-len 9280 + --max-model-len 1048576 --max-num-seqs 64 --max-num-batched-tokens 8192 - --max-cudagraph-capture-size 64 + --max-cudagraph-capture-size 192 --block-size 256 --tokenizer-mode deepseek_v4 --moe-backend deep_gemm_mega_moe @@ -54,19 +69,29 @@ prefill: --no-enable-flashinfer-autotune --numa-bind --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":true,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: + PYTHONHASHSEED: "0" VLLM_USE_RUST_FRONTEND: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml index 6245c9485d..567d95cb20 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -10,24 +10,18 @@ plugins: path: /tmp/endpoints.yaml watchFile: false - - type: prefill-filter - - type: queue-scorer - - type: kv-cache-utilization-scorer - type: active-request-scorer - - type: prefix-cache-scorer + - type: queue-scorer + - type: weighted-random-picker schedulingProfiles: - name: default plugins: - - pluginRef: prefill-filter - - pluginRef: queue-scorer + - pluginRef: active-request-scorer weight: 2 - - pluginRef: kv-cache-utilization-scorer + - pluginRef: queue-scorer weight: 2 - - pluginRef: active-request-scorer - weight: 1 - - pluginRef: prefix-cache-scorer - weight: 3 + - pluginRef: weighted-random-picker dataLayer: discovery: @@ -39,11 +33,11 @@ prefill: enable-expert-parallel: false extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.92 - --max-model-len 9280 + --gpu-memory-utilization 0.9 + --max-model-len 1048576 --max-num-seqs 32 --max-num-batched-tokens 8192 - --max-cudagraph-capture-size 32 + --max-cudagraph-capture-size 96 --block-size 256 --tokenizer-mode deepseek_v4 --disable-custom-all-reduce @@ -63,11 +57,21 @@ prefill: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_NCCL_SYMM_MEM: "0" # TP8 intra-node allreduce; DEP8 recipe uses "1" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" NCCL_P2P_LEVEL: "NVL" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: cuda_copy,cuda_ipc,tcp + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" slurm: time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 4bd13b0572..ea11bbf714 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -69,13 +69,12 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --enforce-eager --gpu-memory-utilization 0.97 - --max-model-len 9280 + --max-model-len 1048576 --max-num-seqs 16 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 - --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --enable-cumem-allocator --block-size 256 --tokenizer-mode deepseek_v4 @@ -88,16 +87,34 @@ prefill: env: PYTHONHASHSEED: "0" VLLM_USE_RUST_FRONTEND: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_RPC_CLIENT_IO_THREADS: "32" + MC_TE_RPC_CLIENT_IO_THREADS: "32" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" MC_TE_METRIC: "0" @@ -112,7 +129,7 @@ decode: --max-cudagraph-capture-size 256 --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --gpu-memory-utilization 0.95 - --max-model-len 9280 + --max-model-len 1048576 --enable-cumem-allocator --block-size 256 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} @@ -126,12 +143,26 @@ decode: env: PYTHONHASHSEED: "0" VLLM_USE_RUST_FRONTEND: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 85e6af68bd..e4be450281 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4502,14 +4502,14 @@ dsv4-fp4-gb200-llmd-vllm: - "DECODE_NODES=2" - "GPUS_PER_NODE=4" -# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, P/D disagg via llmd-vllm. +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, P/D disagg via llmd-vllm. # Always uses Mooncake; DSpark speculative decoding. Requires EPP v0.10.0 # (LLMD_BIN_DIR override in runners/launch_b200-nscale-compat.sh). -dsv4-fp4-b200-llmd-vllm-agentx: +dsv4-fp4-gb200-llmd-vllm-agentx: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 - runner: cluster:b200-nscale + runner: gb200 precision: fp4 framework: llmd-vllm router: { name: llm-d-router, version: "0.10.0" } @@ -4517,10 +4517,8 @@ dsv4-fp4-b200-llmd-vllm-agentx: multinode: true disagg: true scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: + agentic-coding: + - search-space: # 1P DEP8 + 1D DEP8 (2 nodes / 16 GPUs). Always Mooncake. - spec-decoding: "none" conc-list: [64, 128, 192] @@ -4542,7 +4540,7 @@ dsv4-fp4-b200-llmd-vllm-agentx: - "DECODE_NODES=1" - "GPUS_PER_NODE=8" -# Aggregated (TP8 / DEP8) sibling of dsv4-fp4-b200-llmd-vllm-agentx: one +# Aggregated (TP8 / DEP8) sibling of dsv4-fp4-gb200-llmd-vllm-agentx: one # engine per topology does both prefill and decode (no P/D split, no # pd-sidecar, no kv_transfer_config - see the recipe files' header # comments). Separate top-level key because `disagg:` is a whole-key @@ -4551,21 +4549,19 @@ dsv4-fp4-b200-llmd-vllm-agentx: # present there). conc-lists mirror the agentX dspark historical runs: # results/dsv4-pro-dspark/results_ilmarkov-2-ix-agg-tp8-dspark-v3-long # (tp8) and results_ilmarkov-ix-agg-dep8-dspark-1308-long (dep8). -dsv4-fp4-b200-llmd-vllm-agentx-agg: +dsv4-fp4-gb200-llmd-vllm-agentx-agg: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 - runner: cluster:b200-nscale + runner: gb200 precision: fp4 framework: llmd-vllm router: { name: llm-d-router, version: "0.9.0" } multinode: true disagg: false scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: + agentic-coding: + - search-space: # Aggregated TP8 (1 node / 8 GPUs; pure tensor-parallel, no EP). - spec-decoding: "none" conc-list: [1, 2, 4, 8, 12] diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 31bcb9cc5c..bd7f502777 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6608,7 +6608,9 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2787 - config-keys: - - dsv4-fp4-b200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx + scenario-type: + - agentic-coding description: - "Add DeepSeek-V4-Pro-0813 (DSpark) FP4 B200 llmd-vllm P/D disagg benchmark (1P DEP8 + 1D DEP8, 2 nodes / 16 GPUs total); always uses Mooncake RDMA prefix-cache; DSpark speculative decoding (prefill 1 token, decode 5 tokens)" - "Wire a FRAMEWORK==llmd-vllm dispatch into runners/launch_b200-nscale-compat.sh (self-contained, bypasses the srtctl/srt-slurm path)" @@ -6617,7 +6619,9 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 - config-keys: - - dsv4-fp4-b200-llmd-vllm-agentx-agg + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding description: - "Add DeepSeek-V4-Pro-0813 (DSpark) FP4 B200 llmd-vllm aggregated benchmark: TP8 at conc-list [1,2,4,8,12], DEP8 at [12,16,32], DEP8+Mooncake at [52,72] (all 1 node / 8 GPUs, one engine does both prefill and decode)" - "Extend server.sh/job.slurm with aggregated (DECODE_NODES=0) mode: no decode role, no pd-sidecar; rank 0 takes over EPP/Envoy/bench coordinator duties" From 95586e05786cdef85c0b8756366e734fb29571b9 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Wed, 2 Sep 2026 19:03:52 +0200 Subject: [PATCH 17/46] Fix recipe --- benchmarks/multi_node/llm-d/recipe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/llm-d/recipe.py b/benchmarks/multi_node/llm-d/recipe.py index da0d0ffc56..0980afdf73 100644 --- a/benchmarks/multi_node/llm-d/recipe.py +++ b/benchmarks/multi_node/llm-d/recipe.py @@ -52,7 +52,6 @@ def role_assignments(recipe: dict, role: str, env: dict) -> str: config.update( rejection_sample_method="synthetic", synthetic_acceptance_length=al, - enable_adaptive_verification=False, ) print(f"DSpark {role}: K={k}, golden AL={al} ({golden_path.name})", file=sys.stderr) extra = extra[:match.end()] + json.dumps(config, separators=(",", ":")) + extra[match.end() + length:] From 63650150cbd8d2ca74680574398b586ff592c5ca Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Wed, 2 Sep 2026 19:23:10 +0200 Subject: [PATCH 18/46] Fix spec decoding --- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 6a456441bb..8ab90fbaeb 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -72,7 +72,7 @@ prefill: --gpu-memory-utilization 0.97 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --enable-cumem-allocator --block-size 256 --tokenizer-mode deepseek_v4 From 11b8af5b66211a85d8810e1b71036da6a4cdee32 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Wed, 2 Sep 2026 22:29:41 +0200 Subject: [PATCH 19/46] Address cursor's comment --- runners/launch_gb200-nv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 48bcd6f3d7..9fc50c9194 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -79,7 +79,7 @@ import_squash() { echo "Squash file already exists and is valid, skipping import: $squash" else local enroot_runtime - enroot_runtime=$(mktemp -d /tmp/enroot-import.XXXXXX) || exit 1 + enroot_runtime=$(mktemp -d "${TMPDIR:-/tmp}/enroot-import.XXXXXX") || exit 1 trap 'rm -rf -- "$enroot_runtime"' EXIT export ENROOT_RUNTIME_PATH="$enroot_runtime" From 81060c215a4194955da783259bac4361bd43f2f2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 23:14:42 -0500 Subject: [PATCH 20/46] fix: keep GB200 import setup explicit and correct DSpark changelog --- perf-changelog.yaml | 2 +- runners/launch_gb200-nv.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index df11b307dd..2b46ccefd9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6829,6 +6829,6 @@ - agentic-coding description: - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache and llm-d router 0.10.0 for GB200 AgentX trace replay at native context: aggregated TP8/DEP8 (2 nodes, 8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (4 nodes, 16 GPUs)." - - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification." + - "Use DSpark golden AL with adaptive verification: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill." - "Enable Mooncake DRAM offloading for disaggregated and aggregated DEP8+Mooncake variants with dram-utilization=0.60; no offloading for plain TP8/DEP8. SSD offloading remains disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 9fc50c9194..48bcd6f3d7 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -79,7 +79,7 @@ import_squash() { echo "Squash file already exists and is valid, skipping import: $squash" else local enroot_runtime - enroot_runtime=$(mktemp -d "${TMPDIR:-/tmp}/enroot-import.XXXXXX") || exit 1 + enroot_runtime=$(mktemp -d /tmp/enroot-import.XXXXXX) || exit 1 trap 'rm -rf -- "$enroot_runtime"' EXIT export ENROOT_RUNTIME_PATH="$enroot_runtime" From 8493a29c4b97608a00f7025e87640e87e4cc9842 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 23:17:57 -0500 Subject: [PATCH 21/46] fix: preserve fixed DSpark draft budgets for golden AL --- .../llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml | 2 +- .../agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml | 2 +- .../llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml | 2 +- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 4 ++-- docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index bba552a514..7aac8efbc6 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -66,7 +66,7 @@ prefill: --no-enable-flashinfer-autotune --numa-bind --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":true,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: PYTHONHASHSEED: "0" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index d11095a722..296f9b8634 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -69,7 +69,7 @@ prefill: --no-enable-flashinfer-autotune --numa-bind --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":true,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: PYTHONHASHSEED: "0" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml index 1816dfb639..e24d68a69d 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -45,7 +45,7 @@ prefill: --no-disable-hybrid-kv-cache-manager --no-enable-flashinfer-autotune --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":true,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} env: VLLM_USE_RUST_FRONTEND: "1" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 8ab90fbaeb..b8944499e0 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -72,7 +72,7 @@ prefill: --gpu-memory-utilization 0.97 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 - --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --enable-cumem-allocator --block-size 256 --tokenizer-mode deepseek_v4 @@ -126,7 +126,7 @@ decode: extra-args: >- --kv-cache-dtype fp8 --max-num-batched-tokens 8192 - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --gpu-memory-utilization 0.95 --enable-cumem-allocator --block-size 256 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 59755e319f..402148ce83 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -144,7 +144,7 @@ A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. -`recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Plain TP8/DEP8 declares `none`. +`recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Plain TP8/DEP8 declares `none`. Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 0eec3d2a67..77539e00f0 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -144,7 +144,7 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。 -`recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。普通 TP8/DEP8 声明 `none`。 +`recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。普通 TP8/DEP8 声明 `none`。 服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 176d38d845..67d24bbb5d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6839,6 +6839,6 @@ - agentic-coding description: - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache and llm-d router 0.10.0 for GB200 AgentX trace replay at native context: aggregated TP8/DEP8 (2 nodes, 8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (4 nodes, 16 GPUs)." - - "Use DSpark golden AL with adaptive verification: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill." + - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification to preserve the fixed draft budget." - "Enable Mooncake DRAM offloading for disaggregated and aggregated DEP8+Mooncake variants with dram-utilization=0.60; no offloading for plain TP8/DEP8. SSD offloading remains disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 63946097fee562693679d27bbaca5805720e85ab Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 23:29:54 -0500 Subject: [PATCH 22/46] fix: use GB200 InfiniBand adapters for Mooncake transfers --- .../agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml | 2 +- .../llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml | 2 +- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 2 +- docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 296f9b8634..ffbad9a6d8 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -104,7 +104,7 @@ mooncake: metadata_server: "P2PHANDSHAKE" local_buffer_size: "4GB" protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" mode: "embedded" enable_offload: false # SSD only; the embedded DRAM pool is still enabled. diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml index e24d68a69d..ef0a3d1104 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -58,7 +58,7 @@ prefill: VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" # TP8 intra-node allreduce; DEP8 recipe uses "1" + VLLM_USE_NCCL_SYMM_MEM: "0" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index b8944499e0..7c3bf67d9d 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -176,7 +176,7 @@ mooncake: metadata_server: "P2PHANDSHAKE" local_buffer_size: "4GB" protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" mode: "embedded" enable_offload: false # SSD only; the embedded DRAM pool is still enabled. diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 402148ce83..8595067982 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -144,7 +144,7 @@ A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. -`recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Plain TP8/DEP8 declares `none`. +`recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 77539e00f0..229cb8d675 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -144,7 +144,7 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。 -`recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。普通 TP8/DEP8 声明 `none`。 +`recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 67d24bbb5d..83f79166ee 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6840,5 +6840,5 @@ description: - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache and llm-d router 0.10.0 for GB200 AgentX trace replay at native context: aggregated TP8/DEP8 (2 nodes, 8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (4 nodes, 16 GPUs)." - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification to preserve the fixed draft budget." - - "Enable Mooncake DRAM offloading for disaggregated and aggregated DEP8+Mooncake variants with dram-utilization=0.60; no offloading for plain TP8/DEP8. SSD offloading remains disabled." + - "Use Mooncake DRAM offloading (dram-utilization=0.60) over GB200 InfiniBand HCAs mlx5_0,1,3,4 for disaggregated and DEP8+Mooncake variants. Plain TP8/DEP8 has no offload; SSD offload is disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 73476be0d1dcd5a40643a675401a947f0d704fbc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 23:33:41 -0500 Subject: [PATCH 23/46] fix: provide DEP8 router token-load dependencies --- .../llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml | 1 + .../agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml | 1 + docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index 7aac8efbc6..fdd1d1d893 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -10,6 +10,7 @@ plugins: path: /tmp/endpoints.yaml watchFile: false + - type: inflight-load-producer - type: approx-prefix-cache-producer parameters: autoTune: false diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index ffbad9a6d8..5582fbc5b5 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -12,6 +12,7 @@ plugins: path: /tmp/endpoints.yaml watchFile: false + - type: inflight-load-producer - type: approx-prefix-cache-producer parameters: autoTune: false diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 8595067982..6807838836 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -142,7 +142,7 @@ llm-d is not the srt-slurm path: InferenceX owns the Slurm allocation and starts A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml` fallback and removes recipe-specific vLLM flags. Treat that as a validation failure unless fallback is explicitly intended. -The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. +The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. `recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 229cb8d675..a8c5a0e599 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -142,7 +142,7 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 `CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 -GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。 +GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。 `recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 83f79166ee..3d01396726 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6838,7 +6838,7 @@ scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 FP4 with FP8 KV cache and llm-d router 0.10.0 for GB200 AgentX trace replay at native context: aggregated TP8/DEP8 (2 nodes, 8 GPUs) and NIXL disaggregated 1P-DEP8/1D-DEP8 (4 nodes, 16 GPUs)." + - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation and 16-GPU NIXL 1P/1D DEP8 disaggregation. Use llm-d 0.10.0 with prefix-affinity and token-load routing for DEP8." - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification to preserve the fixed draft budget." - "Use Mooncake DRAM offloading (dram-utilization=0.60) over GB200 InfiniBand HCAs mlx5_0,1,3,4 for disaggregated and DEP8+Mooncake variants. Plain TP8/DEP8 has no offload; SSD offload is disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 3ba226bfc674c836b91ba18b84b634fbbe99e6dd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 3 Sep 2026 00:14:41 -0500 Subject: [PATCH 24/46] fix: start a shared Mooncake master for llm-d jobs --- benchmarks/multi_node/llm-d/recipe.py | 1 + benchmarks/multi_node/llm-d/server.sh | 14 ++++++++------ docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/llm-d/recipe.py b/benchmarks/multi_node/llm-d/recipe.py index 0980afdf73..1b3033ccc6 100644 --- a/benchmarks/multi_node/llm-d/recipe.py +++ b/benchmarks/multi_node/llm-d/recipe.py @@ -73,6 +73,7 @@ def mooncake_config(recipe: dict, env: dict) -> str: config = dict(recipe.get("mooncake", {}).get("store_config") or {}) if not config: return "" + config["master_server_address"] = f"{env['ALL_IPS'].split(',')[0]}:50051" if env.get("IS_AGENTIC") == "1": budget_gb = int(env["TOTAL_CPU_DRAM_GB"]) gpus_per_node = int(env["GPUS_PER_NODE"]) diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index b873b62e2f..3a9bc85a93 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -183,12 +183,7 @@ echo "Resolved $ROLE TP_SIZE=$TP_SIZE ROLE_ENABLE_EP=$ROLE_ENABLE_EP" # ---------------------------------------------------------------- # Mooncake KV store (optional, from recipe top-level `mooncake:` key) # ---------------------------------------------------------------- -# When a recipe sets `mooncake.store_config`, write the JSON config to -# /tmp/mooncake_config.json and set MOONCAKE_CONFIG_PATH. The KV transfer -# config section below will then build a MultiConnector chain -# (NixlConnector + MooncakeStoreConnector) instead of plain NixlConnector. -# Mooncake uses P2PHANDSHAKE embedded mode (no external metadata server) -# so no K8s sidecar is required - nodes negotiate directly via RDMA. +# Embedded stores contribute per-rank DRAM to one job-local Mooncake master. MOONCAKE_CONFIG_PATH="" if [[ -n "${CONFIG_FILE}" && -f "/etc/llmd-recipes/${CONFIG_FILE}" ]]; then _MC_JSON=$(python3 /workspace/benchmarks/multi_node/llm-d/recipe.py \ @@ -198,6 +193,13 @@ if [[ -n "${CONFIG_FILE}" && -f "/etc/llmd-recipes/${CONFIG_FILE}" ]]; then MOONCAKE_CONFIG_PATH=/tmp/mooncake_config.json export MOONCAKE_CONFIG_PATH echo "Mooncake enabled: config at $MOONCAKE_CONFIG_PATH" + if [[ "$NODE_RANK" -eq 0 ]]; then + mooncake_master --rpc_port=50051 --metrics_port=50052 \ + > "$BENCHMARK_LOGS_DIR/mooncake_master.log" 2>&1 & + fi + curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ + --retry 30 --retry-connrefused --retry-delay 1 \ + "http://${_ALL_IPS[0]}:50052/metrics" > /dev/null fi fi diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 6807838836..2f913cb140 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -144,7 +144,7 @@ A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. -`recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. +`recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Rank 0 starts a job-local Mooncake master on port 50051 (metrics on 50052); every rank waits for it before starting vLLM. `P2PHANDSHAKE` does not replace the store master. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index a8c5a0e599..18800309d1 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -144,7 +144,7 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。 -`recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 +`recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Rank 0 在端口 50051 启动本次作业专用的 Mooncake master（指标端口为 50052）；所有 rank 等待其就绪后再启动 vLLM。`P2PHANDSHAKE` 不能替代存储 master。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3d01396726..fc1aa1bd58 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6840,5 +6840,5 @@ description: - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation and 16-GPU NIXL 1P/1D DEP8 disaggregation. Use llm-d 0.10.0 with prefix-affinity and token-load routing for DEP8." - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification to preserve the fixed draft budget." - - "Use Mooncake DRAM offloading (dram-utilization=0.60) over GB200 InfiniBand HCAs mlx5_0,1,3,4 for disaggregated and DEP8+Mooncake variants. Plain TP8/DEP8 has no offload; SSD offload is disabled." + - "Use Mooncake DRAM offloading (dram-utilization=0.60) with one job-local master over GB200 InfiniBand HCAs mlx5_0,1,3,4 for disaggregated and DEP8+Mooncake variants. Plain TP8/DEP8 has no offload; SSD offload is disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From bc55cca5a61db7a248135da8b3de7b535bedd92a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 3 Sep 2026 01:00:53 -0500 Subject: [PATCH 25/46] fix: reserve GPU memory for long-context llm-d replay --- .../llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml | 2 +- .../agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml | 2 +- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 1 + docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index fdd1d1d893..fcb48c4905 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -53,7 +53,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.92 + --gpu-memory-utilization 0.9 --max-num-batched-tokens 8192 --block-size 256 --tokenizer-mode deepseek_v4 diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 5582fbc5b5..56cbe0a323 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -56,7 +56,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.92 + --gpu-memory-utilization 0.9 --max-num-batched-tokens 8192 --block-size 256 --tokenizer-mode deepseek_v4 diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 7c3bf67d9d..26a71d3a10 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -69,7 +69,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.97 + --gpu-memory-utilization 0.9 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} @@ -125,9 +125,9 @@ decode: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 + --gpu-memory-utilization 0.9 --max-num-batched-tokens 8192 --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} - --gpu-memory-utilization 0.95 --enable-cumem-allocator --block-size 256 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 26ce8d0f1d..26dfc97b98 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4503,6 +4503,7 @@ dsv4-fp4-gb200-llmd-vllm: - "GPUS_PER_NODE=4" # DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, P/D disagg via llmd-vllm. +# Use a 0.90 GPU-memory budget for long-context workspace headroom. # Always uses Mooncake; DSpark speculative decoding. The DSpark image # bundles EPP/pd-sidecar v0.10.0. dsv4-fp4-gb200-llmd-vllm-agentx: diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 2f913cb140..d5d83a2dd1 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -142,7 +142,7 @@ llm-d is not the srt-slurm path: InferenceX owns the Slurm allocation and starts A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml` fallback and removes recipe-specific vLLM flags. Treat that as a validation failure unless fallback is explicitly intended. -The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. +The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. Set `gpu-memory-utilization=0.90`: the image defaults to 0.92, which exhausted memory in the DEP8 sparse-attention indexer during long-context replay. `recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Rank 0 starts a job-local Mooncake master on port 50051 (metrics on 50052); every rank waits for it before starting vLLM. `P2PHANDSHAKE` does not replace the store master. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 18800309d1..1741de5486 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -142,7 +142,7 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 `CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 -GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。 +GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。设置 `gpu-memory-utilization=0.90`：镜像默认值为 0.92，该设置在 DEP8 长上下文回放时导致稀疏注意力索引器内存不足。 `recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Rank 0 在端口 50051 启动本次作业专用的 Mooncake master（指标端口为 50052）；所有 rank 等待其就绪后再启动 vLLM。`P2PHANDSHAKE` 不能替代存储 master。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fc1aa1bd58..67dcc6d6fe 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6838,7 +6838,7 @@ scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation and 16-GPU NIXL 1P/1D DEP8 disaggregation. Use llm-d 0.10.0 with prefix-affinity and token-load routing for DEP8." + - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation and 16-GPU NIXL 1P/1D DEP8 disaggregation. Use llm-d 0.10.0 with prefix-affinity and token-load routing for DEP8; use a 0.90 GPU-memory budget for long-context workspace headroom." - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification to preserve the fixed draft budget." - "Use Mooncake DRAM offloading (dram-utilization=0.60) with one job-local master over GB200 InfiniBand HCAs mlx5_0,1,3,4 for disaggregated and DEP8+Mooncake variants. Plain TP8/DEP8 has no offload; SSD offload is disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 9c1c0c4ecb21ce808c4cbfea9de966697e583128 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 3 Sep 2026 01:03:00 -0500 Subject: [PATCH 26/46] fix: cover TP8 and DEP8 long-context allocation peaks --- .../llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml | 2 +- .../agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml | 2 +- .../llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml | 2 +- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 2 +- docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index fcb48c4905..0fe05de446 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -53,7 +53,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.9 + --gpu-memory-utilization 0.88 --max-num-batched-tokens 8192 --block-size 256 --tokenizer-mode deepseek_v4 diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 56cbe0a323..3328049329 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -56,7 +56,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.9 + --gpu-memory-utilization 0.88 --max-num-batched-tokens 8192 --block-size 256 --tokenizer-mode deepseek_v4 diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml index ef0a3d1104..b62994470a 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -33,7 +33,7 @@ prefill: enable-expert-parallel: false extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.9 + --gpu-memory-utilization 0.85 --max-num-batched-tokens 8192 --block-size 256 --tokenizer-mode deepseek_v4 diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 26a71d3a10..9798a79f5c 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -69,7 +69,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.9 + --gpu-memory-utilization 0.88 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} @@ -125,7 +125,7 @@ decode: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.9 + --gpu-memory-utilization 0.88 --max-num-batched-tokens 8192 --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --enable-cumem-allocator diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 26dfc97b98..a0968d395d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4503,7 +4503,7 @@ dsv4-fp4-gb200-llmd-vllm: - "GPUS_PER_NODE=4" # DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, P/D disagg via llmd-vllm. -# Use a 0.90 GPU-memory budget for long-context workspace headroom. +# Long-context workspace headroom: GPU-memory budgets are 0.88 for DEP8, 0.85 for TP8. # Always uses Mooncake; DSpark speculative decoding. The DSpark image # bundles EPP/pd-sidecar v0.10.0. dsv4-fp4-gb200-llmd-vllm-agentx: diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index d5d83a2dd1..7450620a85 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -142,7 +142,7 @@ llm-d is not the srt-slurm path: InferenceX owns the Slurm allocation and starts A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml` fallback and removes recipe-specific vLLM flags. Treat that as a validation failure unless fallback is explicitly intended. -The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. Set `gpu-memory-utilization=0.90`: the image defaults to 0.92, which exhausted memory in the DEP8 sparse-attention indexer during long-context replay. +The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. Use `gpu-memory-utilization=0.88` for DEP8 and `0.85` for TP8. Long-context replay exhausted sparse-attention indexer memory at 0.92 and 0.90 respectively; retain headroom without shortening the model context or filtering traces. `recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Rank 0 starts a job-local Mooncake master on port 50051 (metrics on 50052); every rank waits for it before starting vLLM. `P2PHANDSHAKE` does not replace the store master. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 1741de5486..8f7a2ecc80 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -142,7 +142,7 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 `CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 -GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。设置 `gpu-memory-utilization=0.90`：镜像默认值为 0.92，该设置在 DEP8 长上下文回放时导致稀疏注意力索引器内存不足。 +GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。DEP8 使用 `gpu-memory-utilization=0.88`，TP8 使用 `0.85`。长上下文回放分别在 0.92 和 0.90 时耗尽稀疏注意力索引器内存；应保留工作区余量，而不是缩短模型上下文或过滤轨迹。 `recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Rank 0 在端口 50051 启动本次作业专用的 Mooncake master（指标端口为 50052）；所有 rank 等待其就绪后再启动 vLLM。`P2PHANDSHAKE` 不能替代存储 master。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 67dcc6d6fe..ac47b85f31 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6838,7 +6838,7 @@ scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation and 16-GPU NIXL 1P/1D DEP8 disaggregation. Use llm-d 0.10.0 with prefix-affinity and token-load routing for DEP8; use a 0.90 GPU-memory budget for long-context workspace headroom." + - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation and 16-GPU NIXL 1P/1D DEP8 disaggregation. Use llm-d 0.10.0 with prefix-affinity and token-load routing for DEP8; use GPU-memory budgets of 0.88 (DEP8) and 0.85 (TP8) for long-context workspace headroom." - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification to preserve the fixed draft budget." - "Use Mooncake DRAM offloading (dram-utilization=0.60) with one job-local master over GB200 InfiniBand HCAs mlx5_0,1,3,4 for disaggregated and DEP8+Mooncake variants. Plain TP8/DEP8 has no offload; SSD offload is disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 7b65775ade35628ed4ad4ba4aabee861f8e6565b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 3 Sep 2026 09:45:21 -0500 Subject: [PATCH 27/46] fix: prevent duplicate llm-d frontend metric scrapes --- benchmarks/llm-d/envoy.yaml | 7 +++++++ docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 6 +++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/benchmarks/llm-d/envoy.yaml b/benchmarks/llm-d/envoy.yaml index ccba51ba0b..8bc10fe595 100644 --- a/benchmarks/llm-d/envoy.yaml +++ b/benchmarks/llm-d/envoy.yaml @@ -44,6 +44,13 @@ static_resources: - name: vh domains: ["*"] routes: + # Scrape metrics per node, never through load balancing. + - match: { path: "/metrics" } + direct_response: { status: 404 } + typed_per_filter_config: + envoy.filters.http.ext_proc: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute + disabled: true - match: { prefix: "/" } route: cluster: original_dst diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 7450620a85..616a675511 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -146,7 +146,7 @@ The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 `recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Rank 0 starts a job-local Mooncake master on port 50051 (metrics on 50052); every rank waits for it before starting vLLM. `P2PHANDSHAKE` does not replace the store master. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. -Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. +Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy rejects `/metrics` before EPP routing so AIPerf's automatic frontend scrape cannot duplicate worker counters. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. ## Update an image diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 8f7a2ecc80..b11f54883d 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -146,7 +146,7 @@ GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点 `recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Rank 0 在端口 50051 启动本次作业专用的 Mooncake master（指标端口为 50052）；所有 rank 等待其就绪后再启动 vLLM。`P2PHANDSHAKE` 不能替代存储 master。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 -服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 +服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 在 EPP 路由前拒绝 `/metrics`，避免 AIPerf 自动抓取前端时重复统计 worker 计数器。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 ## 更新镜像 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ac47b85f31..b1556eda08 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6838,7 +6838,7 @@ scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation and 16-GPU NIXL 1P/1D DEP8 disaggregation. Use llm-d 0.10.0 with prefix-affinity and token-load routing for DEP8; use GPU-memory budgets of 0.88 (DEP8) and 0.85 (TP8) for long-context workspace headroom." - - "Use DSpark golden AL: K=5, AL=3.61 for aggregated/decode; K=1, AL=1.84 for disaggregated prefill. Disable adaptive verification to preserve the fixed draft budget." - - "Use Mooncake DRAM offloading (dram-utilization=0.60) with one job-local master over GB200 InfiniBand HCAs mlx5_0,1,3,4 for disaggregated and DEP8+Mooncake variants. Plain TP8/DEP8 has no offload; SSD offload is disabled." + - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation, 16-GPU NIXL DEP8 1P/1D, and llm-d 0.10.0 prefix/token-load routing. GPU-memory budgets: TP8 0.85, DEP8 0.88." + - "Use fixed-K DSpark golden AL: K=5/AL=3.61 for aggregated/decode; K=1/AL=1.84 for P/D prefill. Adaptive verification is disabled." + - "Mooncake variants use DRAM offload at 0.60 utilization, one job-local master, and IB HCAs mlx5_0,1,3,4; no SSD offload. Plain TP8/DEP8 has no offload. Scrape vLLM nodes directly without duplicate frontend counters." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 4a6231e46411f10528becb8514c2242058c7a2e3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 3 Sep 2026 13:12:39 -0500 Subject: [PATCH 28/46] fix: retain P/D KV blocks across long decode stalls --- benchmarks/multi_node/llm-d/server.sh | 2 +- docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 3a9bc85a93..84792a134b 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -244,7 +244,7 @@ if [[ "$IS_AGGREGATED" -eq 0 ]]; then _KV_OUTER="kv_consumer" _MC_EXTRA='"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false' fi - KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false,\"kv_lease_duration\":1800}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" else KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" fi diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 616a675511..31f6cfcad6 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -142,7 +142,7 @@ llm-d is not the srt-slurm path: InferenceX owns the Slurm allocation and starts A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml` fallback and removes recipe-specific vLLM flags. Treat that as a validation failure unless fallback is explicitly intended. -The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. Use `gpu-memory-utilization=0.88` for DEP8 and `0.85` for TP8. Long-context replay exhausted sparse-attention indexer memory at 0.92 and 0.90 respectively; retain headroom without shortening the model context or filtering traces. +The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. Use `gpu-memory-utilization=0.88` for DEP8 and `0.85` for TP8. Long-context replay exhausted sparse-attention indexer memory at 0.92 and 0.90 respectively; retain headroom without shortening the model context or filtering traces. NIXL+Mooncake P/D uses a 1,800-second KV lease, matching the model-execution timeout; the default 30-second lease expired during long-context decode stalls at c192. `recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Rank 0 starts a job-local Mooncake master on port 50051 (metrics on 50052); every rank waits for it before starting vLLM. `P2PHANDSHAKE` does not replace the store master. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index b11f54883d..209bb07e03 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -142,7 +142,7 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 `CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 -GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。DEP8 使用 `gpu-memory-utilization=0.88`，TP8 使用 `0.85`。长上下文回放分别在 0.92 和 0.90 时耗尽稀疏注意力索引器内存；应保留工作区余量，而不是缩短模型上下文或过滤轨迹。 +GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。DEP8 使用 `gpu-memory-utilization=0.88`，TP8 使用 `0.85`。长上下文回放分别在 0.92 和 0.90 时耗尽稀疏注意力索引器内存；应保留工作区余量，而不是缩短模型上下文或过滤轨迹。 NIXL+Mooncake P/D 使用 1,800 秒 KV 租约，与模型执行超时一致；默认的 30 秒租约在 c192 长上下文解码停顿期间过期。 `recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Rank 0 在端口 50051 启动本次作业专用的 Mooncake master（指标端口为 50052）；所有 rank 等待其就绪后再启动 vLLM。`P2PHANDSHAKE` 不能替代存储 master。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b1556eda08..f25c25e54e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6838,7 +6838,7 @@ scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation, 16-GPU NIXL DEP8 1P/1D, and llm-d 0.10.0 prefix/token-load routing. GPU-memory budgets: TP8 0.85, DEP8 0.88." + - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation, 16-GPU NIXL DEP8 1P/1D, and llm-d 0.10.0 prefix/token-load routing. GPU-memory budgets: TP8 0.85, DEP8 0.88. P/D KV leases last 1800s to cover long-context decode stalls." - "Use fixed-K DSpark golden AL: K=5/AL=3.61 for aggregated/decode; K=1/AL=1.84 for P/D prefill. Adaptive verification is disabled." - "Mooncake variants use DRAM offload at 0.60 utilization, one job-local master, and IB HCAs mlx5_0,1,3,4; no SSD offload. Plain TP8/DEP8 has no offload. Scrape vLLM nodes directly without duplicate frontend counters." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 9411ed99b27999fe1d2bc6825b2bf533a989a03a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 3 Sep 2026 13:51:19 -0500 Subject: [PATCH 29/46] fix: validate llm-d metrics targets and retain endpoint roles --- benchmarks/multi_node/llm-d/agentic.sh | 32 +++++++++++++++++++++----- docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/llm-d/agentic.sh b/benchmarks/multi_node/llm-d/agentic.sh index d8c793c9b0..14e031744f 100644 --- a/benchmarks/multi_node/llm-d/agentic.sh +++ b/benchmarks/multi_node/llm-d/agentic.sh @@ -12,23 +12,43 @@ export AGENTIC_OUTPUT_DIR="$BENCHMARK_LOGS_DIR" export CONC_LIST="${BENCH_MAX_CONCURRENCY//x/ }" export CONC="${CONC_LIST%% *}" -# Discovery already enumerates every DP-serving node and excludes headless TP -# followers. Scrape vLLM directly, not Envoy or the decode pd-sidecar port. -AIPERF_METRIC_URLS=$(python3 - "$LLMD_ENDPOINTS_FILE" "$VLLM_PORT" <<'PY' +# Use discovery's serving nodes, but scrape vLLM rather than the decode sidecar. +mkdir -p "$RESULT_DIR" +AIPERF_METRIC_URLS=$(python3 - "$LLMD_ENDPOINTS_FILE" "$VLLM_PORT" \ + "$RESULT_DIR/llmd_metrics_endpoints.json" "$DECODE_NODES" <<'PY' +import json import sys import yaml with open(sys.argv[1]) as source: endpoints = yaml.safe_load(source)["endpoints"] -addresses = dict.fromkeys(endpoint["address"] for endpoint in endpoints) -if not addresses: +metrics_endpoints = { + f"http://{endpoint['address']}:{int(sys.argv[2])}/metrics": { + "name": endpoint["name"], + "role": endpoint["labels"]["llm-d.ai/role"] if int(sys.argv[4]) else "combined", + } + for endpoint in endpoints +} +if not metrics_endpoints: raise SystemExit("No llm-d serving endpoints available for metrics") -print(",".join(f"http://{address}:{int(sys.argv[2])}/metrics" for address in addresses)) +with open(sys.argv[3], "w") as output: + json.dump(metrics_endpoints, output, indent=2) + output.write("\n") +print(",".join(metrics_endpoints)) PY ) export AIPERF_METRIC_URLS # benchmark_lib.sh forwards this name to AIPerf's --server-metrics argument. export AIPERF_SERVER_METRICS_URLS="$AIPERF_METRIC_URLS" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" + +# AIPerf also probes the inference URL; it must not return load-balanced counters. +frontend_metrics_status=$(curl --silent --show-error --connect-timeout 5 --max-time 10 \ + --output /dev/null --write-out '%{http_code}' "$AIPERF_SERVER_URL/metrics") +if [[ "$frontend_metrics_status" != "404" ]]; then + echo "ERROR: llm-d frontend /metrics must return 404, got $frontend_metrics_status" >&2 + exit 1 +fi IFS=',' read -r -a metrics_urls <<< "$AIPERF_METRIC_URLS" metrics_probe=$(mktemp /tmp/llmd-metrics.XXXXXX) diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 31f6cfcad6..6abea1b351 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -146,7 +146,7 @@ The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 `recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Rank 0 starts a job-local Mooncake master on port 50051 (metrics on 50052); every rank waits for it before starting vLLM. `P2PHANDSHAKE` does not replace the store master. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. -Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy rejects `/metrics` before EPP routing so AIPerf's automatic frontend scrape cannot duplicate worker counters. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. +Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy rejects `/metrics` before EPP routing so AIPerf's automatic frontend scrape cannot duplicate worker counters. The adapter verifies this 404 before replay and requires exported `vllm:` metrics. Raw AgentX artifacts include `llmd_metrics_endpoints.json`, mapping each scrape URL to its discovery name and `prefill`, `decode`, or `combined` role; engine IDs alone are not unique across P/D groups. This manifest does not add Prometheus labels or change app ingestion. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. ## Update an image diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 209bb07e03..a99485ee7a 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -146,7 +146,7 @@ GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点 `recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Rank 0 在端口 50051 启动本次作业专用的 Mooncake master（指标端口为 50052）；所有 rank 等待其就绪后再启动 vLLM。`P2PHANDSHAKE` 不能替代存储 master。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 -服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 在 EPP 路由前拒绝 `/metrics`，避免 AIPerf 自动抓取前端时重复统计 worker 计数器。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 +服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 在 EPP 路由前拒绝 `/metrics`，避免 AIPerf 自动抓取前端时重复统计 worker 计数器。适配器在回放前验证此端点返回 404，并要求导出结果包含 `vllm:` 指标。原始 AgentX 工件包含 `llmd_metrics_endpoints.json`，将各抓取 URL 映射到服务发现名称及 `prefill`、`decode` 或 `combined` 角色；仅凭 engine ID 无法区分 P/D 组。此清单不会添加 Prometheus 标签或改变应用的摄取逻辑。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 ## 更新镜像 From 1a31973fd597f68f875825b2eb5ee0a70ffffdba Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Thu, 3 Sep 2026 23:55:04 +0200 Subject: [PATCH 30/46] fix: copy configs for p1d1 --- ...g-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 9798a79f5c..7733c3904c 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -69,10 +69,12 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.88 + --gpu-memory-utilization 0.97 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 - --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} + --max-num-seqs 16 + --max-model-len 1048576 + --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --enable-cumem-allocator --block-size 256 --tokenizer-mode deepseek_v4 @@ -103,6 +105,10 @@ prefill: VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED: "1" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx @@ -125,9 +131,12 @@ decode: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.88 - --max-num-batched-tokens 8192 - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} + --gpu-memory-utilization 0.95 + --max-num-batched-tokens 256 + --max-num-seqs 32 + --max-cudagraph-capture-size 256 + --max-model-len 1048576 + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} --enable-cumem-allocator --block-size 256 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} @@ -156,6 +165,9 @@ decode: VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_USE_DEEP_GEMM: "1" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx From 70c8a392798849d18b4aec7d1dd327288152f39d Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Fri, 4 Sep 2026 00:03:46 +0200 Subject: [PATCH 31/46] Add concurrency --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a0968d395d..3cdc6216df 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4525,7 +4525,7 @@ dsv4-fp4-gb200-llmd-vllm-agentx: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake } - conc-list: [64, 128, 192] + conc-list: [64, 128, 160, 192] prefill: num-worker: 1 tp: 8 From 02ed6395f784e894a9cd98ebfc6a5e4218602028 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Fri, 4 Sep 2026 10:01:37 +0200 Subject: [PATCH 32/46] Adjust decode utilization --- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 7733c3904c..085c8e973d 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -131,7 +131,7 @@ decode: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.95 + --gpu-memory-utilization 0.90 --max-num-batched-tokens 256 --max-num-seqs 32 --max-cudagraph-capture-size 256 From 8080915fb5d964dd754bf0042fa90f256d27ee58 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Fri, 4 Sep 2026 10:04:45 +0200 Subject: [PATCH 33/46] fix --- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 085c8e973d..0171367101 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -69,7 +69,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.97 + --gpu-memory-utilization 0.92 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 --max-num-seqs 16 @@ -131,7 +131,7 @@ decode: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.90 + --gpu-memory-utilization 0.95 --max-num-batched-tokens 256 --max-num-seqs 32 --max-cudagraph-capture-size 256 From da74471e27a6704329d4eed7e343261be7f989e5 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Fri, 4 Sep 2026 10:05:01 +0200 Subject: [PATCH 34/46] fix prefill --- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 0171367101..86fe2cbd04 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -69,7 +69,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.92 + --gpu-memory-utilization 0.96 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 --max-num-seqs 16 From 109f59782ed0c9f53cd9206cf9a390a1618fd25b Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Fri, 4 Sep 2026 10:25:33 +0200 Subject: [PATCH 35/46] Disable AV fin sake of golden acceptence --- .../agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 86fe2cbd04..9c27ecbf92 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -74,7 +74,7 @@ prefill: --long-prefill-token-threshold 1024 --max-num-seqs 16 --max-model-len 1048576 - --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --enable-cumem-allocator --block-size 256 --tokenizer-mode deepseek_v4 @@ -136,7 +136,7 @@ decode: --max-num-seqs 32 --max-cudagraph-capture-size 256 --max-model-len 1048576 - --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} --enable-cumem-allocator --block-size 256 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} From 4366bc2ae39f44287e94963eea03cb43431bad72 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 14 Sep 2026 16:17:04 +0200 Subject: [PATCH 36/46] Fix disagg prefill OOM and add SimpleCPUOffloadConnector to mooncake disagg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Lower prefill gpu-memory-utilization 0.96→0.95 to prevent sparse attention indexer OOM under peak concurrency (seen in c128 run: Worker_DP7_EP7 OOM'd on device 3 during FLASHINFER_MLA_SPARSE_DSV4 indexer forward pass) - Add SimpleCPUOffloadConnector (~38 GB CPU DRAM staging) to the disagg+mooncake MultiConnector chain in server.sh, matching the reference production config - Unify both roles to kv_both (decode was incorrectly kv_consumer); decode needs kv_both to serve speculative-decode prefills in DSpark Co-Authored-By: Claude Sonnet 4.6 --- ...disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 2 +- benchmarks/multi_node/llm-d/server.sh | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 9c27ecbf92..4eb2f8b28c 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -69,7 +69,7 @@ prefill: enable-expert-parallel: true extra-args: >- --kv-cache-dtype fp8 - --gpu-memory-utilization 0.96 + --gpu-memory-utilization 0.95 --max-num-batched-tokens 8192 --long-prefill-token-threshold 1024 --max-num-seqs 16 diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 84792a134b..5a5ca32283 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -234,17 +234,11 @@ if [[ "$IS_AGGREGATED" -eq 0 ]]; then fi if [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then # MultiConnector: NixlConnector handles direct P/D KV transfer; + # SimpleCPUOffloadConnector stages KV in CPU DRAM (~38 GB) as a buffer; # MooncakeStoreConnector enables cross-node prefix-cache lookup via RDMA. - # Prefill uses kv_both so it can both store new KV and load cache hits. - # Decode uses kv_consumer with lookup disabled (it only receives from NIXL). - if [[ "$ROLE" == "prefill" ]]; then - _KV_OUTER="kv_both" - _MC_EXTRA='"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false' - else - _KV_OUTER="kv_consumer" - _MC_EXTRA='"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false' - fi - KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false,\"kv_lease_duration\":1800}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"${_KV_OUTER}\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + # Both roles use kv_both so decode can serve speculative-decode prefills. + _MC_EXTRA='"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false' + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false,\"kv_lease_duration\":1800}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":40802189312}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" else KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" fi From 5be3fcfec3463153e954385a74b2cf7b90b43b5a Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 14 Sep 2026 16:39:23 +0200 Subject: [PATCH 37/46] Move dsv4-gb200 changelog entry to end of file Co-Authored-By: Claude Sonnet 4.6 --- perf-changelog.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index af2b7661ba..e40dda4454 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6869,17 +6869,6 @@ - "Refresh the DeepSeek-V4-Pro FP4 B200 vLLM image and search space, pin the Nscale NVFP4 checkpoint, use the FlashInfer CuTeDSL MoE backend for expert-parallel runs with EPLB disabled, set GPU memory utilization to 0.95, and cap the model length at 12288." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2534 -- config-keys: - - dsv4-fp4-gb200-llmd-vllm-agentx - - dsv4-fp4-gb200-llmd-vllm-agentx-agg - scenario-type: - - agentic-coding - description: - - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation, 16-GPU NIXL DEP8 1P/1D, and llm-d 0.10.0 prefix/token-load routing. GPU-memory budgets: TP8 0.85, DEP8 0.88. P/D KV leases last 1800s to cover long-context decode stalls." - - "Use fixed-K DSpark golden AL: K=5/AL=3.61 for aggregated/decode; K=1/AL=1.84 for P/D prefill. Adaptive verification is disabled." - - "Mooncake variants use DRAM offload at 0.60 utilization, one job-local master, and IB HCAs mlx5_0,1,3,4; no SSD offload. Plain TP8/DEP8 has no offload. Scrape vLLM nodes directly without duplicate frontend counters." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 - - config-keys: - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg @@ -7615,3 +7604,14 @@ - "Avoid recursive TileRT eval dispatch and preserve evaluation failures after artifact staging." - "避免 TileRT 评测分发无限递归，并在保存产物后保留评测失败状态。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3067 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation, 16-GPU NIXL DEP8 1P/1D, and llm-d 0.10.0 prefix/token-load routing. GPU-memory budgets: TP8 0.85, DEP8 0.88. P/D KV leases last 1800s to cover long-context decode stalls." + - "Use fixed-K DSpark golden AL: K=5/AL=3.61 for aggregated/decode; K=1/AL=1.84 for P/D prefill. Adaptive verification is disabled." + - "Mooncake variants use DRAM offload at 0.60 utilization, one job-local master, and IB HCAs mlx5_0,1,3,4; no SSD offload. Plain TP8/DEP8 has no offload. Scrape vLLM nodes directly without duplicate frontend counters." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From b089e1664576229079ca1b767932327e07722b48 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 14 Sep 2026 16:58:59 +0200 Subject: [PATCH 38/46] Fix mock curl in test_llmd_agentic_adapter to handle --write-out preflight The previous mock always wrote METRICS_BODY to --output and never printed to stdout, so agentic.sh's envoy preflight check got an empty HTTP status instead of "404" and exited before any assertion could run. Now when --write-out is in args (preflight call), print "404" to stdout. Worker scrape calls (no --write-out) write METRICS_BODY to --output and record the URL in METRICS_REQUESTS as before. Co-Authored-By: Claude Sonnet 4.6 --- runners/test_slurm_utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 80febba516..6ff5599b09 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -234,9 +234,16 @@ def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, met curl.write_text( '#!/usr/bin/env python3\n' 'import os, sys\nfrom pathlib import Path\n' - 'Path(sys.argv[sys.argv.index("--output") + 1]).write_text(os.environ["METRICS_BODY"])\n' - 'with open(os.environ["METRICS_REQUESTS"], "a") as out:\n' - ' out.write(next(arg for arg in sys.argv if arg.startswith("http://")) + "\\n")\n' + 'args = sys.argv[1:]\n' + 'url = next((a for a in args if a.startswith("http://")), "")\n' + 'if "--write-out" in args:\n' + ' print("404", end="")\n' + 'else:\n' + ' out_path = args[args.index("--output") + 1]\n' + ' if out_path != "/dev/null":\n' + ' Path(out_path).write_text(os.environ["METRICS_BODY"])\n' + ' with open(os.environ["METRICS_REQUESTS"], "a") as f:\n' + ' f.write(url + "\\n")\n' ) curl.chmod(0o755) endpoints = tmp_path / "endpoints.yaml" From 884b32bf741dcb22aaf81e550ce997896d12f584 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 14 Sep 2026 17:11:48 +0200 Subject: [PATCH 39/46] Fix test_llmd_agentic_adapter endpoint fixture broken by 9411ed99b Commit 9411ed99b updated agentic.sh to read endpoint['name'], endpoint['labels']['llm-d.ai/role'], and DECODE_NODES (argv[4]) but did not update the test fixture. The endpoint dicts only had address/port so agentic.sh crashed with KeyError: 'name' before reaching the curl calls. Add name and labels to each endpoint, and DECODE_NODES=0 to the env. Co-Authored-By: Claude Sonnet 4.6 --- runners/test_slurm_utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 6ff5599b09..da9b1a9d6f 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -248,9 +248,12 @@ def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, met curl.chmod(0o755) endpoints = tmp_path / "endpoints.yaml" endpoints.write_text(yaml.safe_dump({"endpoints": [ - {"address": "10.0.0.1", "port": "8200"}, - {"address": "10.0.0.2", "port": "8200"}, - {"address": "10.0.0.3", "port": "8000"}, + {"address": "10.0.0.1", "port": "8200", "name": "vllm-node-0", + "labels": {"llm-d.ai/role": "combined"}}, + {"address": "10.0.0.2", "port": "8200", "name": "vllm-node-1", + "labels": {"llm-d.ai/role": "combined"}}, + {"address": "10.0.0.3", "port": "8000", "name": "vllm-node-2", + "labels": {"llm-d.ai/role": "combined"}}, ]})) requests = tmp_path / "metrics-requests.txt" env = dict(os.environ, INFMAX_CONTAINER_WORKSPACE=str(tmp_path), @@ -260,7 +263,7 @@ def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, met LLMD_ENDPOINTS_FILE=str(endpoints), MODEL_NAME="test-model", MODEL_PREFIX="dsv4", FRAMEWORK="llmd-vllm", DURATION="3600", IS_AGENTIC="1", KV_OFFLOADING="none", ENVOY_PORT="8080", VLLM_PORT="8200", BENCHMARK_LOGS_DIR=str(tmp_path / "logs"), - BENCH_MAX_CONCURRENCY="64") + BENCH_MAX_CONCURRENCY="64", DECODE_NODES="0") result = subprocess.run(["bash", str(REPO_ROOT / "benchmarks/multi_node/llm-d/agentic.sh")], env=env, text=True, capture_output=True) if metrics_body.startswith("envoy_"): From 23dae65a36c2efadd89e143e1ebc120044491ad3 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Tue, 15 Sep 2026 16:05:33 +0200 Subject: [PATCH 40/46] Fix mooncake KV pollution and EPP routing for agentic p1d1 and DEP8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disaggregated p1d1 (server.sh): - Remove SimpleCPUOffloadConnector from decode role. In EAGER mode it wrote every decoded KV block to mooncake, polluting the prefix cache with non-reusable decode blocks and evicting prefill prefix entries. Prefill retains SimpleCPUOffload (~38 GB CPU DRAM staging). Aggregated DEP8 mooncake (server.sh): - Add SimpleCPUOffloadConnector (~40 GB) to the IS_AGGREGATED+mooncake path, matching the local DEP8 v1 reference configuration. EPP agentic recipes (lruCapacityPerServer, peakPrefillThroughput, queueThresholdTokens): - p1d1: lruCapacityPerServer 45272→43949 (GPU 10,746 + CPU 33,203 per prefill DP rank, cross-validated against v13 kv_cache_config) - DEP8 no-mooncake: lru 8967→5858 (GPU only, 1,499,703 tokens / 256), peakPrefillThroughput 4783→20000, queueThreshold 2295620→1499703 - DEP8 mooncake: lru 8967→40810 (GPU 5,858 + CPU 34,952 per DP rank), peakPrefillThroughput 4783→20000, queueThreshold 3518020→1499703 All queueThresholdTokens are now calibrated per DP rank (one endpoint = one DP rank), and lruCapacityPerServer reflects actual GPU+CPU KV pool sizes reported by vLLM at the measured gpu_memory_utilization. Co-Authored-By: Claude Sonnet 4.6 --- .../agentic/agg-gb200-dep8-dspark-agentic.yaml | 6 +++--- ...agg-gb200-dep8-dspark-mooncake-agentic.yaml | 8 ++++---- ...gg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 2 +- benchmarks/multi_node/llm-d/server.sh | 18 ++++++++++++++---- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index 0fe05de446..8c8b14e396 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -17,15 +17,15 @@ plugins: blockSizeTokens: 256 maxPrefixTokensToMatch: 1048576 maxPrefixBlocksToMatch: 4096 - lruCapacityPerServer: 8967 + lruCapacityPerServer: 5858 - type: prefix-cache-affinity-filter parameters: - peakPrefillThroughput: 4783 + peakPrefillThroughput: 20000 maxTTFTPenaltyMs: 30000 - type: prefix-cache-scorer - type: token-load-scorer parameters: - queueThresholdTokens: 2295620 + queueThresholdTokens: 1499703 - type: active-request-scorer - type: queue-scorer - type: max-score-picker diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 3328049329..11df6b8a63 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -1,6 +1,6 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated DEP8 with Mooncake prefix-cache. # Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds Mooncake (P2PHANDSHAKE embedded RDMA) -# so server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role). +# so server.sh wires MultiConnector (NixlConnector + SimpleCPUOffloadConnector + MooncakeStoreConnector, kv_both). # DSpark 5 speculative tokens. conc-list [52,72]. apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig @@ -19,15 +19,15 @@ plugins: blockSizeTokens: 256 maxPrefixTokensToMatch: 1048576 maxPrefixBlocksToMatch: 4096 - lruCapacityPerServer: 8967 + lruCapacityPerServer: 40810 - type: prefix-cache-affinity-filter parameters: - peakPrefillThroughput: 4783 + peakPrefillThroughput: 20000 maxTTFTPenaltyMs: 30000 - type: prefix-cache-scorer - type: token-load-scorer parameters: - queueThresholdTokens: 3518020 + queueThresholdTokens: 1499703 - type: active-request-scorer - type: queue-scorer - type: max-score-picker diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 4eb2f8b28c..878d002b24 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -21,7 +21,7 @@ plugins: blockSizeTokens: 256 maxPrefixTokensToMatch: 1048576 maxPrefixBlocksToMatch: 4096 - lruCapacityPerServer: 45272 + lruCapacityPerServer: 43949 - type: prefix-cache-affinity-filter parameters: peakPrefillThroughput: 4783 diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 5a5ca32283..20efdf3b3a 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -234,11 +234,19 @@ if [[ "$IS_AGGREGATED" -eq 0 ]]; then fi if [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then # MultiConnector: NixlConnector handles direct P/D KV transfer; - # SimpleCPUOffloadConnector stages KV in CPU DRAM (~38 GB) as a buffer; # MooncakeStoreConnector enables cross-node prefix-cache lookup via RDMA. - # Both roles use kv_both so decode can serve speculative-decode prefills. + # Prefill additionally uses SimpleCPUOffloadConnector to stage KV in CPU + # DRAM (~38 GB) before writing to mooncake. Decode omits SimpleCPUOffload: + # EAGER mode would store every decoded KV block into mooncake, polluting + # the prefix cache with non-reusable blocks and reducing mooncake hit rate + # for prefill lookups. Both roles use kv_both so decode can serve + # speculative-decode prefills in DSpark. _MC_EXTRA='"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false' - KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false,\"kv_lease_duration\":1800}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":40802189312}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + if [[ "$ROLE" == "prefill" ]]; then + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false,\"kv_lease_duration\":1800}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":40802189312}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + else + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false,\"kv_lease_duration\":1800}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + fi else KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" fi @@ -246,7 +254,9 @@ if [[ "$IS_AGGREGATED" -eq 0 ]]; then elif [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then # Aggregated + Mooncake: single role acts as kv_both (stores new KV and # loads cache hits from the Mooncake RDMA store for prefix-cache sharing). - KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"load_async\":true,\"lookup_async\":true,\"enable_cross_layers_blocks\":false,\"enable_offload\":false}}]}}" + # SimpleCPUOffloadConnector stages freshly computed KV blocks into CPU DRAM + # (~40 GB) before writing to MooncakeStore, matching the local DEP8 v1 setup. + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":42949672960}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"load_async\":true,\"lookup_async\":true,\"enable_cross_layers_blocks\":false,\"enable_offload\":false}}]}}" COMMON_ARGS+=(--kv_transfer_config "$KV_TRANSFER_CONFIG") fi # A single frontend (HTTP + tokenize + DP load-balance) is CPU-bound and caps From d6a6d829e66f71cf2b10a85007663d561fa9efca Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Tue, 15 Sep 2026 16:18:51 +0200 Subject: [PATCH 41/46] Correct after merge --- perf-changelog.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8b9d023275..19ede959d2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7614,17 +7614,6 @@ - "将 GB200 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens；保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3097 -- config-keys: - - dsv4-fp4-gb200-llmd-vllm-agentx - - dsv4-fp4-gb200-llmd-vllm-agentx-agg - scenario-type: - - agentic-coding - description: - - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation, 16-GPU NIXL DEP8 1P/1D, and llm-d 0.10.0 prefix/token-load routing. GPU-memory budgets: TP8 0.85, DEP8 0.88. P/D KV leases last 1800s to cover long-context decode stalls." - - "Use fixed-K DSpark golden AL: K=5/AL=3.61 for aggregated/decode; K=1/AL=1.84 for P/D prefill. Adaptive verification is disabled." - - "Mooncake variants use DRAM offload at 0.60 utilization, one job-local master, and IB HCAs mlx5_0,1,3,4; no SSD offload. Plain TP8/DEP8 has no offload. Scrape vLLM nodes directly without duplicate frontend counters." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 - - config-keys: - qwen3.5-fp4-mi355x-sglang - qwen3.5-fp4-mi355x-sglang-mtp @@ -7800,3 +7789,14 @@ - "Set a 64-token CUDA graph capture floor for H200 DSpark at c1/c2/c4; preserve c8+ sizing, full AgentX traces, AL 3.51, and Engram UVA" - "将 H200 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens；保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3099 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation, 16-GPU NIXL DEP8 1P/1D, and llm-d 0.10.0 prefix/token-load routing. GPU-memory budgets: TP8 0.85, DEP8 0.88. P/D KV leases last 1800s to cover long-context decode stalls." + - "Use fixed-K DSpark golden AL: K=5/AL=3.61 for aggregated/decode; K=1/AL=1.84 for P/D prefill. Adaptive verification is disabled." + - "Mooncake variants use DRAM offload at 0.60 utilization, one job-local master, and IB HCAs mlx5_0,1,3,4; no SSD offload. Plain TP8/DEP8 has no offload. Scrape vLLM nodes directly without duplicate frontend counters." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From ad34e7f4126849e8f8903cc7f9cfc73dd1c5c8a9 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Wed, 16 Sep 2026 13:55:27 +0200 Subject: [PATCH 42/46] fix(llm-d): use external LB and per-rank EPP endpoints for DEP8 AgentX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace hybrid LB and two-endpoint discovery with multi-port external LB, per-DP-rank EPP registration, multi-port pd-sidecar, supervisor health on 8100, Nixl-only decode KV, and rank-aware AgentX metrics scraping. 将 DEP8 AgentX 从 hybrid LB 与双端点发现改为多端口外部 LB、按 DP rank 注册 EPP、多端口 pd-sidecar、8100 supervisor 健康检查、decode 仅 Nixl KV， 以及按 rank 抓取 AgentX 指标。 Co-authored-by: Cursor --- benchmarks/multi_node/llm-d/agentic.sh | 14 ++- benchmarks/multi_node/llm-d/server.sh | 118 ++++++++++++++----------- perf-changelog.yaml | 10 +++ runners/test_slurm_utils.py | 81 ++++++++++++++++- 4 files changed, 163 insertions(+), 60 deletions(-) diff --git a/benchmarks/multi_node/llm-d/agentic.sh b/benchmarks/multi_node/llm-d/agentic.sh index 14e031744f..e95972a01a 100644 --- a/benchmarks/multi_node/llm-d/agentic.sh +++ b/benchmarks/multi_node/llm-d/agentic.sh @@ -15,15 +15,25 @@ export CONC="${CONC_LIST%% *}" # Use discovery's serving nodes, but scrape vLLM rather than the decode sidecar. mkdir -p "$RESULT_DIR" AIPERF_METRIC_URLS=$(python3 - "$LLMD_ENDPOINTS_FILE" "$VLLM_PORT" \ - "$RESULT_DIR/llmd_metrics_endpoints.json" "$DECODE_NODES" <<'PY' + "$RESULT_DIR/llmd_metrics_endpoints.json" "$DECODE_NODES" "${SIDECAR_PORT:-8000}" <<'PY' import json import sys import yaml with open(sys.argv[1]) as source: endpoints = yaml.safe_load(source)["endpoints"] +vllm_base = int(sys.argv[2]) +sidecar_base = int(sys.argv[5]) + +def vllm_metrics_port(endpoint): + role = endpoint["labels"]["llm-d.ai/role"] + endpoint_port = int(endpoint["port"]) + if role == "decode": + return vllm_base + (endpoint_port - sidecar_base) + return endpoint_port + metrics_endpoints = { - f"http://{endpoint['address']}:{int(sys.argv[2])}/metrics": { + f"http://{endpoint['address']}:{vllm_metrics_port(endpoint)}/metrics": { "name": endpoint["name"], "role": endpoint["labels"]["llm-d.ai/role"] if int(sys.argv[4]) else "combined", } diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 20efdf3b3a..b88dca0c89 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -233,19 +233,19 @@ if [[ "$IS_AGGREGATED" -eq 0 ]]; then KV_ROLE="kv_consumer" fi if [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then - # MultiConnector: NixlConnector handles direct P/D KV transfer; - # MooncakeStoreConnector enables cross-node prefix-cache lookup via RDMA. - # Prefill additionally uses SimpleCPUOffloadConnector to stage KV in CPU - # DRAM (~38 GB) before writing to mooncake. Decode omits SimpleCPUOffload: - # EAGER mode would store every decoded KV block into mooncake, polluting - # the prefix cache with non-reusable blocks and reducing mooncake hit rate - # for prefill lookups. Both roles use kv_both so decode can serve - # speculative-decode prefills in DSpark. + # MultiConnector on prefill: NixlConnector handles direct P/D KV transfer; + # SimpleCPUOffloadConnector stages KV in CPU DRAM (~38 GB) before writing + # to MooncakeStoreConnector for cross-node prefix-cache lookup via RDMA. + # Decode uses NixlConnector only (matches agentX v13): Mooncake on decode + # would pollute the prefix cache with non-reusable decode blocks, and + # SimpleCPUOffload in EAGER mode would amplify that. kv_both on decode + # so it can serve speculative-decode prefills in DSpark. _MC_EXTRA='"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false' + _NIXL_EXTRA='"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800' if [[ "$ROLE" == "prefill" ]]; then - KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false,\"kv_lease_duration\":1800}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":40802189312}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{${_NIXL_EXTRA}}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":40802189312}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" else - KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false,\"kv_lease_duration\":1800}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{${_NIXL_EXTRA}}}" fi else KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" @@ -259,21 +259,13 @@ elif [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":42949672960}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"load_async\":true,\"lookup_async\":true,\"enable_cross_layers_blocks\":false,\"enable_offload\":false}}]}}" COMMON_ARGS+=(--kv_transfer_config "$KV_TRANSFER_CONFIG") fi -# A single frontend (HTTP + tokenize + DP load-balance) is CPU-bound and caps -# throughput, so run several. Incompatible with --headless, so it is the one -# flag the headless-worker branch below drops. Overridable via LLMD_API_SERVER_COUNT. -# LB is hybrid: --data-parallel-hybrid-lb; one api-server per node internally -# load-balances its local DP ranks -> ONE serving port (VLLM_PORT) per node, so -# the local rank-0 health port is always VLLM_PORT. +# EP roles use multi-port-external-lb: each local DP rank gets its own serving +# port starting at VLLM_PORT (8200, 8201, ...). The vLLM supervisor binds 8100 +# and serves /health once all engines are ready -> health check uses port 8100. +# Pure-TP roles serve on a single VLLM_PORT with the standard health check. HEALTH_PORT="$VLLM_PORT" -API_SERVER_COUNT="4" -# Multiple frontends only help the DP (wide-EP) path, where they load-balance -# across the node's local DP ranks. A pure-TP engine has a single core with one -# frontend, so it keeps the default count (also avoids --api-server-count -# interacting with the --headless multi-node TP launch below). Every DEP8 node -# gets it; pure-TP nodes get none. if [[ "$ROLE_ENABLE_EP" == "true" ]]; then - COMMON_ARGS+=(--api-server-count "$API_SERVER_COUNT") + HEALTH_PORT="8100" fi # Set to 1 by the pure-TP multi-node branch below on --headless followers, which # run no local api-server; gates the post-launch health wait. @@ -287,10 +279,11 @@ if [[ "$ROLE_ENABLE_EP" == "true" ]]; then COMMON_ARGS+=( --enable-expert-parallel --data-parallel-size "$DP_SIZE" + --data-parallel-multi-port-external-lb + --data-parallel-supervisor-port 8100 ) if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then COMMON_ARGS+=( - --data-parallel-hybrid-lb --data-parallel-size-local "$DP_SIZE_LOCAL" --data-parallel-address "$DP_ADDR" --data-parallel-rpc-port 5555 @@ -347,7 +340,8 @@ if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX SIDECAR_CONNECTOR="nixlv2" SIDECAR_FLAGS=(--port="$SIDECAR_PORT" --vllm-port="$VLLM_PORT" --kv-connector="$SIDECAR_CONNECTOR" --secure-proxy=false - --enable-prefiller-sampling) + --enable-prefiller-sampling + --data-parallel-size="$DP_SIZE_LOCAL") SIDECAR_HEALTH_PORT="$SIDECAR_PORT" echo "Starting pd-sidecar (decode node_rank=$NODE_RANK worker_index=$LWS_WORKER_INDEX): ${SIDECAR_FLAGS[*]}" pd-sidecar "${SIDECAR_FLAGS[@]}" > "$SIDECAR_LOG" 2>&1 & @@ -374,19 +368,27 @@ import os, yaml ips = os.environ['ALL_IPS'].split(',') pn = int(os.environ['PREFILL_NODES']) dn = int(os.environ['DECODE_NODES']) +gpus_per_node = int('$GPUS_PER_NODE') endpoints = [] -def add_role(role, addresses, port, group_size): - for i, address in enumerate(addresses[::group_size]): - endpoints.append({'name': f'{role}-{i}', 'namespace': 'inferencex', - 'address': address, 'port': str(port), - 'labels': {'llm-d.ai/role': role}}) - -prefill_group = 1 if '$PREFILL_ENABLE_EP' == 'true' else pn // int('$PREFILL_WORKERS') -add_role('prefill', ips[:pn], int('$VLLM_PORT'), prefill_group) +def add_role(role, addresses, port, group_size, dp_local=1): + idx = 0 + for address in addresses[::group_size]: + for rank in range(dp_local): + endpoints.append({'name': f'{role}-{idx}', 'namespace': 'inferencex', + 'address': address, 'port': str(port + rank), + 'labels': {'llm-d.ai/role': role}}) + idx += 1 + +prefill_ep = '$PREFILL_ENABLE_EP' == 'true' +prefill_group = 1 if prefill_ep else pn // int('$PREFILL_WORKERS') +prefill_dp_local = gpus_per_node if prefill_ep else 1 +add_role('prefill', ips[:pn], int('$VLLM_PORT'), prefill_group, prefill_dp_local) if dn: - decode_group = 1 if '$ROLE_ENABLE_EP' == 'true' else dn // int('$DECODE_WORKERS') - add_role('decode', ips[pn:pn + dn], int('$SIDECAR_PORT'), decode_group) + decode_ep = '$ROLE_ENABLE_EP' == 'true' + decode_group = 1 if decode_ep else dn // int('$DECODE_WORKERS') + decode_dp_local = gpus_per_node if decode_ep else 1 + add_role('decode', ips[pn:pn + dn], int('$SIDECAR_PORT'), decode_group, decode_dp_local) with open(os.environ['LLMD_ENDPOINTS_FILE'], 'w') as output: yaml.safe_dump({'endpoints': endpoints}, output) print(yaml.safe_dump({'endpoints': endpoints})) @@ -466,19 +468,29 @@ PY # ---- Gate on ALL prefill vLLM /health endpoints (cross-node) ---- # Prefill ranks wait on their own local /health; wait_for_server_ready only - # probes localhost, so the decode leader polls the prefill nodes here. - # endpoints.yaml lists one prefill endpoint per node, so with PREFILL_WORKERS>1 - # (multiple independent DP engines) EVERY prefill node must be probed, not just - # IPS[0]. curl gets an explicit connect/max timeout so a blackholed endpoint - # trips the deadline instead of hanging the whole run (a single timeout-less - # curl once wedged a 2P run for 7h before it was cancelled). + # probes localhost, so the coordinator polls every prefill node here. + # External LB registers one EPP endpoint per DP rank, so dedupe by node IP. + # EP roles expose /health on the DP supervisor (8100), not the serving port. + # curl gets an explicit connect/max timeout so a blackholed endpoint trips the + # deadline instead of hanging the whole run (a single timeout-less curl once + # wedged a 2P run for 7h before it was cancelled). mapfile -t _prefill_ips < <(python3 - "$LLMD_ENDPOINTS_FILE" <<'PY' import sys, yaml +seen = set() for endpoint in yaml.safe_load(open(sys.argv[1]))['endpoints']: - if endpoint['labels']['llm-d.ai/role'] == 'prefill': - print(endpoint['address']) + if endpoint['labels']['llm-d.ai/role'] != 'prefill': + continue + address = endpoint['address'] + if address in seen: + continue + seen.add(address) + print(address) PY ) + _PREFILL_HEALTH_PORT="$VLLM_PORT" + if [[ "$PREFILL_ENABLE_EP" == "true" ]]; then + _PREFILL_HEALTH_PORT="8100" + fi # On failure, dump enough to tell a server-not-ready problem (TCP connects but # /health is slow) apart from a network/subnet problem (TCP connect refused or @@ -550,28 +562,26 @@ PY } >&2 } - # Log the decode->prefill target layout up front so a subnet/interface - # mismatch is visible even on a run that eventually succeeds. Every prefill - # node serves on VLLM_PORT (hybrid LB). - echo "[diag] decode-leader $(hostname 2>/dev/null) local-ips: $(hostname -I 2>/dev/null); prefill targets: ${_prefill_ips[*]}" - echo "Waiting for prefill vLLM /health on ${#_prefill_ips[@]} node(s): ${_prefill_ips[*]}" + # Log the coordinator->prefill target layout up front so a subnet/interface + # mismatch is visible even on a run that eventually succeeds. + echo "[diag] coordinator $(hostname 2>/dev/null) local-ips: $(hostname -I 2>/dev/null); prefill targets: ${_prefill_ips[*]}:${_PREFILL_HEALTH_PORT}" + echo "Waiting for prefill vLLM /health on ${#_prefill_ips[@]} node(s) (port ${_PREFILL_HEALTH_PORT}): ${_prefill_ips[*]}" PREFILL_WAIT_DEADLINE=$(( $(date +%s) + 300 )) for _pidx in "${!_prefill_ips[@]}"; do _pip="${_prefill_ips[$_pidx]}" - _pport="$VLLM_PORT" until curl --output /dev/null --silent --fail \ --connect-timeout 5 --max-time 10 \ - "http://$_pip:$_pport/health"; do + "http://$_pip:${_PREFILL_HEALTH_PORT}/health"; do if [[ "$(date +%s)" -ge "$PREFILL_WAIT_DEADLINE" ]]; then - echo "ERROR: prefill vLLM at $_pip:$_pport not ready within 5 min" >&2 - _diag_prefill_endpoint "$_pip" "$_pport" + echo "ERROR: prefill vLLM at $_pip:${_PREFILL_HEALTH_PORT} not ready within 5 min" >&2 + _diag_prefill_endpoint "$_pip" "$_PREFILL_HEALTH_PORT" exit 1 fi sleep 5 done - echo "Prefill vLLM at $_pip:$_pport is ready" + echo "Prefill vLLM at $_pip:${_PREFILL_HEALTH_PORT} is ready" done - echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready" + echo "All ${#_prefill_ips[@]} prefill vLLM node(s) ready" if [[ "${IS_AGENTIC}" == "1" && "${EVAL_ONLY}" != "true" ]]; then export ENVOY_PORT VLLM_PORT INFMAX_CONTAINER_WORKSPACE diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 19ede959d2..f30d3d6fa6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7800,3 +7800,13 @@ - "Use fixed-K DSpark golden AL: K=5/AL=3.61 for aggregated/decode; K=1/AL=1.84 for P/D prefill. Adaptive verification is disabled." - "Mooncake variants use DRAM offload at 0.60 utilization, one job-local master, and IB HCAs mlx5_0,1,3,4; no SSD offload. Plain TP8/DEP8 has no offload. Scrape vLLM nodes directly without duplicate frontend counters." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Switch DEP8 llm-d wide-EP from hybrid LB to multi-port external LB: register one EPP endpoint per DP rank, open multi-port pd-sidecar on decode, probe DP supervisor /health on 8100, use Nixl-only decode KV, and scrape per-rank vLLM metrics in AgentX." + - "将 DEP8 llm-d 宽 EP 从 hybrid LB 切换为多端口外部 LB：按 DP rank 注册 EPP 端点，decode 侧启用多端口 pd-sidecar，在 8100 探测 DP supervisor /health，decode 仅保留 Nixl KV，并在 AgentX 中按 rank 抓取 vLLM 指标。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 396396ce84..3e28207b1b 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -250,9 +250,9 @@ def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, met endpoints.write_text(yaml.safe_dump({"endpoints": [ {"address": "10.0.0.1", "port": "8200", "name": "vllm-node-0", "labels": {"llm-d.ai/role": "combined"}}, - {"address": "10.0.0.2", "port": "8200", "name": "vllm-node-1", + {"address": "10.0.0.2", "port": "8201", "name": "vllm-node-1", "labels": {"llm-d.ai/role": "combined"}}, - {"address": "10.0.0.3", "port": "8000", "name": "vllm-node-2", + {"address": "10.0.0.3", "port": "8202", "name": "vllm-node-2", "labels": {"llm-d.ai/role": "combined"}}, ]})) requests = tmp_path / "metrics-requests.txt" @@ -262,7 +262,8 @@ def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, met METRICS_BODY=metrics_body, METRICS_REQUESTS=str(requests), LLMD_ENDPOINTS_FILE=str(endpoints), MODEL_NAME="test-model", MODEL_PREFIX="dsv4", FRAMEWORK="llmd-vllm", DURATION="3600", IS_AGENTIC="1", KV_OFFLOADING="none", - ENVOY_PORT="8080", VLLM_PORT="8200", BENCHMARK_LOGS_DIR=str(tmp_path / "logs"), + ENVOY_PORT="8080", VLLM_PORT="8200", SIDECAR_PORT="8000", + BENCHMARK_LOGS_DIR=str(tmp_path / "logs"), BENCH_MAX_CONCURRENCY="64", DECODE_NODES="0") result = subprocess.run(["bash", str(REPO_ROOT / "benchmarks/multi_node/llm-d/agentic.sh")], env=env, text=True, capture_output=True) @@ -272,7 +273,11 @@ def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, met return assert result.returncode == 0, result.stderr recorded = json.loads(result.stdout.splitlines()[-1]) - expected_urls = ["http://10.0.0.1:8200/metrics", "http://10.0.0.2:8200/metrics", "http://10.0.0.3:8200/metrics"] + expected_urls = [ + "http://10.0.0.1:8200/metrics", + "http://10.0.0.2:8201/metrics", + "http://10.0.0.3:8202/metrics", + ] assert requests.read_text().splitlines() == expected_urls assert recorded["AIPERF_METRIC_URLS"].split(",") == expected_urls assert recorded["AIPERF_SERVER_METRICS_URLS"].split(",") == expected_urls @@ -280,6 +285,74 @@ def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, met assert "--server-metrics " + " ".join(expected_urls) + " " in recorded["REPLAY_CMD"] +def test_llmd_agentic_adapter_maps_decode_sidecar_ports_to_vllm_metrics( + tmp_path: Path, +) -> None: + """Disagg decode endpoints list sidecar ports; metrics scrape vLLM DP ranks.""" + client = tmp_path / "benchmarks/multi_node/agentic_srt.sh" + client.parent.mkdir(parents=True) + client.write_text('''source "$REAL_BENCHMARK_LIB" +build_replay_cmd "$RESULT_DIR" +export REPLAY_CMD +python3 - <<'PY' +import json, os +keys = ["AIPERF_METRIC_URLS", "AIPERF_SERVER_METRICS_URLS", "REPLAY_CMD"] +print(json.dumps({key: os.environ[key] for key in keys})) +PY +''') + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + curl = bin_dir / "curl" + curl.write_text( + '#!/usr/bin/env python3\n' + 'import os, sys\nfrom pathlib import Path\n' + 'args = sys.argv[1:]\n' + 'url = next((a for a in args if a.startswith("http://")), "")\n' + 'if "--write-out" in args:\n' + ' print("404", end="")\n' + 'else:\n' + ' out_path = args[args.index("--output") + 1]\n' + ' if out_path != "/dev/null":\n' + ' Path(out_path).write_text(os.environ["METRICS_BODY"])\n' + ' with open(os.environ["METRICS_REQUESTS"], "a") as f:\n' + ' f.write(url + "\\n")\n' + ) + curl.chmod(0o755) + endpoints = tmp_path / "endpoints.yaml" + endpoints.write_text(yaml.safe_dump({"endpoints": [ + {"address": "10.0.0.10", "port": "8200", "name": "prefill-0", + "labels": {"llm-d.ai/role": "prefill"}}, + {"address": "10.0.0.10", "port": "8201", "name": "prefill-1", + "labels": {"llm-d.ai/role": "prefill"}}, + {"address": "10.0.0.20", "port": "8000", "name": "decode-0", + "labels": {"llm-d.ai/role": "decode"}}, + {"address": "10.0.0.20", "port": "8001", "name": "decode-1", + "labels": {"llm-d.ai/role": "decode"}}, + ]})) + requests = tmp_path / "metrics-requests.txt" + env = dict(os.environ, INFMAX_CONTAINER_WORKSPACE=str(tmp_path), + REAL_BENCHMARK_LIB=str(REPO_ROOT / "benchmarks/benchmark_lib.sh"), + PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], + METRICS_BODY="vllm:num_requests_running 0\n", METRICS_REQUESTS=str(requests), + LLMD_ENDPOINTS_FILE=str(endpoints), MODEL_NAME="test-model", MODEL_PREFIX="dsv4", + FRAMEWORK="llmd-vllm", DURATION="3600", IS_AGENTIC="1", KV_OFFLOADING="none", + ENVOY_PORT="8080", VLLM_PORT="8200", SIDECAR_PORT="8000", + BENCHMARK_LOGS_DIR=str(tmp_path / "logs"), + BENCH_MAX_CONCURRENCY="64", DECODE_NODES="2") + result = subprocess.run(["bash", str(REPO_ROOT / "benchmarks/multi_node/llm-d/agentic.sh")], + env=env, text=True, capture_output=True) + assert result.returncode == 0, result.stderr + recorded = json.loads(result.stdout.splitlines()[-1]) + expected_urls = [ + "http://10.0.0.10:8200/metrics", + "http://10.0.0.10:8201/metrics", + "http://10.0.0.20:8200/metrics", + "http://10.0.0.20:8201/metrics", + ] + assert requests.read_text().splitlines() == expected_urls + assert recorded["AIPERF_METRIC_URLS"].split(",") == expected_urls + + def test_b300_dsxe_draft_model_uses_public_paths_and_writable_hf_cache() -> None: launcher = (REPO_ROOT / "runners/launch_b300-dsxe.sh").read_text() From f88fe3cf8213da0a3fa8f8bec61385b352835ebb Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Thu, 17 Sep 2026 11:26:01 +0200 Subject: [PATCH 43/46] fix(llm-d): restore GB200 disagg wrapper and re-deprecate 8k1k key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-remove dsv4-fp4-gb200-llmd-vllm from active master after the upstream merge reintroduced it, and restore the thin AgentX disagg launcher so P/D llmd-vllm jobs can submit again. 重新移除 upstream 合并误带回的 dsv4-fp4-gb200-llmd-vllm 8k1k 主配置项， 并恢复 AgentX P/D llmd-vllm 所需的精简 disagg launcher。 Co-authored-by: Cursor --- .../dsv4_fp4_gb200_llmd-vllm-disagg.sh | 0 configs/nvidia-master.yaml | 82 ------------------- perf-changelog.yaml | 9 ++ 3 files changed, 9 insertions(+), 82 deletions(-) rename benchmarks/multi_node/{deprecated => }/dsv4_fp4_gb200_llmd-vllm-disagg.sh (100%) diff --git a/benchmarks/multi_node/deprecated/dsv4_fp4_gb200_llmd-vllm-disagg.sh b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh similarity index 100% rename from benchmarks/multi_node/deprecated/dsv4_fp4_gb200_llmd-vllm-disagg.sh rename to benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b653d3f608..912b54c735 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4153,88 +4153,6 @@ dsv4-fp4-gb200-dynamo-vllm: ep: 8 dp-attn: true -# TODO: change image to official llmd image. -# Build source: benchmarks/llm-d/Dockerfile. -dsv4-fp4-gb200-llmd-vllm: - image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 - framework: llmd-vllm - router: { name: llm-d-router, version: "0.9.0" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Low latency: 1 prefill DEP8 + 1 decode TP8. - - spec-decoding: "none" - conc-list: [1] - prefill: - num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-low-latency.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" - - # Mid curve: 1 prefill DEP8 + 1 decode DEP8. - - spec-decoding: "none" - conc-list: [256, 512, 1024] - prefill: - num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" - - # Max throughput: 3 prefill DEP8 + 1 decode DEP8. - - spec-decoding: "none" - conc-list: [4096] - prefill: - num-worker: 3 - tp: 1 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=6" - - "PREFILL_WORKERS=3" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" - # DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, P/D disagg via llmd-vllm. # Long-context workspace headroom: GPU-memory budgets are 0.88 for DEP8, 0.85 for TP8. # Always uses Mooncake; DSpark speculative decoding. The DSpark image diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f58eece884..8b5b431d40 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8009,3 +8009,12 @@ - "Switch DEP8 llm-d wide-EP from hybrid LB to multi-port external LB: register one EPP endpoint per DP rank, open multi-port pd-sidecar on decode, probe DP supervisor /health on 8100, use Nixl-only decode KV, and scrape per-rank vLLM metrics in AgentX." - "将 DEP8 llm-d 宽 EP 从 hybrid LB 切换为多端口外部 LB：按 DP rank 注册 EPP 端点，decode 侧启用多端口 pd-sidecar，在 8100 探测 DP supervisor /health，decode 仅保留 Nixl KV，并在 AgentX 中按 rank 抓取 vLLM 指标。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + scenario-type: + - agentic-coding + description: + - "Re-remove archived dsv4-fp4-gb200-llmd-vllm 8k1k master key reintroduced by the upstream merge and restore the thin GB200 llmd-vllm disagg wrapper for AgentX P/D launches." + - "重新移除 upstream 合并误带回的已归档 dsv4-fp4-gb200-llmd-vllm 8k1k 主配置项，并恢复供 AgentX P/D 启动使用的 GB200 llmd-vllm disagg 精简 wrapper。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 3d1ea2b571ac591745236888f0899b1a31ad9466 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Thu, 17 Sep 2026 11:58:10 +0200 Subject: [PATCH 44/46] fix(llm-d): forward INFMAX_CONTAINER_WORKSPACE for agentic throughput --- benchmarks/multi_node/llm-d/job.slurm | 2 ++ benchmarks/multi_node/llm-d/server.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index 5dcfc880f9..3f1e3d0f28 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -67,6 +67,7 @@ DOCKER_CONT_NAME="llmd_bench_${SANITIZED_USER}_${SLURM_JOB_ID}" export DOCKER_CONT_NAME : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" DOCKER_MOUNT_PATH="/workspace" +export INFMAX_CONTAINER_WORKSPACE="$DOCKER_MOUNT_PATH" cleanup() { echo "[${SLURM_JOB_ID}] cleanup on $(hostname)" @@ -109,6 +110,7 @@ done # metadata on both engines. Forward names, not interpolated values: JSON and # HF_TOKEN must never be embedded in the nested shell command or printed. AGENTIC_ENV_NAMES=( + INFMAX_CONTAINER_WORKSPACE IS_AGENTIC SCENARIO_TYPE CONC CONC_LIST DURATION AIPERF_EXPERIMENTAL_FAST IMAGE RECIPE_FINGERPRINT DISAGG HF_TOKEN KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index be3f8ac80f..c042dd23b0 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -586,7 +586,7 @@ PY echo "All ${#_prefill_ips[@]} prefill vLLM node(s) ready" if [[ "${IS_AGENTIC}" == "1" && "${EVAL_ONLY}" != "true" ]]; then - export ENVOY_PORT VLLM_PORT INFMAX_CONTAINER_WORKSPACE + export ENVOY_PORT VLLM_PORT INFMAX_CONTAINER_WORKSPACE=/workspace bash /workspace/benchmarks/multi_node/llm-d/agentic.sh elif [[ "${EVAL_ONLY}" != "true" ]]; then # ---- Benchmark sweep (one run per concurrency level) ---- From 545e5e6acd142cb894eefe7514a2bea8b8f5186c Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Thu, 17 Sep 2026 13:34:17 +0200 Subject: [PATCH 45/46] debug(llm-d): enable VLLM_LOGGING_LEVEL=DEBUG for DSpark AgentX bring-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise vLLM logging on llmd-vllm runtime defaults and GB200 DSpark AgentX recipes so external-LB DEP8 worker deaths leave actionable traces. 将 llmd-vllm 运行时默认与 GB200 DSpark AgentX 配方的 vLLM 日志级别 提升至 DEBUG，以便外部 LB DEP8 worker 异常退出时留下可诊断日志。 Co-authored-by: Cursor --- .../agentic/agg-gb200-dep8-dspark-agentic.yaml | 1 + .../agg-gb200-dep8-dspark-mooncake-agentic.yaml | 1 + .../disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 4 ++-- benchmarks/multi_node/runtime_settings.sh | 2 +- perf-changelog.yaml | 10 ++++++++++ 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index 8c8b14e396..aacd2aaeec 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -81,6 +81,7 @@ prefill: VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_LOGGING_LEVEL: DEBUG VLLM_USE_DEEP_GEMM: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 11df6b8a63..0e310cd6d4 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -84,6 +84,7 @@ prefill: VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_LOGGING_LEVEL: DEBUG VLLM_USE_DEEP_GEMM: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 878d002b24..6718844457 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -107,7 +107,7 @@ prefill: VLLM_USE_BREAKABLE_CUDAGRAPH: "0" FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED: "1" VLLM_NO_USAGE_STATS: "1" - VLLM_LOGGING_LEVEL: INFO + VLLM_LOGGING_LEVEL: DEBUG TQDM_DISABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM @@ -166,7 +166,7 @@ decode: VLLM_USE_DEEP_GEMM: "1" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_NO_USAGE_STATS: "1" - VLLM_LOGGING_LEVEL: INFO + VLLM_LOGGING_LEVEL: DEBUG TQDM_DISABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM diff --git a/benchmarks/multi_node/runtime_settings.sh b/benchmarks/multi_node/runtime_settings.sh index 5bf43deb6e..fe1035b385 100644 --- a/benchmarks/multi_node/runtime_settings.sh +++ b/benchmarks/multi_node/runtime_settings.sh @@ -61,7 +61,7 @@ case "$FRAMEWORK" in ;; llmd-vllm) export LLMD_CONTAINER_ENGINE=docker VLLM_RANDOMIZE_DP_DUMMY_INPUTS=1 - export VLLM_ENGINE_READY_TIMEOUT_S=1800 VLLM_LOGGING_LEVEL=INFO UCX_TLS=cuda_copy,cuda_ipc,rc + export VLLM_ENGINE_READY_TIMEOUT_S=1800 VLLM_LOGGING_LEVEL=DEBUG UCX_TLS=cuda_copy,cuda_ipc,rc export NVSHMEM_REMOTE_TRANSPORT=ibgda NVSHMEM_IB_ENABLE_IBGDA=true NVSHMEM_SYMMETRIC_SIZE=16G export LLMD_API_SERVER_COUNT=4 ;; diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 320e6f313f..2a102b6fe7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8033,3 +8033,13 @@ - "Re-remove archived dsv4-fp4-gb200-llmd-vllm 8k1k master key reintroduced by the upstream merge and restore the thin GB200 llmd-vllm disagg wrapper for AgentX P/D launches." - "重新移除 upstream 合并误带回的已归档 dsv4-fp4-gb200-llmd-vllm 8k1k 主配置项，并恢复供 AgentX P/D 启动使用的 GB200 llmd-vllm disagg 精简 wrapper。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Raise llm-d vLLM logging to DEBUG on GB200 DSpark AgentX recipes to capture worker-death diagnostics during external-LB DEP8 bring-up." + - "将 GB200 DSpark AgentX llm-d 配方的 vLLM 日志级别提升至 DEBUG，以便在外部 LB DEP8 启动阶段捕获 worker 异常退出诊断信息。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 From 061901e8af02d5d06ef5fffd67243953675ce820 Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Thu, 17 Sep 2026 17:05:41 +0200 Subject: [PATCH 46/46] fix(llm-d): shrink Mooncake DRAM budget and restore INFO logging --- .../agentic/agg-gb200-dep8-dspark-agentic.yaml | 1 - .../agg-gb200-dep8-dspark-mooncake-agentic.yaml | 1 - .../disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml | 4 ++-- benchmarks/multi_node/runtime_settings.sh | 2 +- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 10 ++++++++++ 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml index aacd2aaeec..8c8b14e396 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -81,7 +81,6 @@ prefill: VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_LOGGING_LEVEL: DEBUG VLLM_USE_DEEP_GEMM: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml index 0e310cd6d4..11df6b8a63 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -84,7 +84,6 @@ prefill: VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_LOGGING_LEVEL: DEBUG VLLM_USE_DEEP_GEMM: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml index 6718844457..878d002b24 100644 --- a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -107,7 +107,7 @@ prefill: VLLM_USE_BREAKABLE_CUDAGRAPH: "0" FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED: "1" VLLM_NO_USAGE_STATS: "1" - VLLM_LOGGING_LEVEL: DEBUG + VLLM_LOGGING_LEVEL: INFO TQDM_DISABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM @@ -166,7 +166,7 @@ decode: VLLM_USE_DEEP_GEMM: "1" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_NO_USAGE_STATS: "1" - VLLM_LOGGING_LEVEL: DEBUG + VLLM_LOGGING_LEVEL: INFO TQDM_DISABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" TORCH_SYMMMEM: NVSHMEM diff --git a/benchmarks/multi_node/runtime_settings.sh b/benchmarks/multi_node/runtime_settings.sh index fe1035b385..5bf43deb6e 100644 --- a/benchmarks/multi_node/runtime_settings.sh +++ b/benchmarks/multi_node/runtime_settings.sh @@ -61,7 +61,7 @@ case "$FRAMEWORK" in ;; llmd-vllm) export LLMD_CONTAINER_ENGINE=docker VLLM_RANDOMIZE_DP_DUMMY_INPUTS=1 - export VLLM_ENGINE_READY_TIMEOUT_S=1800 VLLM_LOGGING_LEVEL=DEBUG UCX_TLS=cuda_copy,cuda_ipc,rc + export VLLM_ENGINE_READY_TIMEOUT_S=1800 VLLM_LOGGING_LEVEL=INFO UCX_TLS=cuda_copy,cuda_ipc,rc export NVSHMEM_REMOTE_TRANSPORT=ibgda NVSHMEM_IB_ENABLE_IBGDA=true NVSHMEM_SYMMETRIC_SIZE=16G export LLMD_API_SERVER_COUNT=4 ;; diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e192f25f46..84eb73c1a3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4170,7 +4170,7 @@ dsv4-fp4-gb200-llmd-vllm-agentx: disagg: true scenarios: agentic-coding: - - dram-utilization: 0.60 + - dram-utilization: 0.45 search-space: # 1P DEP8 + 1D DEP8 (4 nodes / 16 GPUs). Always Mooncake. - spec-decoding: mtp @@ -4208,7 +4208,7 @@ dsv4-fp4-gb200-llmd-vllm-agentx-agg: disagg: false scenarios: agentic-coding: - - dram-utilization: 0.60 + - dram-utilization: 0.45 search-space: # Aggregated TP8 (2 nodes / 8 GPUs; pure tensor-parallel, no EP). - spec-decoding: mtp diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2a102b6fe7..e8d5adb137 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8043,3 +8043,13 @@ - "Raise llm-d vLLM logging to DEBUG on GB200 DSpark AgentX recipes to capture worker-death diagnostics during external-LB DEP8 bring-up." - "将 GB200 DSpark AgentX llm-d 配方的 vLLM 日志级别提升至 DEBUG，以便在外部 LB DEP8 启动阶段捕获 worker 异常退出诊断信息。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Lower GB200 DSpark AgentX Mooncake embedded segment budget from 60% to 45% node DRAM utilization and restore default vLLM INFO logging after unhelpful DEBUG bring-up." + - "将 GB200 DSpark AgentX Mooncake 嵌入式段预算从节点 DRAM 利用率的 60% 降至 45%，并在 DEBUG 启动诊断无效后恢复默认 vLLM INFO 日志级别。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719