diff --git a/benchmarks/llm-d/envoy.yaml b/benchmarks/llm-d/envoy.yaml index ccba51ba0b..8bc10fe595 100644 --- a/benchmarks/llm-d/envoy.yaml +++ b/benchmarks/llm-d/envoy.yaml @@ -44,6 +44,13 @@ static_resources: - name: vh domains: ["*"] routes: + # Scrape metrics per node, never through load balancing. + - match: { path: "/metrics" } + direct_response: { status: 404 } + typed_per_filter_config: + envoy.filters.http.ext_proc: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute + disabled: true - match: { prefix: "/" } route: cluster: original_dst diff --git a/benchmarks/multi_node/deprecated/dsv4_fp4_gb200_llmd-vllm-disagg.sh b/benchmarks/multi_node/deprecated/dsv4_fp4_gb200_llmd-vllm-disagg.sh deleted file mode 100755 index f3eca3f27b..0000000000 --- a/benchmarks/multi_node/deprecated/dsv4_fp4_gb200_llmd-vllm-disagg.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# -# Wrapper for the DeepSeek-V4-Pro GB200 llmd-vllm P/D disagg benchmark -# (mid-curve 1P1D and high-tpt 2P1D). Sibling of gptoss_fp4_h200_llmd-vllm.sh - -# same shape, different topology (GB200 = 4 GPUs/node, role spans 2 nodes; -# H200 = 8 GPUs/node, role on a single node). The runner resolves this script via -# SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh" -# from launch_gb200-nv.sh. - -set -euo pipefail - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 - -# GB200 = 4 GPUs per node (Grace+Blackwell). The shared submit.sh -# defaults GPUS_PER_NODE to 8, which is wrong for this SKU and would -# overshoot DP_SIZE = nodes * 8. -export GPUS_PER_NODE="${GPUS_PER_NODE:-4}" - -export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# Worker count per role (Option B multi-engine). Prefer an explicit -# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else fall -# back to the matrix num-worker fields (PREFILL_NUM_WORKERS/DECODE_NUM_WORKERS); -# else 1 (single engine = unchanged 1P+1D / mid-curve). submit.sh reads these. -export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}" -export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}" - -JOB_ID=$(bash ./submit.sh \ - "$PREFILL_NODES" \ - "$DECODE_NODES" \ - "$ISL" "$OSL" "${CONC_LIST// /x}" inf \ - "$RANDOM_RANGE_RATIO") - -if [[ -z "$JOB_ID" ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-agg.sh b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-agg.sh new file mode 100644 index 0000000000..772ce13802 --- /dev/null +++ b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-agg.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eo pipefail + +export GPUS_PER_NODE=4 TIME_LIMIT=08:00:00 CONTAINER_IMAGE="$IMAGE" +export PREFILL_WORKERS=1 DECODE_WORKERS=1 + +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" +exec bash ./submit.sh "$PREFILL_NODES" "$DECODE_NODES" \ + "$ISL" "$OSL" "${CONC_LIST// /x}" inf "$RANDOM_RANGE_RATIO" diff --git a/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh new file mode 100755 index 0000000000..58ce0262a5 --- /dev/null +++ b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eo pipefail + +export GPUS_PER_NODE=4 TIME_LIMIT=08:00:00 CONTAINER_IMAGE="$IMAGE" +export PREFILL_WORKERS="$PREFILL_NUM_WORKERS" DECODE_WORKERS="$DECODE_NUM_WORKERS" + +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" +exec bash ./submit.sh "$PREFILL_NODES" "$DECODE_NODES" \ + "$ISL" "$OSL" "${CONC_LIST// /x}" inf "$RANDOM_RANGE_RATIO" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml new file mode 100644 index 0000000000..8c8b14e396 --- /dev/null +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml @@ -0,0 +1,95 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated DEP8, 2 nodes. +# DSpark speculative decoding: 5 tokens. No Mooncake. +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig + +plugins: + - name: file-disc + type: file-discovery + parameters: + path: /tmp/endpoints.yaml + watchFile: false + + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 5858 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 20000 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 1499703 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker + +schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: max-score-picker + +dataLayer: + discovery: + pluginRef: file-disc + +prefill: + tp: 1 + enable-expert-parallel: true + extra-args: >- + --kv-cache-dtype fp8 + --gpu-memory-utilization 0.88 + --max-num-batched-tokens 8192 + --block-size 256 + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 + --moe-backend deep_gemm_mega_moe + --enable-ep-weight-filter + --enable-cumem-allocator + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --numa-bind + --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + PYTHONHASHSEED: "0" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + +slurm: + time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml new file mode 100644 index 0000000000..11df6b8a63 --- /dev/null +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml @@ -0,0 +1,114 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated DEP8 with Mooncake prefix-cache. +# Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds Mooncake (P2PHANDSHAKE embedded RDMA) +# so server.sh wires MultiConnector (NixlConnector + SimpleCPUOffloadConnector + MooncakeStoreConnector, kv_both). +# DSpark 5 speculative tokens. conc-list [52,72]. +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig + +plugins: + - name: file-disc + type: file-discovery + parameters: + path: /tmp/endpoints.yaml + watchFile: false + + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 40810 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 20000 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 1499703 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker + +schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: max-score-picker + +dataLayer: + discovery: + pluginRef: file-disc + +# ---- Per-role vLLM flags ---- +prefill: + tp: 1 + enable-expert-parallel: true + extra-args: >- + --kv-cache-dtype fp8 + --gpu-memory-utilization 0.88 + --max-num-batched-tokens 8192 + --block-size 256 + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 + --moe-backend deep_gemm_mega_moe + --enable-ep-weight-filter + --enable-cumem-allocator + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --numa-bind + --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + PYTHONHASHSEED: "0" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +# ---- Mooncake KV store config ---- +mooncake: + store_config: + metadata_server: "P2PHANDSHAKE" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false # SSD only; the embedded DRAM pool is still enabled. + +# ---- SLURM resource directives ---- +slurm: + time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml new file mode 100644 index 0000000000..b62994470a --- /dev/null +++ b/benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml @@ -0,0 +1,77 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated TP8, 2 nodes. +# DSpark speculative decoding: 5 tokens. No Mooncake (two-node TP8). +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig + +plugins: + - name: file-disc + type: file-discovery + parameters: + path: /tmp/endpoints.yaml + watchFile: false + + - type: active-request-scorer + - type: queue-scorer + - type: weighted-random-picker + +schedulingProfiles: + - name: default + plugins: + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: weighted-random-picker + +dataLayer: + discovery: + pluginRef: file-disc + +prefill: + tp: 8 + # Must be explicit false: server.sh defaults enable-expert-parallel to true. + enable-expert-parallel: false + extra-args: >- + --kv-cache-dtype fp8 + --gpu-memory-utilization 0.85 + --max-num-batched-tokens 8192 + --block-size 256 + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 + --disable-custom-all-reduce + --enable-cumem-allocator + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --speculative-config {"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"} + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + NCCL_P2P_LEVEL: "NVL" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: cuda_copy,cuda_ipc,tcp + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + +slurm: + time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml new file mode 100644 index 0000000000..878d002b24 --- /dev/null +++ b/benchmarks/multi_node/llm-d-recipes/agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml @@ -0,0 +1,196 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, P/D disagg 1P-DEP8/1D-DEP8 (4 nodes). +# Always uses Mooncake (P2PHANDSHAKE embedded RDMA prefix-cache). +# DSpark speculative decoding: prefill 1 token, decode 5 tokens. +# Requires EPP/pd-sidecar v0.10.0 (disagg-profile-handler deciders shape). +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig + +plugins: + - name: file-disc + type: file-discovery + parameters: + path: /tmp/endpoints.yaml + watchFile: false + + - type: prefill-filter + - type: decode-filter + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 43949 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 4783 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 3000000 + - type: active-request-scorer + - type: queue-scorer + - type: always-disagg-pd-decider + - type: disagg-profile-handler + parameters: + deciders: + prefill: always-disagg-pd-decider + - type: max-score-picker + name: prefill-picker + - type: max-score-picker + name: decode-picker + +schedulingProfiles: + - name: prefill + plugins: + - pluginRef: prefill-filter + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: prefill-picker + - name: decode + plugins: + - pluginRef: decode-filter + - pluginRef: active-request-scorer + - pluginRef: decode-picker + +dataLayer: + discovery: + pluginRef: file-disc + +prefill: + tp: 1 + enable-expert-parallel: true + extra-args: >- + --kv-cache-dtype fp8 + --gpu-memory-utilization 0.95 + --max-num-batched-tokens 8192 + --long-prefill-token-threshold 1024 + --max-num-seqs 16 + --max-model-len 1048576 + --speculative-config {"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} + --enable-cumem-allocator + --block-size 256 + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 + --moe-backend deep_gemm_mega_moe + --enable-ep-weight-filter + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --numa-bind + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + PYTHONHASHSEED: "0" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED: "1" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_RPC_CLIENT_IO_THREADS: "32" + MC_TE_RPC_CLIENT_IO_THREADS: "32" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +decode: + tp: 1 + enable-expert-parallel: true + extra-args: >- + --kv-cache-dtype fp8 + --gpu-memory-utilization 0.95 + --max-num-batched-tokens 256 + --max-num-seqs 32 + --max-cudagraph-capture-size 256 + --max-model-len 1048576 + --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false} + --enable-cumem-allocator + --block-size 256 + --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 + --moe-backend deep_gemm_mega_moe + --enable-ep-weight-filter + --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune + --numa-bind + --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true} + env: + PYTHONHASHSEED: "0" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +mooncake: + store_config: + metadata_server: "P2PHANDSHAKE" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false # SSD only; the embedded DRAM pool is still enabled. + +slurm: + time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d/README.md b/benchmarks/multi_node/llm-d/README.md index 81dbd51995..37180d4bf6 100644 --- a/benchmarks/multi_node/llm-d/README.md +++ b/benchmarks/multi_node/llm-d/README.md @@ -25,6 +25,17 @@ the coordinator (EPP + Envoy + bench), exactly like the AMD path's | `xP` | decode leader + pd-sidecar + EPP + Envoy + benchmark client | | `xP+1 .. xP+yD-1` | decode workers | +### Aggregated mode (`yD = 0`) + +`DECODE_NODES=0` runs one engine for prefill and decode. Rank 0 runs EPP, +Envoy, and the client; no P/D sidecar is started. Only the Mooncake variant +uses a KV connector. The master config uses `disagg: false`, `worker`, and +`num-nodes`; discovery labels its serving endpoints `prefill`. + +The GB200 AgentX recipes live under `llm-d-recipes/agentic/`. TP8/DEP8 uses +2 nodes (8 GPUs); 1P-DEP8/1D-DEP8 uses 4 nodes (16 GPUs). Pure TP publishes +only its leader's API endpoint; DEP publishes an endpoint on every node. + Each instance (prefill or decode) is one vLLM engine spanning multiple nodes via `--data-parallel-hybrid-lb`. With `xP=2, yD=2, GPUS_PER_NODE=8` you get DP=16 prefill + DP=16 decode (the wide-EP diff --git a/benchmarks/multi_node/llm-d/agentic.sh b/benchmarks/multi_node/llm-d/agentic.sh new file mode 100644 index 0000000000..e95972a01a --- /dev/null +++ b/benchmarks/multi_node/llm-d/agentic.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Client-only AgentX adapter for an already-ready llm-d Envoy frontend. +set -eo pipefail + +: "${INFMAX_CONTAINER_WORKSPACE:?Set the repository mount path}" +export MODEL="$MODEL_NAME" +export SERVED_MODEL_NAME="$MODEL_NAME" +export PORT="$VLLM_PORT" +export AIPERF_SERVER_URL="http://localhost:$ENVOY_PORT" +export RESULT_DIR="$BENCHMARK_LOGS_DIR/agentic" +export AGENTIC_OUTPUT_DIR="$BENCHMARK_LOGS_DIR" +export CONC_LIST="${BENCH_MAX_CONCURRENCY//x/ }" +export CONC="${CONC_LIST%% *}" + +# Use discovery's serving nodes, but scrape vLLM rather than the decode sidecar. +mkdir -p "$RESULT_DIR" +AIPERF_METRIC_URLS=$(python3 - "$LLMD_ENDPOINTS_FILE" "$VLLM_PORT" \ + "$RESULT_DIR/llmd_metrics_endpoints.json" "$DECODE_NODES" "${SIDECAR_PORT:-8000}" <<'PY' +import json +import sys +import yaml + +with open(sys.argv[1]) as source: + endpoints = yaml.safe_load(source)["endpoints"] +vllm_base = int(sys.argv[2]) +sidecar_base = int(sys.argv[5]) + +def vllm_metrics_port(endpoint): + role = endpoint["labels"]["llm-d.ai/role"] + endpoint_port = int(endpoint["port"]) + if role == "decode": + return vllm_base + (endpoint_port - sidecar_base) + return endpoint_port + +metrics_endpoints = { + f"http://{endpoint['address']}:{vllm_metrics_port(endpoint)}/metrics": { + "name": endpoint["name"], + "role": endpoint["labels"]["llm-d.ai/role"] if int(sys.argv[4]) else "combined", + } + for endpoint in endpoints +} +if not metrics_endpoints: + raise SystemExit("No llm-d serving endpoints available for metrics") +with open(sys.argv[3], "w") as output: + json.dump(metrics_endpoints, output, indent=2) + output.write("\n") +print(",".join(metrics_endpoints)) +PY +) +export AIPERF_METRIC_URLS +# benchmark_lib.sh forwards this name to AIPerf's --server-metrics argument. +export AIPERF_SERVER_METRICS_URLS="$AIPERF_METRIC_URLS" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" + +# AIPerf also probes the inference URL; it must not return load-balanced counters. +frontend_metrics_status=$(curl --silent --show-error --connect-timeout 5 --max-time 10 \ + --output /dev/null --write-out '%{http_code}' "$AIPERF_SERVER_URL/metrics") +if [[ "$frontend_metrics_status" != "404" ]]; then + echo "ERROR: llm-d frontend /metrics must return 404, got $frontend_metrics_status" >&2 + exit 1 +fi + +IFS=',' read -r -a metrics_urls <<< "$AIPERF_METRIC_URLS" +metrics_probe=$(mktemp /tmp/llmd-metrics.XXXXXX) +trap 'rm -f "$metrics_probe"' EXIT +for metrics_url in "${metrics_urls[@]}"; do + curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ + --retry 6 --retry-connrefused --retry-delay 2 \ + "$metrics_url" --output "$metrics_probe" + if ! grep -q '^vllm:' "$metrics_probe"; then + echo "ERROR: no vLLM metrics exposed at $metrics_url" >&2 + exit 1 + fi + echo "vLLM metrics ready: $metrics_url" +done +rm -f "$metrics_probe" +trap - EXIT + +exec bash "$INFMAX_CONTAINER_WORKSPACE/benchmarks/multi_node/agentic_srt.sh" diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index fe36301b6a..3f1e3d0f28 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -48,8 +48,13 @@ fi # prefill workers = ranks 1 .. PREFILL_NODES-1 # decode leader = rank PREFILL_NODES (also coordinator: EPP + Envoy + bench) # decode workers = ranks PREFILL_NODES+1 .. NUM_NODES-1 +# Aggregated mode has no decode address. PREFILL_LEADER_IP="${IPS[0]}" -DECODE_LEADER_IP="${IPS[$PREFILL_NODES]}" +if [[ "$DECODE_NODES" -gt 0 ]]; then + DECODE_LEADER_IP="${IPS[$PREFILL_NODES]}" +else + DECODE_LEADER_IP="" +fi # DP leader addresses for vLLM --data-parallel-address (rank 0 of each instance). PREFILL_DP_ADDR="$PREFILL_LEADER_IP" @@ -62,10 +67,11 @@ DOCKER_CONT_NAME="llmd_bench_${SANITIZED_USER}_${SLURM_JOB_ID}" export DOCKER_CONT_NAME : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" DOCKER_MOUNT_PATH="/workspace" +export INFMAX_CONTAINER_WORKSPACE="$DOCKER_MOUNT_PATH" cleanup() { echo "[${SLURM_JOB_ID}] cleanup on $(hostname)" - [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true + [[ -n "${WATCHER_PID}" ]] && kill "$WATCHER_PID" 2>/dev/null || true } trap cleanup INT TERM HUP EXIT @@ -100,6 +106,27 @@ for setting in $INFERENCEX_RUNTIME_ENV_VARS; do RUNTIME_DOCKER_ENV+=(-e "$setting") done +# Preserve the workflow's AgentX protocol, provenance, offload and physical GPU +# metadata on both engines. Forward names, not interpolated values: JSON and +# HF_TOKEN must never be embedded in the nested shell command or printed. +AGENTIC_ENV_NAMES=( + INFMAX_CONTAINER_WORKSPACE + IS_AGENTIC SCENARIO_TYPE CONC CONC_LIST DURATION AIPERF_EXPERIMENTAL_FAST + IMAGE RECIPE_FINGERPRINT DISAGG HF_TOKEN + KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA + ROUTER_METADATA KV_P2P_TRANSFER TOTAL_CPU_DRAM_GB + PREFILL_NUM_WORKERS PREFILL_TP PREFILL_PP_SIZE PREFILL_PCP_SIZE + PREFILL_DCP_SIZE PREFILL_EP PREFILL_DP_ATTN + DECODE_NUM_WORKERS DECODE_TP DECODE_PP_SIZE DECODE_PCP_SIZE + DECODE_DCP_SIZE DECODE_EP DECODE_DP_ATTN +) +AGENTIC_DOCKER_ENV="" +for env_name in "${AGENTIC_ENV_NAMES[@]}"; do + export "$env_name" + AGENTIC_DOCKER_ENV+=" -e $env_name" +done +AGENTIC_PYXIS_ENV=$(IFS=,; echo "${AGENTIC_ENV_NAMES[*]}") + if [[ "$LLMD_CONTAINER_ENGINE" == "docker" ]]; then # One docker run per node, one task per node. server.sh dispatches by NODE_RANK. srun \ @@ -178,12 +205,14 @@ exec docker run --rm \ -e CONFIG_FILE=$CONFIG_FILE \ ${RUNTIME_DOCKER_ENV[*]} \ -e VLLM_RANDOMIZE_DP_DUMMY_INPUTS=$VLLM_RANDOMIZE_DP_DUMMY_INPUTS \ + -e VLLM_ENGINE_READY_TIMEOUT_S=$VLLM_ENGINE_READY_TIMEOUT_S \ -e VLLM_LOGGING_LEVEL=$VLLM_LOGGING_LEVEL \ -e UCX_TLS=$UCX_TLS \ -e NVSHMEM_REMOTE_TRANSPORT=$NVSHMEM_REMOTE_TRANSPORT \ -e NVSHMEM_IB_ENABLE_IBGDA=$NVSHMEM_IB_ENABLE_IBGDA \ -e NVSHMEM_SYMMETRIC_SIZE=$NVSHMEM_SYMMETRIC_SIZE \ -e LLMD_API_SERVER_COUNT=$LLMD_API_SERVER_COUNT \ + $AGENTIC_DOCKER_ENV \ --name \"${DOCKER_CONT_NAME}_\$SLURM_PROCID\" \ \"\$DOCKER_IMAGE_NAME\" -lc ' set -o pipefail @@ -232,6 +261,7 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then PYXIS_ENV_LIST+=",VLLM_RANDOMIZE_DP_DUMMY_INPUTS,VLLM_ENGINE_READY_TIMEOUT_S,VLLM_LOGGING_LEVEL,UCX_TLS,NVSHMEM_REMOTE_TRANSPORT,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_SYMMETRIC_SIZE,LLMD_API_SERVER_COUNT" PYXIS_ENV_LIST+=",${INFERENCEX_RUNTIME_ENV_VARS// /,}" + PYXIS_ENV_LIST+=",$AGENTIC_PYXIS_ENV" PYXIS_MOUNTS="${MODEL_DIR}:/models:ro" PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs" @@ -240,23 +270,15 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro" PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro" - # Optional: mount the epp / pd-sidecar / envoy binaries from a shared - # filesystem instead of relying on them being baked into the image. - # This lets a STOCK vllm/vllm-openai image be used directly (no - # combined-image rebuild per vLLM version bump) - see - # benchmarks/llm-d/binaries.env + extract-binaries.sh. Each mount is - # gated on the file existing, so this is a no-op when the binaries - # have not been extracted (the baked-image path keeps working), and - # harmless when they have (mounting a binary over the identical one). - # shellcheck source=/dev/null - [[ -f "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" ]] && \ + # AgentX uses the image-bundled router; legacy runs can mount extracted binaries. + if [[ "$IS_AGENTIC" != "1" ]]; then source "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" - for _bin in epp pd-sidecar envoy; do - if [[ -n "${LLMD_BIN_DIR:-}" && -x "${LLMD_BIN_DIR}/${_bin}" ]]; then - PYXIS_MOUNTS+=",${LLMD_BIN_DIR}/${_bin}:/usr/local/bin/${_bin}:ro" - echo "Mounting ${LLMD_BIN_DIR}/${_bin} -> /usr/local/bin/${_bin}" - fi - done + for _bin in epp pd-sidecar envoy; do + if [[ -x "${LLMD_BIN_DIR}/${_bin}" ]]; then + PYXIS_MOUNTS+=",${LLMD_BIN_DIR}/${_bin}:/usr/local/bin/${_bin}:ro" + fi + done + fi # MODEL_DIR / BENCHMARK_LOGS_DIR / NODE_RANK are translated to their # in-container values inside bash -lc (host MODEL_DIR is the source diff --git a/benchmarks/multi_node/llm-d/recipe.py b/benchmarks/multi_node/llm-d/recipe.py new file mode 100644 index 0000000000..1b3033ccc6 --- /dev/null +++ b/benchmarks/multi_node/llm-d/recipe.py @@ -0,0 +1,100 @@ +"""Render llm-d role arguments and enforce AgentX benchmark metadata.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shlex +import sys + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def validate_agentic_offload(recipe: dict, env: dict) -> None: + """An embedded Mooncake store is DRAM offload even with SSD disabled.""" + if env.get("IS_AGENTIC") != "1": + return + store = recipe.get("mooncake", {}).get("store_config") + expected = "dram" if store else "none" + if env.get("KV_OFFLOADING") != expected: + raise ValueError(f"Recipe requires KV_OFFLOADING={expected}; fix the master YAML") + if store and env.get("KV_OFFLOAD_BACKEND") != "mooncake": + raise ValueError("Mooncake recipe requires KV_OFFLOAD_BACKEND=mooncake") + + +def role_assignments(recipe: dict, role: str, env: dict) -> str: + validate_agentic_offload(recipe, env) + section = recipe.get(role) or {} + extra = (section.get("extra-args") or "").strip() + if env.get("IS_AGENTIC") == "1": + match = re.search(r"--speculative-config\s+", extra) + if match: + config, length = json.JSONDecoder().raw_decode(extra[match.end():]) + if config.get("method") == "dspark": + if env.get("SPEC_DECODING") != "mtp": + raise ValueError("DSpark requires SPEC_DECODING=mtp in the master YAML") + if env.get("EVAL_ONLY") == "true": + config.pop("synthetic_acceptance_length", None) + config.pop("rejection_sample_method", None) + else: + if env.get("RUN_EVAL") == "true": + raise ValueError("Run accuracy evals separately with EVAL_ONLY=true, not synthetic AL") + if env.get("MODEL_NAME") != "deepseek-ai/DeepSeek-V4-Pro-0813": + raise ValueError("No registered DSpark golden AL for this model") + golden_path = REPO_ROOT / "golden_al_distribution/dsv4-pro-0813-dspark.yaml" + golden = yaml.safe_load(golden_path.read_text()) + k = config["num_speculative_tokens"] + al = golden["deepseek-v4-pro-0813"]["thinking_on"][k] + config.update( + rejection_sample_method="synthetic", + synthetic_acceptance_length=al, + ) + print(f"DSpark {role}: K={k}, golden AL={al} ({golden_path.name})", file=sys.stderr) + extra = extra[:match.end()] + json.dumps(config, separators=(",", ":")) + extra[match.end() + length:] + assignments = [f"ROLE_EXTRA_ARGS={shlex.quote(extra)}", + f"PREFILL_ENABLE_EP={str(recipe.get('prefill', {}).get('enable-expert-parallel', True)).lower()}"] + if section.get("tp") is not None: + assignments.append(f"TP_SIZE={int(section['tp'])}") + if section.get("enable-expert-parallel") is not None: + assignments.append(f"ROLE_ENABLE_EP={str(section['enable-expert-parallel']).lower()}") + for key, value in (section.get("env") or {}).items(): + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise ValueError(f"Invalid recipe environment variable: {key}") + assignments.append(f"export {key}={shlex.quote(str(value))}") + return "\n".join(assignments) + + +def mooncake_config(recipe: dict, env: dict) -> str: + validate_agentic_offload(recipe, env) + config = dict(recipe.get("mooncake", {}).get("store_config") or {}) + if not config: + return "" + config["master_server_address"] = f"{env['ALL_IPS'].split(',')[0]}:50051" + if env.get("IS_AGENTIC") == "1": + budget_gb = int(env["TOTAL_CPU_DRAM_GB"]) + gpus_per_node = int(env["GPUS_PER_NODE"]) + if budget_gb <= 0 or gpus_per_node <= 0: + raise ValueError("Mooncake requires a positive per-node DRAM budget and GPU count") + # The master budget is per node. Each embedded per-GPU store owns a + # share; transfer buffers are separate from the reusable KV pool. + config["global_segment_size"] = budget_gb * 10**9 // gpus_per_node + return json.dumps(config) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("recipe", type=Path) + output = parser.add_mutually_exclusive_group(required=True) + output.add_argument("--role", choices=("prefill", "decode")) + output.add_argument("--mooncake", action="store_true") + args = parser.parse_args() + recipe = yaml.safe_load(args.recipe.read_text()) + print(mooncake_config(recipe, os.environ) if args.mooncake else role_assignments(recipe, args.role, os.environ)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 4065e6c070..c042dd23b0 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -20,7 +20,8 @@ source /workspace/benchmarks/benchmark_lib.sh check_env_vars \ NODE_RANK PREFILL_NODES DECODE_NODES GPUS_PER_NODE PREFILL_WORKERS \ - DECODE_WORKERS EVAL_ONLY RUN_EVAL + DECODE_WORKERS EVAL_ONLY RUN_EVAL ALL_IPS +IS_AGGREGATED=$(( DECODE_NODES == 0 )) VLLM_PORT=8200 SIDECAR_PORT=8000 ENVOY_PORT=8080 @@ -32,6 +33,9 @@ EPP_METRICS_PORT=9090 # the served-model-name, not a filesystem path. MODEL="${MODEL_DIR}" +# ---------------------------------------------------------------- +# Host IP + default interface +# ---------------------------------------------------------------- # Resolved without iproute2 (`ip` is absent on the arm64 vLLM base); python3's # socket layer exposes the kernel's source-IP / iface choice. _HOST_INFO=$(python3 -c ' @@ -65,6 +69,9 @@ ENVOY_LOG="/benchmark_logs/envoy.log" echo "=== rank=$NODE_RANK host=$HOST_IP model=$MODEL ===" +# ---------------------------------------------------------------- +# Role + topology (Option B engine grouping) +# ---------------------------------------------------------------- # A role's nodes split into PREFILL_WORKERS / DECODE_WORKERS independent DP/EP # engines, each spanning (role_nodes / role_workers) nodes with its own DP # coordinator (leader IP) and rank range. workers=1 => one engine over all role @@ -92,8 +99,7 @@ else exit 1 fi -# Each engine's DP coordinator = its leader node's IP (ALL_IPS[leader rank]); -# fall back to the role leader env when ALL_IPS is unset. +# Each engine's DP coordinator = its leader node's IP; fall back to role leaders. if [[ -n "${_ALL_IPS[${_group_leader_rank}]:-}" ]]; then DP_ADDR="${_ALL_IPS[${_group_leader_rank}]}" elif [[ "$ROLE" == "prefill" ]]; then @@ -108,45 +114,15 @@ START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) # Defaults: TP=1, DP=role_total, EP on (the H200 1P+1D shape). Recipe overrides below. TP_SIZE=1 ROLE_ENABLE_EP=true +PREFILL_ENABLE_EP=true echo "ROLE=$ROLE DP_SIZE=$DP_SIZE DP_ADDR=$DP_ADDR LWS_WORKER_INDEX=$LWS_WORKER_INDEX START_RANK=$START_RANK" -# Per-role keys: tp (int -> --tensor-parallel-size), enable-expert-parallel -# (bool -> --enable-expert-parallel + DP/wide-EP knobs), extra-args (appended -# verbatim), env (map, exported before vllm serve). Absent keys keep the -# defaults above, so a recipe with neither tp nor EP is a plain TP=1 DP+EP run. -ROLE_EXTRA_ARGS="" -if [[ -n "${CONFIG_FILE:-}" ]]; then - RECIPE_PATH="/etc/llmd-recipes/${CONFIG_FILE}" - if [[ -f "$RECIPE_PATH" ]]; then - echo "Loading $ROLE recipe from $RECIPE_PATH" - eval "$(python3 - <&2 - fi -fi -echo "Resolved $ROLE TP_SIZE=$TP_SIZE ROLE_ENABLE_EP=$ROLE_ENABLE_EP" - export GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-$DEFAULT_IFACE} export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-$DEFAULT_IFACE} check_env_vars \ - VLLM_RANDOMIZE_DP_DUMMY_INPUTS VLLM_ENGINE_READY_TIMEOUT_S VLLM_LOGGING_LEVEL UCX_TLS NVSHMEM_REMOTE_TRANSPORT \ - NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_SYMMETRIC_SIZE LLMD_API_SERVER_COUNT + VLLM_RANDOMIZE_DP_DUMMY_INPUTS VLLM_ENGINE_READY_TIMEOUT_S VLLM_LOGGING_LEVEL UCX_TLS \ + NVSHMEM_REMOTE_TRANSPORT NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_SYMMETRIC_SIZE export VLLM_SKIP_P2P_CHECK=1 # Randomized DP dummy inputs make idle DP ranks fan their lockstep dummy passes # across all experts (full MoE all-to-all), wasting prefill bandwidth; a recipe @@ -172,8 +148,7 @@ export VLLM_LOGGING_LEVEL # exposes /dev/infiniband + IPC_LOCK); cuda_copy/cuda_ipc cover intra-node. export UCX_TLS -# Single-node-per-role recipes avoid DeepEP / NVSHMEM ibgda, so leave these off -# there to avoid triggering ibgda code paths that are not needed. + if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then export NVIDIA_GDRCOPY=enabled # ibgda default kept for future DeepEP/wide-EP recipes; a recipe may override @@ -189,34 +164,110 @@ if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then fi fi -if [[ -n "${KV_ROLE_OVERRIDE:-}" ]]; then - KV_ROLE="$KV_ROLE_OVERRIDE" -elif [[ "$ROLE" == "prefill" ]]; then - KV_ROLE="kv_producer" -else - KV_ROLE="kv_consumer" +# ---------------------------------------------------------------- +# Recipe: per-role serve args + env (/etc/llmd-recipes/$CONFIG_FILE) +# ---------------------------------------------------------------- +# Per-role keys: tp (int -> --tensor-parallel-size), enable-expert-parallel +# (bool -> --enable-expert-parallel + DP/wide-EP knobs), extra-args (appended +# verbatim), env (map, exported before vllm serve). Absent keys keep the +# defaults above, so a recipe with neither tp nor EP is a plain TP=1 DP+EP run. +ROLE_EXTRA_ARGS="" +if [[ -n "${CONFIG_FILE}" ]]; then + RECIPE_PATH="/etc/llmd-recipes/${CONFIG_FILE}" + if [[ -f "$RECIPE_PATH" ]]; then + echo "Loading $ROLE recipe from $RECIPE_PATH" + # Keep command substitution separate from eval so renderer failures + # (missing golden AL or incorrect offload metadata) stop server startup. + ROLE_ASSIGNMENTS=$(python3 /workspace/benchmarks/multi_node/llm-d/recipe.py \ + "$RECIPE_PATH" --role "$ROLE") + eval "$ROLE_ASSIGNMENTS" + else + if [[ "${IS_AGENTIC}" == "1" ]]; then + echo "ERROR: AgentX recipe not found: $RECIPE_PATH" >&2 + exit 1 + fi + echo "WARNING: CONFIG_FILE=$CONFIG_FILE but $RECIPE_PATH not found; using defaults" >&2 + fi fi -KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" +echo "Resolved $ROLE TP_SIZE=$TP_SIZE ROLE_ENABLE_EP=$ROLE_ENABLE_EP" +# ---------------------------------------------------------------- +# Mooncake KV store (optional, from recipe top-level `mooncake:` key) +# ---------------------------------------------------------------- +# Embedded stores contribute per-rank DRAM to one job-local Mooncake master. +MOONCAKE_CONFIG_PATH="" +if [[ -n "${CONFIG_FILE}" && -f "/etc/llmd-recipes/${CONFIG_FILE}" ]]; then + _MC_JSON=$(python3 /workspace/benchmarks/multi_node/llm-d/recipe.py \ + "/etc/llmd-recipes/${CONFIG_FILE}" --mooncake) + if [[ -n "$_MC_JSON" ]]; then + echo "$_MC_JSON" > /tmp/mooncake_config.json + MOONCAKE_CONFIG_PATH=/tmp/mooncake_config.json + export MOONCAKE_CONFIG_PATH + echo "Mooncake enabled: config at $MOONCAKE_CONFIG_PATH" + if [[ "$NODE_RANK" -eq 0 ]]; then + mooncake_master --rpc_port=50051 --metrics_port=50052 \ + > "$BENCHMARK_LOGS_DIR/mooncake_master.log" 2>&1 & + fi + curl --fail --silent --show-error --connect-timeout 5 --max-time 10 \ + --retry 30 --retry-connrefused --retry-delay 1 \ + "http://${_ALL_IPS[0]}:50052/metrics" > /dev/null + fi +fi + +# ---------------------------------------------------------------- +# Bring up vLLM engine (every node) +# ---------------------------------------------------------------- COMMON_ARGS=( + --host 0.0.0.0 --port "$VLLM_PORT" --served-model-name "$MODEL_NAME" --trust-remote-code --disable-access-log-for-endpoints=/health,/metrics --tensor-parallel-size "$TP_SIZE" - --kv_transfer_config "$KV_TRANSFER_CONFIG" ) -# One frontend (HTTP + tokenize + DP load-balance) is CPU-bound and caps throughput, -# so run several (LLMD_API_SERVER_COUNT). Incompatible with --headless, so the -# headless-worker branch below drops it. With --data-parallel-hybrid-lb each node's -# api-server balances its local DP ranks, so VLLM_PORT is also the health port. +# Aggregated engines need KV transfer only when Mooncake is enabled. +if [[ "$IS_AGGREGATED" -eq 0 ]]; then + if [[ -n "${KV_ROLE_OVERRIDE}" ]]; then + KV_ROLE="$KV_ROLE_OVERRIDE" + elif [[ "$ROLE" == "prefill" ]]; then + KV_ROLE="kv_producer" + else + KV_ROLE="kv_consumer" + fi + if [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then + # MultiConnector on prefill: NixlConnector handles direct P/D KV transfer; + # SimpleCPUOffloadConnector stages KV in CPU DRAM (~38 GB) before writing + # to MooncakeStoreConnector for cross-node prefix-cache lookup via RDMA. + # Decode uses NixlConnector only (matches agentX v13): Mooncake on decode + # would pollute the prefix cache with non-reusable decode blocks, and + # SimpleCPUOffload in EAGER mode would amplify that. kv_both on decode + # so it can serve speculative-decode prefills in DSpark. + _MC_EXTRA='"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false' + _NIXL_EXTRA='"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800' + if [[ "$ROLE" == "prefill" ]]; then + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{${_NIXL_EXTRA}}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":40802189312}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{${_MC_EXTRA}}}]}}" + else + KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{${_NIXL_EXTRA}}}" + fi + else + KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" + fi + COMMON_ARGS+=(--kv_transfer_config "$KV_TRANSFER_CONFIG") +elif [[ -n "${MOONCAKE_CONFIG_PATH}" ]]; then + # Aggregated + Mooncake: single role acts as kv_both (stores new KV and + # loads cache hits from the Mooncake RDMA store for prefix-cache sharing). + # SimpleCPUOffloadConnector stages freshly computed KV blocks into CPU DRAM + # (~40 GB) before writing to MooncakeStore, matching the local DEP8 v1 setup. + KV_TRANSFER_CONFIG="{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"enable_cross_layers_blocks\":false}},{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":42949672960}},{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"load_async\":true,\"lookup_async\":true,\"enable_cross_layers_blocks\":false,\"enable_offload\":false}}]}}" + COMMON_ARGS+=(--kv_transfer_config "$KV_TRANSFER_CONFIG") +fi +# EP roles use multi-port-external-lb: each local DP rank gets its own serving +# port starting at VLLM_PORT (8200, 8201, ...). The vLLM supervisor binds 8100 +# and serves /health once all engines are ready -> health check uses port 8100. +# Pure-TP roles serve on a single VLLM_PORT with the standard health check. HEALTH_PORT="$VLLM_PORT" -API_SERVER_COUNT="${LLMD_API_SERVER_COUNT}" -# Multiple frontends only help the DP (wide-EP) path. A pure-TP engine has a single -# core with one frontend, so it keeps the default (and avoids --api-server-count -# interacting with the --headless multi-node TP launch below). if [[ "$ROLE_ENABLE_EP" == "true" ]]; then - COMMON_ARGS+=(--api-server-count "$API_SERVER_COUNT") + HEALTH_PORT="8100" fi # Set to 1 by the pure-TP multi-node branch below on --headless followers, which # run no local api-server; gates the post-launch health wait. @@ -230,10 +281,11 @@ if [[ "$ROLE_ENABLE_EP" == "true" ]]; then COMMON_ARGS+=( --enable-expert-parallel --data-parallel-size "$DP_SIZE" + --data-parallel-multi-port-external-lb + --data-parallel-supervisor-port 8100 ) if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then COMMON_ARGS+=( - --data-parallel-hybrid-lb --data-parallel-size-local "$DP_SIZE_LOCAL" --data-parallel-address "$DP_ADDR" --data-parallel-rpc-port 5555 @@ -273,6 +325,9 @@ else echo "vLLM ready on rank $NODE_RANK ($ROLE worker_index=$LWS_WORKER_INDEX, health port $HEALTH_PORT)" fi +# ---------------------------------------------------------------- +# Bring up pd-sidecar (every decode node) +# ---------------------------------------------------------------- # The sidecar forwards a prefill request, reads kv_transfer_params from vLLM's # response, then hits its local decode vLLM, whose NIXLv2 connector pulls KV # directly from prefill vLLM. @@ -282,11 +337,13 @@ fi # endpoint per node. Pure-TP: only the TP-group leader has an api-server # (followers are --headless), so only the leader runs a sidecar and only leaders # are listed as endpoints. +# if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX" -eq 0 ) ]]; then SIDECAR_CONNECTOR="nixlv2" SIDECAR_FLAGS=(--port="$SIDECAR_PORT" --vllm-port="$VLLM_PORT" --kv-connector="$SIDECAR_CONNECTOR" --secure-proxy=false - --enable-prefiller-sampling) + --enable-prefiller-sampling + --data-parallel-size="$DP_SIZE_LOCAL") SIDECAR_HEALTH_PORT="$SIDECAR_PORT" echo "Starting pd-sidecar (decode node_rank=$NODE_RANK worker_index=$LWS_WORKER_INDEX): ${SIDECAR_FLAGS[*]}" pd-sidecar "${SIDECAR_FLAGS[@]}" > "$SIDECAR_LOG" 2>&1 & @@ -295,55 +352,51 @@ if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX echo "pd-sidecar ready on $HOST_IP:$SIDECAR_HEALTH_PORT" fi -# Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then +# ================================================================ +# Coordinator: endpoints, EPP, Envoy, bench, eval +# ================================================================ +# Rank 0 coordinates aggregated runs; the decode leader coordinates P/D runs. +if [[ ( "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ) || \ + ( "$IS_AGGREGATED" -eq 1 && "$ROLE" == "prefill" && "$NODE_RANK" -eq 0 ) ]]; then # Release the allocation whenever the coordinator exits. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT - # namespace must match EPP's --pool-namespace (file-discovery filters by it; - # the schema default 'default' would drop every entry). See README.md. + # DEP registers every node; pure TP registers only each engine's API leader. + export LLMD_ENDPOINTS_FILE=/tmp/endpoints.yaml python3 - < DEP8 hybrid-LB (an api-server per node); -# EP off => pure-TP (only each TP-group leader has an api-server). -decode_ep = ('$ROLE_ENABLE_EP' == 'true') -VLLM_PORT = int('$VLLM_PORT') -SIDECAR_PORT = int('$SIDECAR_PORT') -# ALL_IPS is rank-ordered: ranks [0:pn] are prefill nodes, [pn:pn+dn] decode. -prefill_ips = all_ips[:pn] or [os.environ['PREFILL_LEADER_IP']] -decode_ips = all_ips[pn:pn + dn] or [os.environ['DECODE_LEADER_IP']] +ips = os.environ['ALL_IPS'].split(',') +pn = int(os.environ['PREFILL_NODES']) +dn = int(os.environ['DECODE_NODES']) +gpus_per_node = int('$GPUS_PER_NODE') endpoints = [] -def add_role(role, ips, base_port, group_size=1): - # group_size == 1: one endpoint per node (DEP8 hybrid-LB: each node's - # api-server / sidecar load-balances its local DP ranks). - # group_size > 1: one endpoint per TP-group leader (pure-TP: followers are - # --headless with no api-server), i.e. every group_size-th node IP. - serving_ips = ips[::group_size] if group_size > 1 else ips - for i, ip in enumerate(serving_ips): - endpoints.append({'name': f'{role}-{i}', 'namespace': NS, 'address': ip, - 'port': str(base_port), 'labels': {'llm-d.ai/role': role}}) - -# Prefill (DEP8 in every current recipe): one endpoint per node, EPP hits vLLM -# directly (VLLM_PORT). Decode: EPP hits the pd-sidecar (SIDECAR_PORT); one -# endpoint per node for DEP8, or one per TP-group leader for pure-TP. -add_role('prefill', prefill_ips, VLLM_PORT) -decode_group = 1 if decode_ep else max(1, dn // decode_workers) -add_role('decode', decode_ips, SIDECAR_PORT, group_size=decode_group) -yaml.safe_dump({'endpoints': endpoints}, open('/tmp/endpoints.yaml', 'w')) -print(f'endpoints.yaml ({len(endpoints)} endpoints):') -print(open('/tmp/endpoints.yaml').read()) +def add_role(role, addresses, port, group_size, dp_local=1): + idx = 0 + for address in addresses[::group_size]: + for rank in range(dp_local): + endpoints.append({'name': f'{role}-{idx}', 'namespace': 'inferencex', + 'address': address, 'port': str(port + rank), + 'labels': {'llm-d.ai/role': role}}) + idx += 1 + +prefill_ep = '$PREFILL_ENABLE_EP' == 'true' +prefill_group = 1 if prefill_ep else pn // int('$PREFILL_WORKERS') +prefill_dp_local = gpus_per_node if prefill_ep else 1 +add_role('prefill', ips[:pn], int('$VLLM_PORT'), prefill_group, prefill_dp_local) +if dn: + decode_ep = '$ROLE_ENABLE_EP' == 'true' + decode_group = 1 if decode_ep else dn // int('$DECODE_WORKERS') + decode_dp_local = gpus_per_node if decode_ep else 1 + add_role('decode', ips[pn:pn + dn], int('$SIDECAR_PORT'), decode_group, decode_dp_local) +with open(os.environ['LLMD_ENDPOINTS_FILE'], 'w') as output: + yaml.safe_dump({'endpoints': endpoints}, output) +print(yaml.safe_dump({'endpoints': endpoints})) PY - # EPP + # ---- Bring up EPP ---- # Config: when a recipe is set, project it down to the keys EPP's strict # decoder accepts (it rejects the per-role vLLM / slurm keys); else use the # default mounted at /etc/epp/config.yaml. @@ -391,7 +444,7 @@ PY done echo "EPP listening on $EPP_GRPC_PORT" - # Envoy + # ---- Bring up Envoy ---- envoy -c /etc/envoy/envoy.yaml > "$ENVOY_LOG" 2>&1 & ENVOY_PID=$! @@ -415,13 +468,31 @@ PY done echo "Envoy admin ready; listener should be on $ENVOY_PORT" - # Gate on ALL prefill vLLM /health endpoints. Prefill ranks only wait on their own - # local /health, and with PREFILL_WORKERS>1 every prefill node must be probed, not - # just IPS[0]. curl gets explicit connect/max timeouts so a blackholed endpoint - # trips the deadline instead of hanging the run (a timeout-less curl once wedged - # a 2P run for 7h). - _prefill_ips=( "${_ALL_IPS[@]:0:${PREFILL_NODES}}" ) - [[ ${#_prefill_ips[@]} -gt 0 ]] || _prefill_ips=( "$PREFILL_LEADER_IP" ) + # ---- Gate on ALL prefill vLLM /health endpoints (cross-node) ---- + # Prefill ranks wait on their own local /health; wait_for_server_ready only + # probes localhost, so the coordinator polls every prefill node here. + # External LB registers one EPP endpoint per DP rank, so dedupe by node IP. + # EP roles expose /health on the DP supervisor (8100), not the serving port. + # curl gets an explicit connect/max timeout so a blackholed endpoint trips the + # deadline instead of hanging the whole run (a single timeout-less curl once + # wedged a 2P run for 7h before it was cancelled). + mapfile -t _prefill_ips < <(python3 - "$LLMD_ENDPOINTS_FILE" <<'PY' +import sys, yaml +seen = set() +for endpoint in yaml.safe_load(open(sys.argv[1]))['endpoints']: + if endpoint['labels']['llm-d.ai/role'] != 'prefill': + continue + address = endpoint['address'] + if address in seen: + continue + seen.add(address) + print(address) +PY + ) + _PREFILL_HEALTH_PORT="$VLLM_PORT" + if [[ "$PREFILL_ENABLE_EP" == "true" ]]; then + _PREFILL_HEALTH_PORT="8100" + fi # On failure, dump enough to tell a server-not-ready problem (TCP connects but # /health is slow) apart from a network/subnet problem (TCP connect refused or @@ -432,7 +503,7 @@ PY { echo "=== NET DIAG: decode -> prefill ${ip}:${port} ===" echo "[diag] decode node: $(hostname -f 2>/dev/null || hostname) local-ips: $(hostname -I 2>/dev/null)" - echo "[diag] ifaces: DEFAULT_IFACE=${DEFAULT_IFACE:-} NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-} GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-}" + echo "[diag] ifaces: DEFAULT_IFACE=${DEFAULT_IFACE} NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME} GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME}" # Local source address the kernel would pick to reach ip: reveals which # subnet/interface the route uses, without needing iproute2. python3 - "$ip" <<'PY' 2>&1 || true @@ -483,92 +554,107 @@ PY else echo "[diag] TCP connect ${ip}:${port} FAILED/timed out -> closed, filtered, or unreachable (LIKELY network/subnet/firewall issue)" fi + # L3: ICMP reachability, if ping is present. if command -v ping >/dev/null 2>&1; then ping -c 2 -W 2 "$ip" 2>&1 || echo "[diag] ping ${ip} failed (ICMP blocked or host down)" fi + # Verbose HTTP connect detail (DNS/connect/TLS timing, HTTP status). curl -v --connect-timeout 5 --max-time 8 "http://${ip}:${port}/health" 2>&1 || true echo "=== END NET DIAG ${ip}:${port} ===" } >&2 } - # Log the decode->prefill target layout up front so a subnet/interface - # mismatch is visible even on a run that eventually succeeds. Every prefill - # node serves on VLLM_PORT (hybrid LB). - echo "[diag] decode-leader $(hostname 2>/dev/null) local-ips: $(hostname -I 2>/dev/null); prefill targets: ${_prefill_ips[*]}" - echo "Waiting for prefill vLLM /health on ${#_prefill_ips[@]} node(s): ${_prefill_ips[*]}" + # Log the coordinator->prefill target layout up front so a subnet/interface + # mismatch is visible even on a run that eventually succeeds. + echo "[diag] coordinator $(hostname 2>/dev/null) local-ips: $(hostname -I 2>/dev/null); prefill targets: ${_prefill_ips[*]}:${_PREFILL_HEALTH_PORT}" + echo "Waiting for prefill vLLM /health on ${#_prefill_ips[@]} node(s) (port ${_PREFILL_HEALTH_PORT}): ${_prefill_ips[*]}" PREFILL_WAIT_DEADLINE=$(( $(date +%s) + 300 )) for _pidx in "${!_prefill_ips[@]}"; do _pip="${_prefill_ips[$_pidx]}" - _pport="$VLLM_PORT" until curl --output /dev/null --silent --fail \ --connect-timeout 5 --max-time 10 \ - "http://$_pip:$_pport/health"; do + "http://$_pip:${_PREFILL_HEALTH_PORT}/health"; do if [[ "$(date +%s)" -ge "$PREFILL_WAIT_DEADLINE" ]]; then - echo "ERROR: prefill vLLM at $_pip:$_pport not ready within 5 min" >&2 - _diag_prefill_endpoint "$_pip" "$_pport" + echo "ERROR: prefill vLLM at $_pip:${_PREFILL_HEALTH_PORT} not ready within 5 min" >&2 + _diag_prefill_endpoint "$_pip" "$_PREFILL_HEALTH_PORT" exit 1 fi sleep 5 done - echo "Prefill vLLM at $_pip:$_pport is ready" + echo "Prefill vLLM at $_pip:${_PREFILL_HEALTH_PORT} is ready" done - echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready" - - # Benchmark sweep. BENCH_MAX_CONCURRENCY is 'x'-delimited from submit.sh (e.g. "1024x512"). - IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" - # GPU counts are embedded in the result filename as _gpus_/_ctx_/_gen_ so the CI - # "Process result" step can parse them (same convention as amd_utils/bench.sh). - # ctx = prefill GPUs, gen = decode GPUs. - _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) - _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) - _bench_total_gpus=$(( _bench_prefill_gpus + _bench_decode_gpus )) - if [[ "${EVAL_ONLY}" != "true" ]]; then - for max_concurrency in "${CONCURRENCIES[@]}"; do - num_prompts=$(( max_concurrency * BENCH_NUM_PROMPTS_MULTIPLIER )) - [[ "$num_prompts" -lt 16 ]] && num_prompts=16 - # Bench against Envoy (EPP routes to decode; the sidecar pulls from - # prefill via NIXL). --bench-serving-dir = the /workspace repo bind-mount; - # --tokenizer = /models (served-model-name is not a valid HF repo id). - # DSV4-Pro needs trust-remote-code + tokenizer-mode deepseek_v4 (the older - # transformers wheel does not register it) + chat template / --dsv4 to - # match the dynamo-vllm bench prompt formatting. - bench_extra_args=() - if [[ "${MODEL_NAME,,}" == *"deepseek-v4"* ]]; then - bench_extra_args+=( - --trust-remote-code - --tokenizer-mode deepseek_v4 - --use-chat-template - --dsv4 - ) - fi + echo "All ${#_prefill_ips[@]} prefill vLLM node(s) ready" + + if [[ "${IS_AGENTIC}" == "1" && "${EVAL_ONLY}" != "true" ]]; then + export ENVOY_PORT VLLM_PORT INFMAX_CONTAINER_WORKSPACE=/workspace + bash /workspace/benchmarks/multi_node/llm-d/agentic.sh + elif [[ "${EVAL_ONLY}" != "true" ]]; then + # ---- Benchmark sweep (one run per concurrency level) ---- + # BENCH_MAX_CONCURRENCY is an 'x'-delimited list from submit.sh (e.g. "1024x512"). + IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" + # GPU counts embedded in the result filename as _gpus_/_ctx_/_gen_ tokens so the + # CI "Process result" step (benchmark-multinode-tmpl.yml) can parse them and run + # process_result.py for llm-d -- same filename convention as amd_utils/bench.sh. + # ctx = prefill GPUs, gen = decode GPUs; nodes*GPUS_PER_NODE is correct for any + # PREFILL_WORKERS/DECODE_WORKERS split (e.g. high-tpt 2P -> 16 prefill GPUs). + _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) + _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) + _bench_total_gpus=$(( _bench_prefill_gpus + _bench_decode_gpus )) + for max_concurrency in "${CONCURRENCIES[@]}"; do + num_prompts=$(( max_concurrency * BENCH_NUM_PROMPTS_MULTIPLIER )) + [[ "$num_prompts" -lt 16 ]] && num_prompts=16 + # Bench against Envoy (EPP routes to decode; the sidecar pulls from + # prefill via NIXL). --bench-serving-dir = the /workspace repo bind-mount; + # --tokenizer = /models (served-model-name is not a valid HF repo id). + # DSV4-Pro needs trust-remote-code + tokenizer-mode deepseek_v4 (the older + # transformers wheel does not register it) + chat template / --dsv4 to + # match the dynamo-vllm bench prompt formatting. + bench_extra_args=() + if [[ "${MODEL_NAME,,}" == *"deepseek-v4"* ]]; then + bench_extra_args+=( + --trust-remote-code + --tokenizer-mode deepseek_v4 + --use-chat-template + --dsv4 + ) + fi + + # Non-fatal: a failed or timed-out conc point must not abort the sweep + # or (under set -e) skip the allocation release below. The EXIT trap + # releases the allocation regardless, but continuing here lets a + # multi-conc sweep record every point it can. + run_benchmark_serving \ + --bench-serving-dir /workspace \ + --tokenizer /models \ + --model "$MODEL_NAME" \ + --port "$ENVOY_PORT" \ + --backend openai \ + --input-len "$BENCH_INPUT_LEN" \ + --output-len "$BENCH_OUTPUT_LEN" \ + --random-range-ratio "$BENCH_RANDOM_RANGE_RATIO" \ + --num-prompts "$num_prompts" \ + --max-concurrency "$max_concurrency" \ + --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ + --result-dir "$BENCHMARK_LOGS_DIR/" \ + "${bench_extra_args[@]}" \ + || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" + done - # Non-fatal: a failed or timed-out conc point must not abort the sweep or (under - # set -e) skip the allocation release below. - run_benchmark_serving \ - --bench-serving-dir /workspace \ - --tokenizer /models \ - --model "$MODEL_NAME" \ - --port "$ENVOY_PORT" \ - --backend openai \ - --input-len "$BENCH_INPUT_LEN" \ - --output-len "$BENCH_OUTPUT_LEN" \ - --random-range-ratio "$BENCH_RANDOM_RANGE_RATIO" \ - --num-prompts "$num_prompts" \ - --max-concurrency "$max_concurrency" \ - --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ - --result-dir "$BENCHMARK_LOGS_DIR/" \ - "${bench_extra_args[@]}" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" - done fi - # Eval (optional) + # ---- Eval (optional) ---- if [[ "${RUN_EVAL}" == "true" ]]; then - # run_eval/append_lm_eval_summary read EVAL_CONCURRENT_REQUESTS and CONC (not - # EVAL_CONC). Exporting CONC makes meta_env.json's "conc" match what - # utils/evals/validate_scores.py --expected-concs verifies; without it the - # metadata records conc=1 and score verification fails even when accuracy passes. - if [[ -n "${EVAL_CONC:-}" ]]; then + # Concurrency for the eval and, crucially, for the concurrency stamped + # into meta_env.json. run_eval/append_lm_eval_summary read + # EVAL_CONCURRENT_REQUESTS and CONC (not EVAL_CONC), so mirror the AMD + # multi-node servers: use the workflow-provided EVAL_CONC when set, else + # fall back to the max of the (x-delimited) BENCH_MAX_CONCURRENCY list. + # Exporting CONC makes meta_env.json's "conc" match what + # utils/evals/validate_scores.py --expected-concs verifies; without it + # CONC is empty, the metadata records conc=1, and score verification + # fails ("eval metadata concurrency does not match workflow request") + # even when accuracy passes. + if [[ -n "${EVAL_CONC}" ]]; then export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" else export EVAL_CONCURRENT_REQUESTS=$(printf '%s' "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) diff --git a/benchmarks/multi_node/llm-d/test_agentic_adapter.py b/benchmarks/multi_node/llm-d/test_agentic_adapter.py new file mode 100644 index 0000000000..d3ff2ce76b --- /dev/null +++ b/benchmarks/multi_node/llm-d/test_agentic_adapter.py @@ -0,0 +1,148 @@ +import json +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +@pytest.mark.parametrize("metrics_body", ["vllm:num_requests_running 0\n", "envoy_http_requests_total 0\n"]) +def test_llmd_agentic_adapter_uses_discovered_worker_metrics(tmp_path: Path, metrics_body: str) -> None: + """Check endpoint selection, preflight failure, and the real AIPerf CLI builder.""" + client = tmp_path / "benchmarks/multi_node/agentic_srt.sh" + client.parent.mkdir(parents=True) + client.write_text('''source "$REAL_BENCHMARK_LIB" +build_replay_cmd "$RESULT_DIR" +export REPLAY_CMD +python3 - <<'PY' +import json, os +keys = ["AIPERF_METRIC_URLS", "AIPERF_SERVER_METRICS_URLS", "REPLAY_CMD"] +print(json.dumps({key: os.environ[key] for key in keys})) +PY +''') + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + curl = bin_dir / "curl" + curl.write_text( + '#!/usr/bin/env python3\n' + 'import os, sys\nfrom pathlib import Path\n' + 'args = sys.argv[1:]\n' + 'url = next((a for a in args if a.startswith("http://")), "")\n' + 'if "--write-out" in args:\n' + ' print("404", end="")\n' + 'else:\n' + ' out_path = args[args.index("--output") + 1]\n' + ' if out_path != "/dev/null":\n' + ' Path(out_path).write_text(os.environ["METRICS_BODY"])\n' + ' with open(os.environ["METRICS_REQUESTS"], "a") as f:\n' + ' f.write(url + "\\n")\n' + ) + curl.chmod(0o755) + endpoints = tmp_path / "endpoints.yaml" + endpoints.write_text(yaml.safe_dump({"endpoints": [ + {"address": "10.0.0.1", "port": "8200", "name": "vllm-node-0", + "labels": {"llm-d.ai/role": "combined"}}, + {"address": "10.0.0.2", "port": "8201", "name": "vllm-node-1", + "labels": {"llm-d.ai/role": "combined"}}, + {"address": "10.0.0.3", "port": "8202", "name": "vllm-node-2", + "labels": {"llm-d.ai/role": "combined"}}, + ]})) + requests = tmp_path / "metrics-requests.txt" + env = dict(os.environ, INFMAX_CONTAINER_WORKSPACE=str(tmp_path), + REAL_BENCHMARK_LIB=str(REPO_ROOT / "benchmarks/benchmark_lib.sh"), + PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], + METRICS_BODY=metrics_body, METRICS_REQUESTS=str(requests), + LLMD_ENDPOINTS_FILE=str(endpoints), MODEL_NAME="test-model", MODEL_PREFIX="dsv4", + FRAMEWORK="llmd-vllm", DURATION="3600", IS_AGENTIC="1", KV_OFFLOADING="none", + ENVOY_PORT="8080", VLLM_PORT="8200", SIDECAR_PORT="8000", + BENCHMARK_LOGS_DIR=str(tmp_path / "logs"), + BENCH_MAX_CONCURRENCY="64", DECODE_NODES="0") + result = subprocess.run(["bash", str(REPO_ROOT / "benchmarks/multi_node/llm-d/agentic.sh")], + env=env, text=True, capture_output=True) + if metrics_body.startswith("envoy_"): + assert result.returncode != 0 + assert "no vLLM metrics exposed" in result.stderr + return + assert result.returncode == 0, result.stderr + recorded = json.loads(result.stdout.splitlines()[-1]) + expected_urls = [ + "http://10.0.0.1:8200/metrics", + "http://10.0.0.2:8201/metrics", + "http://10.0.0.3:8202/metrics", + ] + assert requests.read_text().splitlines() == expected_urls + assert recorded["AIPERF_METRIC_URLS"].split(",") == expected_urls + assert recorded["AIPERF_SERVER_METRICS_URLS"].split(",") == expected_urls + assert "--url http://localhost:8080 " in recorded["REPLAY_CMD"] + assert "--server-metrics " + " ".join(expected_urls) + " " in recorded["REPLAY_CMD"] + + +def test_llmd_agentic_adapter_maps_decode_sidecar_ports_to_vllm_metrics( + tmp_path: Path, +) -> None: + """Disagg decode endpoints list sidecar ports; metrics scrape vLLM DP ranks.""" + client = tmp_path / "benchmarks/multi_node/agentic_srt.sh" + client.parent.mkdir(parents=True) + client.write_text('''source "$REAL_BENCHMARK_LIB" +build_replay_cmd "$RESULT_DIR" +export REPLAY_CMD +python3 - <<'PY' +import json, os +keys = ["AIPERF_METRIC_URLS", "AIPERF_SERVER_METRICS_URLS", "REPLAY_CMD"] +print(json.dumps({key: os.environ[key] for key in keys})) +PY +''') + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + curl = bin_dir / "curl" + curl.write_text( + '#!/usr/bin/env python3\n' + 'import os, sys\nfrom pathlib import Path\n' + 'args = sys.argv[1:]\n' + 'url = next((a for a in args if a.startswith("http://")), "")\n' + 'if "--write-out" in args:\n' + ' print("404", end="")\n' + 'else:\n' + ' out_path = args[args.index("--output") + 1]\n' + ' if out_path != "/dev/null":\n' + ' Path(out_path).write_text(os.environ["METRICS_BODY"])\n' + ' with open(os.environ["METRICS_REQUESTS"], "a") as f:\n' + ' f.write(url + "\\n")\n' + ) + curl.chmod(0o755) + endpoints = tmp_path / "endpoints.yaml" + endpoints.write_text(yaml.safe_dump({"endpoints": [ + {"address": "10.0.0.10", "port": "8200", "name": "prefill-0", + "labels": {"llm-d.ai/role": "prefill"}}, + {"address": "10.0.0.10", "port": "8201", "name": "prefill-1", + "labels": {"llm-d.ai/role": "prefill"}}, + {"address": "10.0.0.20", "port": "8000", "name": "decode-0", + "labels": {"llm-d.ai/role": "decode"}}, + {"address": "10.0.0.20", "port": "8001", "name": "decode-1", + "labels": {"llm-d.ai/role": "decode"}}, + ]})) + requests = tmp_path / "metrics-requests.txt" + env = dict(os.environ, INFMAX_CONTAINER_WORKSPACE=str(tmp_path), + REAL_BENCHMARK_LIB=str(REPO_ROOT / "benchmarks/benchmark_lib.sh"), + PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], + METRICS_BODY="vllm:num_requests_running 0\n", METRICS_REQUESTS=str(requests), + LLMD_ENDPOINTS_FILE=str(endpoints), MODEL_NAME="test-model", MODEL_PREFIX="dsv4", + FRAMEWORK="llmd-vllm", DURATION="3600", IS_AGENTIC="1", KV_OFFLOADING="none", + ENVOY_PORT="8080", VLLM_PORT="8200", SIDECAR_PORT="8000", + BENCHMARK_LOGS_DIR=str(tmp_path / "logs"), + BENCH_MAX_CONCURRENCY="64", DECODE_NODES="2") + result = subprocess.run(["bash", str(REPO_ROOT / "benchmarks/multi_node/llm-d/agentic.sh")], + env=env, text=True, capture_output=True) + assert result.returncode == 0, result.stderr + recorded = json.loads(result.stdout.splitlines()[-1]) + expected_urls = [ + "http://10.0.0.10:8200/metrics", + "http://10.0.0.10:8201/metrics", + "http://10.0.0.20:8200/metrics", + "http://10.0.0.20:8201/metrics", + ] + assert requests.read_text().splitlines() == expected_urls + assert recorded["AIPERF_METRIC_URLS"].split(",") == expected_urls diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36e171688e..76753efc01 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -3874,6 +3874,110 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 8 dp-attn: true +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, P/D disagg via llmd-vllm. +# Long-context workspace headroom: GPU-memory budgets are 0.88 for DEP8, 0.85 for TP8. +# Always uses Mooncake; DSpark speculative decoding. The DSpark image +# bundles EPP/pd-sidecar v0.10.0. +dsv4-fp4-gb200-llmd-vllm-agentx: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: llmd-vllm + router: { name: llm-d-router, version: "0.10.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.45 + search-space: + # 1P DEP8 + 1D DEP8 (4 nodes / 16 GPUs). Always Mooncake. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [64, 128, 160, 192] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=agentic/disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=2" + - "GPUS_PER_NODE=4" + +# Aggregated TP8/DEP8; only the Mooncake arm enables DRAM offloading. +dsv4-fp4-gb200-llmd-vllm-agentx-agg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: llmd-vllm + router: { name: llm-d-router, version: "0.10.0" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.45 + search-space: + # Aggregated TP8 (2 nodes / 8 GPUs; pure tensor-parallel, no EP). + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 4, 8, 12] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=2" + - "DECODE_NODES=0" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=agentic/agg-gb200-tp8-dspark-agentic.yaml" + # Aggregated DEP8 (2 nodes / 8 GPUs; DP=8 + EP), no Mooncake. + - spec-decoding: mtp + kv-offloading: none + conc-list: [12, 16, 32] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + - "DECODE_NODES=0" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-agentic.yaml" + # Aggregated DEP8 with Mooncake prefix-cache KV store. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [52, 72] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + - "DECODE_NODES=0" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml" + qwen3.5-fp8-gb200-dynamo-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 4c6abfccfa..6252c9de7c 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -197,6 +197,12 @@ llm-d is not the srt-slurm path: InferenceX owns the Slurm allocation and starts A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml` fallback and removes recipe-specific vLLM flags. Treat that as a validation failure unless fallback is explicitly intended. +The GB200 DSpark AgentX keys use `cluster:gb200-nv`: TP8/DEP8 spans two nodes (8 GPUs), and 1P-DEP8/1D-DEP8 spans four (16 GPUs). The ARM64 image digest includes router v0.10.0; AgentX does not mount the legacy router binaries. Recipes using `token-load-scorer` must explicitly include `inflight-load-producer` to supply its uncached-token dependency. The launcher uses the staged `DeepSeek-V4-Pro-0813` checkpoint, not the older V4-Pro weights. Use `gpu-memory-utilization=0.88` for DEP8 and `0.85` for TP8. Long-context replay exhausted sparse-attention indexer memory at 0.92 and 0.90 respectively; retain headroom without shortening the model context or filtering traces. NIXL+Mooncake P/D uses a 1,800-second KV lease, matching the model-execution timeout; the default 30-second lease expired during long-context decode stalls at c192. + +`recipe.py` injects DSpark golden AL for throughput; `EVAL_ONLY=true` keeps real verification. Keep adaptive verification disabled: its confidence-based draft-budget trimming changes the fixed-K golden AL. Mooncake's embedded store is DRAM offloading even with `enable_offload: false` (the flag controls SSD). Declare `kv-offloading: dram`, `kv-offload-backend: { name: mooncake }`, and `dram-utilization`; the runtime divides each node's budget among its four GPU ranks. Rank 0 starts a job-local Mooncake master on port 50051 (metrics on 50052); every rank waits for it before starting vLLM. `P2PHANDSHAKE` does not replace the store master. Mooncake uses InfiniBand HCAs `mlx5_0,mlx5_1,mlx5_3,mlx5_4`; `mlx5_2` and `mlx5_5` are Ethernet. Plain TP8/DEP8 declares `none`. + +Discovery excludes headless TP followers. `agentic.sh` checks every serving node's vLLM `/metrics`, exports `AIPERF_METRIC_URLS` and `AIPERF_SERVER_METRICS_URLS`, and forwards them through AIPerf's `--server-metrics`. Envoy rejects `/metrics` before EPP routing so AIPerf's automatic frontend scrape cannot duplicate worker counters. The adapter verifies this 404 before replay and requires exported `vllm:` metrics. Raw AgentX artifacts include `llmd_metrics_endpoints.json`, mapping each scrape URL to its discovery name and `prefill`, `decode`, or `combined` role; engine IDs alone are not unique across P/D groups. This manifest does not add Prometheus labels or change app ingestion. Envoy and P/D-sidecar metrics are not substitutes for vLLM metrics. + ## Update an image Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants), the matching master configs, runtime scripts, and checked-in recipes. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index f877035f18..e6f0bd7ee8 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -174,6 +174,12 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 `CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 +GB200 DSpark AgentX 配置使用 `cluster:gb200-nv`：TP8/DEP8 跨两个节点（8 GPU），1P-DEP8/1D-DEP8 跨四个节点（16 GPU）。ARM64 镜像摘要包含 router v0.10.0；AgentX 不挂载旧版路由器二进制文件。使用 `token-load-scorer` 的配置必须显式声明 `inflight-load-producer`，以提供其依赖的未缓存 token 数据。launcher 使用预先存储的 `DeepSeek-V4-Pro-0813` checkpoint，而非旧版 V4-Pro 权重。DEP8 使用 `gpu-memory-utilization=0.88`，TP8 使用 `0.85`。长上下文回放分别在 0.92 和 0.90 时耗尽稀疏注意力索引器内存；应保留工作区余量，而不是缩短模型上下文或过滤轨迹。 NIXL+Mooncake P/D 使用 1,800 秒 KV 租约，与模型执行超时一致；默认的 30 秒租约在 c192 长上下文解码停顿期间过期。 + +`recipe.py` 为吞吐测试注入 DSpark golden AL；`EVAL_ONLY=true` 保留真实验证。保持自适应验证关闭：它根据置信度缩减草稿 token 预算，会改变固定 K 的 golden AL。Mooncake 嵌入式存储即使设置 `enable_offload: false` 也属于 DRAM 卸载（该开关控制 SSD）。需声明 `kv-offloading: dram`、`kv-offload-backend: { name: mooncake }` 和 `dram-utilization`；运行时将每节点预算均分给四个 GPU rank。Rank 0 在端口 50051 启动本次作业专用的 Mooncake master（指标端口为 50052）；所有 rank 等待其就绪后再启动 vLLM。`P2PHANDSHAKE` 不能替代存储 master。Mooncake 使用 InfiniBand HCA `mlx5_0,mlx5_1,mlx5_3,mlx5_4`；`mlx5_2` 和 `mlx5_5` 为以太网设备。普通 TP8/DEP8 声明 `none`。 + +服务发现排除无 API 的 TP follower。`agentic.sh` 检查各服务节点的 vLLM `/metrics`，导出 `AIPERF_METRIC_URLS` 和 `AIPERF_SERVER_METRICS_URLS`，再通过 AIPerf 的 `--server-metrics` 传递。Envoy 在 EPP 路由前拒绝 `/metrics`，避免 AIPerf 自动抓取前端时重复统计 worker 计数器。适配器在回放前验证此端点返回 404，并要求导出结果包含 `vllm:` 指标。原始 AgentX 工件包含 `llmd_metrics_endpoints.json`，将各抓取 URL 映射到服务发现名称及 `prefill`、`decode` 或 `combined` 角色；仅凭 engine ID 无法区分 P/D 组。此清单不会添加 Prometheus 标签或改变应用的摄取逻辑。Envoy 和 P/D sidecar 的指标不能替代 vLLM 指标。 + ## 更新镜像 来源：[`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants)、对应主配置、运行时脚本与检入的 Recipe。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 43fbc3590d..39ffc88685 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8359,3 +8359,53 @@ - "Replace the decode-role prefill-round-robin-balance: true with load-balance-method: follow_bootstrap_room in the 5 srt-slurm recipes of this key that set it: SGLang nightly-dev-cu13-20260918-20518d85 no longer has --prefill-round-robin-balance (run 35319100186, every decode worker of the DEP16 arm exited at argument parsing) and its --load-balance-method choices are auto, round_robin, follow_bootstrap_room, total_requests and total_tokens; the prefill roles keep load-balance-method: round_robin" - "在该 key 中设置了该参数的 5 个 srt-slurm 配方里，将 decode 角色的 prefill-round-robin-balance: true 替换为 load-balance-method: follow_bootstrap_room：SGLang nightly-dev-cu13-20260918-20518d85 已没有 --prefill-round-robin-balance（运行 35319100186 中 DEP16 分支的所有 decode worker 在参数解析阶段退出），其 --load-balance-method 可选值为 auto、round_robin、follow_bootstrap_room、total_requests 与 total_tokens；prefill 角色保持 load-balance-method: round_robin" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3253 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro-0813 FP4/FP8-KV on GB200: 8-GPU TP8/DEP8 aggregation, 16-GPU NIXL DEP8 1P/1D, and llm-d 0.10.0 prefix/token-load routing. GPU-memory budgets: TP8 0.85, DEP8 0.88. P/D KV leases last 1800s to cover long-context decode stalls." + - "Use fixed-K DSpark golden AL: K=5/AL=3.61 for aggregated/decode; K=1/AL=1.84 for P/D prefill. Adaptive verification is disabled." + - "Mooncake variants use DRAM offload at 0.60 utilization, one job-local master, and IB HCAs mlx5_0,1,3,4; no SSD offload. Plain TP8/DEP8 has no offload. Scrape vLLM nodes directly without duplicate frontend counters." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Switch DEP8 llm-d wide-EP from hybrid LB to multi-port external LB: register one EPP endpoint per DP rank, open multi-port pd-sidecar on decode, probe DP supervisor /health on 8100, use Nixl-only decode KV, and scrape per-rank vLLM metrics in AgentX." + - "将 DEP8 llm-d 宽 EP 从 hybrid LB 切换为多端口外部 LB：按 DP rank 注册 EPP 端点，decode 侧启用多端口 pd-sidecar，在 8100 探测 DP supervisor /health，decode 仅保留 Nixl KV，并在 AgentX 中按 rank 抓取 vLLM 指标。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + scenario-type: + - agentic-coding + description: + - "Re-remove archived dsv4-fp4-gb200-llmd-vllm 8k1k master key reintroduced by the upstream merge and restore the thin GB200 llmd-vllm disagg wrapper for AgentX P/D launches." + - "重新移除 upstream 合并误带回的已归档 dsv4-fp4-gb200-llmd-vllm 8k1k 主配置项，并恢复供 AgentX P/D 启动使用的 GB200 llmd-vllm disagg 精简 wrapper。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Raise llm-d vLLM logging to DEBUG on GB200 DSpark AgentX recipes to capture worker-death diagnostics during external-LB DEP8 bring-up." + - "将 GB200 DSpark AgentX llm-d 配方的 vLLM 日志级别提升至 DEBUG，以便在外部 LB DEP8 启动阶段捕获 worker 异常退出诊断信息。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentx + - dsv4-fp4-gb200-llmd-vllm-agentx-agg + scenario-type: + - agentic-coding + description: + - "Lower GB200 DSpark AgentX Mooncake embedded segment budget from 60% to 45% node DRAM utilization and restore default vLLM INFO logging after unhelpful DEBUG bring-up." + - "将 GB200 DSpark AgentX Mooncake 嵌入式段预算从节点 DRAM 利用率的 60% 降至 45%，并在 DEBUG 启动诊断无效后恢复默认 vLLM INFO 日志级别。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2719 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 52a05900ce..7e8902395a 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -75,7 +75,7 @@ import_squash() { echo "Squash file already exists and is valid, skipping import: $squash" else local enroot_runtime - enroot_runtime=$(mktemp -d "${TMPDIR:-/tmp}/enroot-import.XXXXXX") || exit 1 + enroot_runtime=$(mktemp -d /tmp/enroot-import.XXXXXX) || exit 1 trap 'rm -rf -- "$enroot_runtime"' EXIT export ENROOT_RUNTIME_PATH="$enroot_runtime" @@ -122,7 +122,10 @@ if [[ "$MODEL_PREFIX" == "dsv41flash" && ( "$FRAMEWORK" == "vllm" || "$FRAMEWORK fi if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then - if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then + if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$MODEL" == *-0813 ]]; then + export MODEL_PATH="/mnt/lustre01/users-public/sa-shared/models/DeepSeek-V4-Pro-0813" + export MODEL_NAME="$MODEL" + elif [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" export MODEL_NAME="deepseek-ai/DeepSeek-V4-Pro" else @@ -139,14 +142,18 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" mkdir -p "$BENCHMARK_LOGS_DIR" - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh" + if [[ "$DISAGG" == "true" ]]; then + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh" + else + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-agg.sh" + fi BENCH_SCRIPT="benchmarks/multi_node/${SCRIPT_NAME}" if [[ ! -f "$BENCH_SCRIPT" ]]; then echo "Error: llm-d wrapper not found: $BENCH_SCRIPT" >&2 exit 1 fi - JOB_ID=$(bash "$BENCH_SCRIPT") + JOB_ID=$(bash "$BENCH_SCRIPT") || exit 1 if [[ -z "$JOB_ID" ]]; then echo "Error: failed to submit llm-d job" >&2 exit 1 @@ -162,6 +169,11 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 done < <(find "$BENCHMARK_LOGS_DIR" -name "${RESULT_FILENAME}*.json" -print0 2>/dev/null) + if [[ "$IS_AGENTIC" == "1" && "$EVAL_ONLY" != "true" ]]; then + mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" + cp -R "$BENCHMARK_LOGS_DIR/agentic/." "$GITHUB_WORKSPACE/LOGS/agentic/" || exit 1 + fi + if [[ "${RUN_EVAL}" == "true" ]]; then EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR" -type d -name eval_results -print -quit 2>/dev/null) if [[ -z "$EVAL_DIR" ]]; then