diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh index d5d00eb8a..f8989d129 100644 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh @@ -46,6 +46,33 @@ while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) done +# TP2 leaves ~145 GiB of weights on each 180 GB B200 even with the Engram +# tables offloaded. At the upstream 16384 batched tokens the sparse-attention +# indexer's [batched-tokens, 1M] fp8 logits buffer is 32 GiB, and graph capture +# for c32-c128 pushed the KV budget to -10.8 GiB (run 35180394796: c1-c16 +# served, c32/c64/c128 died in memory profiling). Cap batched tokens at 4096 +# (8 GiB, as the H100 arm does), bound the scheduler batch to the AgentX +# fan-out, and stop capturing above 512 tokens; TP4 and TP8 keep the defaults. +TP2_ARGS=() +if (( TP == 2 )); then + MAX_NUM_SEQS=$((2 * CONC)) + if (( MAX_NUM_SEQS > 256 )); then + MAX_NUM_SEQS=256 + fi + # FlashInfer's autotune dummy run batches max-num-seqs requests through + # the DSpark draft head; with 2-8 requests on TP2 it selected an invalid + # MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4) and the engine + # never started (run 35320655804: c1/c2/c4 failed, c8 with 16 seqs and + # every larger point served). 16 is the smallest value that has passed. + if (( MAX_NUM_SEQS < 16 )); then + MAX_NUM_SEQS=16 + fi + if (( CAPTURE_SIZE > 512 )); then + CAPTURE_SIZE=512 + fi + TP2_ARGS=(--max-num-batched-tokens 4096 --max-num-seqs "$MAX_NUM_SEQS") +fi + # Pyxis shares the host network; port 8888 can already belong to a host service. select_available_server_port export AIPERF_SERVER_URL="http://localhost:${PORT}" @@ -71,6 +98,7 @@ VLLM_CMD=( --speculative-config "$SPEC_CONFIG" --max-model-len 1048576 --max-cudagraph-capture-size "$CAPTURE_SIZE" + "${TP2_ARGS[@]}" --disable-uvicorn-access-log "${LOAD_ARGS[@]}" ) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c5d5edac6..c01649f91 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8162,6 +8162,7 @@ dsv41flash-fp4-b200-vllm-agentic-dspark: search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } # SGLang arm for DeepSeek-V4.1-Flash AgentX on B200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5b4309f5..cfd1762ef 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8455,3 +8455,29 @@ description: - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + description: + - "Add B200 TP2 DeepSeek-V4.1-Flash with --engram-config '{\"cpu_offload\":true}' for AgentX, at concurrency 1, 2, 4, 8, 16, 32, 64, 128; retain the existing TP4 AgentX arm and pinned vLLM image." + - "Keep DSpark5, golden AL 3.51 for throughput, real block rejection with adaptive verification for evals, and GPU-resident KV." + - "新增 B200 TP2 DeepSeek-V4.1-Flash AgentX 配方,启用 Engram CPU 卸载,并发为 1、2、4、8、16、32、64、128;保留现有 TP4 AgentX 分支和固定 vLLM 镜像。吞吐使用 DSpark5 和黄金 AL 3.51,eval 使用真实块拒绝采样和自适应验证,KV 保留在 GPU 上。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "TP2 only: cap --max-num-batched-tokens at 4096, bound --max-num-seqs to 2*CONC (max 256) and stop CUDA graph capture at 512 tokens in the shared AgentX script, after run 35180394796 served TP2 c1-c16 but c32, c64 and c128 died in memory profiling with -10.8 GiB of KV budget (145 GiB of weights per GPU plus the 32 GiB indexer buffer at 16384 batched tokens); TP4 and TP8 arms keep the upstream defaults" + - "仅 TP2:在共享 AgentX 脚本中将 --max-num-batched-tokens 上限设为 4096,--max-num-seqs 限制为 2*CONC(最多 256),CUDA graph 捕获止于 512 token;运行 35180394796 中 TP2 c1-c16 可服务,但 c32、c64、c128 在显存分析阶段因 KV 预算为 -10.8 GiB 而失败(每 GPU 145 GiB 权重加 16384 批处理 token 时 32 GiB 的 indexer 缓冲区);TP4 与 TP8 配方保持上游默认" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Floor the TP2 --max-num-seqs at 16 (it stays 2x the concurrency above that, capped at 256): in run 35320655804 c1, c2 and c4 never started because FlashInfer's autotune dummy run batches max-num-seqs requests through the DSpark draft head and with 2-8 requests selected an invalid MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4), while c8 with 16 seqs and every larger point served" + - "将 TP2 的 --max-num-seqs 下限设为 16(高于该值时仍为并发数的 2 倍,上限 256):运行 35320655804 中 c1、c2、c4 未能启动,原因是 FlashInfer 的 autotune dummy run 会让 max-num-seqs 个请求经过 DSpark draft head,在 2-8 个请求时选中了无效的 MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4),而 c8(16 seqs)及更大的点均正常服务" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216