Add B200 TP2 DSv4.1 Flash CPU offload / 新增 B200 TP2 DSv4.1 Flash CPU 卸载

benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh

@@ -46,6 +46,33 @@ while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 ));
     CAPTURE_SIZE=$((CAPTURE_SIZE * 2))
 done
 
+# TP2 leaves ~145 GiB of weights on each 180 GB B200 even with the Engram
+# tables offloaded. At the upstream 16384 batched tokens the sparse-attention
+# indexer's [batched-tokens, 1M] fp8 logits buffer is 32 GiB, and graph capture
+# for c32-c128 pushed the KV budget to -10.8 GiB (run 35180394796: c1-c16
+# served, c32/c64/c128 died in memory profiling). Cap batched tokens at 4096
+# (8 GiB, as the H100 arm does), bound the scheduler batch to the AgentX
+# fan-out, and stop capturing above 512 tokens; TP4 and TP8 keep the defaults.
+TP2_ARGS=()
+if (( TP == 2 )); then
+    MAX_NUM_SEQS=$((2 * CONC))
+    if (( MAX_NUM_SEQS > 256 )); then
+        MAX_NUM_SEQS=256
+    fi
+    # FlashInfer's autotune dummy run batches max-num-seqs requests through
+    # the DSpark draft head; with 2-8 requests on TP2 it selected an invalid
+    # MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4) and the engine
+    # never started (run 35320655804: c1/c2/c4 failed, c8 with 16 seqs and
+    # every larger point served). 16 is the smallest value that has passed.
+    if (( MAX_NUM_SEQS < 16 )); then
+        MAX_NUM_SEQS=16
+    fi
+    if (( CAPTURE_SIZE > 512 )); then
+        CAPTURE_SIZE=512
+    fi
+    TP2_ARGS=(--max-num-batched-tokens 4096 --max-num-seqs "$MAX_NUM_SEQS")
+fi
+
 # Pyxis shares the host network; port 8888 can already belong to a host service.
 select_available_server_port
 export AIPERF_SERVER_URL="http://localhost:${PORT}"
@@ -71,6 +98,7 @@ VLLM_CMD=(
     --speculative-config "$SPEC_CONFIG"
     --max-model-len 1048576
     --max-cudagraph-capture-size "$CAPTURE_SIZE"
+    "${TP2_ARGS[@]}"
     --disable-uvicorn-access-log
     "${LOAD_ARGS[@]}"
 )

benchmarks/single_node/fixed_seq_len/dsv41flash_fp4_b200_vllm_mtp.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# B200 fixed-sequence counterpart of the native DSpark AgentX recipe.
+# https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash
+source "$(dirname "$0")/../../benchmark_lib.sh"
+check_env_vars MODEL TP CONC ISL OSL RANDOM_RANGE_RATIO RESULT_FILENAME RESULT_DIR MAX_MODEL_LEN
+check_env_vars INFMAX_CONTAINER_WORKSPACE
+check_env_vars DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE EVAL_ONLY VLLM_ENGINE_READY_TIMEOUT_S
+export GPU_COUNT="$TP"
+
+if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then
+    hf download "$MODEL" --local-dir "$MODEL_PATH"
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+fi
+
+nvidia-smi
+mkdir -p "$RESULT_DIR"
+SERVER_LOG="$RESULT_DIR/server.log"
+export VLLM_USE_RUST_FRONTEND=1
+export PYTHONUNBUFFERED=1
+
+LOAD_ARGS=()
+if [[ -n "${VLLM_SAFETENSORS_LOAD_STRATEGY:-}" ]]; then
+    LOAD_ARGS=(--safetensors-load-strategy "$VLLM_SAFETENSORS_LOAD_STRATEGY")
+fi
+
+NUM_SPEC_TOKENS=5
+CAPTURE_SIZE="${DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE}"
+while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do
+    CAPTURE_SIZE=$((CAPTURE_SIZE * 2))
+done
+
+# TP2 leaves ~145 GiB of weights on each 180 GB B200 even with the Engram
+# tables offloaded, and vLLM's memory profiling counts the captured graphs
+# against the KV budget: c1-c32 served, but c64 (capture 512) ended with
+# -2.65 GiB and c128 (capture 1024) with -10.8 GiB of KV memory in run
+# 35316389982, while --max-model-len was still the 1M context. With the
+# matrix-supplied context below, stop capturing above 512 tokens on TP2;
+# larger DSpark verify batches decode eagerly. TP4 keeps the default.
+if (( TP == 2 && CAPTURE_SIZE > 512 )); then
+    CAPTURE_SIZE=512
+fi
+# Real DSpark verification (block rejection with adaptive verification) profiles
+# roughly 10 GiB more than the synthetic-acceptance throughput points: on TP2 the
+# c128 throughput point still had 8.13 GiB for KV at capture 512, while the
+# c64 and c128 evals ended at -2.22 GiB and never started (run 35355746550).
+# Halve the graph tier and let the eval use 0.95 of HBM; GSM8K prompts are short.
+EVAL_MEM_ARGS=()
+if [[ "${EVAL_ONLY}" == true ]] && (( TP == 2 )); then
+    CAPTURE_SIZE=256
+    EVAL_MEM_ARGS=(--gpu-memory-utilization 0.95)
+    # 256 captured tokens cover 42 requests at 1 + 5 draft tokens each. The c128
+    # eval kept ~125 GSM8K requests running, so every DSpark verify step ran
+    # above the graph tier through the untuned eager path (FlashInfer logged
+    # "No tuned config covers mxfp8_gemm" for each odd draft-head shape) and
+    # the draft lm_head GEMM died with cudaErrorIllegalAddress 70 s into the
+    # eval (run 35399984613). The TP4 eval at capture 1024 and the TP2 c1-c32
+    # throughput points, all inside their graph tiers, were clean. Keep the
+    # eval's request concurrency inside the tier; GSM8K is short enough that
+    # 32 concurrent requests finish the 1319 prompts within the job budget.
+    EVAL_CONC_CAP=32
+    if ! [[ "${EVAL_CONCURRENT_REQUESTS:-}" =~ ^[0-9]+$ ]] || (( EVAL_CONCURRENT_REQUESTS > EVAL_CONC_CAP )); then
+        export EVAL_CONCURRENT_REQUESTS="$EVAL_CONC_CAP"
+    fi
+fi
+select_available_server_port
+
+# Match AgentX's golden AL 3.51; accuracy evals use real target verification.
+if [[ "${EVAL_ONLY}" == true ]]; then
+    SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":true}'
+else
+    SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}'
+fi
+
+start_gpu_monitor
+# Fixed-sequence runs serve the matrix-supplied context (isl + osl + slack),
+# not the checkpoint's 1M: the sparse-attention indexer allocates a
+# [batched-tokens, max-model-len] fp8 buffer and the profiler reserves KV for
+# one full-context request, which at 1M left 0.97 GiB of KV at TP2 c32 in run
+# 35316389982. Accuracy evals use the eval context instead.
+MODEL_LEN="$MAX_MODEL_LEN"
+if [[ "${EVAL_ONLY}" == true ]]; then
+    # benchmark_lib derives EVAL_MAX_MODEL_LEN (isl + osl + 256, capped at the
+    # checkpoint's context) as the other fixed-seq arms do; the workflow does
+    # not export it (run 35320655804: both eval jobs exited at check_env_vars).
+    setup_eval_context
+    MODEL_LEN="$EVAL_MAX_MODEL_LEN"
+fi
+VLLM_CMD=(
+    vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
+    --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP"
+    --language-model-only
+    --tokenizer-mode deepseek_v41
+    --tool-call-parser deepseek_v41 --enable-auto-tool-choice
+    --reasoning-parser deepseek_v41
+    --engram-config '{"cpu_offload":true}'
+    --speculative-config "$SPEC_CONFIG"
+    --max-model-len "$MODEL_LEN"
+    --max-cudagraph-capture-size "$CAPTURE_SIZE"
+    "${EVAL_MEM_ARGS[@]}"
+    --disable-uvicorn-access-log
+    "${LOAD_ARGS[@]}"
+)
+printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
+"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [[ "${EVAL_ONLY}" == true ]]; then
+    run_eval --framework lm-eval --port "$PORT"
+    # run_eval leaves lm-eval output in its temp directory for non-agentic
+    # scenarios; staging into the workspace root, where the workflow looks for
+    # results*.json, is the recipe's job (run 35387151990: GSM8K scored 0.97 and
+    # the job still failed with "no results*.json files found").
+    append_lm_eval_summary
+else
+    run_benchmark_serving \
+        --model "$MODEL" --port "$PORT" --backend vllm \
+        --input-len "$ISL" --output-len "$OSL" \
+        --random-range-ratio "$RANDOM_RANGE_RATIO" \
+        --num-prompts "$((CONC * 10))" --max-concurrency "$CONC" \
+        `# The workflow reads $RESULT_FILENAME.json from the repository root, which the` \
+        `# dsv41flash launchers mount at /ix rather than /workspace (run 35314631817` \
+        `# wrote it under RESULT_DIR and the canary reported the result missing).` \
+        --result-filename "$RESULT_FILENAME" --result-dir "$INFMAX_CONTAINER_WORKSPACE/" \
+        --use-chat-template --tokenizer-mode deepseek_v41 \
+        --server-pid "$SERVER_PID"
+fi

configs/nvidia-master.yaml

@@ -8162,6 +8162,23 @@ dsv41flash-fp4-b200-vllm-agentic-dspark:
       search-space:
       # Engram weights use UVA DRAM; the KV cache stays GPU-resident.
       - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] }
+      - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] }
+
+dsv41flash-fp4-b200-vllm-dspark:
+  image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3
+  model: deepseek-ai/DeepSeek-V4.1-Flash
+  model-prefix: dsv41flash
+  runner: cluster:b200-nscale
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 8192
+      osl: 1024
+      search-space:
+      # Engram CPU offload is weight placement, not KV-cache offload.
+      - { tp: 2, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] }
 
 # SGLang arm for DeepSeek-V4.1-Flash AgentX on B200, from the SGLang cookbook
 # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1).

docs/configuration-procedures.md

@@ -254,6 +254,15 @@ at 2046 or 8190 tokens. It sets `--max-num-batched-tokens` to 2048 for concurren
 TP2 concurrency-128 variant also sets `--gpu-memory-utilization 0.97`. Other SKUs
 continue to use the shared script.
 
+B200 retains its TP4 AgentX arm and adds TP2 at concurrency
+`[1, 2, 4, 8, 16, 32, 64, 128]`. The new
+`dsv41flash-fp4-b200-vllm-dspark` fixed-sequence recipe runs 8K/1K at TP2 over
+the same concurrency list. Both use the existing B200 vLLM image and
+`--engram-config '{"cpu_offload":true}'`, with GPU-resident KV.
+The fixed-sequence script preserves the AgentX serving settings and golden
+DSpark AL, uses chat-formatted prompts, and uses real block rejection for evals.
+TP2 memory fit, correctness, and throughput require GPU sweep evidence.
+
 The GB300 launcher allows 7200 seconds for engine readiness. In [run 34504969146](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34504969146), the Rust frontend exhausted its 3600-second deadline while the engine was still capturing graphs; model loading alone took 18–23 minutes. This extends startup time without changing the benchmark duration or decoding settings.
 
 GPU sweep and eval evidence is required before calling any recipe validated.

docs/configuration-procedures_zh.md

@@ -228,6 +228,15 @@ TP2 并发 128 使用 `--max-num-batched-tokens 2048`，其余情况使用 8192
 `--max-num-seqs` 固定为 256。TP2 并发 128 还设置
 `--gpu-memory-utilization 0.97`。其他 SKU 继续使用共享脚本。
 
+B200 保留 TP4 AgentX 分支，并新增 TP2，并发为
+`[1, 2, 4, 8, 16, 32, 64, 128]`。新增的
+`dsv41flash-fp4-b200-vllm-dspark` 固定序列配方以 TP2 运行 8K/1K，
+使用相同并发列表。两个场景均沿用现有 B200 vLLM 镜像和
+`--engram-config '{"cpu_offload":true}'`，KV 保留在 GPU 上。
+固定序列脚本保留 AgentX 的服务设置和 DSpark 黄金 AL，使用聊天模板编码
+提示词，并在 eval 中使用真实块拒绝采样。TP2 的显存容量、正确性和吞吐量
+仍需 GPU 扫描验证。
+
 GB300 launcher 将引擎就绪等待时间设为 7200 秒。在[运行 34504969146](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34504969146) 中，仅模型加载就耗时 18–23 分钟；Rust frontend 达到 3600 秒期限时，引擎仍在捕获 CUDA graph。此次仅延长启动等待时间，基准测试时长和解码设置保持不变。
 
 来源：[上游配方](https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash)。

infx/bench_serving/benchmark_serving.py

@@ -115,8 +115,8 @@ def _load_tokenizer(
     transformers). Prefer backend_request_func.get_tokenizer on fallback so
     client tokenization stays aligned with the sglang server (#1381, #1428).
     """
-    if tokenizer_mode == "deepseek_v4":
-        # HF AutoTokenizer may not recognize deepseek_v4; use vLLM's loader.
+    if tokenizer_mode in {"deepseek_v4", "deepseek_v41"}:
+        # HF AutoTokenizer may not recognize DeepSeek's modes; use vLLM's loader.
         try:
             from vllm.tokenizers import get_tokenizer as _vllm_get_tokenizer
         except ImportError:
@@ -1316,7 +1316,7 @@ def main(args: argparse.Namespace) -> None:
         "--tokenizer-mode",
         type=str,
         default="auto",
-        choices=["auto", "slow", "mistral", "custom", "deepseek_v4"],
+        choices=["auto", "slow", "mistral", "custom", "deepseek_v4", "deepseek_v41"],
         help='The tokenizer mode.\n\n* "auto" will use the '
         'fast tokenizer if available.\n* "slow" will '
         "always use the slow tokenizer. \n* "