From 3ba6b7c8b25fae68a1c2320a8e822fdf4fc47ef8 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Sat, 19 Sep 2026 21:11:22 -0400
Subject: [PATCH 1/2] =?UTF-8?q?Add=20B200=20TP2=20DSv4.1=20Flash=20CPU=20o?=
 =?UTF-8?q?ffload=20/=20=E6=96=B0=E5=A2=9E=20B200=20TP2=20DSv4.1=20Flash?=
 =?UTF-8?q?=20CPU=20=E5=8D=B8=E8=BD=BD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add B200 TP2 with --engram-config '{"cpu_offload":true}' for AgentX at
concurrency 1-128, keeping the TP4 AgentX arm and the pinned vLLM image.
Rebased onto origin/main as a single commit; the fixed-sequence 8k1k arm and
the client/test/doc changes that served it are dropped.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 .../agentic/dsv41flash_fp4_vllm_mtp.sh        | 28 +++++++++++++++++++
 configs/nvidia-master.yaml                    |  1 +
 perf-changelog.yaml                           | 26 +++++++++++++++++
 3 files changed, 55 insertions(+)

diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh
index d5d00eb8a9..f8989d1296 100644
--- a/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh
@@ -46,6 +46,33 @@ while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 ));
     CAPTURE_SIZE=$((CAPTURE_SIZE * 2))
 done
 
+# TP2 leaves ~145 GiB of weights on each 180 GB B200 even with the Engram
+# tables offloaded. At the upstream 16384 batched tokens the sparse-attention
+# indexer's [batched-tokens, 1M] fp8 logits buffer is 32 GiB, and graph capture
+# for c32-c128 pushed the KV budget to -10.8 GiB (run 35180394796: c1-c16
+# served, c32/c64/c128 died in memory profiling). Cap batched tokens at 4096
+# (8 GiB, as the H100 arm does), bound the scheduler batch to the AgentX
+# fan-out, and stop capturing above 512 tokens; TP4 and TP8 keep the defaults.
+TP2_ARGS=()
+if (( TP == 2 )); then
+    MAX_NUM_SEQS=$((2 * CONC))
+    if (( MAX_NUM_SEQS > 256 )); then
+        MAX_NUM_SEQS=256
+    fi
+    # FlashInfer's autotune dummy run batches max-num-seqs requests through
+    # the DSpark draft head; with 2-8 requests on TP2 it selected an invalid
+    # MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4) and the engine
+    # never started (run 35320655804: c1/c2/c4 failed, c8 with 16 seqs and
+    # every larger point served). 16 is the smallest value that has passed.
+    if (( MAX_NUM_SEQS < 16 )); then
+        MAX_NUM_SEQS=16
+    fi
+    if (( CAPTURE_SIZE > 512 )); then
+        CAPTURE_SIZE=512
+    fi
+    TP2_ARGS=(--max-num-batched-tokens 4096 --max-num-seqs "$MAX_NUM_SEQS")
+fi
+
 # Pyxis shares the host network; port 8888 can already belong to a host service.
 select_available_server_port
 export AIPERF_SERVER_URL="http://localhost:${PORT}"
@@ -71,6 +98,7 @@ VLLM_CMD=(
     --speculative-config "$SPEC_CONFIG"
     --max-model-len 1048576
     --max-cudagraph-capture-size "$CAPTURE_SIZE"
+    "${TP2_ARGS[@]}"
     --disable-uvicorn-access-log
     "${LOAD_ARGS[@]}"
 )
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index c5d5edac68..c01649f918 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8162,6 +8162,7 @@ dsv41flash-fp4-b200-vllm-agentic-dspark:
       search-space:
       # Engram weights use UVA DRAM; the KV cache stays GPU-resident.
       - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] }
+      - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] }
 
 # SGLang arm for DeepSeek-V4.1-Flash AgentX on B200, from the SGLang cookbook
 # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1).
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index c5b4309f5f..cfd1762efd 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -8455,3 +8455,29 @@
   description:
     - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274
+
+- config-keys:
+    - dsv41flash-fp4-b200-vllm-agentic-dspark
+  description:
+    - "Add B200 TP2 DeepSeek-V4.1-Flash with --engram-config '{\"cpu_offload\":true}' for AgentX, at concurrency 1, 2, 4, 8, 16, 32, 64, 128; retain the existing TP4 AgentX arm and pinned vLLM image."
+    - "Keep DSpark5, golden AL 3.51 for throughput, real block rejection with adaptive verification for evals, and GPU-resident KV."
+    - "新增 B200 TP2 DeepSeek-V4.1-Flash AgentX 配方，启用 Engram CPU 卸载，并发为 1、2、4、8、16、32、64、128；保留现有 TP4 AgentX 分支和固定 vLLM 镜像。吞吐使用 DSpark5 和黄金 AL 3.51，eval 使用真实块拒绝采样和自适应验证，KV 保留在 GPU 上。"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216
+
+- config-keys:
+    - dsv41flash-fp4-b200-vllm-agentic-dspark
+  scenario-type:
+    - agentic-coding
+  description:
+    - "TP2 only: cap --max-num-batched-tokens at 4096, bound --max-num-seqs to 2*CONC (max 256) and stop CUDA graph capture at 512 tokens in the shared AgentX script, after run 35180394796 served TP2 c1-c16 but c32, c64 and c128 died in memory profiling with -10.8 GiB of KV budget (145 GiB of weights per GPU plus the 32 GiB indexer buffer at 16384 batched tokens); TP4 and TP8 arms keep the upstream defaults"
+    - "仅 TP2：在共享 AgentX 脚本中将 --max-num-batched-tokens 上限设为 4096，--max-num-seqs 限制为 2*CONC（最多 256），CUDA graph 捕获止于 512 token；运行 35180394796 中 TP2 c1-c16 可服务，但 c32、c64、c128 在显存分析阶段因 KV 预算为 -10.8 GiB 而失败（每 GPU 145 GiB 权重加 16384 批处理 token 时 32 GiB 的 indexer 缓冲区）；TP4 与 TP8 配方保持上游默认"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216
+
+- config-keys:
+    - dsv41flash-fp4-b200-vllm-agentic-dspark
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Floor the TP2 --max-num-seqs at 16 (it stays 2x the concurrency above that, capped at 256): in run 35320655804 c1, c2 and c4 never started because FlashInfer's autotune dummy run batches max-num-seqs requests through the DSpark draft head and with 2-8 requests selected an invalid MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4), while c8 with 16 seqs and every larger point served"
+    - "将 TP2 的 --max-num-seqs 下限设为 16（高于该值时仍为并发数的 2 倍，上限 256）：运行 35320655804 中 c1、c2、c4 未能启动，原因是 FlashInfer 的 autotune dummy run 会让 max-num-seqs 个请求经过 DSpark draft head，在 2-8 个请求时选中了无效的 MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4)，而 c8（16 seqs）及更大的点均正常服务"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216

From f79215fc5faeb5094478ca3fa58f479eb82a9889 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Sun, 20 Sep 2026 01:45:12 -0400
Subject: [PATCH 2/2] chore: refresh PR #3216 for sweep reuse [skip-sweep]
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sync with origin/main after the green sweep run 35480787586; the reuse gate
authorizes that run on this head.

在绿色 sweep 运行 35480787586 之后与 origin/main 同步；reuse gate 在此 head 上授权该运行。

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>