From ddaa3a97727901e4014e715b5306a8ae948be7ba Mon Sep 17 00:00:00 2001 From: yhyang201 Date: Fri, 4 Sep 2026 07:29:28 +0800 Subject: [PATCH 1/3] fix(dsv4-b300): restore MegaMoE FP4 acts via --enable-w4a4-mxfp4-megamoe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sglang#35918 deprecated the SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS / _USE_MXF4_KIND env vars to warn-only, so the nightly-20260827 image runs MegaMoE on the default FP8-acts path: -8.6% at conc 384 and the conc-128 OOM. Pass the flag, and restore conc 64/128 to the hicache row. sglang#35918 将 SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS / _USE_MXF4_KIND 两个环境变量废弃为仅告警，nightly-20260827 镜像因此回退到默认 FP8 激活 路径：conc 384 下降 8.6%，conc 128 OOM。改为显式传 --enable-w4a4-mxfp4-megamoe，并把 conc 64/128 加回 hicache 搜索网格。 --- .../single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh | 11 +++++++++-- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 13 +++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh index 231fd481b8..6a75172857 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh @@ -128,6 +128,11 @@ if [ "$DP_ATTENTION" = "true" ]; then --dist-init-addr "127.0.0.1:$((PORT + 2000))" --ep-size "$EP_SIZE" --moe-a2a-backend megamoe + # sgl-project/sglang#35918 replaced SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + # /_USE_MXF4_KIND with this flag; on newer builds the env vars only print a + # deprecation warning and forward nothing, so without it MegaMoE silently + # falls back from the FP4-acts kernel to the default FP8-acts JIT path. + --enable-w4a4-mxfp4-megamoe --enable-deepseek-v4-fp4-indexer --disable-flashinfer-autotune ) @@ -221,8 +226,10 @@ export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 if [ "$DP_ATTENTION" = "true" ]; then # MegaMoE's FP4/MXF4 activation path is opt-in -- both flags default False, # so --moe-a2a-backend megamoe alone runs a different kernel than the one - # measured. DG_USE_FP4_ACTS / DG_USE_MXF4_KIND are forwarded to DeepGEMM - # automatically from these two. + # measured. On builds predating sgl-project/sglang#35918 these two env vars + # are what forwards DG_USE_FP4_ACTS / DG_USE_MXF4_KIND to DeepGEMM; on newer + # builds they are deprecated no-ops and --enable-w4a4-mxfp4-megamoe (passed + # above) carries the forwarding. Export both so the recipe works on either. export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 # Must cover the per-rank prefill budget (8192) or startup raises; the diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0a6c86df6d..80e1cbc8f0 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1158,7 +1158,7 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260827-20621aa1 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: sglang multinode: false @@ -1167,7 +1167,7 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: - dram-utilization: 0.95 search-space: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [32, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" } } # DeepSeek-V4-Pro on B300 with EAGLE/MTP speculative decoding. Recipe is # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6b249ea15e..5be36ed37b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6921,3 +6921,16 @@ - "Pick up the latest automatic ROCm DeepSeek-V4 optimizations, including fused mHC post/pre plus RMSNorm, gfx950 C4A top-k dispatch, fused C4 compressor GEMMs, fused SWA q/kv RMSNorm plus q FP8 quantization, and medium-batch cooperative top-k tuning." - "Keep the existing VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, and --moe-backend aiter settings, and explicitly add VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 plus VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 to both STP and MTP paths. The current checkpoint's shared-expert path does not satisfy the latest vLLM fusion conditions, so that fusion flag self-disables while preserving recipe parity." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2792 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Pass --enable-w4a4-mxfp4-megamoe on the DP-attention path. sgl-project/sglang#35918 (first shipped in lmsysorg/sglang:nightly-dev-cu13-20260827-20621aa1, the image pinned by #2759) replaced SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS / _USE_MXF4_KIND with this flag, and the _DeprecatedEnv entries it registered carry no replacement forwarding: on the new build the two env exports only print a deprecation warning and set nothing, so os.getenv('DG_USE_FP4_ACTS') in mega_moe.py stays unset and MegaMoE silently falls back from the FP4-acts kernel to the default FP8-acts JIT path. Confirmed in the run 33051183882 server log: 60 'Environment variable SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS is deprecated' warnings and enable_w4a4_mxfp4_megamoe: False in server_args, while the recipe exported both env vars." + - "Measured cost of the fallback, run 33051183882 (FP8-acts) vs run 32695861783 (FP4-acts), same recipe flags and same 3,630 s closed-loop window: conc 384 output throughput 5,351 -> 4,890 tok/s (-8.6%) and conc 256 -5.7%, with per-decode-step time +7.8% at c384 at identical batch size (24.3 vs 24.4 req/rank), identical accept length (2.485 vs 2.492), 100% CUDA-graph coverage, and ~95% prefix-cache hit in both runs; prefill input throughput on >=8192-token batches fell 4,659 -> 4,277 tok/s (-8.2%). The TP8 no-offload row (flashinfer_mxfp4, no MegaMoE) is unchanged within +/-0.3% at c1/c8/c16/c32 across the same two runs, isolating the regression to the MegaMoE path, and dcgm telemetry shows SM clocks within 1% (2,021 vs 2,003 MHz), ruling out the b300-019 -> b300-001 node swap." + - "The same fallback explains the conc-128 OOM that #2759 cut from the grid: FP8 activations double the MegaMoE dispatch workspace versus FP4, exhausting the 0.93 mem-fraction tier's headroom until deep_gemm fp8_fp4_paged_mqa_logits fails a 5.35 GiB allocation against 4.55 GiB free. With FP4 acts restored the workspace shrinks back, so this also restores concurrency 64 and 128 to the DEP8 hicache row, returning the search space from 10 points to 12; conc 64 had never been measured on this image, and both points are re-validated by this sweep." + - "Why the published c512/c576 points looked fine despite the fallback: per-step time rose there too (+17% at c512, +9% at c576) and prefill fell -6.5%, but both points are queue-bound (52-72 queued req/rank vs ~23 at c384), so slower steps pushed decode occupancy up (18.6 -> 23.8 req/rank at c512, full-KV usage 0.371 -> 0.473) and the larger batches absorbed the penalty; c384/c256 are admission-pinned at ~24 req/rank by the prefill-delayer / prefill-decode-interval 20 policy, so the step-time regression passed through 1:1. Restoring FP4 acts should shorten steps at every point; e2e at c512/c576 is expected to hold while occupancy floats back down." + - "Keep exporting the two deprecated env vars so the recipe still selects FP4 acts on pre-#35918 images; SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 is still a live EnvInt on the new build and is unchanged." + - "Also retarget the lane from the decommissioned cluster:b300-nv (#2818) to cluster:b300-dsxe (#2826). The DSXE nodes expose available-cpu-dram-mib 3,977,095 vs 2,964,436 on the retired NV fleet; the HiCache ratio-3 host tier is sized off device KV, not total DRAM, so the tier stays ~2 TB and the extra budget is headroom only. All reference numbers above were measured on the NV fleet; this sweep is the first DSXE run for this key and doubles as the cluster A/B check." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From b68f2498de45764b2beea971f0123730e2d16c22 Mon Sep 17 00:00:00 2001 From: yhyang201 Date: Tue, 8 Sep 2026 09:15:27 +0800 Subject: [PATCH 2/3] chore(changelog): point the entry at PR #2878 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将条目指向 PR #2878 --- perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5be36ed37b..f58896c0c4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2878 - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -6933,4 +6933,4 @@ - "Why the published c512/c576 points looked fine despite the fallback: per-step time rose there too (+17% at c512, +9% at c576) and prefill fell -6.5%, but both points are queue-bound (52-72 queued req/rank vs ~23 at c384), so slower steps pushed decode occupancy up (18.6 -> 23.8 req/rank at c512, full-KV usage 0.371 -> 0.473) and the larger batches absorbed the penalty; c384/c256 are admission-pinned at ~24 req/rank by the prefill-delayer / prefill-decode-interval 20 policy, so the step-time regression passed through 1:1. Restoring FP4 acts should shorten steps at every point; e2e at c512/c576 is expected to hold while occupancy floats back down." - "Keep exporting the two deprecated env vars so the recipe still selects FP4 acts on pre-#35918 images; SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 is still a live EnvInt on the new build and is unchanged." - "Also retarget the lane from the decommissioned cluster:b300-nv (#2818) to cluster:b300-dsxe (#2826). The DSXE nodes expose available-cpu-dram-mib 3,977,095 vs 2,964,436 on the retired NV fleet; the HiCache ratio-3 host tier is sized off device KV, not total DRAM, so the tier stays ~2 TB and the extra budget is headroom only. All reference numbers above were measured on the NV fleet; this sweep is the first DSXE run for this key and doubles as the cluster A/B check." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2878 From 3ccb09a46afe5ab6015e6ccf8da268c618c2818e Mon Sep 17 00:00:00 2001 From: yhyang201 Date: Tue, 8 Sep 2026 09:15:45 +0800 Subject: [PATCH 3/3] fix(changelog): keep the TileRT entry on its own placeholder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅把本 PR 的条目指向 #2878，TileRT 条目保留原占位符 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f58896c0c4..e75fcbe35e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2878 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp