From 2f17ca077f706d3a80d9303693ee92bb38cb4cd5 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Wed, 9 Sep 2026 05:24:20 -0700 Subject: [PATCH 1/2] perf(sglang): use DSpark6 for B300 DSV4 AgentX --- .../agentic/dsv4_fp4_b300_sglang_mtp.sh | 28 ++++++++----------- configs/nvidia-master.yaml | 15 +++++----- perf-changelog.yaml | 11 ++++++++ runners/launch_b300-dsxe.sh | 20 +++++++++---- runners/test_slurm_utils.py | 20 +++++++++++++ 5 files changed, 66 insertions(+), 28 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh index 231fd481b8..889363fef3 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh @@ -2,7 +2,7 @@ set -eo pipefail set -x -# Agentic trace replay for DeepSeek-V4-Pro FP4 on B300 with native EAGLE MTP. +# Agentic trace replay for DeepSeek-V4-Pro-0813 FP4 on B300 with DSpark K=6. # Throughput uses the committed golden synthetic AL; eval retains real target # verification. # @@ -122,12 +122,14 @@ if [ "$DP_ATTENTION" = "true" ]; then --enable-prefill-delayer --prefill-decode-interval 20 --enable-dp-attention + --enable-dp-lm-head --enable-dp-attention-local-control-broadcast --incremental-streaming-output --stream-interval 20 --dist-init-addr "127.0.0.1:$((PORT + 2000))" --ep-size "$EP_SIZE" --moe-a2a-backend megamoe + --enable-w4a4-mxfp4-megamoe --enable-deepseek-v4-fp4-indexer --disable-flashinfer-autotune ) @@ -147,8 +149,8 @@ if [ "$DP_ATTENTION" = "true" ]; then MEM_FRACTION_STATIC=0.86 elif [ "$CONC" -ge 384 ]; then MEM_FRACTION_STATIC=0.88 - elif [ "$CONC" -ge 256 ]; then - MEM_FRACTION_STATIC=0.9 + elif [ "$CONC" -ge 32 ]; then + MEM_FRACTION_STATIC=0.90 fi else # DEP4 is squeezed from both sides: weights occupy ~90% of each GPU when @@ -156,8 +158,7 @@ if [ "$DP_ATTENTION" = "true" ]; then # below ~0.902 (no KV left), while megamoe still needs its ~7 GB # workspace above the static budget. 0.95 leaves only ~11 GB free -- the # same margin that OOM'd a rank at DEP8 conc 256 -- so use 0.93, which - # gives the ~16 GB that DEP8 conc 128 runs with at the same per-rank load - # (max-running-requests/dp = 32 in both cases). + # leaves approximately 16 GB for the MegaMoE workspace. MEM_FRACTION_STATIC=0.93 fi # --chunked-prefill-size is a GLOBAL budget: server_args.py divides it by @@ -194,7 +195,7 @@ CUDA_GRAPH_MAX_BS=$((CONC * 4)) CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") SWA_FULL_TOKENS_RATIO=0.1 if [ "$DP_ATTENTION" = "true" ]; then - # Decode graphs must cover the padded MTP batch across all DP ranks, which + # Decode graphs must cover the padded speculative batch across all DP ranks, which # exceeds CONC; capping at 64 would fall back to eager decode. CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) SWA_FULL_TOKENS_RATIO=0.075 @@ -219,18 +220,12 @@ export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 export SGLANG_OPT_USE_TOPK_V2=1 export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 if [ "$DP_ATTENTION" = "true" ]; then - # MegaMoE's FP4/MXF4 activation path is opt-in -- both flags default False, - # so --moe-a2a-backend megamoe alone runs a different kernel than the one - # measured. DG_USE_FP4_ACTS / DG_USE_MXF4_KIND are forwarded to DeepGEMM - # automatically from these two. - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 # Must cover the per-rank prefill budget (8192) or startup raises; the # extra 128 is headroom over the exact-fit boundary. export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 fi if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.49 + export SGLANG_SIMULATE_ACC_LEN=3.77 export SGLANG_SIMULATE_ACC_METHOD=match-expected export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token fi @@ -261,10 +256,11 @@ SGLANG_CMD=( --reasoning-parser deepseek-v4 --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" --watchdog-timeout 1800 - --speculative-algorithm EAGLE - --speculative-num-steps 3 + --speculative-algorithm DSPARK + --speculative-dspark-block-size 6 + --speculative-num-steps 1 --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 7 "${MODEL_ARGS[@]}" "${METRICS_ARGS[@]}" "${CACHE_ARGS[@]}" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1d0144da48..21e840b690 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1155,10 +1155,10 @@ dsv4-fp4-b300-sglang: - { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 4096 } dsv4-fp4-b300-sglang-agentic-hicache-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260827-20621aa1 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-20260901-07c8f729 + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: sglang multinode: false @@ -1166,10 +1166,10 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.95 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [32, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" } } - # DeepSeek-V4-Pro on B300 with EAGLE/MTP speculative decoding. Recipe is + # DeepSeek-V4-Pro-0813 on B300 with DSpark speculative decoding. Recipe is # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by # DP_ATTENTION: # dp-attn: false -> TP-only + flashinfer_mxfp4 + chunked-prefill 8192 @@ -1178,7 +1178,8 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: # + chunked-prefill 65536 + mem-fraction 0.90 # + swa-full-tokens-ratio 0.075 # + prefill-decode-interval 20 - # Both paths share EAGLE (3,1,4) and max-running-requests 2*CONC. + # Both paths share DSpark K=6 (1 step, 7 draft tokens) and + # max-running-requests 2*CONC. dsv4-fp4-b300-sglang-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260610-f332e526 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9dfeb4a02a..04ebc7b417 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6984,3 +6984,14 @@ - "Export OMP_NUM_THREADS=1 from all three scripts. vLLM clamps torch threads itself, but only after the weights are loaded: the 'Reducing Torch threads from 16 to 1 for serving' line is emitted after 'Model loading took', so the checkpoint is read with 16 torch threads per worker across 8 workers. Setting it externally caps threads from process start, which is what that log line recommends." - "Weight loading currently costs 5760-5946 s (96-99 min) per job on this runner, measured across the c1, c4, c56 and c64 AgentX jobs of run 34087124574 - roughly half of each ~3 h job. It does not affect any measured number: loading completes before warmup and the profiling phase. The effect is also only observable on this runner. A local EXT4 box serves the checkpoint from page cache and loads in ~40 s with or without the cap, so it never exercises the cold-read path. CI reads cold from WEKAFS, where each of the 8 ranks streams the full 805 GiB to slice its own shards - roughly 6.4 TB of aggregate network reads per job - and that is where oversubscribed reader threads cost real time." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2851 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Replace the B300 DeepSeek-V4-Pro AgentX native EAGLE/MTP serving path with DSpark block size 6 on the DeepSeek-V4-Pro-0813 checkpoint." + - "Use one speculative step, seven draft tokens, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." + - "Use SGLang's current W4A4 MegaMoE and DP LM-head flags for DP attention, with additional MegaMoE workspace headroom at concurrency 32, 64, and 128." + - "Route the workload through the B300 DSXE launcher, read weights from /data/models, and mount a writable Hugging Face/Xet cache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 70254deb65..2dcda83435 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -19,6 +19,7 @@ SQUASH_DIR="/data/home/sa-gha-runner/squash" # it is read-only from the job's point of view. Anything not in STAGED_MODELS is # downloaded into WRITABLE_MODELS_DIR (shared Lustre) by the single-node scripts. MODEL_ROOT="/scratch/models" +SHARED_MODEL_ROOT="/data/models" WRITABLE_MODELS_DIR="/data/home/sa-gha-runner/models" # Official power (dcgm-power) runs use a separate, pinned producer; CI derives @@ -394,15 +395,23 @@ done find . -name '.nfs*' -delete 2>/dev/null || true else - # HF_HUB_CACHE is set to help with dataset download inside the container - # for eval jobs. - export HF_HUB_CACHE="$HOME/.cache/huggingface" + # AgentX trace datasets need a writable persistent cache. Keep the host and + # container paths separate so the cache remains valid with + # --no-container-mount-home. + HF_CACHE_HOST_DIR="${B300_HF_CACHE_HOST_DIR:-$HOME/.cache/huggingface}" + HF_CACHE_CONTAINER_DIR="${B300_HF_CACHE_CONTAINER_DIR:-/hf_hub_cache}" + mkdir -p "$HF_CACHE_HOST_DIR/hub" "$HF_CACHE_HOST_DIR/xet" + export HF_HOME="$HF_CACHE_CONTAINER_DIR" + export HF_HUB_CACHE="$HF_CACHE_CONTAINER_DIR/hub" + export HF_XET_CACHE="$HF_CACHE_CONTAINER_DIR/xet" # MODEL stays the HF id for the client; MODEL_PATH is where the server reads # weights. Only the root holding MODEL_PATH is mounted -- mounting both roots # makes pyxis fail whenever the unused one is absent on the node. MODEL_BASENAME="${MODEL##*/}" - if [[ " ${STAGED_MODELS[*]} " == *" ${MODEL_BASENAME} "* ]]; then + if [[ "$MODEL_BASENAME" == "DeepSeek-V4-Pro-0813" ]]; then + MODEL_MOUNT_DIR="$SHARED_MODEL_ROOT" + elif [[ " ${STAGED_MODELS[*]} " == *" ${MODEL_BASENAME} "* ]]; then MODEL_MOUNT_DIR="$MODEL_ROOT" else MODEL_MOUNT_DIR="$WRITABLE_MODELS_DIR" @@ -411,7 +420,7 @@ else export MODEL_PATH="${MODEL_MOUNT_DIR}/${MODEL_BASENAME}" SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') + SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" || "$SPEC_DECODING" == "draft_model" ]] && printf '_mtp' || printf '') # Prefer a framework-tagged script (e.g. dsv4_fp4_b300_sglang.sh); fall back to # the untagged historical name for scripts that haven't been retagged yet. BENCH_BASE="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_b300" @@ -461,6 +470,7 @@ else CONTAINER_MOUNTS=( "$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR" "$MODEL_MOUNT_DIR:$MODEL_MOUNT_DIR" + "$HF_CACHE_HOST_DIR:$HF_CACHE_CONTAINER_DIR" ) CONTAINER_MOUNTS_ARG=$(IFS=,; printf '%s' "${CONTAINER_MOUNTS[*]}") diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 649f52c313..3015b63496 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -148,6 +148,26 @@ def test_copy_agentic_results_fails_when_aggregate_is_missing( assert "no run_conc*.json results found" in result.stderr +def test_b300_dsxe_draft_model_uses_public_paths_and_writable_hf_cache() -> None: + launcher = (REPO_ROOT / "runners/launch_b300-dsxe.sh").read_text() + + assert 'SLURM_ACCOUNT="benchmark"' in launcher + assert 'SQUASH_DIR="/data/home/sa-gha-runner/squash"' in launcher + assert 'SHARED_MODEL_ROOT="/data/models"' in launcher + assert 'WRITABLE_MODELS_DIR="/data/home/sa-gha-runner/models"' in launcher + assert '[[ "$MODEL_BASENAME" == "DeepSeek-V4-Pro-0813" ]]' in launcher + assert 'MODEL_MOUNT_DIR="$SHARED_MODEL_ROOT"' in launcher + assert "nv-gha-runner" not in launcher + assert ( + '[[ "$SPEC_DECODING" == "mtp" || "$SPEC_DECODING" == "draft_model" ]]' + in launcher + ) + assert 'export HF_HOME="$HF_CACHE_CONTAINER_DIR"' in launcher + assert 'export HF_HUB_CACHE="$HF_CACHE_CONTAINER_DIR/hub"' in launcher + assert 'export HF_XET_CACHE="$HF_CACHE_CONTAINER_DIR/xet"' in launcher + assert '"$HF_CACHE_HOST_DIR:$HF_CACHE_CONTAINER_DIR"' in launcher + + def test_patch_srt_eval_dispatch_forwards_selection_and_is_idempotent( tmp_path: Path, ) -> None: From b98a821426f73cee101e567ebccf632ab9003a51 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Wed, 9 Sep 2026 05:25:51 -0700 Subject: [PATCH 2/2] chore: link perf changelog to #2918 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 04ebc7b417..57b178b079 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6994,4 +6994,4 @@ - "Use one speculative step, seven draft tokens, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." - "Use SGLang's current W4A4 MegaMoE and DP LM-head flags for DP attention, with additional MegaMoE workspace headroom at concurrency 32, 64, and 128." - "Route the workload through the B300 DSXE launcher, read weights from /data/models, and mount a writable Hugging Face/Xet cache." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2918