diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh index 231fd481b..889363fef 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh @@ -2,7 +2,7 @@ set -eo pipefail set -x -# Agentic trace replay for DeepSeek-V4-Pro FP4 on B300 with native EAGLE MTP. +# Agentic trace replay for DeepSeek-V4-Pro-0813 FP4 on B300 with DSpark K=6. # Throughput uses the committed golden synthetic AL; eval retains real target # verification. # @@ -122,12 +122,14 @@ if [ "$DP_ATTENTION" = "true" ]; then --enable-prefill-delayer --prefill-decode-interval 20 --enable-dp-attention + --enable-dp-lm-head --enable-dp-attention-local-control-broadcast --incremental-streaming-output --stream-interval 20 --dist-init-addr "127.0.0.1:$((PORT + 2000))" --ep-size "$EP_SIZE" --moe-a2a-backend megamoe + --enable-w4a4-mxfp4-megamoe --enable-deepseek-v4-fp4-indexer --disable-flashinfer-autotune ) @@ -147,8 +149,8 @@ if [ "$DP_ATTENTION" = "true" ]; then MEM_FRACTION_STATIC=0.86 elif [ "$CONC" -ge 384 ]; then MEM_FRACTION_STATIC=0.88 - elif [ "$CONC" -ge 256 ]; then - MEM_FRACTION_STATIC=0.9 + elif [ "$CONC" -ge 32 ]; then + MEM_FRACTION_STATIC=0.90 fi else # DEP4 is squeezed from both sides: weights occupy ~90% of each GPU when @@ -156,8 +158,7 @@ if [ "$DP_ATTENTION" = "true" ]; then # below ~0.902 (no KV left), while megamoe still needs its ~7 GB # workspace above the static budget. 0.95 leaves only ~11 GB free -- the # same margin that OOM'd a rank at DEP8 conc 256 -- so use 0.93, which - # gives the ~16 GB that DEP8 conc 128 runs with at the same per-rank load - # (max-running-requests/dp = 32 in both cases). + # leaves approximately 16 GB for the MegaMoE workspace. MEM_FRACTION_STATIC=0.93 fi # --chunked-prefill-size is a GLOBAL budget: server_args.py divides it by @@ -194,7 +195,7 @@ CUDA_GRAPH_MAX_BS=$((CONC * 4)) CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") SWA_FULL_TOKENS_RATIO=0.1 if [ "$DP_ATTENTION" = "true" ]; then - # Decode graphs must cover the padded MTP batch across all DP ranks, which + # Decode graphs must cover the padded speculative batch across all DP ranks, which # exceeds CONC; capping at 64 would fall back to eager decode. CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) SWA_FULL_TOKENS_RATIO=0.075 @@ -219,18 +220,12 @@ export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 export SGLANG_OPT_USE_TOPK_V2=1 export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 if [ "$DP_ATTENTION" = "true" ]; then - # MegaMoE's FP4/MXF4 activation path is opt-in -- both flags default False, - # so --moe-a2a-backend megamoe alone runs a different kernel than the one - # measured. DG_USE_FP4_ACTS / DG_USE_MXF4_KIND are forwarded to DeepGEMM - # automatically from these two. - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 # Must cover the per-rank prefill budget (8192) or startup raises; the # extra 128 is headroom over the exact-fit boundary. export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 fi if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.49 + export SGLANG_SIMULATE_ACC_LEN=3.77 export SGLANG_SIMULATE_ACC_METHOD=match-expected export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token fi @@ -261,10 +256,11 @@ SGLANG_CMD=( --reasoning-parser deepseek-v4 --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" --watchdog-timeout 1800 - --speculative-algorithm EAGLE - --speculative-num-steps 3 + --speculative-algorithm DSPARK + --speculative-dspark-block-size 6 + --speculative-num-steps 1 --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 7 "${MODEL_ARGS[@]}" "${METRICS_ARGS[@]}" "${CACHE_ARGS[@]}" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ecfd9bffc..a41d1da91 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1155,10 +1155,10 @@ dsv4-fp4-b300-sglang: - { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 4096 } dsv4-fp4-b300-sglang-agentic-hicache-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260827-20621aa1 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-20260901-07c8f729 + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: sglang multinode: false @@ -1166,10 +1166,10 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.95 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [32, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" } } - # DeepSeek-V4-Pro on B300 with EAGLE/MTP speculative decoding. Recipe is + # DeepSeek-V4-Pro-0813 on B300 with DSpark speculative decoding. Recipe is # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by # DP_ATTENTION: # dp-attn: false -> TP-only + flashinfer_mxfp4 + chunked-prefill 8192 @@ -1178,7 +1178,8 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: # + chunked-prefill 65536 + mem-fraction 0.90 # + swa-full-tokens-ratio 0.075 # + prefill-decode-interval 20 - # Both paths share EAGLE (3,1,4) and max-running-requests 2*CONC. + # Both paths share DSpark K=6 (1 step, 7 draft tokens) and + # max-running-requests 2*CONC. dsv4-fp4-b300-sglang-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260610-f332e526 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f69af977a..91d173487 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7286,3 +7286,14 @@ description: - "Use thinking-on golden synthetic AL 3.51 for five-token DSpark throughput; disable adaptive verification and retain real verification for evals" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2974 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Replace the B300 DeepSeek-V4-Pro AgentX native EAGLE/MTP serving path with DSpark block size 6 on the DeepSeek-V4-Pro-0813 checkpoint." + - "Use one speculative step, seven draft tokens, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." + - "Use SGLang's current W4A4 MegaMoE and DP LM-head flags for DP attention, with additional MegaMoE workspace headroom at concurrency 32, 64, and 128." + - "Route the workload through the B300 DSXE launcher, read weights from /data/models, and mount a writable Hugging Face/Xet cache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2918 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index ec9082545..f7747304e 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -19,6 +19,7 @@ SQUASH_DIR="/data/home/sa-gha-runner/squash" # it is read-only from the job's point of view. Anything not in STAGED_MODELS is # downloaded into WRITABLE_MODELS_DIR (shared Lustre) by the single-node scripts. MODEL_ROOT="/scratch/models" +SHARED_MODEL_ROOT="/data/models" WRITABLE_MODELS_DIR="/data/home/sa-gha-runner/models" # Official power (dcgm-power) runs use a separate, pinned producer; CI derives @@ -394,15 +395,23 @@ done find . -name '.nfs*' -delete 2>/dev/null || true else - # HF_HUB_CACHE is set to help with dataset download inside the container - # for eval jobs. - export HF_HUB_CACHE="$HOME/.cache/huggingface" + # AgentX trace datasets need a writable persistent cache. Keep the host and + # container paths separate so the cache remains valid with + # --no-container-mount-home. + HF_CACHE_HOST_DIR="${B300_HF_CACHE_HOST_DIR:-$HOME/.cache/huggingface}" + HF_CACHE_CONTAINER_DIR="${B300_HF_CACHE_CONTAINER_DIR:-/hf_hub_cache}" + mkdir -p "$HF_CACHE_HOST_DIR/hub" "$HF_CACHE_HOST_DIR/xet" + export HF_HOME="$HF_CACHE_CONTAINER_DIR" + export HF_HUB_CACHE="$HF_CACHE_CONTAINER_DIR/hub" + export HF_XET_CACHE="$HF_CACHE_CONTAINER_DIR/xet" # MODEL stays the HF id for the client; MODEL_PATH is where the server reads # weights. Only the root holding MODEL_PATH is mounted -- mounting both roots # makes pyxis fail whenever the unused one is absent on the node. MODEL_BASENAME="${MODEL##*/}" - if [[ " ${STAGED_MODELS[*]} " == *" ${MODEL_BASENAME} "* ]]; then + if [[ "$MODEL_BASENAME" == "DeepSeek-V4-Pro-0813" ]]; then + MODEL_MOUNT_DIR="$SHARED_MODEL_ROOT" + elif [[ " ${STAGED_MODELS[*]} " == *" ${MODEL_BASENAME} "* ]]; then MODEL_MOUNT_DIR="$MODEL_ROOT" else MODEL_MOUNT_DIR="$WRITABLE_MODELS_DIR" @@ -411,7 +420,7 @@ else export MODEL_PATH="${MODEL_MOUNT_DIR}/${MODEL_BASENAME}" SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') + SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" || "$SPEC_DECODING" == "draft_model" ]] && printf '_mtp' || printf '') # Prefer a framework-tagged script (e.g. dsv4_fp4_b300_sglang.sh); fall back to # the untagged historical name for scripts that haven't been retagged yet. BENCH_BASE="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_b300" @@ -468,6 +477,7 @@ else CONTAINER_MOUNTS=( "$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR" "$MODEL_MOUNT_DIR:$MODEL_MOUNT_DIR" + "$HF_CACHE_HOST_DIR:$HF_CACHE_CONTAINER_DIR" ) CONTAINER_MOUNTS_ARG=$(IFS=,; printf '%s' "${CONTAINER_MOUNTS[*]}") diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 778b35732..254c1a41f 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -169,6 +169,26 @@ def test_copy_agentic_results_fails_when_aggregate_is_missing( assert "no run_conc*.json results found" in result.stderr +def test_b300_dsxe_draft_model_uses_public_paths_and_writable_hf_cache() -> None: + launcher = (REPO_ROOT / "runners/launch_b300-dsxe.sh").read_text() + + assert 'SLURM_ACCOUNT="benchmark"' in launcher + assert 'SQUASH_DIR="/data/home/sa-gha-runner/squash"' in launcher + assert 'SHARED_MODEL_ROOT="/data/models"' in launcher + assert 'WRITABLE_MODELS_DIR="/data/home/sa-gha-runner/models"' in launcher + assert '[[ "$MODEL_BASENAME" == "DeepSeek-V4-Pro-0813" ]]' in launcher + assert 'MODEL_MOUNT_DIR="$SHARED_MODEL_ROOT"' in launcher + assert "nv-gha-runner" not in launcher + assert ( + '[[ "$SPEC_DECODING" == "mtp" || "$SPEC_DECODING" == "draft_model" ]]' + in launcher + ) + assert 'export HF_HOME="$HF_CACHE_CONTAINER_DIR"' in launcher + assert 'export HF_HUB_CACHE="$HF_CACHE_CONTAINER_DIR/hub"' in launcher + assert 'export HF_XET_CACHE="$HF_CACHE_CONTAINER_DIR/xet"' in launcher + assert '"$HF_CACHE_HOST_DIR:$HF_CACHE_CONTAINER_DIR"' in launcher + + def test_patch_srt_eval_dispatch_forwards_selection_and_is_idempotent( tmp_path: Path, ) -> None: