diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index ccceb2b1d1..b701fd0819 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -121,25 +121,24 @@ if agentic_kv_offload_enabled; then fi # ---- Parallelism ------------------------------------------------------------ -# NOTE: the DP-attention path below is currently DORMANT (no dp-attn arms in -# amd-master.yaml for this key). It is kept so a future arm can enable it -# without rebuilding the router plumbing: sglang-router fronts the DP ranks -# with consistent hashing on the AIPerf correlation id, keeping multi-turn -# sessions on the DP rank that holds their radix/hicache prefix. +# The DP-attention path below is live: sglang-router fronts the DP ranks with +# consistent hashing on the AIPerf correlation id, keeping multi-turn sessions +# on the DP rank that holds their radix/hicache prefix. USE_SGLANG_ROUTER=false SGLANG_BACKEND_PORT="$PORT" -# Small prefill chunks interleave long-context agentic prefills across -# requests instead of letting one ~100K-token prefill monopolize the engine -# (the conc>=16 queue-saturation / decode-stall failure mode). 8192 = 32*256, -# a page-size multiple well under the dsv4 compressor kernel's uint16 token -# cap; same value the multi-node DeepSeek-V4-Pro-AgentX no_dp profile uses. -if [ "$TP" -eq 8 ]; then +# Small prefill chunks interleave long-context agentic prefills. The flag is +# engine-wide and DP divides it by dp_size (=TP), so DP uses 8192*TP to keep +# 8192 per rank. TP-only: 8192 at TP4, 16384 at TP8. +case "$TP" in + 4|8) ;; + *) echo "Error: unsupported TP '$TP' (expected: 4 or 8)" >&2; exit 1 ;; +esac +if [ "$DP_ATTENTION" = "true" ]; then + CHUNKED_PREFILL_SIZE=$((8192 * TP)) +elif [ "$TP" -eq 8 ]; then CHUNKED_PREFILL_SIZE=16384 -elif [ "$TP" -eq 4 ]; then - CHUNKED_PREFILL_SIZE=8192 else - echo "Error: unsupported TP '$TP' (expected: 4 or 8)" >&2 - exit 1 + CHUNKED_PREFILL_SIZE=8192 fi MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.86}" PARALLEL_ARGS=(--tensor-parallel-size "$TP") @@ -158,32 +157,29 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES_DP:-5}" - SHARED_EXPERTS_ARGS=(--disable-shared-experts-fusion) - SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO_DP:-0.15}" + MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC_DP:-0.92}" - # Chunked prefill is a whole-engine budget, so widen it by the DP degree. - CHUNKED_PREFILL_SIZE=$((CHUNKED_PREFILL_SIZE * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention --enable-prefill-delayer - --enable-two-batch-overlap --enable-dp-attention-local-control-broadcast --tokenizer-worker-num "$TP" --stream-interval 20 --prefill-decode-interval 10 + --prefill-delayer-token-usage-low-watermark "${DP_PREFILL_DELAYER_LOW_WATERMARK:-0.7}" ) fi if [ "$EP_SIZE" -gt 1 ]; then PARALLEL_ARGS+=(--ep-size "$EP_SIZE") + SHARED_EXPERTS_ARGS=(--disable-shared-experts-fusion) fi # AgentX concurrency counts live session trees, not individual requests. # Subagent fan-out can push instantaneous request concurrency above CONC, so # leave 2x headroom rather than clipping those bursts at the scheduler. MAX_RUNNING_REQUESTS=$((2 * CONC)) -[ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 @@ -196,24 +192,32 @@ fi # ---- Speculative decoding --------------------------------------------------- # DeepSeek-V4 ships a built-in MTP head, loaded through the EAGLE spec path # with eagle-topk 1 (a single MTP chain); NOT NEXTN, whose V3/R1 loader -# crashes on the V4 architecture. Depth 3 matches the vLLM agentic sibling -# (dsv4-fp4-mi355x-vllm-agentic-mtp) and the fixed-seq-len SGLang MTP recipe. +# crashes on the V4 architecture. +if [ "$CONC" -ge 256 ]; then + DSV4_SPEC_NUM_STEPS=1 + DSV4_GOLDEN_AL=1.79 +else + DSV4_SPEC_NUM_STEPS=3 + DSV4_GOLDEN_AL=2.49 +fi + SPEC_ARGS=( --speculative-algorithm EAGLE - --speculative-num-steps 3 + --speculative-num-steps "$DSV4_SPEC_NUM_STEPS" --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens $((DSV4_SPEC_NUM_STEPS + 1)) ) # Throughput runs pin acceptance to the committed golden AL for this model, -# thinking mode, and draft length (golden_al_distribution/dsv4_mtp.yaml: -# thinking_on, 3 -> 2.49). Eval-only runs keep real target verification so -# accuracy stays meaningful. +# thinking mode, and draft length (golden_al_distribution/dsv4_mtp.yaml, +# thinking_on column: 3 -> 2.49, 1 -> 1.79). Eval-only runs keep real target +# verification so accuracy stays meaningful. if [ "${EVAL_ONLY:-false}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.49 + export SGLANG_SIMULATE_ACC_LEN="$DSV4_GOLDEN_AL" export SGLANG_SIMULATE_ACC_METHOD=match-expected export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token fi +echo "MTP draft length: num_steps=$DSV4_SPEC_NUM_STEPS (conc $CONC), golden AL=$DSV4_GOLDEN_AL" # ---- Launch ----------------------------------------------------------------- # No --chat-template: the AgentX traces are tool-heavy, and diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 8f6d246d2e..9a9fe7eb8b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1743,7 +1743,7 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260902 + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1754,7 +1754,6 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 8], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: mtp } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 96, 128, 160], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e9fab1e0d8..6d695e5b01 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6957,7 +6957,6 @@ - "Tune the 8k/1k serving recipe with --mamba-full-memory-ratio 0.37 and --linear-attn-prefill-backend flashinfer; raise max-prefill-tokens and chunked-prefill-size from 16384 to 32768 and mem-fraction-static from 0.8 to 0.86." - "Expand the TP8 and TP4/EP1 sweep coverage through concurrency 640." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2866 - - config-keys: - dsv4-fp4-b200-sglang-agentic-hicache-mtp @@ -7058,3 +7057,16 @@ description: - "Remove the unstable concurrency-1152 2P DEP8 prefill plus DEP12 decode point after repeated benchmark failures." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." + - "Keep TP-only chunked-prefill at 8192 for TP4 and 16384 for TP8. The DP-attention path uses an 8192 per-rank budget (engine-wide 8192*TP, 65536 at TP8) instead of scaling the TP8 16384 base." + - "DP-attention serving: drop --enable-two-batch-overlap; set mem-fraction-static to 0.92 (pure TP stays 0.86); keep --enforce-shared-experts-fusion and move --disable-shared-experts-fusion to the EP branch only; keep swa-full-tokens-ratio at 0.10; add --prefill-delayer-token-usage-low-watermark 0.7." + - "Remove the 256 cap on max-running-requests." + - "MTP draft length by concurrency: speculative-num-steps 3 (golden AL 2.49) below conc 256, and 1 (golden AL 1.79) at and above it." + - "Trim the search space to TP8 no-offload conc [1, 4, 16], TP8 hicache conc [32, 48], and TP8 DP-attention hicache conc [128, 256]." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2885