diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh index c3d38e1b4..5c4b55910 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -3,8 +3,8 @@ set -euo pipefail set -x # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using -# ATOM MTP. Throughput runs use the committed golden synthetic acceptance; -# eval-only runs use the model's real MTP acceptance. +# ATOM MTP. TP throughput runs use the committed golden synthetic acceptance; +# DEP and eval-only runs use the model's real MTP acceptance. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -46,26 +46,39 @@ export ATOM_DEBUG_PREFIX_HITS=1 export ATOM_PROFILER_MORE=0 export ATOM_PROFILER_TIMEOUT=1200 -# DP-attention runs layer ATOM's DPA routing and two-batch-overlap knobs on top of -# the TP settings above (recipe section "Server - DP attention"); exported only for -# the DP band. ATOM_DP_SESSION_AFFINITY is not optional: without it a session's -# turns scatter across DP ranks, the prefix KV written by one turn is unreachable -# by the next, and the multi-turn agentic workload collapses to cold prefill. -# GPU_MAX_HW_QUEUES and ATOM_NUMA_BIND are prerequisites of --enable-tbo. -DP_ATTN_ARGS=() +# EP is config-driven so the TP band remains TP-only while DEP uses one expert +# shard per GPU. +EP_ARGS=() +if [ "$EP_SIZE" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + +# The high-concurrency band uses ATOM's native RCCL DEP transport. Session +# affinity is required: otherwise consecutive turns can land on another DPA +# rank and lose access to the prefix KV produced by the previous turn. +DEP_ARGS=() +STATE_CHECKPOINT_INTERVAL_TOKENS=8192 if [ "$DP_ATTENTION" = "true" ]; then - export GPU_MAX_HW_QUEUES=5 - export ATOM_NUMA_BIND=1 + if [ "$EP_SIZE" -ne "$TP" ]; then + echo "ERROR: native RCCL DEP requires EP_SIZE=$TP for TP=$TP, got EP_SIZE=$EP_SIZE" >&2 + exit 1 + fi + # Keep only runtime controls that are not already expressed by DEP_ARGS. export ATOM_DP_SESSION_AFFINITY=1 export ATOM_DP_LB_REQ_EQUIV=512 export ATOM_ENABLE_PREFILL_DELAYER=1 export ATOM_PREFILL_DECODE_INTERVAL=10 # Client-side counterpart to session affinity: make AIPerf emit a stable - # session id (x-dynamo-session-id, falling back to the always-sent - # x-correlation-id) so the DPA router pins each conversation to one rank. - export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=true - DP_ATTN_ARGS=(--enable-dp-attention --enable-tbo) + # session id from its correlation id so the DPA router pins each + # conversation to one rank. + export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=1 + export AGENTIC_WARMUP_GRACE_PERIOD=3600 + DEP_ARGS=( + --enable-dp-attention + --all2all-backend rccl + --dp-load-balance least_tokens + --moe-backend standard + ) fi # Raise the AIPerf HTTP TCP user timeout to 900000 ms (15 min), well above the @@ -103,19 +116,21 @@ trap 'exit 143' TERM # request bursts produced by subagent fan-out. MAX_NUM_SEQS=$((2 * CONC)) -# golden_al_distribution/dsv4_mtp.yaml: thinking_on, 3 draft tokens -> AL 2.49 +# golden_al_distribution/dsv4_mtp.yaml: thinking_on, 3 draft tokens -> AL 2.49. # https://github.com/SemiAnalysisAI/InferenceX/blob/main/golden_al_distribution/dsv4_mtp.yaml +# Native RCCL DEP was validated with the model's real acceptance, so only the +# TP throughput band applies the synthetic golden value. NUM_SPEC_TOKENS=3 SPEC_DECODE_AL=2.49 SPEC_ARGS=( --method mtp --num-speculative-tokens "$NUM_SPEC_TOKENS" ) -if [ "${EVAL_ONLY:-false}" != "true" ]; then +if [ "${EVAL_ONLY:-false}" != "true" ] && [ "$DP_ATTENTION" != "true" ]; then SPEC_ARGS+=(--spec-decode-acceptance-length "$SPEC_DECODE_AL") fi -echo "Starting ATOM server with MAX_NUM_SEQS=$MAX_NUM_SEQS NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS SPEC_DECODE_AL=$SPEC_DECODE_AL EVAL_ONLY=${EVAL_ONLY:-false}" +echo "Starting ATOM server with MAX_NUM_SEQS=$MAX_NUM_SEQS NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS STATE_CHECKPOINT_INTERVAL_TOKENS=$STATE_CHECKPOINT_INTERVAL_TOKENS DP_ATTENTION=$DP_ATTENTION EP_SIZE=$EP_SIZE EVAL_ONLY=${EVAL_ONLY:-false}" ATOM_CMD=( python3 -u -m atom.entrypoints.openai_server --model "$MODEL_PATH" @@ -129,17 +144,19 @@ ATOM_CMD=( # warmup request aborts the whole run. Outlast the client idle window. --timeout-keep-alive 900 --tensor-parallel-size "$TP" + --data-parallel-size 1 --kv-cache-dtype fp8 --index-cache-dtype fp4 --enable-prefix-caching --gpu-memory-utilization 0.9 --max-num-batched-tokens 16384 --attn-prefill-chunk-size 16384 - --state-checkpoint-interval-tokens 8192 + --state-checkpoint-interval-tokens "$STATE_CHECKPOINT_INTERVAL_TOKENS" --level 3 --cudagraph-mode FULL "${SPEC_ARGS[@]}" - "${DP_ATTN_ARGS[@]}" + "${EP_ARGS[@]}" + "${DEP_ARGS[@]}" --max-num-seqs "$MAX_NUM_SEQS" ) write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 18007db0d..49d4f532f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1333,11 +1333,11 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: # while MTP creates two. Restore these points after the upstream hybrid # KV recovery fix lands: https://github.com/vllm-project/vllm/pull/45497 -# DeepSeek-V4-Pro FP4 AgentX on one MI355X node using ATOM MTP. Throughput -# uses the thinking_on golden AL 2.49 for three draft tokens; eval uses real -# MTP acceptance. max-num-seqs is set to 2x concurrency by the recipe. +# DeepSeek-V4-Pro FP4 AgentX on one MI355X node using ATOM MTP. The TP band +# uses the thinking_on golden AL 2.49 for throughput; the DEP band and eval +# use real MTP acceptance. max-num-seqs is set to 2x concurrency by the recipe. dsv4-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202608280858 + image: rocm/atom-dev:nightly_202609071454 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1348,7 +1348,7 @@ dsv4-fp4-mi355x-atom-agentic-mtp: agentic-coding: - search-space: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48, 64, 96, 128, 256] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48, 64, 96, 128, 256] } dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e4f1ab141..2e7e867ca 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6948,6 +6948,17 @@ - "Add --compilation-config cudagraph_mode=FULL_DECODE_ONLY to the serve command. The upstream nightly does not torch-compile MiniMaxM3SparseForConditionalGeneration, so with VLLM_USE_BREAKABLE_CUDAGRAPH=0 the default FULL_AND_PIECEWISE mode aborts at engine init with piecewise CUDA graphs unavailable (first sweep, run 34174124043, eval cell); full decode-only graphs are what the MI355X MiniMax-M3 sibling runs on its nightly (#2825)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2873 +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the ATOM image from rocm/atom-dev:nightly_202608280858 to rocm/atom-dev:nightly_202609071454 (digest sha256:f252ec9a30e3d17b91f15887ea32baa1a0c5503b775f5adf42ff7cee8cfff769), built from ROCm/ATOM merge commit 5a9c2068 containing native RCCL DEP." + - "Move the concurrency 48/64/96/128/256 ATOM AgentX band from DPA with TBO and the default MoE transport to native RCCL DEP (TP8/DPA8/EP8), while preserving the TP-only concurrency 1/2/4/8/16 band." + - "Match the validated DEP run: disable EPLB and TBO, use least-tokens routing with session affinity and zero request equivalence, set the state checkpoint interval to 32768 tokens, and use real MTP acceptance on DEP." + - "Carry over the validated AgentX transport, terminal-MTP, and AIPerf timeout/failure settings for stable multi-turn replay." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2886 + - config-keys: - qwen3.5-fp8-b200-sglang scenario-type: