From 1de59121bee3850290903080fac92a8e5f03358a Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 8 Sep 2026 03:00:06 -0500 Subject: [PATCH 1/6] perf(amd): switch DSV4 AgentX to native RCCL DEP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 DeepSeek-V4-Pro MI355X ATOM AgentX 的 c48 及以上测试切换到本地验证过的原生 RCCL DEP 配置，并保持低并发 TP 测试不变。 同步固定的 post-merge ATOM 镜像、EP8 元数据、关闭 TBO/EPLB、真实 MTP 接受率以及本地验证过的路由和 AIPerf 参数。 --- .../agentic/dsv4_fp4_mi355x_atom_mtp.sh | 82 +++++++++++++------ configs/amd-master.yaml | 10 +-- perf-changelog.yaml | 11 +++ 3 files changed, 74 insertions(+), 29 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh index c3d38e1b48..6cda34475f 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -3,8 +3,8 @@ set -euo pipefail set -x # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using -# ATOM MTP. Throughput runs use the committed golden synthetic acceptance; -# eval-only runs use the model's real MTP acceptance. +# ATOM MTP. TP throughput runs use the committed golden synthetic acceptance; +# DEP and eval-only runs use the model's real MTP acceptance. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -43,29 +43,59 @@ export AITER_LOG_LEVEL=WARNING export ATOM_MOE_GU_ITLV=1 export ATOM_DISABLE_MMAP=true export ATOM_DEBUG_PREFIX_HITS=1 +export ATOM_ENABLE_EPLB=0 +export ATOM_ENABLE_TBO=0 export ATOM_PROFILER_MORE=0 export ATOM_PROFILER_TIMEOUT=1200 -# DP-attention runs layer ATOM's DPA routing and two-batch-overlap knobs on top of -# the TP settings above (recipe section "Server - DP attention"); exported only for -# the DP band. ATOM_DP_SESSION_AFFINITY is not optional: without it a session's -# turns scatter across DP ranks, the prefix KV written by one turn is unreachable -# by the next, and the multi-turn agentic workload collapses to cold prefill. -# GPU_MAX_HW_QUEUES and ATOM_NUMA_BIND are prerequisites of --enable-tbo. -DP_ATTN_ARGS=() +# EP is config-driven so the TP band remains TP-only while DEP uses one expert +# shard per GPU. +EP_ARGS=() +if [ "$EP_SIZE" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + +# The high-concurrency band uses ATOM's native RCCL DEP transport. Session +# affinity is required: otherwise consecutive turns can land on another DPA +# rank and lose access to the prefix KV produced by the previous turn. +DEP_ARGS=() +STATE_CHECKPOINT_INTERVAL_TOKENS=8192 if [ "$DP_ATTENTION" = "true" ]; then - export GPU_MAX_HW_QUEUES=5 - export ATOM_NUMA_BIND=1 + if [ "$EP_SIZE" -ne "$TP" ]; then + echo "ERROR: native RCCL DEP requires EP_SIZE=$TP for TP=$TP, got EP_SIZE=$EP_SIZE" >&2 + exit 1 + fi + export ATOM_ALL2ALL_BACKEND=rccl + export ATOM_ENABLE_DP_ATTENTION=1 + export ATOM_ENABLE_EXPERT_PARALLEL=1 export ATOM_DP_SESSION_AFFINITY=1 - export ATOM_DP_LB_REQ_EQUIV=512 - export ATOM_ENABLE_PREFILL_DELAYER=1 - export ATOM_PREFILL_DECODE_INTERVAL=10 + export ATOM_DP_LOAD_BALANCE=least_tokens + export ATOM_DP_LB_REQ_EQUIV=0 + export ATOM_DP_AFFINITY_MAX_TOKEN_SKEW=32768 + export ATOM_MAX_CONSECUTIVE_PREFILL_BATCHES=0 + export ATOM_DECODE_ITL_CREDIT_CAP_PERIODS=1 + export ATOM_DECODE_ITL_ENTER_MARGIN_MS=0 + export ATOM_DECODE_ITL_SLA_MS=0 + export ATOM_CANCEL_TERMINAL_MTP_PROPOSAL=1 + export ATOM_DEFER_MTP_PROPOSAL=1 + export ATOM_TERMINAL_MTP_FAST_PATH=1 # Client-side counterpart to session affinity: make AIPerf emit a stable - # session id (x-dynamo-session-id, falling back to the always-sent - # x-correlation-id) so the DPA router pins each conversation to one rank. - export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=true - DP_ATTN_ARGS=(--enable-dp-attention --enable-tbo) + # session id from its correlation id so the DPA router pins each + # conversation to one rank. + export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=1 + export AIPERF_HTTP_KEEPALIVE_TIMEOUT=4 + export AIPERF_FAILED_REQUEST_THRESHOLD=0.10 + export AIPERF_LIVE_FAILED_REQUEST_THRESHOLD=0.10 + export AIPERF_TRACE_IDLE_GAP_CAP_SECONDS=300 + export AIPERF_WARMUP_REQUESTS_PER_LANE=10 + export AIPERF_BENCHMARK_GRACE_PERIOD=30 + STATE_CHECKPOINT_INTERVAL_TOKENS=32768 + DEP_ARGS=( + --enable-dp-attention + --all2all-backend rccl + --dp-load-balance least_tokens + --moe-backend standard + ) fi # Raise the AIPerf HTTP TCP user timeout to 900000 ms (15 min), well above the @@ -103,19 +133,21 @@ trap 'exit 143' TERM # request bursts produced by subagent fan-out. MAX_NUM_SEQS=$((2 * CONC)) -# golden_al_distribution/dsv4_mtp.yaml: thinking_on, 3 draft tokens -> AL 2.49 +# golden_al_distribution/dsv4_mtp.yaml: thinking_on, 3 draft tokens -> AL 2.49. # https://github.com/SemiAnalysisAI/InferenceX/blob/main/golden_al_distribution/dsv4_mtp.yaml +# Native RCCL DEP was validated with the model's real acceptance, so only the +# TP throughput band applies the synthetic golden value. NUM_SPEC_TOKENS=3 SPEC_DECODE_AL=2.49 SPEC_ARGS=( --method mtp --num-speculative-tokens "$NUM_SPEC_TOKENS" ) -if [ "${EVAL_ONLY:-false}" != "true" ]; then +if [ "${EVAL_ONLY:-false}" != "true" ] && [ "$DP_ATTENTION" != "true" ]; then SPEC_ARGS+=(--spec-decode-acceptance-length "$SPEC_DECODE_AL") fi -echo "Starting ATOM server with MAX_NUM_SEQS=$MAX_NUM_SEQS NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS SPEC_DECODE_AL=$SPEC_DECODE_AL EVAL_ONLY=${EVAL_ONLY:-false}" +echo "Starting ATOM server with MAX_NUM_SEQS=$MAX_NUM_SEQS NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS STATE_CHECKPOINT_INTERVAL_TOKENS=$STATE_CHECKPOINT_INTERVAL_TOKENS DP_ATTENTION=$DP_ATTENTION EP_SIZE=$EP_SIZE EVAL_ONLY=${EVAL_ONLY:-false}" ATOM_CMD=( python3 -u -m atom.entrypoints.openai_server --model "$MODEL_PATH" @@ -129,17 +161,19 @@ ATOM_CMD=( # warmup request aborts the whole run. Outlast the client idle window. --timeout-keep-alive 900 --tensor-parallel-size "$TP" + --data-parallel-size 1 --kv-cache-dtype fp8 --index-cache-dtype fp4 --enable-prefix-caching --gpu-memory-utilization 0.9 --max-num-batched-tokens 16384 --attn-prefill-chunk-size 16384 - --state-checkpoint-interval-tokens 8192 + --state-checkpoint-interval-tokens "$STATE_CHECKPOINT_INTERVAL_TOKENS" --level 3 --cudagraph-mode FULL "${SPEC_ARGS[@]}" - "${DP_ATTN_ARGS[@]}" + "${EP_ARGS[@]}" + "${DEP_ARGS[@]}" --max-num-seqs "$MAX_NUM_SEQS" ) write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 2510dc3b8d..6a48067c7e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1332,11 +1332,11 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: # while MTP creates two. Restore these points after the upstream hybrid # KV recovery fix lands: https://github.com/vllm-project/vllm/pull/45497 -# DeepSeek-V4-Pro FP4 AgentX on one MI355X node using ATOM MTP. Throughput -# uses the thinking_on golden AL 2.49 for three draft tokens; eval uses real -# MTP acceptance. max-num-seqs is set to 2x concurrency by the recipe. +# DeepSeek-V4-Pro FP4 AgentX on one MI355X node using ATOM MTP. The TP band +# uses the thinking_on golden AL 2.49 for throughput; the DEP band and eval +# use real MTP acceptance. max-num-seqs is set to 2x concurrency by the recipe. dsv4-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202608280858 + image: rocm/atom-dev:nightly_202609071454 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1347,7 +1347,7 @@ dsv4-fp4-mi355x-atom-agentic-mtp: agentic-coding: - search-space: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48, 64, 96, 128, 256] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48, 64, 96, 128, 256] } dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e0ef448195..f8fe7ec8fa 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6947,3 +6947,14 @@ - "Update vLLM ROCm image from vllm/vllm-openai-rocm:v0.27.1 (v0.27.1 release) to vllm/vllm-openai-rocm:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 (2026-09-07 upstream ROCm nightly, digest sha256:74d4a95f3ae672ecddf9acb7296917def82d9eca51687fa2862ae72b03ff1907, tag commit vllm-project/vllm@d9105ea8; Docker Hub last pushed 2026-09-07T05:26:48Z). benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh is unchanged: TRITON_ATTN attention, fp8 KV, block-size 128, EAGLE3 speculative decoding with the Inferact MiniMax-M3 EAGLE3-GQA draft and the committed golden synthetic acceptance length, minimax_m3 tool-call and reasoning parsers; the search space is unchanged. The upstream commit-pinned ROCm nightly is the same vllm commit the B200 MiniMax-M3 AgentX recipe moved to in #2860. Note that vllm-openai-rocm commit-nightly tags have expired from Docker Hub within days in the past; node squash caches keep merged configs running, but a re-pin to a durable tag may be needed later." - "Add --compilation-config cudagraph_mode=FULL_DECODE_ONLY to the serve command. The upstream nightly does not torch-compile MiniMaxM3SparseForConditionalGeneration, so with VLLM_USE_BREAKABLE_CUDAGRAPH=0 the default FULL_AND_PIECEWISE mode aborts at engine init with piecewise CUDA graphs unavailable (first sweep, run 34174124043, eval cell); full decode-only graphs are what the MI355X MiniMax-M3 sibling runs on its nightly (#2825)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2873 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the ATOM image from rocm/atom-dev:nightly_202608280858 to rocm/atom-dev:nightly_202609071454 (digest sha256:f252ec9a30e3d17b91f15887ea32baa1a0c5503b775f5adf42ff7cee8cfff769), built from ROCm/ATOM merge commit 5a9c2068 containing native RCCL DEP." + - "Move the concurrency 48/64/96/128/256 ATOM AgentX band from DPA with TBO and the default MoE transport to native RCCL DEP (TP8/DPA8/EP8), while preserving the TP-only concurrency 1/2/4/8/16 band." + - "Match the validated DEP run: disable EPLB and TBO, use least-tokens routing with session affinity and zero request equivalence, set the state checkpoint interval to 32768 tokens, and use real MTP acceptance on DEP." + - "Carry over the validated AgentX transport, terminal-MTP, and AIPerf timeout/failure settings for stable multi-turn replay." + pr-link: TBD From ee11ce503650c4802dc8301f4fec6233a4992fcb Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 8 Sep 2026 03:01:22 -0500 Subject: [PATCH 2/6] docs(perf): link DSV4 RCCL DEP pull request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 DeepSeek-V4-Pro RCCL DEP 性能变更记录中的占位链接替换为实际的 InferenceX PR 链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f8fe7ec8fa..a7c3bcfd8b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6957,4 +6957,4 @@ - "Move the concurrency 48/64/96/128/256 ATOM AgentX band from DPA with TBO and the default MoE transport to native RCCL DEP (TP8/DPA8/EP8), while preserving the TP-only concurrency 1/2/4/8/16 band." - "Match the validated DEP run: disable EPLB and TBO, use least-tokens routing with session affinity and zero request equivalence, set the state checkpoint interval to 32768 tokens, and use real MTP acceptance on DEP." - "Carry over the validated AgentX transport, terminal-MTP, and AIPerf timeout/failure settings for stable multi-turn replay." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2886 From 07d6235ae43b0c0673bbfeb5335f5bd5f4b7a18d Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 8 Sep 2026 05:23:17 -0500 Subject: [PATCH 3/6] fix(amd): trim DSV4 AgentX DEP overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 精简 DeepSeek-V4-Pro AgentX RCCL DEP 配置，移除与 CLI 或公共默认值重复的环境变量，并补齐本地验证使用的 3600 秒 warmup grace。 --- .../agentic/dsv4_fp4_mi355x_atom_mtp.sh | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh index 6cda34475f..a513b90a74 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -43,8 +43,6 @@ export AITER_LOG_LEVEL=WARNING export ATOM_MOE_GU_ITLV=1 export ATOM_DISABLE_MMAP=true export ATOM_DEBUG_PREFIX_HITS=1 -export ATOM_ENABLE_EPLB=0 -export ATOM_ENABLE_TBO=0 export ATOM_PROFILER_MORE=0 export ATOM_PROFILER_TIMEOUT=1200 @@ -65,17 +63,9 @@ if [ "$DP_ATTENTION" = "true" ]; then echo "ERROR: native RCCL DEP requires EP_SIZE=$TP for TP=$TP, got EP_SIZE=$EP_SIZE" >&2 exit 1 fi - export ATOM_ALL2ALL_BACKEND=rccl - export ATOM_ENABLE_DP_ATTENTION=1 - export ATOM_ENABLE_EXPERT_PARALLEL=1 + # Keep only runtime controls that are not already expressed by DEP_ARGS. export ATOM_DP_SESSION_AFFINITY=1 - export ATOM_DP_LOAD_BALANCE=least_tokens export ATOM_DP_LB_REQ_EQUIV=0 - export ATOM_DP_AFFINITY_MAX_TOKEN_SKEW=32768 - export ATOM_MAX_CONSECUTIVE_PREFILL_BATCHES=0 - export ATOM_DECODE_ITL_CREDIT_CAP_PERIODS=1 - export ATOM_DECODE_ITL_ENTER_MARGIN_MS=0 - export ATOM_DECODE_ITL_SLA_MS=0 export ATOM_CANCEL_TERMINAL_MTP_PROPOSAL=1 export ATOM_DEFER_MTP_PROPOSAL=1 export ATOM_TERMINAL_MTP_FAST_PATH=1 @@ -84,10 +74,7 @@ if [ "$DP_ATTENTION" = "true" ]; then # conversation to one rank. export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=1 export AIPERF_HTTP_KEEPALIVE_TIMEOUT=4 - export AIPERF_FAILED_REQUEST_THRESHOLD=0.10 - export AIPERF_LIVE_FAILED_REQUEST_THRESHOLD=0.10 - export AIPERF_TRACE_IDLE_GAP_CAP_SECONDS=300 - export AIPERF_WARMUP_REQUESTS_PER_LANE=10 + export AGENTIC_WARMUP_GRACE_PERIOD=3600 export AIPERF_BENCHMARK_GRACE_PERIOD=30 STATE_CHECKPOINT_INTERVAL_TOKENS=32768 DEP_ARGS=( From 252e081de11314bebd7e85fb76309966bf2c29d8 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 8 Sep 2026 07:08:10 -0500 Subject: [PATCH 4/6] fix(amd): restore DSV4 AgentX scheduling controls Restore the request-equivalent weight, prefill delayer, and decode interval requested for the AgentX run. Remove the newly introduced terminal MTP overrides while keeping the rest of the cleanup unchanged. --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh index a513b90a74..c3a3d231b1 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -65,10 +65,9 @@ if [ "$DP_ATTENTION" = "true" ]; then fi # Keep only runtime controls that are not already expressed by DEP_ARGS. export ATOM_DP_SESSION_AFFINITY=1 - export ATOM_DP_LB_REQ_EQUIV=0 - export ATOM_CANCEL_TERMINAL_MTP_PROPOSAL=1 - export ATOM_DEFER_MTP_PROPOSAL=1 - export ATOM_TERMINAL_MTP_FAST_PATH=1 + export ATOM_DP_LB_REQ_EQUIV=512 + export ATOM_ENABLE_PREFILL_DELAYER=1 + export ATOM_PREFILL_DECODE_INTERVAL=10 # Client-side counterpart to session affinity: make AIPerf emit a stable # session id from its correlation id so the DPA router pins each # conversation to one rank. From 292074558de07ead3c1b48a53a7584bb3d6023e3 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 8 Sep 2026 07:41:31 -0500 Subject: [PATCH 5/6] fix(amd): remove redundant AgentX timeout overrides Keep the 3600-second agentic warmup allowance, but rely on the server keep-alive setting and AIPerf default benchmark grace period. --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh index c3a3d231b1..62e0c46415 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -72,9 +72,7 @@ if [ "$DP_ATTENTION" = "true" ]; then # session id from its correlation id so the DPA router pins each # conversation to one rank. export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=1 - export AIPERF_HTTP_KEEPALIVE_TIMEOUT=4 export AGENTIC_WARMUP_GRACE_PERIOD=3600 - export AIPERF_BENCHMARK_GRACE_PERIOD=30 STATE_CHECKPOINT_INTERVAL_TOKENS=32768 DEP_ARGS=( --enable-dp-attention From 5c91ae849d4623c9726d88a5634fad7c16c05c67 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 8 Sep 2026 10:39:37 -0500 Subject: [PATCH 6/6] fix(amd): keep DSV4 state checkpoints at 8K Remove the DEP-only 32K override so both TP and DEP retain the original 8192-token state checkpoint interval. --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh index 62e0c46415..5c4b559108 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -73,7 +73,6 @@ if [ "$DP_ATTENTION" = "true" ]; then # conversation to one rank. export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=1 export AGENTIC_WARMUP_GRACE_PERIOD=3600 - STATE_CHECKPOINT_INTERVAL_TOKENS=32768 DEP_ARGS=( --enable-dp-attention --all2all-backend rccl