[AMD] improve dsr1 fp4 disagg perf on mi355x - rapid follow up PR incoming to quant correction on fp8 combine

.github/configs/amd-master.yaml

@@ -1145,10 +1145,9 @@ dsr1-fp8-mi355x-sglang-disagg-mtp:
           - "DECODE_NODES=1"
           - "DECODE_MTP_SIZE=2"
 
-
 dsr1-fp4-mi355x-sglang-disagg:
-  image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-3
-  model: amd/DeepSeek-R1-0528-MXFP4
+  image: lmsysorg/sglang-rocm:v0.5.10.post1-rocm720-mi35x-20260501
+  model: amd/DeepSeek-R1-0528-MXFP4-v2
   model-prefix: dsr1
   runner: mi355x-disagg
   precision: fp4
@@ -1239,7 +1238,7 @@ dsr1-fp4-mi355x-sglang-disagg:
 
       # 1*DEP4+ 1*DEP8
       - spec-decoding: "none"
-        conc-list: [ 1024, 2048 ]
+        conc-list: [ 1024, 2048, 4096 ]
         prefill:
           num-worker: 1
           tp: 4
@@ -1336,16 +1335,16 @@ dsr1-fp4-mi355x-sglang-disagg:
           - "DECODE_NODES=2"
           - "DECODE_MTP_SIZE=0"
 
-      # 4*DEP4 + 1*DEP8
+      # 2*DEP8 + 1*DEP8
       - spec-decoding: "none"
         conc-list: [ 1024, 2048, 4096 ]
         prefill:
-          num-worker: 4
-          tp: 4
-          ep: 4
+          num-worker: 2
+          tp: 8
+          ep: 8
           dp-attn: true
           additional-settings:
-          - "PREFILL_NODES=4"
+          - "PREFILL_NODES=2"
         decode:
           num-worker: 1
           tp: 8
@@ -1355,9 +1354,10 @@ dsr1-fp4-mi355x-sglang-disagg:
           - "DECODE_NODES=1"
           - "DECODE_MTP_SIZE=0"
 
+
 dsr1-fp4-mi355x-sglang-disagg-mtp:
-  image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-3
-  model: amd/DeepSeek-R1-0528-MXFP4
+  image: lmsysorg/sglang-rocm:v0.5.10.post1-rocm720-mi35x-20260501
+  model: amd/DeepSeek-R1-0528-MXFP4-v2
   model-prefix: dsr1
   runner: mi355x-disagg
   precision: fp4
@@ -1425,7 +1425,7 @@ dsr1-fp4-mi355x-sglang-disagg-mtp:
           dp-attn: false
           additional-settings:
           - "DECODE_NODES=2"
-          - "DECODE_MTP_SIZE=1"
+          - "DECODE_MTP_SIZE=2"
 
       # 1P2D TP4
       - spec-decoding: "mtp" 
@@ -1444,11 +1444,11 @@ dsr1-fp4-mi355x-sglang-disagg-mtp:
           dp-attn: false
           additional-settings:
           - "DECODE_NODES=2"
-          - "DECODE_MTP_SIZE=1"
+          - "DECODE_MTP_SIZE=2"
 
       # 1*DEP4+ 1*DEP8
       - spec-decoding: "mtp"
-        conc-list: [ 1024, 2048 ]
+        conc-list: [ 1024, 2048, 4096 ]
         prefill:
           num-worker: 1
           tp: 4
@@ -1526,7 +1526,7 @@ dsr1-fp4-mi355x-sglang-disagg-mtp:
           dp-attn: false
           additional-settings:
           - "DECODE_NODES=2"
-          - "DECODE_MTP_SIZE=1"
+          - "DECODE_MTP_SIZE=2"
 
       # 1P2D TP4
       - spec-decoding: "mtp"
@@ -1545,18 +1545,18 @@ dsr1-fp4-mi355x-sglang-disagg-mtp:
           dp-attn: false
           additional-settings:
           - "DECODE_NODES=2"
-          - "DECODE_MTP_SIZE=1"
+          - "DECODE_MTP_SIZE=2"
 
-      # 4*DEP4 + 1*DEP8
+      # 2*DEP8 + 1*DEP8
       - spec-decoding: "mtp"
         conc-list: [ 1024, 2048, 4096 ]
         prefill:
-          num-worker: 4
-          tp: 4
-          ep: 4
+          num-worker: 2
+          tp: 8
+          ep: 8
           dp-attn: true
           additional-settings:
-          - "PREFILL_NODES=4"
+          - "PREFILL_NODES=2"
         decode:
           num-worker: 1
           tp: 8
@@ -1565,6 +1565,7 @@ dsr1-fp4-mi355x-sglang-disagg-mtp:
           additional-settings:
           - "DECODE_NODES=1"
           - "DECODE_MTP_SIZE=1"
+
 
 dsv4-fp8-mi355x-sglang:
   image: rocm/sgl-dev:deepseek-v4-mi35x

benchmarks/multi_node/amd_utils/env.sh

@@ -34,39 +34,47 @@ export IBDEVICES
 export GLOO_SOCKET_IFNAME=$(ip route | grep '^default' | awk '{print $5}' | head -n 1)
 export NCCL_SOCKET_IFNAME=$(ip route | grep '^default' | awk '{print $5}' | head -n 1)
 
-set +x
 
 export NCCL_IB_HCA=$IBDEVICES
 
 export SGLANG_USE_AITER=1
-export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=1200
-export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=1200
+
+export SGLANG_MORI_DISPATCH_DTYPE=auto
+export SGLANG_MORI_FP8_COMB=true
+export SGLANG_MORI_QP_PER_TRANSFER=4
+export SGLANG_MORI_NUM_WORKERS=4
+export MORI_IO_SQ_BACKOFF_TIMEOUT_US=50000
+
+export MORI_IO_QP_MAX_SEND_WR=16384
+export MORI_IO_QP_MAX_CQE=32768 
+export MORI_IO_QP_MAX_SGE=4
+
+export MORI_IO_TC_DISABLE=0
+
+export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
+export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
 
 # Disable allocating memory in one pass
 export MORI_SHMEM_MODE=ISOLATION
-export SGLANG_MORI_FP8_DISP=True
 
-if [[ "$MODEL_NAME" == *mxfp4* ]]; then
-export SGLANG_MORI_FP8_DISP=False
-fi
+# Enable spec v2 
+export SGLANG_ENABLE_SPEC_V2=1
+export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
 
-export SGLANG_MORI_FP4_DISP=False
-export SGLANG_MORI_FP8_COMB=False
+export SGLANG_LOG_MS=true
+export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=32
 
-# Per-role dispatch token limits (prefill uses higher throughput, decode uses lower)
-export MORI_MAX_DISPATCH_TOKENS_PREFILL=16384
-if [[ "$MODEL_NAME" == *mxfp4* ]]; then
-    export MORI_MAX_DISPATCH_TOKENS_PREFILL=12288
-fi
-export MORI_MAX_DISPATCH_TOKENS_DECODE=160
+export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192
+export MORI_MAX_DISPATCH_TOKENS_DECODE=512
+
+export MORI_MOE_MAX_INPUT_TOKENS_PREFILL=32768
+export MORI_MOE_MAX_INPUT_TOKENS_DECODE=2703
 
 # set MTP size=1 when EP16
 export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2))
 
 export MORI_EP_LAUNCH_CONFIG_MODE=AUTO
-export MORI_IO_QP_MAX_SEND_WR=16384
-export MORI_IO_QP_MAX_CQE=32768
-export MORI_IO_QP_MAX_SGE=4
+
 
 export MORI_APP_LOG_LEVEL=INFO
 
@@ -89,17 +97,21 @@ $1 == "DSCP" && $2 == ":" && $NF == p {
     if [[ -n "$ND_DSCP" ]] && [[ -n "$ND_PRIO" ]]; then
         TC=$(( 4 * ND_DSCP ))
         export MORI_RDMA_SL=$ND_PRIO
+        export MORI_IO_SL=$ND_PRIO
         export MORI_RDMA_TC=$TC
-        echo "[INFO] Detected QoS config from nicctl: MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL"
+        export MORI_IO_TC=$TC
+        echo "[INFO] Detected QoS config from nicctl: MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL, MORI_IO_TC=$MORI_IO_TC, MORI_IO_SL=$MORI_IO_SL"
     else
         echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection."
         # Fall back to hostname-based detection
         NODENAME=$(hostname -s)
         if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then
             export MORI_RDMA_TC=96
+            export MORI_IO_TC=96
             echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME"
         elif [[ $NODENAME == mia1* ]]; then
             export MORI_RDMA_TC=104
+            export MORI_IO_TC=104
             echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME"
         else
             echo "[INFO] Unable to detect MORI_RDMA_TC from hostname. Skipping RDMA QoS configuration."
@@ -110,9 +122,11 @@ else
     NODENAME=$(hostname -s)
     if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then
         export MORI_RDMA_TC=96
+        export MORI_IO_TC=96
         echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME"
     elif [[ $NODENAME == mia1* ]]; then
         export MORI_RDMA_TC=104
+        export MORI_IO_TC=104
         echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME"
     else
         echo "[INFO] nicctl not found and unable to detect from hostname. Skipping RDMA QoS configuration."
@@ -124,3 +138,4 @@ fi
 export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH}
 
 
+set +x

benchmarks/multi_node/amd_utils/models.yaml

@@ -222,3 +222,37 @@ DeepSeek-R1-0528-MXFP4:
       max_running_requests: 128
       chunked_prefill_size: 262144
       cuda_graph_bs_range: "1-128"
+
+DeepSeek-R1-0528-MXFP4-v2:
+  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
+  mtp_flags: "--speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-eagle-topk 1 --speculative-attention-mode decode "
+  dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --stream-interval 100 --tokenizer-worker-num 32 "
+  prefill:
+    mem_fraction_static: 0.8
+    disable_radix_cache: true
+    dp:
+      max_running_requests: 4096
+      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"
+      cuda_graph_bs: "1 2 3"
+      context_length: 9217
+      max_total_tokens: 131072
+      enable_two_batch_overlap: true
+    no_dp:
+      max_running_requests: 128
+      chunked_prefill_size: 16384
+      cuda_graph_bs_range: "1-128"
+  decode:
+    mem_fraction_static: 0.85
+    prefill_round_robin_balance: true
+    dp:
+      max_running_requests: 4096
+      chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE"
+      cuda_graph_bs_range: "1-512"
+    ep_only:
+      max_running_requests: 256
+      chunked_prefill_size: 262144
+      cuda_graph_bs_range: "1-256"
+    no_dp:
+      max_running_requests: 128
+      chunked_prefill_size: 262144
+      cuda_graph_bs_range: "1-128"