perf(amd): port MI355X Qwen3.5 FP8 disagg sweep to srt-slurm

benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml

@@ -0,0 +1,32 @@
+# srt-slurm cluster profile for the MI355X AMD Slurm cluster. The login and
+# compute nodes share /it-share, so source, output, image, and result paths do
+# not require node-local transport.
+
+cluster: mi355x-amds
+default_partition: compute
+default_time_limit: "01:00:00"
+output_dir: /it-share/gharunners2/srt-slurm/outputs
+
+gpus_per_node: 8
+accelerator_vendor: amd
+network_interface: eno0
+
+gpu_sbatch_directive: gres
+use_segment_sbatch_directive: false
+use_exclusive_sbatch_directive: false
+runtime_config_transport: shared-filesystem
+
+default_sbatch_directives:
+  cpus-per-task: "32"
+  mem: "128G"
+
+default_mounts:
+  /dev/kfd: /dev/kfd
+  /dev/dri: /dev/dri
+  /it-share/hf-hub-cache: /hf_hub_cache
+  # The launcher appends the exact shared workspace and result roots.
+
+containers:
+  sglang-v0.5.16-mi35x: /it-share/gharunners2/srt-slurm/containers/sglang-rocm-v0.5.16-mi35x-20260728.sqsh
+
+nginx_raise_ulimit: false

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml

@@ -0,0 +1,149 @@
+# Production-scale MI355X port of the existing Qwen3.5 FP8 1P1D recipe.
+# Each role owns one full 8-GPU node; the native SGLang Router provides the
+# request plane and AMD MoRI moves KV directly between the P/D workers.
+
+name: "mi355x-sglang-qwen3.5-fp8-disagg-1p1d-tp4p-tp8d-fixed-seq"
+
+model:
+  path: "hf:Qwen/Qwen3.5-397B-A17B-FP8"
+  container: "sglang-v0.5.16-mi35x"
+  precision: "fp8"
+
+identity:
+  model:
+    repo: "Qwen/Qwen3.5-397B-A17B-FP8"
+  container:
+    image: "lmsysorg/sglang:v0.5.16-rocm720-mi35x"
+  frameworks:
+    sglang: "0.5.16"
+    sglang-router: "0.3.2"
+
+slurm:
+  time_limit: "08:00:00"
+
+# This topology owns every GPU on each MI355X node. Give SGLang the complete
+# 128-core cpuset as well: its ROCm image enables per-GPU CPU affinity, which
+# maps the eight TP ranks across the full dual-socket CPU topology.
+sbatch_directives:
+  cpus-per-task: "128"
+  mem: "0"
+
+resources:
+  gpu_type: "mi355x"
+  gpus_per_node: 8
+  prefill_nodes: 1
+  decode_nodes: 1
+  prefill_workers: 1
+  decode_workers: 1
+  gpus_per_prefill: 4
+  gpus_per_decode: 8
+
+frontend:
+  type: sglang
+  enable_multiple_frontends: false
+  args:
+    policy: round_robin
+    prefill-policy: round_robin
+    decode-policy: round_robin
+
+backend:
+  type: sglang
+  prefill_environment: &common_environment
+    HF_HOME: "/hf_hub_cache"
+    # Hugging Face stores hub snapshots under $HF_HOME/hub. Keep the explicit
+    # cache variables on that same path so srt-slurm's prefetch and every
+    # backend process resolve the identical, current snapshot.
+    HF_HUB_CACHE: "/hf_hub_cache/hub"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub"
+    PYTHONUNBUFFERED: "1"
+    IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
+    GLOO_SOCKET_IFNAME: "eno0"
+    NCCL_SOCKET_IFNAME: "eno0"
+    MORI_RDMA_TC: "104"
+    MORI_IO_TC: "104"
+    MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000"
+    MORI_IO_QP_MAX_SEND_WR: "16384"
+    MORI_IO_QP_MAX_CQE: "32768"
+    MORI_IO_QP_MAX_SGE: "2"
+    MORI_SHMEM_MODE: "ISOLATION"
+    SGLANG_USE_AITER: "1"
+    AITER_LOG_LEVEL: "ERROR"
+    SGLANG_MORI_DISPATCH_DTYPE: "auto"
+    SGLANG_MORI_QP_PER_TRANSFER: "4"
+    SGLANG_MORI_NUM_WORKERS: "4"
+    SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600"
+    SGLANG_HEALTH_CHECK_TIMEOUT: "600"
+  decode_environment: *common_environment
+  sglang_config:
+    prefill: &common_config
+      served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8"
+      tensor-parallel-size: 4
+      disaggregation-transfer-backend: mori
+      disaggregation-ib-device: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
+      kv-cache-dtype: "fp8_e4m3"
+      attention-backend: aiter
+      moe-dense-tp-size: 1
+      load-balance-method: round_robin
+      watchdog-timeout: 3600
+      decode-log-interval: 1000
+      log-level: warning
+      mem-fraction-static: 0.80
+      max-running-requests: 128
+      chunked-prefill-size: 262144
+      context-length: 16384
+      disable-radix-cache: true
+      cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8]
+      enable-metrics: true
+    decode:
+      <<: *common_config
+      mem-fraction-static: 0.85
+      prefill-round-robin-balance: true
+
+srun_options:
+  container-writable: ""
+  container-remap-root: ""
+  mem: "0"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 5
+
+benchmark:
+  type: custom
+  command: |
+    set -euo pipefail
+    result_root="/results/${SLURM_JOB_ID}"
+    mkdir -p "${result_root}/fixed-seq"
+    trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT
+    for concurrency in 8 16 32 64 128; do
+      num_prompts=$((concurrency * 10))
+      if ((num_prompts < 16)); then
+        num_prompts=16
+      fi
+      python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
+        --backend openai-chat \
+        --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
+        --endpoint /v1/chat/completions \
+        --model Qwen/Qwen3.5-397B-A17B-FP8 \
+        --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \
+        --dataset-name random \
+        --random-input-len 8192 \
+        --random-output-len 1024 \
+        --random-range-ratio 1.0 \
+        --random-num-workers 1 \
+        --num-warmups "$((concurrency * 2))" \
+        --num-prompts "${num_prompts}" \
+        --max-concurrency "${concurrency}" \
+        --request-rate inf \
+        --ignore-eos \
+        --disable-tqdm \
+        --save-result \
+        --result-dir "${result_root}/fixed-seq" \
+        --result-filename "qwen3.5-fp8-disagg-isl8192-osl1024-c${concurrency}.json"
+    done
+  env:
+    HF_HOME: /hf_hub_cache
+    HF_HUB_CACHE: /hf_hub_cache/hub
+    HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub

configs/amd-master.yaml

@@ -285,24 +285,25 @@ qwen3.5-fp8-mi355x-sglang-disagg:
     - isl: 8192
       osl: 1024
       search-space:
-      # 1P+1D TP4P+TP8D/EP1 baseline (no speculative decoding).
-      # TP4 prefill saves 4 GPUs vs TP8P while delivering identical decode
-      # interactivity and 24-31% better throughput/GPU (12 vs 16 GPUs).
+      # The srt-slurm recipe owns the complete c8-c128 sweep in one Slurm
+      # allocation so model initialization is paid once instead of per point.
+      # 1P+1D TP4P+TP8D/EP1 remains the current baseline: TP4 prefill saves
+      # four GPUs without changing the TP8 decode service.
       # dp-attn intentionally false: with --enable-dp-attention +
       # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size,
       # but is_deepep_class_backend() excludes MoRI, so
       # num_shared_slots stays at the global value (1) and the
       # (num_experts - num_shared_slots) % moe_ep_size assertion in
       # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared).
       - spec-decoding: "none"
-        conc-list: [ 8, 16, 32, 64, 128 ]
+        conc-list: [1]
         prefill:
           num-worker: 1
           tp: 4
           ep: 1
           dp-attn: false
           additional-settings:
-          - "PREFILL_NODES=1"
+          - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml"
         decode:
           num-worker: 1
           tp: 8

perf-changelog.yaml

@@ -6526,7 +6526,6 @@
     - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification."
     - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2758
-
 - config-keys:
     - qwen3.8next-fp8-h100-sglang-agentic-mtp
   scenario-type:
@@ -6537,3 +6536,16 @@
     - "Use a float32 Mamba SSM state, as Hopper's flashinfer verify kernel requires, unlike the bfloat16 the Blackwell arms must use."
     - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2756
+
+- config-keys:
+    - qwen3.5-fp8-mi355x-sglang-disagg
+  scenario-type:
+    - fixed-seq-len
+  description:
+    - "Replace the model-specific amd_utils launcher with native srt-slurm orchestration using SGLang Router and AMD MoRI."
+    - "Preserve the current one-prefill TP4 plus one-decode TP8 topology and the 8k1k concurrency 8/16/32/64/128 search space."
+    - "Run all five points in one eight-hour Slurm allocation through the unchanged benchmark_serving.py custom benchmark."
+    - "Use the existing lmsysorg/sglang:v0.5.16-rocm720-mi35x image, FP8 KV cache, and AITer attention settings."
+    - "Pin the launcher to SemiAnalysisAI/srt-slurm PR #7 revision 10a58d2ebb4d756c423424049c736b909b72e14f, including DP-aware SGLang P/D rank routing, multi-node rank-offset preservation, and ATOM recipe flag normalization."
+    - "Reuse validated MI355X Hugging Face cache trees through the standard $HF_HOME/hub layout under a per-model lock."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2628