diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index a3dc16d1b9..cdb706b94f 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -369,6 +369,11 @@ jobs: # git dir whose HEAD does not resolve to a commit is dropped # together with its worktree so checkout re-clones it. repo_dir="${{ github.workspace }}" + legacy_amd_dir="${repo_dir}/benchmarks/multi_node/amd_utils" + if [ -d "${legacy_amd_dir}" ] && [ -n "$(find "${legacy_amd_dir}" -xdev \! -user "$(id -u)" -print -quit)" ]; then + echo "[git-repair] Reclaiming legacy AMD workspace files created by containers" + sudo -n chown -R "$(id -u):$(id -g)" "${legacy_amd_dir}" + fi if [ -d "${repo_dir}/.git" ]; then find "${repo_dir}/.git" -name index.lock -type f -delete || true for gitdir in "${repo_dir}"/.git/modules/* "${repo_dir}"/.git/modules/*/*; do diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 11ea880236..d007767d59 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -296,6 +296,11 @@ jobs: # git dir whose HEAD does not resolve to a commit is dropped # together with its worktree so checkout re-clones it. repo_dir="${{ github.workspace }}" + legacy_amd_dir="${repo_dir}/benchmarks/multi_node/amd_utils" + if [ -d "${legacy_amd_dir}" ] && [ -n "$(find "${legacy_amd_dir}" -xdev \! -user "$(id -u)" -print -quit)" ]; then + echo "[git-repair] Reclaiming legacy AMD workspace files created by containers" + sudo -n chown -R "$(id -u):$(id -g)" "${legacy_amd_dir}" + fi if [ -d "${repo_dir}/.git" ]; then find "${repo_dir}/.git" -name index.lock -type f -delete || true for gitdir in "${repo_dir}"/.git/modules/* "${repo_dir}"/.git/modules/*/*; do diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 4819f50599..cf8853b341 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1010,7 +1010,11 @@ run_lm_eval() { export INFERENCEX_LM_EVAL_RUNTIME_READY=true fi - local openai_server_base="http://0.0.0.0:${port}" + # Most launchers run eval beside the API process and keep the historical + # 0.0.0.0 default. Orchestrators such as srt-slurm can place the benchmark + # client on a different node, so allow them to provide the routed host. + local openai_server_host="${EVAL_SERVER_HOST:-0.0.0.0}" + local openai_server_base="http://${openai_server_host}:${port}" local openai_chat_base="${openai_server_base}/v1/chat/completions" export OPENAI_API_KEY=${OPENAI_API_KEY:-EMPTY} MODEL_NAME=${MODEL_NAME:-$MODEL} # Prefer MODEL_NAME, else MODEL diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 95653207b6..0000000000 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bash - -# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X -# (DeepSeek-V4-Pro FP4, 1P1D TP8). -# -# CI-style sibling of dsr1_fp4_mi355x_sglang-disagg.sh: driven entirely by -# environment variables and submits a SLURM job via submit.sh. The agentic / -# HiCache-offload configuration mirrors the DSR1 recipe but uses DSV4-Pro -# specific flags (dsv4 attention backend, page-size 256, SWA settings). - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - DURATION \ - KV_OFFLOADING \ - IS_AGENTIC \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# ── Identity / result naming ── -export MODEL_PREFIX="${MODEL_PREFIX:-dsv4}" -export PRECISION="${PRECISION:-fp4}" -export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-dsv4-fp4-agentic}}" - -# ── Agentic benchmark params ── -export DURATION="${DURATION:-1800}" -# DSV4-Pro max model len for agentic traces (matches single-node recipe). -export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}" - -# ── In-tree sglang patches ── -# mori_conn.py targets hybrid-state bugs (GLM-5, Qwen3.5). DSV4-Pro uses a -# pure MoE/DSA architecture without hybrid state; skip to avoid interference. -export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}" - -# ── Aiter fault mitigation ── -# --disable-custom-all-reduce avoids a known aiter fault on MI355X. -export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" - -# ── KV cache offloading (HiCache) ── -# KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). -# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe -# only implements HiCache, so "hicache" is the only supported value. -# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. -export KV_OFFLOADING="${KV_OFFLOADING:-none}" -if [[ "$KV_OFFLOADING" != "none" ]]; then - export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" -fi -# HiCache/Mooncake tunables only matter when KV offloading is enabled. -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then - export HICACHE_TIER="${HICACHE_TIER:-L2}" - export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" - # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. - export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" - # HiCache ratio (host pool = ratio * GPU KV pool). - export HICACHE_RATIO="${HICACHE_RATIO:-3}" - # DSv4 wants the ratio-based pool, but server_sglang.sh prefers - # --hicache-size over --hicache-ratio when TOTAL_CPU_DRAM_GB is set. - # Opt out via FORCE_HICACHE_RATIO instead of unsetting TOTAL_CPU_DRAM_GB - # (also required client-side by benchmark_lib.sh when KV_OFFLOADING=dram). - export FORCE_HICACHE_RATIO=1 - - # ── HiCache layout/backend by tier ── - # L3 (Mooncake): page_first + direct + write_through + storage=mooncake - # L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none - # NOTE: write_through_selective evicts only under GPU memory pressure, avoiding - # the mori RDMA race that causes GPU memory access faults with write_through. - if [[ "${HICACHE_TIER^^}" == "L3" ]]; then - export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}" - else - export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - fi - export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}" - # Shared nodes: use non-default Mooncake ports to avoid collisions. - export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}" - export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" - export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}" - export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" - export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" - export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}" - export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" - export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" - export MC_DEVICE="${MC_DEVICE:-}" - export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" - export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" -fi - -# ── MoRIIO RDMA Send Queue tuning ── -export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" -export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" - -# ── SGLang PD router policy + server metrics ── -export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}" -export ENABLE_METRICS="${ENABLE_METRICS:-1}" - -# ── MTP ── -export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" - -# Derive EP/DP enable flags from the topology inputs. -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 0ea94e0bde..41a60e152e 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -42,6 +42,60 @@ done resolve_trace_source install_agentic_deps +# Preserve the legacy DP-attention replay contract. The SGLang router uses this +# header to keep every request in one AgentX correlation tree on a stable DP +# route, which is important for both session continuity and prefix-cache reuse. +if [[ "${PREFILL_DP_ATTN:-false}" == "true" ]]; then + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true +fi + +# Reset every advertised SGLang worker before each concurrency point, matching +# the retired amd_utils trace replay. /flush_cache covers GPU radix + host +# HiCache; the storage-backend endpoint is best-effort because L3 is optional. +clear_agentic_worker_caches() { + local timeout_seconds="${FLUSH_DRAIN_TIMEOUT:-120}" + local metrics_csv="${AIPERF_SERVER_METRICS_URLS:-}" + if [[ -z "$metrics_csv" ]]; then + echo "[clear_caches] WARN: AIPERF_SERVER_METRICS_URLS unset; skipping cache flush" >&2 + return 0 + fi + + local -a metrics_urls + IFS=',' read -r -a metrics_urls <<< "$metrics_csv" + local metrics_url base_url start response code flushed + for metrics_url in "${metrics_urls[@]}"; do + [[ -n "$metrics_url" ]] || continue + base_url="${metrics_url%/metrics}" + start=$(date +%s) + flushed=0 + response="" + while :; do + response=$(curl -sf -m 10 -X POST "${base_url}/flush_cache" 2>/dev/null || true) + if grep -qi "Cache flushed" <<< "$response"; then + flushed=1 + break + fi + if (( $(date +%s) - start >= timeout_seconds )); then + break + fi + sleep 3 + done + if (( flushed )); then + echo "[clear_caches] ${base_url}: L1+L2 flushed" + else + echo "[clear_caches] WARN ${base_url}: L1+L2 flush not confirmed after ${timeout_seconds}s" >&2 + fi + + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' \ + -X POST "${base_url}/hicache/storage-backend/clear" 2>/dev/null || true) + if [[ "$code" == "200" ]]; then + echo "[clear_caches] ${base_url}: L3 store cleared" + else + echo "[clear_caches] ${base_url}: L3 clear http=${code:-000} (optional backend unavailable)" + fi + done +} + wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" @@ -125,6 +179,9 @@ for index in "${!CONCURRENCIES[@]}"; do mkdir -p "$RESULT_DIR" echo "Running agentic concurrency $concurrency of: ${CONCURRENCIES[*]}" + if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then + clear_agentic_worker_caches + fi build_replay_cmd "$RESULT_DIR" run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/multi_node/amd_utils/bench.sh b/benchmarks/multi_node/amd_utils/bench.sh deleted file mode 100755 index 3dde0a68a5..0000000000 --- a/benchmarks/multi_node/amd_utils/bench.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash -# Dual-Engine Disaggregated Benchmark Runner -# -# ENGINE=sglang (default): SGLang benchmark -# ENGINE=vllm: vLLM benchmark -# -# Produces JSON result files via benchmark_serving.py so that the CI pipeline -# can collect and process results. -# -# Usage: bash bench.sh \ -# \ -# - -ENGINE="${ENGINE:-sglang-disagg}" - -n_prefill=$1 -n_decode=$2 -prefill_gpus=$3 -decode_gpus=$4 -model_path=$5 -model_name=$6 -MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - BENCH_MODEL="${MODEL_NAME:-${MODEL_PATH}}" -else - BENCH_MODEL="${MODEL_PATH}" -fi -log_path=$7 - -chosen_isl=${8:-1024} -chosen_osl=${9:-1024} -concurrency_list=${10:-"512x1"} -if [[ "$ENGINE" == "vllm-disagg" ]]; then - chosen_req_rate=${11:-inf} -else - chosen_req_rate=${11:-1} -fi -random_range_ratio=${12:-0.8} -num_prompts_multiplier=${13:-10} - -IFS='x' read -r -a chosen_concurrencies <<< "$concurrency_list" - -ROUTER_PORT="${ROUTER_PORT:-30000}" - -export TRANSFORMERS_VERBOSITY=error -export TOKENIZERS_PARALLELISM=false - -echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" - -profile_folder="${log_path}/${ENGINE}_isl_${chosen_isl}_osl_${chosen_osl}" -mkdir -p "$profile_folder" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -for max_concurrency in "${chosen_concurrencies[@]}"; do - - export_file="${profile_folder}/concurrency_${max_concurrency}_req_rate_${chosen_req_rate}_gpus_$((prefill_gpus+decode_gpus))_ctx_${prefill_gpus}_gen_${decode_gpus}" - - num_prompts=$(( max_concurrency * num_prompts_multiplier )) - if [[ "$num_prompts" -lt 16 ]]; then - num_prompts=16 - fi - - echo "profile_folder: $profile_folder" - echo "max_concurrency: $max_concurrency" - echo "chosen_req_rate: $chosen_req_rate" - echo "MODEL_PATH: $MODEL_PATH" - echo "ROUTER_PORT: $ROUTER_PORT" - echo "chosen_isl: $chosen_isl" - echo "chosen_osl: $chosen_osl" - echo "num_prompts: $num_prompts" - echo "export_file: $export_file" - - # Engine-specific extra flags - extra_flags="" - # vllm - if [[ "$ENGINE" == "vllm-disagg" ]]; then - extra_flags="--trust-remote-code --tokenizer $MODEL_PATH" - # atom - elif [[ "$ENGINE" == "atom-disagg" ]]; then - extra_flags="--trust-remote-code --tokenizer $MODEL_PATH" - if [ "$IS_MTP" = "true" ]; then - # just override extra_flags as dsv3 use different tokenizer path - if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then - extra_flags="--dsv4" - else - extra_flags="--use-chat-template" - fi - fi - # sglang - else - if [ "$IS_MTP" = "true" ]; then - if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then - extra_flags="--dsv4" - else - extra_flags="--use-chat-template" - fi - fi - fi - - run_benchmark_serving \ - --bench-serving-dir "$REPO_ROOT" \ - --model "$BENCH_MODEL" \ - --port "$ROUTER_PORT" \ - --backend openai \ - --input-len "$chosen_isl" \ - --output-len "$chosen_osl" \ - --random-range-ratio "$random_range_ratio" \ - --num-prompts "$num_prompts" \ - --max-concurrency "$max_concurrency" \ - --result-filename "$export_file" \ - --result-dir /workspace/ \ - $extra_flags - - echo "-----------------------------------------" - - # vLLM: cooldown between rounds for idle KV block reaper - if [[ "$ENGINE" == "vllm-disagg" ]]; then - echo "[BENCH] Cooldown: waiting 10s for idle KV block reaper..." - sleep 10 - fi -done diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh deleted file mode 100755 index 0cf3978bbf..0000000000 --- a/benchmarks/multi_node/amd_utils/env.sh +++ /dev/null @@ -1,377 +0,0 @@ -#!/bin/bash -# Dual-engine environment setup for multi-node disaggregated serving. -# -# ENGINE=sglang (default): SGLang/MoRI environment -# ENGINE=vllm: vLLM/Nixl environment -# -# REQUIRED ENVIRONMENT VARIABLES: -# IBDEVICES - RDMA/InfiniBand device names (e.g., ionic_0,ionic_1,... or mlx5_0,mlx5_1,...) -# Set by runner or auto-detected from hostname. -set -x - -ENGINE="${ENGINE:-sglang-disagg}" -export PYTHONDONTWRITEBYTECODE=1 - -# ============================================================================= -# HiCache / Mooncake settings from job.slurm -# ============================================================================= -# job.slurm writes the recipe-provided HiCache/Mooncake tunables to -# hicache_mc_.env and mounts it read-only at /config/hicache_mc.env. Source -# it here (auto-export) so values like HICACHE_PAGE_SIZE=256 reach the container -# before server_sglang.sh applies its "${VAR:-default}" fallbacks. Without this -# the vars arrive unset and server_sglang.sh defaults HICACHE_PAGE_SIZE to 1, -# overriding the recipe's --page-size. Empty values in the file are harmless: -# the "${VAR:-default}" fallbacks still treat "" as unset. -if [[ -f /config/hicache_mc.env ]]; then - set -a - source /config/hicache_mc.env - set +a - echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})" -fi - -# ============================================================================= -# Shared: IBDEVICES detection -# ============================================================================= - -# Prefer IBDEVICES set by runner (runners/launch_mi355x-amds.sh) -# Fall back to hostname detection if not set (for direct script execution) -if [[ -z "$IBDEVICES" ]]; then - DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') - if [[ -n "$DETECTED" ]]; then - export IBDEVICES="$DETECTED" - echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" - else - echo "ERROR: Unable to detect RDMA devices. Set IBDEVICES explicitly." >&2 - exit 1 - fi -else - echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" -fi -export IBDEVICES - -# Shared: Auto-detect default network interface (portable across clusters) -# Only auto-detect if not already set by the runner/environment -if [[ -z "$GLOO_SOCKET_IFNAME" ]]; then - export GLOO_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) -fi -if [[ -z "$NCCL_SOCKET_IFNAME" ]]; then - export NCCL_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) -fi - -set +x - -export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} - -# ============================================================================= -# MoRI-specific environment -# ============================================================================= -# Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. - -export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}" -export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" -export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}" -export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" -export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" - -# QoS/DSCP configuration -# Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname -if [[ -n "$MORI_RDMA_TC" ]]; then - echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC (set by runner or environment)" -elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null| awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - TC=$(( 4 * ND_DSCP )) - export MORI_RDMA_SL=$ND_PRIO - export MORI_IO_SL=$ND_PRIO - export MORI_RDMA_TC=$TC - export MORI_IO_TC=$TC - echo "[INFO] Detected QoS config from nicctl: MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL, MORI_IO_TC=$MORI_IO_TC, MORI_IO_SL=$MORI_IO_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - # Fall back to hostname-based detection - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export MORI_RDMA_TC=96 - export MORI_IO_TC=96 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export MORI_RDMA_TC=104 - export MORI_IO_TC=104 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - else - echo "[INFO] Unable to detect MORI_RDMA_TC from hostname. Skipping RDMA QoS configuration." - fi - fi -else - # nicctl not available, try hostname-based detection - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export MORI_RDMA_TC=96 - export MORI_IO_TC=96 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export MORI_RDMA_TC=104 - export MORI_IO_TC=104 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - else - echo "[INFO] nicctl not found and unable to detect from hostname. Skipping RDMA QoS configuration." - echo " This is normal for clusters without QoS or outside Docker containers." - fi -fi - -# ============================================================================= -# Engine-specific environment -# ============================================================================= - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - # ========================================================================= - # vLLM/Nixl-specific environment - # ========================================================================= - export VLLM_USE_V1=1 - export VLLM_SERVER_DEV_MODE=0 - export VLLM_DISABLE_REQUEST_ID_RANDOMIZATION=1 - - set -x - - # UCX_NET_DEVICES: Use the first tw-eth interface for UCX TCP transport - if [[ -z "$UCX_NET_DEVICES" ]]; then - UCX_NET_DEV=$(ip -o link show 2>/dev/null | awk -F': ' '/tw-eth/{print $2}' | head -1) - if [[ -n "$UCX_NET_DEV" ]]; then - export UCX_NET_DEVICES="$UCX_NET_DEV" - else - FIRST_IB=$(echo "$IBDEVICES" | cut -d',' -f1) - if [[ -n "$FIRST_IB" ]]; then - export UCX_NET_DEVICES="${FIRST_IB}:1" - fi - fi - echo "[INFO] Auto-set UCX_NET_DEVICES=$UCX_NET_DEVICES" - else - echo "[INFO] Using UCX_NET_DEVICES=$UCX_NET_DEVICES (set by environment)" - fi - - # RoCEv2: use IPv4-mapped GID (index 1) for inter-node RDMA routing - export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-1} - - # QoS/DSCP configuration for lossless RoCEv2 fabric. - if [[ -n "$UCX_IB_TRAFFIC_CLASS" ]]; then - echo "[INFO] Using UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS (set by environment)" - elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null | awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - export UCX_IB_TRAFFIC_CLASS=$(( 4 * ND_DSCP )) - export UCX_IB_SL=$ND_PRIO - echo "[INFO] Detected QoS from nicctl: UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS, UCX_IB_SL=$UCX_IB_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - fi - fi - else - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - else - echo "[INFO] No nicctl and unable to detect from hostname. Skipping QoS configuration." - fi - fi - - set +x - echo "[INFO] IBDEVICES=$IBDEVICES UCX_NET_DEVICES=$UCX_NET_DEVICES NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME UCX_IB_GID_INDEX=$UCX_IB_GID_INDEX UCX_IB_TRAFFIC_CLASS=${UCX_IB_TRAFFIC_CLASS:-unset}" - -else - # ========================================================================= - # SGLang-specific environment - # ========================================================================= - - export SGLANG_USE_AITER=1 - export AITER_LOG_LEVEL=ERROR - - export SGLANG_MORI_DISPATCH_DTYPE=auto - # export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast - # export MORI_COMBINE_DTYPE_DECODE=fp8 - export MORI_COMBINE_DTYPE_PREFILL="" - export MORI_COMBINE_DTYPE_DECODE="" - export SGLANG_MORI_QP_PER_TRANSFER=4 - export SGLANG_MORI_NUM_WORKERS=4 - # Keep these as overridable defaults (not hard assignments), otherwise - # later tuning blocks cannot raise them for high-concurrency runs. - # export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" - - # export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" - # export MORI_IO_QP_MAX_CQE=32768 - # export MORI_IO_QP_MAX_SGE=1 - - # export MORI_IO_TC_DISABLE=0 - - export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 - export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 - - export SGLANG_HEALTH_CHECK_TIMEOUT=600 - - # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading - if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then - export SGLANG_ROCM_FUSED_DECODE_MLA=0 - export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - export SAFETENSORS_FAST_GPU=1 - fi - - # Disable allocating memory in one pass - export MORI_SHMEM_MODE=ISOLATION - - # Enable spec v2 - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - - export SGLANG_LOG_MS=true - export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=32 - - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=512 - - export MORI_MOE_MAX_INPUT_TOKENS_PREFILL=32768 - export MORI_MOE_MAX_INPUT_TOKENS_DECODE=2703 - - # set MTP size=1 when EP16 - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) - - export MORI_EP_LAUNCH_CONFIG_MODE=AUTO - - # Default to WARNING to cut per-op MoRI log spam on long multinode/eval - # runs; override with MORI_APP_LOG_LEVEL=INFO when debugging. - export MORI_APP_LOG_LEVEL="${MORI_APP_LOG_LEVEL:-WARNING}" - - # Router logging control: - # 0 (default) keeps noisy per-request access logs out of stdout while still logging to file. - # 1 mirrors router logs to stdout via tee (useful for live debugging). - export SGLANG_ROUTER_STDOUT_LOGS="${SGLANG_ROUTER_STDOUT_LOGS:-0}" - - # FIXME: WA for latest upstream 0305 image - export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} - - # Decode CUDA-graph capture crash on ROCm 7.2.0 (TP8+EP8, mori a2a). - # Symptom: during decode cuda-graph capture, the torch ProcessGroupNCCL - # *watchdog* thread calls hipEventQuery() to poll in-flight NCCL work. - # ROCm <= 7.2.0's HIP runtime does NOT honor cudaStreamCaptureModeThreadLocal, - # so the watchdog's cross-thread query touches the main thread's active - # capture and invalidates it -> "HIP error: operation not permitted on an - # event last recorded in a capturing stream (hipErrorCapturedEvent)" -> - # watchdog aborts -> "Rank 0 scheduler died during initialization" (-6). - # This is a HIP runtime bug, not OOM and not a mori/deepep-mode bug (it - # fires for --deepep-mode normal and auto alike; EP8+mori just adds NCCL - # PGs that make the watchdog race fire). Refs: sgl-project/sglang#29235, - # #24011; ROCm/hip#3876; pytorch/pytorch#176251. - # Real fix = ROCm 7.2.2+ (honors THREAD_LOCAL). Until the base image is - # bumped, TORCH_NCCL_BLOCKING_WAIT=true makes NCCL work completion use a - # blocking wait instead of the async watchdog hipEventQuery poll, so no - # event is queried during capture. CUDA graph stays fully enabled. - export TORCH_NCCL_BLOCKING_WAIT="${TORCH_NCCL_BLOCKING_WAIT:-1}" - export NCCL_BLOCKING_WAIT="${NCCL_BLOCKING_WAIT:-1}" - # export NCCL_DEBUG="${NCCL_DEBUG:-INFO}" - - # ========================================================================= - # DeepSeek-V4-Pro PD recipe overrides - # Placed at the end of the SGLang env block so it wins over the global - # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD - # commands (ported from InferenceX amd/dsv4_sgl_di). These SGLANG_OPT_* / - # AITER_* kernel-routing knobs steer DSv4 away from the default aiter CK - # fused-MoE path, which raises "Unsupported kernel config for moe heuristic - # dispatch" at decode time on this fp4 model (job 19034 crash). Only the - # SGLang/MoRI env knobs are pinned here; CLI flags live in models.yaml and - # the cluster NIC/socket vars stay runner-derived. - # ========================================================================= - if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then - export SGLANG_AITER_MLA_PERSIST=0 - ## resolve the OOR issue - export HSA_NO_SCRATCH_RECLAIM=0 - # MoRI RDMA send-queue depth for DSv4 (overrides the global default above). - export MORI_IO_QP_MAX_SEND_WR=32767 - # Unified radix tree: cache impl with per-component (full-attn / SWA) - # management for hybrid-attention models. Set unconditionally (not gated on - # hicache) so all SGLang runs use it. - export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 - # Proactively free out-of-window SWA KV slots during chunked prefill. - # Without it, in-flight requests pin SWA KV for their whole context, keeping - # the SWA pool under constant eviction pressure; under LRU the trailing - # window of cached sessions gets flushed, making prefix-cache hits bimodal - # and collapsing the effective hit rate on multi-turn agentic workloads. - export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 - - # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default) - export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL=auto - export MORI_COMBINE_DTYPE_DECODE=auto - - # Per-role MoRI dispatch sizing (used by the harness chunked/MoE math) - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=64 - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE - - # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 - # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* - # coupling when set. - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 - - # Fixed inter-kernel switch threshold (not derived). - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 - - # Overlap plan stream on for DSv4 (global default is 0) - # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - - # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) - export SGLANG_DEFAULT_THINKING=1 - export SGLANG_DSV4_REASONING_EFFORT=high - export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false - export SGLANG_USE_AITER=1 - export SGLANG_USE_ROCM700A=0 - export SGLANG_OPT_USE_FUSED_COMPRESS=true - export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton - export SGLANG_OPT_FP8_WO_A_GEMM=false - export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false - export SGLANG_OPT_USE_TOPK_V2=false - export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} - export SGLANG_OPT_USE_TILELANG_INDEXER=false - export SGLANG_OPT_USE_TILELANG_MHC_PRE=false - export SGLANG_OPT_USE_TILELANG_MHC_POST=false - export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 - export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true - export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false - export SGLANG_ROCM_USE_MULTI_STREAM=false - export AITER_BF16_FP8_MOE_BOUND=0 - export SGLANG_EAGER_INPUT_NO_COPY=true - export SGLANG_SHARED_EXPERT_TP1=1 - export SGLANG_DP_SHARED_EXPERT_LOCAL=1 - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 - fi - -fi \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/env_atom.sh b/benchmarks/multi_node/amd_utils/env_atom.sh deleted file mode 100644 index 089594cbe4..0000000000 --- a/benchmarks/multi_node/amd_utils/env_atom.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# ATOM/mooncake-specific environment setup for multi-node disaggregated serving. -# -# Sourced by server_atom.sh in place of env.sh (which is SGLang/MoRI-specific). -# -# REQUIRED ENVIRONMENT VARIABLES: -# IBDEVICES - RDMA/InfiniBand device names (e.g., ionic_0,ionic_1,...) -# Set by runner or auto-detected from hostname. - -set -x - -export PYTHONUNBUFFERED=1 -export PYTHONDONTWRITEBYTECODE=1 - -# ============================================================================= -# IBDEVICES detection (same as env.sh) -# ============================================================================= - -if [[ -z "$IBDEVICES" ]]; then - DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') - if [[ -n "$DETECTED" ]]; then - export IBDEVICES="$DETECTED" - echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" - else - # ATOM uses mooncake proxy_ip/handshake_port for KV transfer — IBDEVICES is - # not passed as a server argument (unlike SGLang --disaggregation-ib-device). - # Log a warning but do not fail; mooncake will use its own RDMA device selection. - echo "[WARN] Unable to detect RDMA devices via ibv_devinfo; IBDEVICES unset (non-fatal for ATOM/mooncake)" >&2 - fi -else - echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" -fi -export IBDEVICES - -# ============================================================================= -# ATOM/mooncake-specific environment -# ============================================================================= - -# mooncake RDMA KV transfer library path -export LD_LIBRARY_PATH=/opt/venv/lib/python3.10/site-packages/mooncake:/opt/rocm/lib:${LD_LIBRARY_PATH:-} - -# faster model loading (safetensors only) -export SAFETENSORS_FAST_GPU=1 - -# aiter logging (WARNING to reduce noise; use DEBUG for troubleshooting) -export VLLM_LOG_LEVEL=WARNING -export ATOM_LOG_LEVEL=WARNING -export AITER_LOG_LEVEL=WARNING -export LOG_LEVEL=WARNING -export LOGLEVEL=WARNING - -set +x - -# ATOM_HOST_IP is set per-node in server_atom.sh (= host_ip, used as handshake IP) -echo "[INFO] ATOM env: IBDEVICES=$IBDEVICES LD_LIBRARY_PATH includes mooncake" \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh b/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh deleted file mode 100755 index 86acabbab0..0000000000 --- a/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash -# Pre-flight RDMA QoS/DCQCN validation for a single node, run by job.slurm via -# `srun` across every allocated node BEFORE any container/GPU time is spent. -# -# A misconfigured NIC (PFC not covering the RoCE priority, DCQCN disabled, ...) -# doesn't make MoRI's cross-node EP/RDMA transfers error out -- it just quietly -# degrades, and the only symptom is unexplained tail latency or throughput -# variance in the benchmark numbers hours later. Failing fast here, before the -# job burns node-hours, is much cheaper than debugging that after the fact. -# -# check_qos()/check_dcqcn() below are trimmed/adapted from ROCm/mori's -# tools/env_check.sh: -# https://github.com/ROCm/mori/blob/main/tools/env_check.sh -# The upstream script's expensive ib_write_bw/ib_write_lat full-mesh bandwidth -# and latency tests are intentionally NOT ported here -- this runs before -# every single job, so it must be fast (seconds), not a multi-minute -# fabric-wide benchmark of its own. -# -# Scoped to AMD Pollara (ionic) NICs via nicctl, matching this repo's current -# fleet (see env.sh's nicctl-based MORI_RDMA_TC/SL detection). If bnxt_re or -# mlx5 NICs are added to the fleet, port the bnxt_*/mlx5_* check_* functions -# from the upstream script following the same pattern. -# -# Exit code: 0 = OK (or gracefully skipped, e.g. no ionic NICs on this host), -# 1 = hard QoS/DCQCN misconfiguration -- do not proceed with the run. -set -uo pipefail - -AINIC_MIN_VER="1.117.5-a-45" # minimum recommended AINIC firmware for IBGDA - -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[0;33m' -NC='\033[0m' - -log_ok() { echo -e "[$(hostname -s)] ${GREEN}[OK]${NC} $*"; } -log_fail() { echo -e "[$(hostname -s)] ${RED}[FAIL]${NC} $*"; } -log_warn() { echo -e "[$(hostname -s)] ${YELLOW}[WARN]${NC} $*"; } -die() { log_fail "$@"; exit 1; } - -# version_ge -> true if >= (dotted/hyphenated, via sort -V) -version_ge() { - local cand="$1" min="$2" - [[ "$cand" == "$min" ]] && return 0 - [[ "$(printf '%s\n%s\n' "$cand" "$min" | sort -V | head -1)" == "$min" ]] -} - -# check_ainic_version_recommendation -# warns if firmware is on the IBGDA-incapable 1.117.1 branch, or below the -# recommended minimum for cross-node MORI (EP over RDMA / IBGDA). -check_ainic_version_recommendation() { - local ver="$1" - [[ -n "$ver" ]] || { log_warn "cannot verify AINIC firmware version against recommendation (empty)"; return; } - if [[ "$ver" =~ ^1\.117\.1([.-]|$) ]]; then - log_warn "AINIC firmware $ver is on the 1.117.1 branch, which does NOT support IBGDA -- upgrade to >= $AINIC_MIN_VER" - elif version_ge "$ver" "$AINIC_MIN_VER"; then - log_ok "AINIC firmware $ver meets the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" - else - log_warn "AINIC firmware $ver is below the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" - fi -} - -# check_versions() -- informational only (never hard-fails the job). -check_versions() { - local fw_output sw_output - fw_output=$(sudo nicctl show version firmware 2>/dev/null) - sw_output=$(sudo nicctl show version host-software 2>/dev/null) - - local fw_versions fw_count - fw_versions=$(echo "$fw_output" | grep -i "firmware" | awk '{print $NF}' | sort -u) - fw_count=$(echo "$fw_versions" | grep -c . || true) - if [[ "$fw_count" -ne 1 ]]; then - log_warn "firmware versions not consistent across NICs:" - echo "$fw_versions" - local v - while read -r v; do [[ -n "$v" ]] && check_ainic_version_recommendation "$v"; done <<< "$fw_versions" - else - log_ok "firmware : $fw_versions" - check_ainic_version_recommendation "$fw_versions" - fi - - local nicctl_ver - nicctl_ver=$(echo "$sw_output" | grep "nicctl" | awk '{print $NF}') - [[ -n "$nicctl_ver" ]] && log_ok "nicctl : $nicctl_ver" || log_warn "cannot determine nicctl version" -} - -# check_qos() -- HARD gate: classification type must be DSCP, and PFC no-drop -# must be enabled and cover every no-drop priority. Dies (exit 1) otherwise. -check_qos() { - local qos_output - qos_output=$(sudo nicctl show qos 2>/dev/null) - [[ -n "$qos_output" ]] || die "sudo nicctl show qos returned nothing" - - local class_type - class_type=$(echo "$qos_output" | grep "Classification type" | head -1 | awk '{print $NF}') - [[ "$class_type" == "DSCP" ]] || die "classification type is '$class_type', expected 'DSCP'" - log_ok "classification type : DSCP" - - local nd_prio_raw - nd_prio_raw=$(echo "$qos_output" | grep "PFC no-drop priorities" | head -1 | awk '{print $NF}') - [[ -n "$nd_prio_raw" ]] || die "cannot find PFC no-drop priority" - local nd_prios=() - IFS=',' read -ra nd_prios <<< "$nd_prio_raw" - log_ok "no-drop priorities : ${nd_prios[*]}" - - local pfc_bitmap - pfc_bitmap=$(echo "$qos_output" | grep "PFC priority bitmap" | head -1 | awk '{print $NF}') - [[ -n "$pfc_bitmap" && "$pfc_bitmap" != "0x0" ]] || die "PFC is not enabled (bitmap=$pfc_bitmap)" - local p - for p in "${nd_prios[@]}"; do - (( pfc_bitmap & (1 << p) )) || die "PFC bitmap $pfc_bitmap does not cover priority $p" - done - log_ok "PFC enabled for priorities ${nd_prios[*]} (bitmap=$pfc_bitmap)" -} - -# check_dcqcn() -- HARD gate: DCQCN must be enabled on every ROCE device, and -# the CNP DSCP must be consistent across NICs. Exits 1 otherwise. -check_dcqcn() { - local dcqcn_output - dcqcn_output=$(sudo nicctl show dcqcn 2>/dev/null) - [[ -n "$dcqcn_output" ]] || die "sudo nicctl show dcqcn returned nothing" - - local total - total=$(echo "$dcqcn_output" | grep -c "ROCE device") - [[ "$total" -gt 0 ]] || die "no ROCE devices found in dcqcn output" - - local disabled - disabled=$(echo "$dcqcn_output" | grep "Status" | grep -v "Enabled" || true) - if [[ -n "$disabled" ]]; then - log_fail "some ROCE devices have DCQCN disabled:" - echo "$disabled" - exit 1 - fi - log_ok "DCQCN enabled on all $total ROCE devices" - - local cnp_values cnp_count - cnp_values=$(echo "$dcqcn_output" | grep "DSCP value used for CNP" | awk '{print $NF}' | sort -u) - cnp_count=$(echo "$cnp_values" | grep -c . || true) - [[ "$cnp_count" -eq 1 ]] || die "CNP DSCP not consistent across NICs: $cnp_values" - log_ok "CNP DSCP = $cnp_values (consistent across all NICs)" -} - -# ============================= main ============================= - -if ! command -v nicctl &>/dev/null; then - log_warn "nicctl not found on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check (not an ionic NIC host, or nicctl not on PATH)" - exit 0 -fi - -# nicctl exits 0 even with no NIC present, so check its output rather than its exit code. -_nicctl_probe=$(sudo nicctl show version firmware 2>&1 || true) -if echo "$_nicctl_probe" | grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC'; then - log_warn "nicctl present but no ionic NIC detected/accessible on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check" - exit 0 -fi - -check_versions -check_qos -check_dcqcn - -log_ok "RDMA QoS/DCQCN pre-flight check passed on $(hostname -s)" diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm deleted file mode 100755 index edf83015a9..0000000000 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ /dev/null @@ -1,789 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=disagg-bench -#SBATCH -N 3 # Overridden by submit.sh -N flag -#SBATCH -n 3 # Overridden by submit.sh -n flag -#SBATCH --ntasks-per-node=1 -#SBATCH --spread-job -#SBATCH --gres=gpu:8 -#SBATCH --time=24:00:00 -# --output and --error are set by submit.sh via BENCHMARK_LOGS_DIR - -ENGINE="${ENGINE:-sglang-disagg}" - -echo "=== Job Start Time ===" -echo "UTC Time: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" -echo "PST Time: $(TZ=America/Los_Angeles date '+%Y-%m-%d %H:%M:%S %Z')" -echo "ENGINE: $ENGINE" -echo "=======================" -echo "" - -# ============================================================================= -# Model Validation -# ============================================================================= - -# Use $(pwd) not BASH_SOURCE — sbatch copies the script to /var/spool/slurmd/ -# at runtime, but the CWD remains the submit-time directory (amd_utils/). -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODELS_YAML="$(pwd)/models_vllm.yaml" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - MODELS_YAML="$(pwd)/models_atom.yaml" -else - MODELS_YAML="$(pwd)/models.yaml" -fi - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "Error: models YAML not found at $MODELS_YAML" - exit 1 -fi - -if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then - echo "Error: DOCKER_IMAGE_NAME is not set." - exit 1 -fi - -MODEL_NAME="${MODEL_NAME:-None}" -# Resolve the models.yaml entry the same way server_sglang.sh does: agentic runs -# (IS_AGENTIC) use the '-AgentX' recipe, non-agentic disaggregated runs use -# '-DI'. Fall back to the bare model name if the variant key is absent. -# MODEL_NAME itself is left unchanged so env.sh/server_sglang.sh still see the base name. -if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - MODEL_YAML_KEY="${MODEL_NAME}-AgentX" -else - MODEL_YAML_KEY="${MODEL_NAME}-DI" -fi -if ! grep -q "^${MODEL_YAML_KEY}:" "$MODELS_YAML"; then - if grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then - MODEL_YAML_KEY="$MODEL_NAME" - else - echo "Error: Model '$MODEL_YAML_KEY' (nor bare '$MODEL_NAME') not found in $MODELS_YAML" - echo "Available models:" - grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' - exit 1 - fi -fi -echo "Model found: $MODEL_YAML_KEY (MODEL_NAME=$MODEL_NAME, IS_AGENTIC=${IS_AGENTIC:-0})" - -RUN_FILE="server.sh" -echo "Runfile set: $RUN_FILE" - -# DI_REPO_DIR points to the repo root. -# $(pwd) is amd_utils/ (the sbatch submit dir); go up 3 levels to reach the repo root. -export DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) - -xP="${xP:-1}" -yD="${yD:-1}" - -# Benchmark configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" - -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -# Engine-specific defaults -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP:-false}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP:-false}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP:-false}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP:-false}" -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" -DECODE_MTP_SIZE=${DECODE_MTP_SIZE:-0} - -# Router selection: "vllm-router" (external container) or "moriio" (in-container proxy) -ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" -ROUTER_PORT="${ROUTER_PORT:-30000}" -PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" - -# ============================================================================= -# Model Path Resolution -# ============================================================================= - -# MODEL_DIR detection: prefer env var, fall back to hostname detection -if [[ -z "$MODEL_DIR" ]]; then - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - MODEL_DIR="/nfsdata" - elif [[ $NODENAME == mia1* ]]; then - MODEL_DIR="/it-share/data" - else - MODEL_DIR="/nfsdata" - fi - echo "[INFO] Auto-detected MODEL_DIR=$MODEL_DIR from hostname $(hostname -s)" -fi -export MODEL_DIR - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - # vLLM: Extract hf_dir from models.yaml, search multiple paths, resolve HF cache snapshots - DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - DISK_DIR_NAME="${DISK_DIR_NAME:-$MODEL_NAME}" - echo "Looking for model: $MODEL_NAME (disk dir: $DISK_DIR_NAME)" - - resolve_hf_cache_path() { - local base_path=$1 - if [[ -d "${base_path}/snapshots" ]]; then - local snapshot=$(ls -1 "${base_path}/snapshots" 2>/dev/null | head -1) - if [[ -n "$snapshot" ]]; then - echo "${base_path}/snapshots/${snapshot}" - return 0 - fi - fi - echo "$base_path" - return 1 - } - - MODEL_PATH="" - SEARCH_PATHS=( - "${MODEL_DIR}/${DISK_DIR_NAME}" - "${MODEL_DIR}/${MODEL_NAME}" - "/nfsdata/hf_hub_cache-0/${DISK_DIR_NAME}" - "/nfsdata/hf_hub_cache-0/${MODEL_NAME}" - ) - - for search_path in "${SEARCH_PATHS[@]}"; do - if [[ -d "$search_path" ]]; then - RESOLVED=$(resolve_hf_cache_path "$search_path") - MODEL_PATH="$RESOLVED" - echo "Found MODEL_PATH: $MODEL_PATH" - break - fi - done - - if [[ -z "$MODEL_PATH" ]]; then - echo "FATAL: Model '$MODEL_NAME' not found. Searched:" - for p in "${SEARCH_PATHS[@]}"; do echo " - $p"; done - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -else - # SGLang: Validate model path across all allocated nodes - echo "Looking for model: $MODEL_NAME" - echo "Checking model availability across all allocated nodes..." - - ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") - TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) - echo "Total allocated nodes: $TOTAL_NODES" - echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" - - check_model_path() { - local path=$1 - local check_name=$2 - echo "Checking $check_name: $path" - srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " - if [ -d '$path' ]; then - echo \"\$(hostname): Found $path\" - exit 0 - else - echo \"\$(hostname): Missing $path\" - exit 1 - fi - " - local exit_code=$? - if [ $exit_code -eq 0 ]; then - echo "$check_name available on ALL nodes" - return 0 - else - echo "$check_name NOT available on all nodes" - return 1 - fi - } - - # Extract hf_dir from models.yaml (same as vllm-disagg path above) - SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" - - # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); - # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. - if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then - # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) - # Use it directly if it exists on all nodes, otherwise try subdirectory combos. - if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then - echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" - elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then - MODEL_PATH="$MODEL_PATH/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_PATH" - echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" - echo " - $MODEL_PATH/$MODEL_NAME" - exit 1 - fi - elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then - MODEL_PATH="$MODEL_DIR/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" - echo " - $MODEL_DIR/$MODEL_NAME" - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -fi - -# ============================================================================= -# Node Selection -# ============================================================================= - -NUM_NODES=$((xP + yD)) -echo "NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD)" - -FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) -SELECTED_NODELIST_STR=$(echo "$SELECTED_NODES" | tr '\n' ',' | sed 's/,$//') - -# Docker privilege detection — evaluated per-node since group membership varies. -# Exported as a snippet so every srun participant resolves it locally. -export DOCKER_CMD_DETECT='if docker ps &>/dev/null 2>&1; then DOCKER_CMD=docker; else DOCKER_CMD="sudo docker"; fi' - -# Update SLURM environment variables -export SLURM_NNODES=$NUM_NODES -export SLURM_NTASKS=$NUM_NODES -export SLURM_JOB_NUM_NODES=$NUM_NODES -export SLURM_NPROCS=$NUM_NODES -export SLURM_JOB_NODELIST="$SELECTED_NODELIST_STR" -export SLURM_NODELIST="$SELECTED_NODELIST_STR" -export SLURM_TASKS_PER_NODE="1(x$NUM_NODES)" -export SLURM_NTASKS_PER_NODE=1 - -echo "" -echo "Selected nodes: $SELECTED_NODELIST_STR" - -# ============================================================================= -# IP Resolution -# ============================================================================= - -USER_NAME=$(whoami) -MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) -NODE0_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'ip route get 1.1.1.1') -NODE0_ADDR=$(echo "$NODE0_ADDR" | awk '/src/ {print $7}') - -IPS=() -for NODE in $SELECTED_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'ip route get 1.1.1.1') - IP=$(echo "$IP" | awk '/src/ {print $7}') - IPS+=("$IP") -done - -echo "Node IPs: ${IPS[*]}" - -DOCKER_MOUNT_PATH="/workspace" -WS_PATH="${DOCKER_MOUNT_PATH}/benchmarks/multi_node/amd_utils" - -NNODES=$NUM_NODES - -echo "MASTER_NODE: ${MASTER_NODE}" -echo "NODE0_ADDR: ${NODE0_ADDR}" -echo "NNODES: ${NNODES}" -echo "REPO DIR: ${DI_REPO_DIR}" -echo "USER: ${USER_NAME}" - -# Reduce log spam -export TQDM_MININTERVAL=20 - -# Translate the host-resolved MODEL_PATH to the Docker mount namespace -DOCKER_MODEL_PATH="${MODEL_PATH/#$MODEL_DIR//models}" - -export DI_REPO_DIR=$DI_REPO_DIR -export WS_PATH=$WS_PATH -export NNODES=$NNODES -export NODE0_ADDR=$NODE0_ADDR -export MODEL_PATH=$MODEL_PATH -export MODEL_DIR=$MODEL_DIR -export xP=$xP -export yD=$yD -export MODEL_NAME=$MODEL_NAME -export USER_NAME=$USER_NAME -export IPADDRS="$(echo "${IPS[*]}" | sed 's/ /,/g')" -export GPUS_PER_NODE=$GPUS_PER_NODE -export BENCH_INPUT_LEN=$BENCH_INPUT_LEN -export BENCH_OUTPUT_LEN=$BENCH_OUTPUT_LEN -export BENCH_RANDOM_RANGE_RATIO=$BENCH_RANDOM_RANGE_RATIO -export BENCH_NUM_PROMPTS_MULTIPLIER=$BENCH_NUM_PROMPTS_MULTIPLIER -export BENCH_MAX_CONCURRENCY=$BENCH_MAX_CONCURRENCY -export BENCH_REQUEST_RATE=$BENCH_REQUEST_RATE -export DRY_RUN="${DRY_RUN:-0}" -export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" -export KEEP_CONTAINERS="${KEEP_CONTAINERS:-0}" -export ENGINE=$ENGINE - -# Eval-related env vars (threaded from submit.sh) -export RUN_EVAL="${RUN_EVAL:-false}" -export EVAL_ONLY="${EVAL_ONLY:-false}" -export EVAL_CONC="${EVAL_CONC:-}" -export FRAMEWORK="${FRAMEWORK:-}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-}" -export IS_MULTINODE="${IS_MULTINODE:-false}" -export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" -export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" -export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" -export HF_TOKEN="${HF_TOKEN:-}" -export SCENARIO_TYPE="${SCENARIO_TYPE:-}" -export EVAL_LIMIT="${EVAL_LIMIT:-}" - -SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') -export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" - -# vLLM external router container. -# NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older -# dated tags are garbage-collected (manifest unknown) -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260716-1fbcde7}" -ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" -# Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). -CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" -export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" - -SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) - -# ============================================================================= -# RDMA QoS / DCQCN Pre-flight Check -# ============================================================================= -# Gate the run on NIC QoS (PFC/DSCP) and DCQCN config on every node before -# any container/GPU time is spent. Runs on the bare host (nicctl is a host -# tool). See rdma_check.sh for details. -RDMA_CHECK_SCRIPT="$(pwd)/helpers/rdma_check.sh" -if [[ "${SKIP_RDMA_CHECK:-0}" == "1" ]]; then - echo "[INFO] SKIP_RDMA_CHECK=1 set; skipping RDMA QoS/DCQCN pre-flight check" -elif [[ -f "$RDMA_CHECK_SCRIPT" ]]; then - echo "Checking RDMA QoS/DCQCN configuration on all $NUM_NODES allocated node(s)..." - srun --nodelist="$SELECTED_NODELIST_SRUN" --ntasks=$NUM_NODES bash "$RDMA_CHECK_SCRIPT" - RDMA_CHECK_RC=$? - if [[ $RDMA_CHECK_RC -ne 0 ]]; then - echo "FATAL: RDMA QoS/DCQCN pre-flight check failed on one or more nodes (see [FAIL] lines above)." - echo " Set SKIP_RDMA_CHECK=1 to bypass (not recommended -- MoRI cross-node transfers would run unprotected by PFC/DCQCN)." - exit 1 - fi - echo "RDMA QoS/DCQCN pre-flight check passed on all $NUM_NODES node(s)" -else - echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check" -fi - -cleanup() { - echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..." - # Backstop: on scancel/timeout/step-hang the foreground `exec docker run` - # client is killed before --rm can fire, so the container (and its GPU/ports/ - # :30000) leaks and the step never returns. Force-remove THIS job's container - # on every allocated node. Scoped to $DOCKER_CONT_NAME so it never touches - # other users' containers. (Ported from InferenceY 51ebfa88.) - srun --nodelist="$SELECTED_NODELIST_SRUN" \ - bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' 2>/dev/null || true - rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true - echo "[${SLURM_JOB_ID}] cleanup done." -} - -trap cleanup INT TERM HUP - -# Force NFS cache refresh on all nodes -echo "Refreshing NFS caches on all nodes..." -srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c ' - sync - ls -la '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils > /dev/null 2>&1 - stat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server.sh > /dev/null 2>&1 - cat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server.sh > /dev/null 2>&1 - echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null 2>&1 || true - echo "NFS cache refreshed on $(hostname)" -' - -# ============================================================================= -# Build engine-specific Docker environment variables -# ============================================================================= - -# Common env vars (always passed) -DOCKER_ENV_COMMON=( - -e SLURM_JOB_ID=\$SLURM_JOB_ID - -e SLURM_JOB_NODELIST=\$SLURM_JOB_NODELIST - -e NNODES=\$NNODES - -e NODE_RANK=\$SLURM_PROCID - -e NODE0_ADDR=\$NODE0_ADDR - -e MODEL_DIR=/models - -e MODEL_NAME=\$MODEL_NAME - -e GPUS_PER_NODE=\$GPUS_PER_NODE - -e xP=\$xP - -e yD=\$yD - -e IPADDRS=\$IPADDRS - -e BENCH_INPUT_LEN=\$BENCH_INPUT_LEN - -e BENCH_OUTPUT_LEN=\$BENCH_OUTPUT_LEN - -e BENCH_RANDOM_RANGE_RATIO=\$BENCH_RANDOM_RANGE_RATIO - -e BENCH_NUM_PROMPTS_MULTIPLIER=\$BENCH_NUM_PROMPTS_MULTIPLIER - -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY - -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE - -e TQDM_MININTERVAL=\$TQDM_MININTERVAL - -e BENCHMARK_LOGS_DIR=/benchmark_logs - -e ENGINE=\$ENGINE - -e WS_PATH=${WS_PATH} - -e RUN_EVAL=\$RUN_EVAL - -e EVAL_ONLY=\$EVAL_ONLY - -e \"EVAL_CONC=\$EVAL_CONC\" - -e FRAMEWORK=\$FRAMEWORK - -e PRECISION=\$PRECISION - -e MODEL_PREFIX=\$MODEL_PREFIX - -e RUNNER_TYPE=\$RUNNER_TYPE - -e RESULT_FILENAME=\$RESULT_FILENAME - -e SPEC_DECODING=\$SPEC_DECODING - # DISAGG was never forwarded into the container at all (not even under a - # different name), so process_agentic_result.py's env_bool("DISAGG") always - # defaulted to false in the result JSON regardless of the actual topology. - -e DISAGG=\${DISAGG:-false} - -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE - # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the - # *_SIZE vars server_sglang.sh uses for launch args) are what - # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without - # these the container never sees them (only *_SIZE was passed), so agentic - # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 - # for every multinode run. - -e PREFILL_TP=\$PREFILL_TP - -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS - -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP - -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP - # process_agentic_result.py reads these (distinct from the ENABLE_* launch - # booleans above) for agg_bmk.json's parallelism metadata; without them it - # defaulted to ep=1/dp_attention=false even for DEP8 arms. - -e PREFILL_EP=\${PREFILL_EP:-1} - -e PREFILL_DP_ATTN=\${PREFILL_DP_ATTN:-false} - -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} - -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} - -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-} - -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} - -e DURATION=\${DURATION:-1800} - -e IS_AGENTIC=\${IS_AGENTIC:-0} - -e KV_OFFLOADING=\${KV_OFFLOADING:-none} - -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} - -e KV_OFFLOAD_BACKEND_METADATA=\"\${KV_OFFLOAD_BACKEND_METADATA:-}\" - -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} - -e ENABLE_METRICS=\${ENABLE_METRICS:-0} - -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} - -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY:-random} - -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} - -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} - -e DECODE_TP_SIZE=\$DECODE_TP_SIZE - -e DECODE_TP=\$DECODE_TP - -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS - -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP - -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP - # See PREFILL_EP/PREFILL_DP_ATTN comment above -- decode counterpart. - -e DECODE_EP=\${DECODE_EP:-1} - -e DECODE_DP_ATTN=\${DECODE_DP_ATTN:-false} - -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE - -e IS_MULTINODE=\$IS_MULTINODE - -e DRY_RUN=\${DRY_RUN:-0} - # SWE-bench agentic eval runs inside this container and needs Modal/HF - # credentials to launch sandboxes and download datasets. - -e SWEBENCH_USE_MODAL=\${SWEBENCH_USE_MODAL:-false} - -e MODAL_TOKEN_ID=\${MODAL_TOKEN_ID:-} - -e MODAL_TOKEN_SECRET=\${MODAL_TOKEN_SECRET:-} - -e HF_TOKEN=\${HF_TOKEN:-} - -e SCENARIO_TYPE=\${SCENARIO_TYPE:-} - -e \"EVAL_LIMIT=\${EVAL_LIMIT:-}\" -) - -# Engine-specific env vars -if [[ "$ENGINE" == "vllm-disagg" ]]; then - DOCKER_ENV_ENGINE=( - -e VLLM_WS_PATH=${WS_PATH} - -e MODEL_PATH=$DOCKER_MODEL_PATH - -e UCX_TLS=tcp,self,shm,rocm_ipc,rocm_copy,cma - -e UCX_SOCKADDR_TLS_PRIORITY=tcp - -e UCX_MEMTYPE_CACHE=y - -e UCX_RNDV_SCHEME=get_zcopy - -e UCX_RNDV_THRESH=4k - -e UCX_ROCM_IPC_MIN_ZCOPY=0 - -e UCX_LOG_LEVEL=warn - -e HSA_ENABLE_SDMA=1 - -e PROXY_STREAM_IDLE_TIMEOUT=\${PROXY_STREAM_IDLE_TIMEOUT:-300} - -e PYTHONPYCACHEPREFIX=/tmp/pycache - ) -elif [[ "$ENGINE" == "atom-disagg" ]]; then - DOCKER_ENV_ENGINE=( - -e ATOM_WS_PATH=${WS_PATH} - -e PREFILL_PORT=${PREFILL_PORT:-8010} - -e DECODE_PORT=${DECODE_PORT:-8020} - -e ROUTER_PORT=${ROUTER_PORT:-30000} - -e HANDSHAKE_PORT=${HANDSHAKE_PORT:-6301} - -e MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.85} - -e KV_CACHE_DTYPE=${KV_CACHE_DTYPE:-fp8} - -e BLOCK_SIZE=${BLOCK_SIZE:-16} - -e MAX_NUM_SEQS=${MAX_NUM_SEQS:-256} - -e MAX_MODEL_LEN=${MAX_MODEL_LEN:-} - -e MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-} - -e EXTRA_SERVER_ARGS=\${EXTRA_SERVER_ARGS:-} - -e IBDEVICES=${IBDEVICES:-} - ) -else - DOCKER_ENV_ENGINE=( - -e SGLANG_WS_PATH=${WS_PATH} - ) -fi - -# HiCache / Mooncake settings are delivered via a bind-mounted config file rather -# than a long list of docker -e flags. Write it once to the shared benchmark-logs -# dir (already a host path, visible on every node) and mount it read-only at -# /config/hicache_mc.env, where env.sh sources it before applying its defaults. -# Empty values are preserved so env.sh's "${VAR:-default}" fallbacks still apply. -HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" -cat > "$HICACHE_MC_CONFIG" < $HICACHE_MC_CONFIG" - -# Engine-specific container filter for pre-clean -CONT_FILTER="name=^container_${ENGINE}_" - -# ============================================================================= -# Optional: separate benchmark-client image (agentic runs) — node-0 sibling -# ============================================================================= -# When CLIENT_IMAGE is set, node 0 runs the aiperf trace replay in its own -# sibling container built from CLIENT_IMAGE (which ships a pre-baked aiperf + -# deps), instead of rebuilding the aiperf venv inside the server container every -# run. Give the server container access to the host docker socket + CLI and the -# host paths the sibling needs for its bind mounts. These fragments are expanded -# at submit time and injected into the server `docker run` below; empty (no-op) -# when CLIENT_IMAGE is unset, so the in-container aiperf path is unchanged. -CLIENT_DOCKER_MOUNTS="" -CLIENT_DOCKER_ENV="" -if [[ -n "${CLIENT_IMAGE:-}" ]]; then - HOST_DOCKER_BIN="$(command -v docker || echo /usr/bin/docker)" - CLIENT_DOCKER_MOUNTS="-v /var/run/docker.sock:/var/run/docker.sock -v ${HOST_DOCKER_BIN}:/usr/bin/docker" - CLIENT_DOCKER_ENV="-e CLIENT_IMAGE=${CLIENT_IMAGE} -e HOST_REPO_DIR=${DI_REPO_DIR} -e HOST_MODEL_DIR=${MODEL_DIR} -e HOST_BENCH_LOGS=${BENCHMARK_LOGS_DIR} -e CLIENT_CONT_NAME=${CLIENT_CONT_NAME}" - echo "[client] node-0 sibling benchmark-client image enabled: ${CLIENT_IMAGE}" - # Best-effort pre-pull on all nodes so node 0's sibling launch doesn't stall. - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD pull '"$CLIENT_IMAGE"' >/dev/null 2>&1 || true' 2>/dev/null || true -fi - -srun \ - --nodelist="$SELECTED_NODELIST_SRUN" \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - bash -lc " -set -euo pipefail - -echo \"Rank \$SLURM_PROCID on \$(hostname)\" - -# Per-node docker privilege detection -eval \"\$DOCKER_CMD_DETECT\" -echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" - -# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack) -RDMA_MOUNTS=() -if [[ "$ENGINE" == "atom-disagg" ]]; then - -# When the container base OS differs from the host (e.g. Ubuntu 24.04 image -# on a 22.04 host), the container's bundled libibverbs/libionic may be -# ABI-incompatible with the host kernel drivers. Detect the NIC type and -# bind-mount the host's out-of-tree RDMA userspace libraries into the -# container so the RDMA stack always matches the running kernel. -_detect_nic_type() { - if [[ -n \"\${MORI_NIC_TYPE:-}\" ]]; then echo \"\$MORI_NIC_TYPE\"; return; fi - local bnxt=0 mlx5=0 ionic=0 - if [[ -d /sys/class/infiniband ]]; then - for dev in /sys/class/infiniband/*; do - local name; name=\$(basename \"\$dev\") - case \"\$name\" in - bnxt_re*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; - *) - local drv; drv=\$(basename \"\$(readlink -f \"\$dev/device/driver\" 2>/dev/null)\" 2>/dev/null || true) - case \"\$drv\" in bnxt*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; esac ;; - esac - done - fi - if (( bnxt >= mlx5 && bnxt >= ionic && bnxt > 0 )); then echo bnxt - elif (( ionic >= mlx5 && ionic > 0 )); then echo ionic - else echo mlx5; fi -} - -_find_host_ibverbs() { - for c in /usr/lib64/libibverbs.so.1 /lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1.14.39.0 /usr/lib/x86_64-linux-gnu/libibverbs.so.1; do - local r; r=\$(readlink -f \"\$c\" 2>/dev/null || true) - [[ \"\$r\" == *libibverbs.so.1.14.57.0 ]] && continue - if [[ -f \"\$r\" ]]; then echo \"\$r\"; return; fi - done -} - -_NIC_TYPE=\$(_detect_nic_type) -echo \"[rdma] NIC type: \${_NIC_TYPE} on \$(hostname)\" - -if [[ \"\$_NIC_TYPE\" == \"ionic\" || \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - _host_ibv=\$(_find_host_ibverbs) - if [[ -n \"\$_host_ibv\" ]]; then - RDMA_MOUNTS+=(-v \"\$_host_ibv:/lib/x86_64-linux-gnu/libibverbs.so.1\") - fi -fi - -if [[ \"\$_NIC_TYPE\" == \"ionic\" ]]; then - for _dir in /usr/local/lib /usr/lib/x86_64-linux-gnu; do - for _lib in \"\$_dir\"/libionic*.so; do - [[ -f \"\$_lib\" ]] || continue - _real=\$(readlink -f \"\$_lib\") - [[ -f \"\$_real\" ]] && RDMA_MOUNTS+=(-v \"\$_real:\$_real\") - RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - done - if [[ -d /usr/lib/x86_64-linux-gnu/libibverbs ]]; then - for _lib in /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:\$_lib\") - done - fi - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -elif [[ \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - for _lib in /usr/local/lib/libbnxt_re-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/libibverbs/\$(basename \"\$_lib\")\") - done - for _lib in /usr/local/lib/libbnxt_re.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -fi - -if [[ \${#RDMA_MOUNTS[@]} -gt 0 ]]; then - echo \"[rdma] bind-mounts: \${RDMA_MOUNTS[*]}\" -else - echo \"[rdma] no out-of-tree RDMA mounts needed\" -fi -fi # end: if ENGINE == atom-disagg - -# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released -# before the drain gate. stop-only left containers in Created/Exited state -# on some nodes. -\$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true -\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop -t 15 || true -\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD rm -f || true -sleep 2 - -# GPU drain gate: fail fast on leftover VRAM use instead of OOMing in model -# load ~15 min later. Reuses wait_for_amd_gpu_clean from benchmark_lib.sh. -if [[ \"${SKIP_GPU_SANITY:-0}\" == \"1\" ]]; then - echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU pre-flight drain check\" -else - # Unset so benchmark_lib.sh's unrelated agentic KV_OFFLOADING check doesn't exit 1 here. - bash -c \"unset IS_AGENTIC SCENARIO_TYPE; source $DI_REPO_DIR/benchmarks/benchmark_lib.sh && wait_for_amd_gpu_clean\" -fi - -# Start vLLM external router container on node 0 -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true - \$DOCKER_CMD run -d \ - --name \"$ROUTER_CONT_NAME\" \ - --network host \ - --ulimit nofile=1048576:1048576 \ - -v /tmp:/run_logs \ - \"$VLLM_ROUTER_IMAGE\" \ - bash -lc \"mkdir -p /run_logs/slurm_job-${SLURM_JOB_ID} && exec vllm-router \ - --vllm-pd-disaggregation \ - --kv-connector moriio \ - --vllm-discovery-address 0.0.0.0:${PROXY_PING_PORT} \ - --port ${ROUTER_PORT} \ - --host 0.0.0.0 \ - --policy consistent_hash \ - --prefill-policy consistent_hash \ - --decode-policy consistent_hash \ - --log-level info 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/vllm_router_\$(hostname).log \" -fi - -# Skip exec on vllm-disagg rank 0 so we can stop the router after the main -# container exits. Without this, decode nodes block forever waiting for the -# router port to close (the router is a separate container). -MAYBE_EXEC=exec -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - MAYBE_EXEC= - set +e -fi - -\$MAYBE_EXEC \$DOCKER_CMD run \ - --init \ - --stop-timeout 10 \ - --device /dev/dri \ - --device /dev/kfd \ - --device /dev/infiniband \ - --device=/dev/infiniband/rdma_cm \ - --device=/dev/infiniband/uverbs0 \ - --device=/dev/infiniband/uverbs1 \ - --device=/dev/infiniband/uverbs2 \ - --device=/dev/infiniband/uverbs3 \ - --device=/dev/infiniband/uverbs4 \ - --device=/dev/infiniband/uverbs5 \ - --device=/dev/infiniband/uverbs6 \ - --device=/dev/infiniband/uverbs7 \ - --ulimit memlock=-1 \ - --ulimit stack=67108864 \ - --ulimit nofile=1048576:1048576 \ - --network host \ - --ipc host \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v /sys:/sys \ - $(command -v nicctl >/dev/null 2>&1 && echo "-v $(which nicctl):/usr/sbin/nicctl") \ - -v ${MODEL_DIR}:/models \ - -v \$HOME/.ssh:/root/.ssh \ - --shm-size 128G \ - -v /tmp:/run_logs \ - -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ - -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ - -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ - ${EXTRA_DOCKER_MOUNTS:-} \ - ${CLIENT_DOCKER_MOUNTS} \ - \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ - ${DOCKER_ENV_COMMON[*]} \ - ${DOCKER_ENV_ENGINE[*]} \ - ${CLIENT_DOCKER_ENV} \ - --name \"$DOCKER_CONT_NAME\" \ - --entrypoint \"\" \ - \"$DOCKER_IMAGE_NAME\" bash -lc ' - set -o pipefail - mkdir -p /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"' - '"$RUN_FILE_FULL"' 2>&1 | tee /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"'/server_\$(hostname).log - ' - -# Only reached when exec was skipped (vllm-disagg rank 0) -DOCKER_EXIT_CODE=\$? -echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-router...\" -\$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true -exit \$DOCKER_EXIT_CODE -" - -if [[ "${KEEP_CONTAINERS}" != "1" ]]; then - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' - - # Clean up vLLM external router container on node 0 - if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then - srun --nodes=1 --ntasks=1 --nodelist="$MASTER_NODE" bash -c ' - eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$ROUTER_CONT_NAME"' 2>/dev/null || true - ' - fi -fi diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml deleted file mode 100644 index 88a61ccc10..0000000000 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ /dev/null @@ -1,436 +0,0 @@ -# Model-specific SGLang server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# base_flags: str # Common flags for both prefill and decode -# mtp_flags: str # Appended to decode when DECODE_MTP_SIZE > 0 -# dp_flags: str # Appended when DP attention is enabled (prefill or decode) -# ep_flags: str # Appended when EP is enabled. EP-specific MoE knobs only -# # (a2a backend, deepep mode, ep-dispatch algorithm). With -# # ep=1 these are dropped so the MoE runs tensor-parallel (TP). -# prefill: -# mem_fraction_static: float -# disable_radix_cache: bool -# dp: # Config when data-parallel attention is enabled -# max_running_requests: int -# chunked_prefill_size: str # Can be integer or bash arithmetic expression -# cuda_graph_bs: str # Space-separated values -# no_dp: # Config when data-parallel attention is disabled -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str # "start-end" expanded via seq -# decode: -# mem_fraction_static: float -# prefill_round_robin_balance: bool -# dp: -# max_running_requests: int -# chunked_prefill_size: str -# cuda_graph_bs_range: str -# ep_only: # Config when EP is enabled but DP is disabled -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str -# no_dp: -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str - -DeepSeek-V3: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-V3-0324: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--enable-dp-attention --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--enable-dp-attention --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -GLM-5-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'" - mtp_flags: "" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-Preview: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: 16384 - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-v2: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-eagle-topk 1 --speculative-attention-mode decode " - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --stream-interval 100 --tokenizer-worker-num 32 " - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - context_length: 9217 - max_total_tokens: 131072 - enable_two_batch_overlap: true - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-512" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-V4-Pro-AgentX: - base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" - dp_flags: "--enable-dp-attention --enable-prefill-delayer" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" - prefill: - mem_fraction_static: 0.85 - disable_radix_cache: false - disable_cuda_graph: true - dp: - max_running_requests: 1024 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" # dsv4 compressor kernel uint16 token cap (255*256) - context_length: 1048576 - # max_total_tokens: 1048576 - no_dp: - max_running_requests: 64 - # Small prefill chunks interleave long-context agentic prefills across - # requests instead of letting one ~100K-token prefill monopolize the - # engine (the conc>=16 queue-saturation / decode-stall failure mode). - # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192). - # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192 - # (32*256) stays a page-size multiple well under that cap. - chunked_prefill_size: 8192 - context_length: 1048576 - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - disagg_decode_enable_radix_cache: false - dp: - max_running_requests: 1024 - cuda_graph_bs_range: "1-128" - no_dp: - max_running_requests: 128 - cuda_graph_bs_range: "1-128" - -DeepSeek-V4-Pro-DI: - base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori" - # DeepSeek-V4 built-in MTP: load via the EAGLE spec path with eagle-topk 1 (a single - # MTP chain). NOT NEXTN -- the V3/R1 NEXTN loader crashes the dsv4 decode server at - # init on the V4 architecture. Mirrors the sglang dsv4 recipes - # (srt-slurm-recipes/sglang/deepseek-v4/*: EAGLE + eagle-topk 1, in-checkpoint draft, - # no draft-model-path). Consumed by build_server_config only when DECODE_MTP_SIZE > 0 - # (spec-decoding: mtp); a no-op for the spec-none base scenario (DECODE_MTP_SIZE=0). - # build_server_config appends --speculative-num-steps/--speculative-num-draft-tokens - # from DECODE_MTP_SIZE. - mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - disable_cuda_graph: true - dp: - max_running_requests: 1024 - chunked_prefill_size: 131072 - context_length: 9217 - max_total_tokens: 2097152 - no_dp: - max_running_requests: 128 - chunked_prefill_size: 131072 - context_length: 9217 - max_total_tokens: 2097152 - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 1024 - cuda_graph_bs_range: "1-128" - no_dp: - max_running_requests: 128 - cuda_graph_bs_range: "1-128" diff --git a/benchmarks/multi_node/amd_utils/models_atom.yaml b/benchmarks/multi_node/amd_utils/models_atom.yaml deleted file mode 100644 index 485320dffa..0000000000 --- a/benchmarks/multi_node/amd_utils/models_atom.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Model-specific ATOM server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# env: str # Space-separated KEY=VALUE pairs exported unconditionally -# tp_dp_flags: str # Shared TP+DPA flags (fallback when prefill/decode-specific keys are absent) -# prefill_tp_dp_flags: str # TP+DPA flags for prefill only (overrides tp_dp_flags) -# decode_tp_dp_flags: str # TP+DPA flags for decode only (overrides tp_dp_flags) -# tp_dp_env: str # Space-separated KEY=VALUE pairs exported only in TP+DPA mode -# ep_dp_flags: str # Shared EP+DPA flags (fallback when prefill/decode-specific keys are absent) -# prefill_ep_dp_flags: str # EP+DPA flags for prefill only (overrides ep_dp_flags) -# decode_ep_dp_flags: str # EP+DPA flags for decode only (overrides ep_dp_flags) -# ep_dp_env: str # Space-separated KEY=VALUE pairs exported only in EP+DPA mode -# mtp_flags: str # Flags passed to SPEC_ARGS before $DECODE_MTP_SIZE (e.g. "--method mtp --num-speculative-tokens") -# kv_cache_flags: str # Full --kv_cache_dtype flag string (e.g. "--kv_cache_dtype fp8", or "" for none) -# online_quant_config: str # JSON string passed to --online_quant_config (used when DPA is disabled) -# online_quant_dpa_config: str # JSON string passed to --online_quant_config when DPA is enabled (falls back to online_quant_config) -# block_size: str # --block-size value (overrides server_atom.sh default of 16) -# mem_frac_static: str # --gpu-memory-utilization value (overrides default of 0.85) -# max_model_len: str # --max-model-len value (overrides default of unset) -# max_num_seqs: str # --max-num-seqs value (overrides default of 256) -# max_num_batched_tokens: str # --max-num-batched-tokens value (overrides default of unset) -# scheduler_delay_factor: str # --scheduler-delay-factor value (overrides default of unset) - -DeepSeek-V4-Pro: - env: "ATOM_MOE_GU_ITLV=1 AITER_BF16_FP8_MOE_BOUND=0" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention --enable-tbo" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo" - decode_tp_dp_flags: "--enable-dp-attention --enable-tbo" - tp_dp_env: "GPU_MAX_HW_QUEUES=5 ATOM_CPU_AFFINITY=1" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method mtp --num-speculative-tokens" - -MiniMax-M3-MXFP4: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_FORCE_ATTN_TRITON=1" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo prefill" - decode_tp_dp_flags: "--enable-dp-attention" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens" - online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - online_quant_dpa_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - block_size: "128" - mem_frac_static: "0.8" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" - -Kimi-K2.5-MXFP4: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 AITER_MXFP4_INTERMEDIATE=1" - kv_cache_flags: "--kv_cache_dtype fp8" - mem_frac_static: "0.9" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" - -MiniMax-M3-MXFP8: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_FORCE_ATTN_TRITON=1" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo prefill" - decode_tp_dp_flags: "--enable-dp-attention" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens" - online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - online_quant_dpa_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*.gate.*","*.block_sparse_moe.experts*"]}' - block_size: "128" - mem_frac_static: "0.8" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml deleted file mode 100644 index 79d4f4d8f8..0000000000 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Model-specific vLLM server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the model identifier -# used in amd-master.yaml and the directory/HF-cache name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# prefill_flags: str # vLLM CLI flags for prefill workers -# decode_flags: str # vLLM CLI flags for decode workers -# env: str # Space-separated KEY=VALUE pairs exported before vllm serve -# hf_dir: str # (optional) On-disk directory name if it differs from the key -# # e.g. HF cache layout: models--amd--Kimi-K2.5-MXFP4 - -Llama-3.1-405B-Instruct-FP8-KV: - prefill_flags: "--tensor-parallel-size 8 --kv-cache-dtype fp8" - decode_flags: "--tensor-parallel-size 8 --kv-cache-dtype fp8" - env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" - -amd-Llama-3.3-70B-Instruct-FP8-KV: - prefill_flags: "--tensor-parallel-size 8 --max-model-len 65536 --kv-cache-dtype fp8" - decode_flags: "--tensor-parallel-size 8 --max-model-len 65536 --kv-cache-dtype fp8" - env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" - -Kimi-K2.5-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" - decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" - hf_dir: "models--amd--Kimi-K2.5-MXFP4" - -MiniMax-M2.5: - # AITER fused-MoE kernel fmoe_bf16_blockscaleFp8_g1u1_vs_silu_32x384 for gfx950 writes OOB when run with MiniMax's shapes at M=8K(=num batched tokens), crashing vllm during AITER warmup. - # Set token budget to 4k to avoid using that shape, instead of disabling AITER_MOE. - prefill_flags: "--max-num-batched-tokens 4K --tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --no-enable-prefix-caching --gpu-memory-utilization 0.95 --block-size 32" - decode_flags: "--max-num-batched-tokens 4K --tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --no-enable-prefix-caching --gpu-memory-utilization 0.95 --block-size 32" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ENGINE_READY_TIMEOUT_S=3600 VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1" - hf_dir: "models--MiniMaxAI--MiniMax-M2.5" - -MiniMax-M3-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --max-num-batched-tokens 32768 --max-num-seqs 512 --block-size 128 --language-model-only --attention-backend TRITON_ATTN --moe-backend aiter --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - decode_flags: "--tensor-parallel-size 8 --max-num-batched-tokens 32768 --max-num-seqs 512 --block-size 128 --language-model-only --attention-backend TRITON_ATTN --moe-backend aiter --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" - prefill_env: "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048" - hf_dir: "models--amd--MiniMax-M3-MXFP4" - -gpt-oss-120b: - prefill_flags: "--tensor-parallel-size 8" - decode_flags: "--tensor-parallel-size 8" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_TRITON_BF16_GEMM=0 VLLM_USE_AITER_UNIFIED_ATTENTION=1 VLLM_ROCM_USE_AITER_MHA=0 ROCM_TRITON_MOE_PRESHUFFLE_SCALES=0" - -MiniMax-M3-MXFP8: - # MiniMax-M3 MXFP8 disagg, no EP. The --tensor-parallel-size 8 below is just a - # placeholder: server_vllm.sh sed-rewrites it to PREFILL_TP_SIZE/DECODE_TP_SIZE - # from the master-config prefill/decode tp (the sweep uses TP4 workers only, - # varying the prefill:decode worker ratio rather than TP). - # --block-size 128 is mandatory (MSA sparse/index cache); text-only benchmark - # so --language-model-only frees the vision encoder. gfx950 uses FP8 KV cache. - # Serve flags + env kept in sync with the single-node recipe - # benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x.sh: - # --moe-backend aiter (AITER fused MoE), --linear-backend emulation, and the - # larger --max-num-batched-tokens 32768 prefill budget; plus the AITER - # router-append shared-experts fusion (self-disables under EP; the sweep is - # EP1 so it stays active) and INT6 quick all-reduce quantization. - prefill_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend aiter --linear-backend emulation --max-num-batched-tokens 32768 --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - decode_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend aiter --linear-backend emulation --max-num-batched-tokens 32768 --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" - hf_dir: "models--MiniMaxAI--MiniMax-M3-MXFP8" diff --git a/benchmarks/multi_node/amd_utils/node_excludes.yaml b/benchmarks/multi_node/amd_utils/node_excludes.yaml deleted file mode 100644 index 6ba840328e..0000000000 --- a/benchmarks/multi_node/amd_utils/node_excludes.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Per (framework, model) SLURM node exclusions, consumed by submit.sh. -# -# A rule applies when FRAMEWORK matches "framework" exactly and MODEL_NAME is -# in "models"; its exclude_nodes are then passed to sbatch --exclude. Any -# (framework, model) combination with no matching rule gets an empty exclude -# list (i.e. no nodes excluded). -# -# SLURM_EXCLUDE_NODES env var, if set, always overrides this file. - -rules: - - framework: vllm-disagg - models: - - Kimi-K2.5-MXFP4 - exclude_nodes: mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12 diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh deleted file mode 100755 index b62ca58160..0000000000 --- a/benchmarks/multi_node/amd_utils/server.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Multi-Engine Disaggregated Server Dispatcher -# ============================================================================= -# Dispatches to the engine-specific server launcher based on ENGINE env var. -# ENGINE=sglang-disagg (default) -> server_sglang.sh (SGLang + MoRI) -# ENGINE=vllm-disagg -> server_vllm.sh (vLLM + Nixl/MoRI-IO) -# ENGINE=atom-disagg -> server_atom.sh (ATOM + mooncake) -# ============================================================================= - -ENGINE="${ENGINE:-sglang-disagg}" -WS_PATH="${WS_PATH:-${SGLANG_WS_PATH:-${VLLM_WS_PATH:-${ATOM_WS_PATH:-$(dirname "${BASH_SOURCE[0]}")}}}}" -export WS_PATH ENGINE - -echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - source "$WS_PATH/server_vllm.sh" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - export ATOM_WS_PATH="$WS_PATH" - source "$WS_PATH/server_atom.sh" -else - source "$WS_PATH/server_sglang.sh" -fi diff --git a/benchmarks/multi_node/amd_utils/server_atom.sh b/benchmarks/multi_node/amd_utils/server_atom.sh deleted file mode 100755 index 2fee5679ca..0000000000 --- a/benchmarks/multi_node/amd_utils/server_atom.sh +++ /dev/null @@ -1,622 +0,0 @@ -#!/bin/bash -# ATOM Disaggregated Server Launcher -# ============================================================================= -# Uses atom.entrypoints.openai_server with mooncake RDMA KV transfer. -# Mirrors server_sglang.sh topology (dynamic xP/yD) but adapts to ATOM's -# explicit kv-transfer-config and atomesh router. -# -# Key differences from server_sglang.sh: -# - Engine: atom.entrypoints.openai_server (not sglang.launch_server) -# - KV transfer: mooncake (--kv-transfer-config JSON) -# - Router: atomesh (not sglang_router) -# - Prefill port: $PREFILL_PORT (default 8010) / Decode port: $DECODE_PORT (default 8020) -# - Router port: $ROUTER_PORT (default 8000) -# ============================================================================= - -# ============================================================================= -# Environment Configuration -# ============================================================================= - -NODE0_ADDR="${NODE0_ADDR:-localhost}" -NODE_RANK="${NODE_RANK:-0}" -MODEL_DIR="${MODEL_DIR:-}" -MODEL_NAME="${MODEL_NAME:-}" - -xP="${xP:-1}" -yD="${yD:-1}" - -IPADDRS="${IPADDRS:-localhost}" - -# Parallelism -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP}" - -# MTP -DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" - -# ATOM server ports (different from SGLang which uses 8000 for all) -PREFILL_PORT="${PREFILL_PORT:-8010}" -DECODE_PORT="${DECODE_PORT:-8020}" -ROUTER_PORT="${ROUTER_PORT:-8000}" -HANDSHAKE_PORT="${HANDSHAKE_PORT:-6301}" - -# ATOM server tuning — defaults applied after YAML load (env var > YAML > shell default) -EXTRA_SERVER_ARGS="${EXTRA_SERVER_ARGS:-}" - -# Benchmark Configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" - -DRY_RUN="${DRY_RUN:-0}" -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -# ============================================================================= -# Dependencies and Environment Setup -# ============================================================================= - -source $ATOM_WS_PATH/setup_deps.sh -source $ATOM_WS_PATH/env_atom.sh - -# Raise FD limit — lm-eval with high num_concurrent can exhaust the default 1024 -ulimit -n 65536 2>/dev/null || ulimit -n 8192 2>/dev/null || true -echo "ulimit -n (open files): $(ulimit -n)" - -host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}') -if [[ -z "$host_ip" ]]; then - host_ip=$(hostname -I 2>/dev/null | awk '{print $1}') -fi -host_name=$(hostname) - -# ============================================================================= -# Model-Specific Configuration from YAML -# ============================================================================= -# Load model-specific config from YAML (single parse for all fields) -set -x -_yaml_tmp=$(mktemp) -python3 << PYEOF > "$_yaml_tmp" -import yaml -with open('${ATOM_WS_PATH}/models_atom.yaml') as f: - m = yaml.safe_load(f).get('${MODEL_NAME}', {}) -def sh(v): return v.replace("'", "'\\''") -print(f"MODEL_ENVS='{sh(m.get('env', ''))}'") -_tp_dp = m.get('tp_dp_flags', '') -print(f"PREFILL_MODEL_TP_DP_FLAGS='{sh(m.get('prefill_tp_dp_flags', _tp_dp))}'") -print(f"DECODE_MODEL_TP_DP_FLAGS='{sh(m.get('decode_tp_dp_flags', _tp_dp))}'") -_ep_dp = m.get('ep_dp_flags', '') -print(f"PREFILL_MODEL_EP_DP_FLAGS='{sh(m.get('prefill_ep_dp_flags', _ep_dp))}'") -print(f"DECODE_MODEL_EP_DP_FLAGS='{sh(m.get('decode_ep_dp_flags', _ep_dp))}'") -print(f"MODEL_TP_DP_ENV='{sh(m.get('tp_dp_env', ''))}'") -print(f"MODEL_EP_DP_ENV='{sh(m.get('ep_dp_env', ''))}'") -print(f"MODEL_MTP_FLAGS='{sh(m.get('mtp_flags', ''))}'") -print(f"MODEL_KV_ARG='{sh(m.get('kv_cache_flags', ''))}'") -print(f"_ONLINE_QUANT_CONFIG='{sh(m.get('online_quant_config', ''))}'") -print(f"_ONLINE_QUANT_DPA_CONFIG='{sh(m.get('online_quant_dpa_config', m.get('online_quant_config', '')))}'") -print(f"_YAML_BLOCK_SIZE='{sh(m.get('block_size', ''))}'") -print(f"_YAML_MEM_FRAC_STATIC='{sh(m.get('mem_frac_static', ''))}'") -print(f"_YAML_MAX_MODEL_LEN='{sh(m.get('max_model_len', ''))}'") -print(f"_YAML_MAX_NUM_SEQS='{sh(m.get('max_num_seqs', ''))}'") -print(f"_YAML_MAX_NUM_BATCHED_TOKENS='{sh(m.get('max_num_batched_tokens', ''))}'") -print(f"_YAML_SCHEDULER_DELAY_FACTOR='{sh(m.get('scheduler_delay_factor', ''))}'") -PYEOF -# shellcheck source=/dev/null -source "$_yaml_tmp" -rm -f "$_yaml_tmp" -unset _yaml_tmp - -# Apply server-tuning: YAML > env var > shell default -# (job.slurm injects BLOCK_SIZE/MEM_FRAC_STATIC/MAX_NUM_SEQS with hardcoded -# defaults into the Docker env, so env-first would always shadow the YAML.) -BLOCK_SIZE="${_YAML_BLOCK_SIZE:-${BLOCK_SIZE:-16}}" -MEM_FRAC_STATIC="${_YAML_MEM_FRAC_STATIC:-${MEM_FRAC_STATIC:-0.85}}" -MAX_MODEL_LEN="${_YAML_MAX_MODEL_LEN:-${MAX_MODEL_LEN:-}}" -MAX_NUM_SEQS="${_YAML_MAX_NUM_SEQS:-${MAX_NUM_SEQS:-256}}" -MAX_NUM_BATCHED_TOKENS="${_YAML_MAX_NUM_BATCHED_TOKENS:-${MAX_NUM_BATCHED_TOKENS:-}}" -SCHEDULER_DELAY_FACTOR="${_YAML_SCHEDULER_DELAY_FACTOR:-${SCHEDULER_DELAY_FACTOR:-}}" -unset _YAML_BLOCK_SIZE _YAML_MEM_FRAC_STATIC _YAML_MAX_MODEL_LEN _YAML_MAX_NUM_SEQS _YAML_MAX_NUM_BATCHED_TOKENS _YAML_SCHEDULER_DELAY_FACTOR - -# ============================================================================= -# Cluster Topology Configuration -# ============================================================================= - -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE)) -DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE)) -NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) - -# Build prefill IP list and atomesh --prefill args -PREFILL_ARGS="" -PREFILL_IPS=() -for i in $(seq 0 $((xP - 1))); do - idx=$((i * PREFILL_NODES_PER_WORKER)) - PREFILL_IPS[$i]="${IP_ARRAY[$idx]}" - PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$idx]}:${PREFILL_PORT}" -done - -# Build decode IP list and atomesh --decode args -DECODE_ARGS="" -DECODE_IPS=() -for i in $(seq 0 $((yD - 1))); do - idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) - DECODE_IPS[$i]="${IP_ARRAY[$idx]}" - DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$idx]}:${DECODE_PORT}" -done - -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP}" - - - - -# Parallel args -PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE") #TP -ONLINE_QUANT_ARG="" -if [ "$PREFILL_ENABLE_DP" = "true" ]; then - if [ "$PREFILL_ENABLE_EP" = "true" ]; then #EP+DPA - PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE" ${PREFILL_MODEL_EP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_EP_DP_ENV}; do export "$_dp_env_pair"; done - else #TP+DPA - PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE" ${PREFILL_MODEL_TP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_TP_DP_ENV}; do export "$_dp_env_pair"; done - fi - if [[ -n "$_ONLINE_QUANT_DPA_CONFIG" ]]; then - ONLINE_QUANT_ARG="--online_quant_config '${_ONLINE_QUANT_DPA_CONFIG}'" - fi -else - if [[ -n "$_ONLINE_QUANT_CONFIG" ]]; then - ONLINE_QUANT_ARG="--online_quant_config '${_ONLINE_QUANT_CONFIG}'" - fi -fi - -DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE") #TP -if [ "$DECODE_ENABLE_DP" = "true" ]; then - if [ "$DECODE_ENABLE_EP" = "true" ]; then #EP+DPA - DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE" ${DECODE_MODEL_EP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_EP_DP_ENV}; do export "$_dp_env_pair"; done - else #TP+DPA - DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE" ${DECODE_MODEL_TP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_TP_DP_ENV}; do export "$_dp_env_pair"; done - fi -fi -unset _dp_env_pair -unset _ONLINE_QUANT_CONFIG _ONLINE_QUANT_DPA_CONFIG - -for _env_pair in ${MODEL_ENVS}; do - export "$_env_pair" -done -unset _env_pair - -# MTP args -SPEC_ARGS=() -if [[ -n "$MODEL_MTP_FLAGS" && "${DECODE_MTP_SIZE:-0}" -gt 0 ]]; then - SPEC_ARGS=(${MODEL_MTP_FLAGS} "$DECODE_MTP_SIZE") -fi - -# KV cache arg - full flag string from YAML -KV_CACHE_ARG="${MODEL_KV_ARG}" - -# Optional model length / batched-token cap -MODEL_LEN_ARGS="" -if [[ -n "$MAX_MODEL_LEN" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --max-model-len ${MAX_MODEL_LEN}" -fi -if [[ -n "$MAX_NUM_BATCHED_TOKENS" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS}" -fi -if [[ -n "$SCHEDULER_DELAY_FACTOR" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --scheduler-delay-factor ${SCHEDULER_DELAY_FACTOR}" -fi - - -cat < prefill node 0 + router -# rank 1 .. (NODE_OFFSET-1) -> remaining prefill nodes -# rank NODE_OFFSET .. -> decode nodes -# ============================================================================= -if [ "$NODE_RANK" -eq 0 ]; then - # ────────────────────────────────────────────────────────────────────────── - # Node 0: prefill server (producer) + atomesh router - # ────────────────────────────────────────────────────────────────────────── - echo "NODE INFO =======================================" - echo "${host_name}:${host_ip} is Prefill Node 0 + Router" - echo "Prefill TP=${PREFILL_TP_SIZE}, Decode TP=${DECODE_TP_SIZE}" - echo "Prefill servers: ${PREFILL_ARGS}" - echo "Decode servers: ${DECODE_ARGS}" - echo "================================================" - - PREFILL_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${PREFILL_PORT} \ - --trust-remote-code \ - ${PREFILL_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_producer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill0_${host_name}.log & - set +x - prefill0_pid=$! - fi - - # Wait for all prefill and decode servers to be ready - WAIT_SERVER_TIMEOUT="${WAIT_SERVER_TIMEOUT:-2500}" - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for all servers to be up (timeout=${WAIT_SERVER_TIMEOUT}s)..." - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for prefill/decode /health endpoints" - else - _deadline=$(( $(date +%s) + WAIT_SERVER_TIMEOUT )) - for _ip in "${PREFILL_IPS[@]}"; do - echo "[wait] prefill http://${_ip}:${PREFILL_PORT}/health" - while ! curl -sf --max-time 10 "http://${_ip}:${PREFILL_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_deadline ]]; then - echo "[wait][FAIL] prefill ${_ip}:${PREFILL_PORT} not ready after ${WAIT_SERVER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] prefill ${_ip}:${PREFILL_PORT} ready" - done - for _ip in "${DECODE_IPS[@]}"; do - echo "[wait] decode http://${_ip}:${DECODE_PORT}/health" - while ! curl -sf --max-time 10 "http://${_ip}:${DECODE_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_deadline ]]; then - echo "[wait][FAIL] decode ${_ip}:${DECODE_PORT} not ready after ${WAIT_SERVER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] decode ${_ip}:${DECODE_PORT} ready" - done - fi - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "All servers up. Starting atomesh router..." - - ROUTER_CMD="/usr/local/bin/atomesh launch \ - --host 0.0.0.0 --port ${ROUTER_PORT} \ - --pd-disaggregation \ - ${PREFILL_ARGS} \ - ${DECODE_ARGS} \ - --policy random \ - --backend atom \ - --log-level info \ - --disable-health-check \ - --disable-circuit-breaker \ - --prometheus-port 29100" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $ROUTER_CMD" - else - ROUTER_LOG_FILE="/tmp/slurm_job-${SLURM_JOB_ID}_router_${host_name}.log" - set -x - eval "$ROUTER_CMD" 2>&1 | tee "$ROUTER_LOG_FILE" & - set +x - proxy_pid=$! - - # Wait for router to accept connections - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-300}" - echo "[wait] router http://0.0.0.0:${ROUTER_PORT}/v1/models (timeout=${WAIT_ROUTER_TIMEOUT}s)" - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/v1/models" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${ROUTER_PORT}/v1/models not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router /v1/models ready" - - echo "Router is ready for benchmarking" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Ready for benchmarking on ${host_name}:${host_ip}" - - cd $ATOM_WS_PATH - - export IS_MTP="false" - if [[ -n "$MODEL_MTP_FLAGS" && "${DECODE_MTP_SIZE:-0}" -gt 0 ]]; then - export IS_MTP="true" - fi - - BENCH_CMD="bash $ATOM_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - # Run evaluation if requested (before killing router) - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - # Health check: verify the router is still serving before running eval. - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/health" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --framework lm-eval --port ${ROUTER_PORT} (conc=${EVAL_CONCURRENT_REQUESTS})" - else - MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" run_eval --framework lm-eval --port ${ROUTER_PORT} - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 - EVAL_FAILED=1 - else - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - export DECODE_NUM_WORKERS="${yD}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - - MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" append_lm_eval_summary - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - mkdir -p "$EVAL_COPY_DIR" - for f in meta_env.json; do - [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" - done - find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - - echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" - fi - fi - - popd - fi - fi - - # Copy results - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR:-/run_logs}/logs" - mkdir -p "$LOGS_OUTPUT" - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Waiting 60s before killing router and prefill server..." - sleep 60 - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing router and prefill server" - if [[ "$DRY_RUN" -eq 0 ]]; then - kill $proxy_pid - kill $prefill0_pid - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then - # ────────────────────────────────────────────────────────────────────────── - # Prefill nodes 1..N (kv_producer) - # ────────────────────────────────────────────────────────────────────────── - echo "${host_name}:${host_ip} is Prefill Node (rank ${NODE_RANK})" - - # Determine which prefill worker this node belongs to, and its headnode IP - prefill_worker_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) - PREFILL_HEADNODE_IP="${PREFILL_IPS[$prefill_worker_idx]}" - - PREFILL_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${PREFILL_PORT} \ - --trust-remote-code \ - ${PREFILL_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_producer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log & - set +x - prefill_pid=$! - trap 'echo "Caught signal, killing prefill (pid=$prefill_pid)"; kill $prefill_pid 2>/dev/null; exit 0' SIGTERM SIGINT - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for router to be up..." - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-2800}" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for router ${NODE0_ADDR}:${ROUTER_PORT}/health" - else - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${NODE0_ADDR}:${ROUTER_PORT} not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router ${NODE0_ADDR}:${ROUTER_PORT} ready" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting until router closes..." - trap 'echo "Caught signal, killing prefill (pid=$prefill_pid)"; kill $prefill_pid 2>/dev/null; exit 0' SIGTERM SIGINT - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait until router ${NODE0_ADDR}:${ROUTER_PORT} closes" - else - while curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - sleep 10 & - wait $! - done - echo "[wait] router ${NODE0_ADDR}:${ROUTER_PORT} closed" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing prefill server (rank ${NODE_RANK})" - if [[ "$DRY_RUN" -eq 0 ]]; then kill $prefill_pid 2>/dev/null; fi - -else - # ────────────────────────────────────────────────────────────────────────── - # Decode nodes (kv_consumer) - # ────────────────────────────────────────────────────────────────────────── - RANK=$((NODE_RANK - NODE_OFFSET)) - echo "${host_name}:${host_ip} is Decode Node (rank ${RANK})" - - _MAX_CONC=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - CUDAGRAPH_SIZES='[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' - - DECODE_MAX_NUM_SEQS="${_MAX_CONC}" - - DECODE_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${DECODE_PORT} \ - --trust-remote-code \ - ${DECODE_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${DECODE_MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_consumer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - --cudagraph-capture-sizes "${CUDAGRAPH_SIZES}" \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - set -x - eval "$DECODE_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log & - set +x - decode_pid=$! - trap 'echo "Caught signal, killing decode (pid=$decode_pid)"; kill $decode_pid 2>/dev/null; exit 0' SIGTERM SIGINT - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for router to be up..." - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-2800}" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for router ${NODE0_ADDR}:${ROUTER_PORT}/health" - else - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${NODE0_ADDR}:${ROUTER_PORT} not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router ${NODE0_ADDR}:${ROUTER_PORT} ready" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting until router closes..." - trap 'echo "Caught signal, killing decode (pid=$decode_pid)"; kill $decode_pid 2>/dev/null; exit 0' SIGTERM SIGINT - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait until router ${NODE0_ADDR}:${ROUTER_PORT} closes" - else - while curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - sleep 10 & - wait $! - done - echo "[wait] router ${NODE0_ADDR}:${ROUTER_PORT} closed" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing decode server (rank ${RANK})" - if [[ "$DRY_RUN" -eq 0 ]]; then kill $decode_pid 2>/dev/null; fi -fi - -echo "Script completed successfully" -exit 0 \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh deleted file mode 100755 index aaaca61ef5..0000000000 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ /dev/null @@ -1,1374 +0,0 @@ -#!/bin/bash -# SGLang Disaggregated Server Launcher with Model-Specific Configurations -# ============================================================================= - -# ============================================================================= -# Environment Configuration -# ============================================================================= - -NODE0_ADDR="${NODE0_ADDR:-localhost}" -NODE_RANK="${NODE_RANK:-0}" -MODEL_DIR="${MODEL_DIR:-}" -MODEL_NAME="${MODEL_NAME:-}" - -xP="${xP:-1}" #-> Number of Prefill Workers -yD="${yD:-1}" #-> Number of Decode Workers - -IPADDRS="${IPADDRS:-localhost}" -HEADNODE_PORT="${HEADNODE_PORT:-20000}" -# Parallelism Configuration -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP:-true}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP:-true}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP:-true}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP:-true}" -DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" - -# Benchmark Configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" - -# Extract the maximum concurrency from the x-delimited list -BENCH_MAX_CONC_VALUE=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - -# Dry Run for debugging purpose -DRY_RUN="${DRY_RUN:-0}" - -# GPU count (expandable for different hardware) -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - - -# ============================================================================= -# Dependencies and Environment Setup -# ============================================================================= -source $SGLANG_WS_PATH/setup_deps.sh -source $SGLANG_WS_PATH/env.sh - -host_ip=$(ip route get 1.1.1.1 | awk '/src/ {print $7}') -host_name=$(hostname) - -# MORI_RDMA_TC configuration (optional) -# If set by runner, use it for RDMA traffic class configuration -# If not set, RDMA operations will proceed without QoS/traffic class settings -if [[ -n "${MORI_RDMA_TC}" ]]; then - echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC for RDMA traffic class configuration" - echo "[INFO] Host '$host_name' configured with MORI_RDMA_TC=$MORI_RDMA_TC" -else - echo "[INFO] MORI_RDMA_TC not set. Skipping RDMA traffic class configuration." - echo "[INFO] This is normal for clusters without QoS requirements." -fi - -# ============================================================================= -# Model-Specific Configuration from YAML -# ============================================================================= -MODELS_YAML="${SGLANG_WS_PATH}/models.yaml" - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "ERROR: models.yaml not found at $MODELS_YAML" - exit 1 -fi - -# Load model config via inline Python (PyYAML is available in SGLang containers) -# Formula evaluation (e.g. "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK * TP * xP") -# is done here in Python to avoid bash glob-expanding the * characters. -eval "$(python3 -c " -import yaml, sys, os - -config_path = '${MODELS_YAML}' -model_name = '${MODEL_NAME}' - -# Select the models.yaml recipe variant by run type: agentic runs (IS_AGENTIC) -# use the '-AgentX' entry, non-agentic disaggregated runs use '-DI'. -# Fall back to the bare model name if the variant-specific key is absent. -is_agentic = '${IS_AGENTIC:-0}'.strip().lower() in ('1', 'true') -model_key = f'{model_name}-AgentX' if is_agentic else f'{model_name}-DI' - -with open(config_path) as f: - models = yaml.safe_load(f) - -if model_key not in models: - if model_name in models: - model_key = model_name - else: - print(f'echo \"ERROR: Model {model_key} not in models.yaml\"; exit 1') - sys.exit(0) - -m = models[model_key] -print(f'echo \"Selected models.yaml entry: {model_key} (IS_AGENTIC={is_agentic})\"') - -def eval_formula(val): - \"\"\"Evaluate chunked_prefill_size: if string, resolve variable names from env and compute.\"\"\" - if isinstance(val, (int, float)): - return int(val) - s = str(val) - # Build a namespace from env vars (convert numeric values to int) - ns = {} - for k, v in os.environ.items(): - try: - ns[k] = int(v) - except (ValueError, TypeError): - pass - try: - return int(eval(s, {'__builtins__': {}}, ns)) - except Exception as e: - print(f'echo \"WARNING: Cannot evaluate formula: {s} ({e})\"', file=sys.stderr) - return val - -def parse_range(cuda_range, default_start, default_end): - if '-' in str(cuda_range): - s, e = str(cuda_range).split('-') - return s, e - return str(default_start), str(default_end) - -# Output shell variables -print(f'MODEL_BASE_FLAGS=\"{m.get(\"base_flags\", \"\")}\"') -print(f'MODEL_MTP_FLAGS=\"{m.get(\"mtp_flags\", \"\")}\"') -print(f'MODEL_DP_FLAGS=\"{m.get(\"dp_flags\", \"\")}\"') -print(f'MODEL_EP_FLAGS=\"{m.get(\"ep_flags\", \"\")}\"') - -prefill = m.get('prefill', {}) -decode = m.get('decode', {}) - -print(f'PREFILL_MEM_FRACTION_STATIC=\"{prefill.get(\"mem_fraction_static\", 0.8)}\"') -print(f'PREFILL_DISABLE_RADIX_CACHE=\"{prefill.get(\"disable_radix_cache\", True)}\"') -print(f'PREFILL_DISABLE_CUDA_GRAPH=\"{prefill.get(\"disable_cuda_graph\", False)}\"') - -dp = prefill.get('dp', {}) -no_dp = prefill.get('no_dp', {}) -print(f'PREFILL_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 24)}\"') -print(f'PREFILL_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'PREFILL_CUDA_GRAPH_BS_DP=\"{dp.get(\"cuda_graph_bs\", \"1 2 3\")}\"') -print(f'PREFILL_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') -print(f'PREFILL_MAX_TOTAL_TOKENS_DP=\"{dp.get(\"max_total_tokens\", \"\")}\"') -print(f'PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP=\"{dp.get(\"enable_two_batch_overlap\", False)}\"') -print(f'PREFILL_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') -print(f'PREFILL_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'PREFILL_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') -print(f'PREFILL_MAX_TOTAL_TOKENS_NO_DP=\"{no_dp.get(\"max_total_tokens\", \"\")}\"') -s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) -print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') -print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') - -print(f'DECODE_MEM_FRACTION_STATIC=\"{decode.get(\"mem_fraction_static\", 0.85)}\"') -print(f'DECODE_PREFILL_ROUND_ROBIN_BALANCE=\"{decode.get(\"prefill_round_robin_balance\", True)}\"') -print(f'DECODE_DISAGG_ENABLE_RADIX_CACHE=\"{decode.get(\"disagg_decode_enable_radix_cache\", False)}\"') - -dp = decode.get('dp', {}) -ep_only = decode.get('ep_only', {}) -no_dp = decode.get('no_dp', {}) - -# Decode DP config -print(f'DECODE_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 4096)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') -s, e = parse_range(dp.get('cuda_graph_bs_range', '1-160'), 1, 160) -print(f'DECODE_CUDA_GRAPH_BS_DP_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') - -# Decode EP-only config (EP enabled but DP disabled) -print(f'DECODE_MAX_RUNNING_REQUESTS_EP_ONLY=\"{ep_only.get(\"max_running_requests\", 256)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_EP_ONLY=\"{eval_formula(ep_only.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_EP_ONLY=\"{ep_only.get(\"context_length\", \"\")}\"') -s, e = parse_range(ep_only.get('cuda_graph_bs_range', '1-256'), 1, 256) -print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') - -# Decode no-DP config -print(f'DECODE_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') -s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) -print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') -")" - -echo "Loaded model configuration for: $MODEL_NAME" - -# Compute DP-dependent prefill parameters -if [[ "$PREFILL_ENABLE_DP" == "true" ]]; then - prefill_cuda_graph_bs=($PREFILL_CUDA_GRAPH_BS_DP) - prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_DP - prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_DP - prefill_context_length=$PREFILL_CONTEXT_LENGTH_DP - prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_DP - prefill_enable_two_batch_overlap=$PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP -else - prefill_cuda_graph_bs=($(seq $PREFILL_CUDA_GRAPH_BS_NO_DP_START $PREFILL_CUDA_GRAPH_BS_NO_DP_END)) - prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_NO_DP - prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_NO_DP - prefill_context_length=$PREFILL_CONTEXT_LENGTH_NO_DP - prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_NO_DP - prefill_enable_two_batch_overlap="false" -fi - -# When both DP and EP are enabled, override max-running-requests with max bench concurrency -if [[ "$PREFILL_ENABLE_DP" == "true" ]] && [[ "$PREFILL_ENABLE_EP" == "true" ]]; then - prefill_max_running_requests=$BENCH_MAX_CONC_VALUE - prefill_dp_ranks=$PREFILL_TP_SIZE - # MORI_MAX_DISPATCH_TOKENS_PREFILL stays at 8192 (no change) - # MORI_MOE_MAX_INPUT_TOKENS_PREFILL=$((MORI_MAX_DISPATCH_TOKENS_PREFILL * prefill_dp_ranks / 2)) - echo "[DP+EP override] Prefill: max-running-requests=$prefill_max_running_requests, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" -fi - -# Compute DP-dependent decode parameters (3-way: DP > EP-only > no_dp) -if [[ "$DECODE_ENABLE_DP" == "true" ]]; then - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_DP_START $DECODE_CUDA_GRAPH_BS_DP_END)) - decode_max_running_requests=$((DECODE_CUDA_GRAPH_BS_DP_END * DECODE_TP_SIZE)) - decode_context_length=$DECODE_CONTEXT_LENGTH_DP -elif [[ "$DECODE_ENABLE_EP" == "true" ]]; then - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_EP_ONLY_START $DECODE_CUDA_GRAPH_BS_EP_ONLY_END)) - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_EP_ONLY - decode_context_length=$DECODE_CONTEXT_LENGTH_EP_ONLY -else - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_NO_DP_START $DECODE_CUDA_GRAPH_BS_NO_DP_END)) - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_NO_DP - decode_context_length=$DECODE_CONTEXT_LENGTH_NO_DP -fi -# In PD-disaggregation the decode must admit requests against the SAME context -# length as prefill; otherwise decode accepts over-length requests that prefill -# rejects, and those requests hang forever waiting for a KV transfer that never -# comes (the 8k1k conc-500 straggler). Fall back to the prefill value if the -# decode context_length is not set in the model config, so the two always agree. -if [[ -z "$decode_context_length" ]]; then - decode_context_length=$prefill_context_length -fi - -# When both DP and EP are enabled, override max-running-requests and dispatch tokens -if [[ "$DECODE_ENABLE_DP" == "true" ]] && [[ "$DECODE_ENABLE_EP" == "true" ]]; then - decode_max_running_requests=$BENCH_MAX_CONC_VALUE - decode_dp_ranks=$DECODE_TP_SIZE - MORI_MAX_DISPATCH_TOKENS_DECODE=$((BENCH_MAX_CONC_VALUE / decode_dp_ranks)) - # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * decode_dp_ranks * 7 / 10)) - # Update derived variable - SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD - echo "[DP+EP override] Decode: max-running-requests=$decode_max_running_requests, DISPATCH_TOKENS=$MORI_MAX_DISPATCH_TOKENS_DECODE, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_DECODE, INTER_KERNEL_SWITCH=$SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD" -fi - -# Build the composed config strings (equivalent to the old MODEL_PREFILL_CONFIGS / MODEL_DECODE_CONFIGS) -# disable_cuda_graph (model-level) routes prefill to --disable-cuda-graph instead of --cuda-graph-bs. -if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "true" ]]; then - PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --disable-cuda-graph " -else - PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} " -fi - -if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" -fi -# Agentic runs: keep radix/prefix cache enabled by replacing --disable-radix-cache with empty. -if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}" -fi -if [[ -n "$prefill_context_length" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}" -fi -if [[ -n "$prefill_max_total_tokens" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --max-total-tokens ${prefill_max_total_tokens}" -fi -if [[ "$prefill_enable_two_batch_overlap" == "True" ]] || [[ "$prefill_enable_two_batch_overlap" == "true" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --enable-two-batch-overlap" - PREFILL_SDMA_ENV="MORI_ENABLE_SDMA=true" -fi - -DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-running-requests ${decode_max_running_requests} --cuda-graph-bs ${decode_cuda_graph_bs[*]} " - -if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" -fi -if [[ -n "$decode_context_length" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --context-length ${decode_context_length}" -fi - -if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" -fi - -if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then - MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) - # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MOE_MAX_INPUT_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) -fi - -# ============================================================================= -# Cluster Topology Configuration -# ============================================================================= -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -# Ceiling division by GPUS_PER_NODE for nodes-per-worker -PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + 7) / GPUS_PER_NODE)) -DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + 7) / GPUS_PER_NODE)) -NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) - -# Build prefill arguments dynamically based on xP -PREFILL_HEADNODE_URLS=() -PREFILL_ARGS="" -# Per-worker Prometheus /metrics endpoints (port 8000) for aiperf's -# --server-metrics scrape. The router on :30000 does not serve Prometheus, so -# aiperf must scrape each prefill/decode worker directly (see ENABLE_METRICS). -SERVER_METRICS_URLS=() -# Per-worker base URLs (port 8000) for direct cache flushing between -# concurrency points. The router (:30000) does not fan /flush_cache out, so -# trace_replay.sh must POST to each prefill/decode worker directly. -SERVER_FLUSH_URLS=() -for i in $(seq 0 $((xP - 1))); do - prefill_idx=$((i * PREFILL_NODES_PER_WORKER)) - PREFILL_HEADNODE_URLS[$i]="${IP_ARRAY[$prefill_idx]}:${HEADNODE_PORT}" - PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$prefill_idx]}:8000" - SERVER_METRICS_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000/metrics") - SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000") -done - -# Build decode arguments dynamically based on yD -DECODE_HEADNODE_URLS=() -DECODE_ARGS="" -for i in $(seq 0 $((yD - 1))); do - decode_idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) - DECODE_HEADNODE_URLS[$i]="${IP_ARRAY[$decode_idx]}:${HEADNODE_PORT}" - DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$decode_idx]}:8000" - SERVER_METRICS_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000/metrics") - SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000") -done - -echo "Prefill worker headnode list: ${PREFILL_HEADNODE_URLS[@]}" -echo "Decode worker headnode list: ${DECODE_HEADNODE_URLS[@]}" -echo "Server metrics endpoints: ${SERVER_METRICS_URLS[@]}" -echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" - -# ============================================================================= -# Configuration Builder Functions -# ============================================================================= - -# KV_P2P_TRANSFER (from amd-master.yaml kv-p2p-transfer) overrides the -# --disaggregation-transfer-backend baked into models.yaml base_flags. -apply_kv_p2p_transfer_override() { - local flags="$1" - if [[ -z "${KV_P2P_TRANSFER:-}" ]]; then - printf '%s' "$flags" - return 0 - fi - local stripped - stripped="$(echo "$flags" | sed -E 's/--disaggregation-transfer-backend[[:space:]]+[^[:space:]]+//g')" - stripped="${stripped#"${stripped%%[![:space:]]*}"}" - stripped="${stripped%"${stripped##*[![:space:]]}"}" - echo "[KV_P2P] Using disaggregation-transfer-backend=${KV_P2P_TRANSFER} (KV_P2P_TRANSFER env)" >&2 - printf '%s --disaggregation-transfer-backend %s' "$stripped" "$KV_P2P_TRANSFER" -} - -build_server_config() { - local mode="$1" - local model_name="$2" - local tp_size="$3" - local enable_ep="$4" - local enable_dp="$5" - local decode_mtp_size="$6" - - # Calculate EP and DP sizes based on enable flags - local ep_size=1 - local dp_size=1 - - if [[ "$enable_ep" == "true" ]]; then - ep_size=$tp_size - fi - - if [[ "$enable_dp" == "true" ]]; then - dp_size=$tp_size - fi - - # Build parallelism arguments - local parallel_args="--tp-size ${tp_size}" - - if [[ "$enable_ep" == "true" ]]; then - parallel_args="$parallel_args --ep-size ${ep_size}" - fi - - if [[ "$enable_dp" == "true" ]]; then - parallel_args="$parallel_args --dp-size ${dp_size}" - fi - - # Get model-specific configuration from YAML-loaded variables - local base_config - base_config="$(apply_kv_p2p_transfer_override "$MODEL_BASE_FLAGS")" - local mtp_config="" - local dp_config="" - local ep_config="" - local specific_config="" - - # MTP config (only if MTP is enabled and mode is decode) - if [ "$decode_mtp_size" -gt 0 ]; then - mtp_config="${MODEL_MTP_FLAGS} --speculative-num-steps ${decode_mtp_size} --speculative-num-draft-tokens $((decode_mtp_size + 1))" - fi - - # DP config (only if DP is enabled) - if [[ "$enable_dp" == "true" ]]; then - dp_config="$MODEL_DP_FLAGS" - fi - - # EP config (only if EP is enabled): a2a backend, deepep mode, ep-dispatch algo. - # With ep=1 (EP disabled) these are dropped, so the MoE runs tensor-parallel (TP) - # instead of expert-parallel — even when dp-attention is on. - if [[ "$enable_ep" == "true" ]]; then - ep_config="$MODEL_EP_FLAGS" - fi - - # Mode-specific config - if [[ "$mode" == "prefill" ]]; then - specific_config="$PREFILL_MODE_FLAGS" - elif [[ "$mode" == "decode" ]]; then - specific_config="$DECODE_MODE_FLAGS" - fi - - # Combine: parallel args + base config + ep config + mtp config + dp config + specific config - local full_config="$parallel_args" - if [[ -n "$base_config" ]]; then - full_config="$full_config $base_config" - fi - if [[ -n "$ep_config" ]]; then - full_config="$full_config $ep_config" - fi - # MTP/speculative flags go to BOTH prefill and decode. In PD-disaggregation the - # draft (nextn) layers participate in prefill KV computation as well as decode - # verification, so the speculative config must match on both roles. Gating this to - # decode only left prefill without the nextn KV layer: prefill registered one fewer - # PD state component than decode, which newer sglang (v0.5.15+) rejects outright - # ("state component count mismatch") and older builds tolerated silently while - # feeding the decode's nextn verification uninitialized state (lossy greedy MTP). - if [[ -n "$mtp_config" ]]; then - full_config="$full_config $mtp_config" - fi - if [[ -n "$dp_config" ]]; then - full_config="$full_config $dp_config" - fi - if [[ -n "$specific_config" ]]; then - full_config="$full_config $specific_config" - fi - - echo "$full_config" -} - -# Build complete server configurations -PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE") -DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE") - -# Expose Prometheus /metrics on the servers when requested (ENABLE_METRICS=1). -if [[ "${ENABLE_METRICS:-0}" == "1" ]]; then - [[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics" - [[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics" -fi - -if [[ -n "$MODEL_NAME" ]]; then - echo "Using model-specific configuration for: $MODEL_NAME" -fi - -# sync.py barrier timeout for server-up (port 8000). DSV4 needs more headroom. -# Override via SYNC_BARRIER_TIMEOUT if needed. -if [[ -z "${SYNC_BARRIER_TIMEOUT:-}" ]]; then - case "${MODEL_NAME}" in - *DeepSeek-V4*) SYNC_BARRIER_TIMEOUT=3000 ;; - *) SYNC_BARRIER_TIMEOUT=1800 ;; - esac -fi -echo "SYNC_BARRIER_TIMEOUT=${SYNC_BARRIER_TIMEOUT}s (model=${MODEL_NAME:-unset})" - -# ============================================================================= -# Optional KV cache offloading (HiCache) — enabled when -# KV_OFFLOADING != none AND KV_OFFLOAD_BACKEND == hicache. -# HiCache extends RadixAttention, so radix cache MUST stay on (drop -# --disable-radix-cache). The --hicache-* flags are appended to BOTH the -# prefill and decode server configs. -# ============================================================================= -KV_OFFLOADING="${KV_OFFLOADING:-none}" -KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" -if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" - HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" - HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-wait_complete}" - - # Optional L3 storage tier behind the CPU-DRAM (L2) cache. - # "" -> CPU DRAM only (default) - # "mooncake"-> Mooncake distributed KV store (needs a mooncake_master) - HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - - # Layout / IO backend / write policy are backend-specific: - # mooncake L3: page_first_direct + the "direct" IO backend (the Mooncake - # store maps a page-contiguous segment for RDMA/zero-copy). This layout - # asserts host_pool > device_pool, so it needs a large CPU-DRAM budget. - # L2-only (CPU DRAM): layer_first + the "kernel" IO backend. layer_first - # has no host>device constraint (the "direct" IO backend REQUIRES a - # page_first layout, so it cannot be paired with layer_first). - if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - else - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - fi - - # Mooncake master/connection settings (used only when storage=mooncake). - # The master runs once on node 0; every prefill/decode server connects to - # it via NODE0_ADDR so it is reachable across nodes. - MC_MASTER_PORT="${MC_MASTER_PORT:-50061}" - MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" - MC_METRICS_PORT="${MC_METRICS_PORT:-9003}" - MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" - MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" - MC_PROTOCOL="${MC_PROTOCOL:-tcp}" - MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" - MC_DEVICE="${MC_DEVICE:-$IBDEVICES}" - MC_MASTER_ADDR="${MC_MASTER_ADDR:-${NODE0_ADDR}:${MC_MASTER_PORT}}" - MC_METADATA_SERVER="${MC_METADATA_SERVER:-http://${NODE0_ADDR}:${MC_METADATA_PORT}/metadata}" - - # Emit the --hicache-storage-backend flags (empty unless mooncake). The - # extra-config JSON is single-quoted so it survives the later `eval` of the - # launch command as a single argument. - build_storage_flags() { - [[ "$HICACHE_STORAGE_BACKEND" != "mooncake" ]] && return 0 - local extra="{\"master_server_address\": \"${MC_MASTER_ADDR}\", \"protocol\": \"${MC_PROTOCOL}\", \"device_name\": \"${MC_DEVICE}\", \"local_hostname\": \"${host_ip}\", \"global_segment_size\": \"${MC_GLOBAL_SEG}\", \"metadata_server\": \"${MC_METADATA_SERVER}\", \"check_server\": false}" - echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" - } - - # HiCache capacity. Prefer an absolute per-rank pool derived from the - # per-node DRAM budget computed by the sweep generator (enforcement); fall - # back to --hicache-ratio (relative to the GPU KV pool) when no budget is - # provided, keeping configs that predate the budget unchanged. - # FORCE_HICACHE_RATIO lets a recipe opt into ratio-based sizing without - # unsetting TOTAL_CPU_DRAM_GB — that var is also the shared client-side - # gate (benchmark_lib.sh requires it whenever KV_OFFLOADING=dram) and is - # forwarded verbatim into client.env below, so unsetting it here would - # make the aiperf client container fail its own env validation before - # ever sending a request. - HICACHE_RATIO="${HICACHE_RATIO:-5}" - HICACHE_SIZING_FLAGS="--hicache-ratio ${HICACHE_RATIO}" - # DeepSeek V4's hybrid HiCache pool rejects --hicache-size (requires - # --hicache-ratio), so the absolute per-node budget cannot be applied to it. - # See sglang _deepseek_v4_num_host_pages() (raises ValueError when - # server_args.hicache_size > 0): - # https://github.com/sgl-project/sglang/blob/9dd57ef8c48e2cd82292d849f01e2130c5203e67/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py#L262-L266 - # FORCE_HICACHE_RATIO additionally lets a recipe opt into ratio-based sizing - # for any other model without unsetting TOTAL_CPU_DRAM_GB (see comment above). - if [[ "${FORCE_HICACHE_RATIO:-0}" != "1" && -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* ]]; then - # TOTAL_CPU_DRAM_GB is the prefill worker's per-node budget (only prefill - # offloads KV to CPU DRAM today); --hicache-size is per rank per host - # pool. A prefill server may span nodes (PREFILL_TP_SIZE is its total - # ranks), so divide by the ranks that land on one node. - prefill_ranks_per_node=$(( PREFILL_TP_SIZE < GPUS_PER_NODE ? PREFILL_TP_SIZE : GPUS_PER_NODE )) - prefill_hicache_size_gb=$(( TOTAL_CPU_DRAM_GB / prefill_ranks_per_node / HICACHE_HOST_POOL_COUNT )) - if (( prefill_hicache_size_gb < 1 )); then - echo "Error: TOTAL_CPU_DRAM_GB=${TOTAL_CPU_DRAM_GB} / ranks_per_node=${prefill_ranks_per_node} / host_pools=${HICACHE_HOST_POOL_COUNT} rounds below 1 GB" >&2 - exit 1 - fi - HICACHE_SIZING_FLAGS="--hicache-size ${prefill_hicache_size_gb}" - echo "[HiCache] prefill CPU pool capped at ${prefill_hicache_size_gb} GB/rank (budget ${TOTAL_CPU_DRAM_GB} GB / ranks_per_node ${prefill_ranks_per_node} / host_pools ${HICACHE_HOST_POOL_COUNT})" - fi - - build_hicache_flags() { - echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache ${HICACHE_SIZING_FLAGS} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" - } - - # HiCache requires RadixAttention; strip any --disable-radix-cache. - PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" - DECODE_SERVER_CONFIG="${DECODE_SERVER_CONFIG//--disable-radix-cache/}" - - # Prefill always gets HiCache. - PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG $(build_hicache_flags "$PREFILL_TP_SIZE")" - - - DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --page-size ${HICACHE_PAGE_SIZE}" - echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND} applied to prefill only; decode mirrors --page-size ${HICACHE_PAGE_SIZE} for transfer compatibility (chunk cache under the mori transfer backend)" - echo "[HiCache] params: io_backend=${HICACHE_IO_BACKEND}, mem_layout=${HICACHE_MEM_LAYOUT}, page_size=${HICACHE_PAGE_SIZE}, write_policy=${HICACHE_WRITE_POLICY}, prefetch_policy=${HICACHE_PREFETCH_POLICY}, storage_backend=${HICACHE_STORAGE_BACKEND:-none}" - if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then - echo "[HiCache] Mooncake store: master=${MC_MASTER_ADDR} metadata=${MC_METADATA_SERVER} protocol=${MC_PROTOCOL} device=${MC_DEVICE} segment=${MC_GLOBAL_SEG} threads=${MC_MASTER_THREADS} eviction_watermark=${MC_EVICTION_HIGH_WATERMARK}" - fi -else - echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND:-none} (HiCache disabled)" -fi - -if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then - PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') - DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE -fi - -# ============================================================================= -# Container Synchronization -# ============================================================================= - -# sync.py barrier/health-barrier exits 1 on timeout (and prints which -# node/port never became ready), but without an explicit check here the -# script would silently continue past a timed-out barrier -- printing a -# misleading "success" message and launching the next stage against -# servers/routers that never actually came up, instead of failing fast. -run_barrier_or_die() { - local desc="$1" cmd="$2" - if ! eval "$cmd"; then - echo "FATAL: ${desc} failed — see the sync.py timeout output above for which node/port never became ready." >&2 - exit 1 - fi -} - -echo "Waiting at the container creation barrier on $host_name" -run_barrier_or_die "container creation barrier" "python3 $SGLANG_WS_PATH/sync.py barrier \ - --local-ip ${host_ip} \ - --local-port 5000 \ - --enable-port \ - --node-ips ${IPADDRS} \ - --node-ports 5000 \ - --wait-for-all-ports \ - --timeout 300" - - -# ============================================================================= -# Node Role Assignment and Server Launch -# ============================================================================= - -# Run a blocking command while watching the local server PID. If the server dies -# (crash / OOM / killed) the blocking command is aborted and we return non-zero, -# so the srun task exits non-zero and SLURM's --kill-on-bad-exit tears the whole -# job down in seconds instead of waiting out the ~1800s barrier timeout. -wait_or_die() { # $1 = server pid to watch; rest = blocking command - local watch=$1; shift - "$@" & local cmd=$! - while kill -0 "$cmd" 2>/dev/null; do - kill -0 "$watch" 2>/dev/null || { - echo "FATAL: $(hostname) local sglang server (pid $watch) died; tearing down job" >&2 - kill "$cmd" 2>/dev/null || true - return 1 - } - sleep 5 - done - wait "$cmd" -} - -if [ "$NODE_RANK" -eq 0 ]; then - echo "NODE INFO =======================================" - echo "================================================" - echo "Node List : ${SLURM_JOB_NODELIST}" - echo "Node IPs : ${IPADDRS}" - echo "Model Name : ${MODEL_NAME:-'Not specified'}" - echo "================================================" - - echo "CLUSTER INFO ====================================" - echo "================================================" - echo "${host_name}:${host_ip} is Proxy Node and Prefill Node" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" - echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" - echo "Prefill servers ($((PREFILL_TP_SIZE/GPUS_PER_NODE)) nodes): ${PREFILL_ARGS}" - echo "Decode servers ($((DECODE_TP_SIZE/GPUS_PER_NODE)) nodes): ${DECODE_ARGS}" - echo "Prefill env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_PREFILL}" - echo "Decode env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_DECODE} " - echo "Decode env: SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE} " - - echo "================================================" - - # Dump all resolved commands to a text file for debugging / reproducibility. - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - echo "# Model: ${MODEL_NAME} Image: ${DOCKER_IMAGE_NAME:-unknown}" >> "$CMD_DUMP" - - # Start the Mooncake store master (L3 HiCache backend) on node 0 only. - # All prefill/decode servers connect to it via NODE0_ADDR:MC_MASTER_PORT. - if [[ "${KV_OFFLOADING:-none}" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" && "${HICACHE_STORAGE_BACKEND:-}" == "mooncake" ]]; then - echo "Starting Mooncake master on ${host_ip}:${MC_MASTER_PORT} (metadata :${MC_METADATA_PORT}, metrics :${MC_METRICS_PORT})" - MC_MASTER_CMD="mooncake_master \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port=${MC_METADATA_PORT} \ - --rpc_port=${MC_MASTER_PORT} \ - --rpc_thread_num=${MC_MASTER_THREADS} \ - --metrics_port=${MC_METRICS_PORT} \ - --enable_metric_reporting=true \ - --eviction_high_watermark_ratio=${MC_EVICTION_HIGH_WATERMARK}" - dump_cmd "MOONCAKE MASTER" "$MC_MASTER_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $MC_MASTER_CMD" - else - MC_MASTER_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/mooncake_master_${host_name}.log" - mooncake_master \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port="${MC_METADATA_PORT}" \ - --rpc_port="${MC_MASTER_PORT}" \ - --rpc_thread_num="${MC_MASTER_THREADS}" \ - --metrics_port="${MC_METRICS_PORT}" \ - --enable_metric_reporting=true \ - --eviction_high_watermark_ratio="${MC_EVICTION_HIGH_WATERMARK}" \ - > "${MC_MASTER_LOG}" 2>&1 & - mc_master_pid=$! - sleep 3 - # Fail loudly on a port collision. On shared nodes the Mooncake RPC - # port may already be taken by another user's master; in that case the - # metrics-port health check below can still pass against the foreign - # master while our RPC port is dead, and the prefill then hangs. - if grep -qiE "Address already in use|bind .*error" "${MC_MASTER_LOG}" 2>/dev/null; then - echo "ERROR: mooncake_master failed to bind port ${MC_MASTER_PORT} (already in use)." - echo " Set MC_MASTER_PORT/MC_METRICS_PORT to free ports and resubmit." - grep -iE "Address already in use|bind .*error" "${MC_MASTER_LOG}" | tail -3 - exit 1 - fi - for ((i=3; i<=60; i+=3)); do - if curl -sf "http://127.0.0.1:${MC_METRICS_PORT}/get_all_segments" >/dev/null 2>&1; then - echo " mooncake master OK at ${i}s" - break - fi - sleep 3 - done - fi - fi - - # start the head prefill server - PREFILL_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then - PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" - fi - set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path $MODEL_DIR/$MODEL_NAME \ - --disaggregation-mode prefill \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${PREFILL_SERVER_CONFIG} " - - if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then - PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[0]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank 0" - fi - - - dump_cmd "PREFILL (node 0)" "$PREFILL_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - # Launch under `setsid` so the server (python + its TP-scheduler - # children) sits in a dedicated process group; teardown can then - # `kill -- -$pgid` the WHOLE tree. Killing $prefill0_pid alone leaves - # children holding the process-sub tee's pipe, so the container's outer - # `| tee` never gets EOF and the container never exits (srun/CI hangs). - # Process substitution (not `| tee`) keeps $! as the setsid group leader, - # not tee's. Mirrors the router launch below. - setsid bash -c "$PREFILL_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log >/dev/null) 2>&1 & - set +x - prefill0_pid=$! - prefill0_pgid=$(ps -o pgid= -p "$prefill0_pid" 2>/dev/null | tr -d ' ') - : "${prefill0_pgid:=$prefill0_pid}" - fi - - - echo "Waiting for all prefill and decode servers to be up . . ." - - - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${IPADDRS} \ - --node-ports 8000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$prefill0_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - echo "Congratulations!!! All prefill and decode servers are up . . ." - - if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - # Agentic router config (main): long-context prefills can look unhealthy to - # the default circuit breaker during a concurrent burst. Disable the breaker - # and relax health-check sensitivity so a busy-but-alive worker is not - # ejected. cache_aware prefill routing exploits HiCache/radix prefix reuse - # across the agentic trace; round_robin decode keeps the single decode worker - # fed evenly. Override via ROUTER_RESILIENCE_FLAGS / ROUTER_POLICY_FLAGS. - ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" - # server_sglang.sh previously read ROUTER_PREFILL_POLICY, but the recipe - # scripts export PREFILL_ROUTER_POLICY, so the recipe's policy override was - # silently ignored and the router always fell back to this hardcoded - # default. Also comment out ROUTER_DECODE_POLICY for now (superseded by - # --dp-aware below). - ROUTER_PREFILL_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}" - # ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" - ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" - ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" - ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --dp-aware --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" - else - # DI router config (8k1k branch, run 28696443568): with defaults the per-worker - # circuit stays OPEN for cb-timeout-duration-secs=60 before a half-open retrial. - # In that run every request 503'd from request #1 ("all circuits open or - # unhealthy") for ~31s and lm_eval (max_retries=5) then gave up -- i.e. the - # circuit was still open when the client budget ran out, so 0 result files were - # produced. Shortening the open->half-open window (and letting the router itself - # retry a failed worker selection) lets a transient trip re-close INSIDE the - # client retry budget instead of nuking the whole eval. The breaker stays fully - # ENABLED (thresholds unchanged); this only speeds recovery. Override via - # ROUTER_CB_ARGS / ROUTER_POLICY_FLAGS. - ROUTER_CB_ARGS="${ROUTER_CB_ARGS:---cb-timeout-duration-secs 15 --retry-max-retries 3}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy random --prefill-policy random --decode-policy random}" - ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:-${ROUTER_CB_ARGS}}" - fi - - echo "Router config: IS_AGENTIC=${IS_AGENTIC:-0} policy/resilience=${ROUTER_POLICY_FLAGS} ${ROUTER_RESILIENCE_FLAGS}" - - ROUTER_CMD="python -m sglang_router.launch_router \ - --pd-disaggregation \ - --port 30000 \ - ${ROUTER_POLICY_FLAGS} \ - ${ROUTER_RESILIENCE_FLAGS} \ - ${PREFILL_ARGS} \ - ${DECODE_ARGS}" - - - dump_cmd "ROUTER" "$ROUTER_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $ROUTER_CMD" - else - ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/router_${host_name}.log" - # sgl-router (Rust/tracing) emits ANSI color codes. NO_COLOR asks it to - # skip them at the source; the sed strip guarantees a clean file even if - # it doesn't honor NO_COLOR. Both branches use process substitution so - # $! stays the router pid, not sed's/tee's pid. - # - # Newer sglang-router (>=0.5.14) spawns the actual Rust worker - # (`sglang::router`, which binds :30000) as a child and lets the python - # launcher exit, so the worker reparents to init. It KEEPS its process - # group, though. We therefore launch under `setsid` to isolate the - # launcher+worker in a dedicated process group and record that pgid, so - # teardown can `kill -- -$proxy_pgid` the whole group even after the - # launcher is gone. `kill $proxy_pid` alone would miss the worker. - set -x - if [[ "${SGLANG_ROUTER_STDOUT_LOGS:-0}" == "1" ]]; then - NO_COLOR=1 setsid bash -c "exec $ROUTER_CMD" > >(sed -u -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' | tee "$ROUTER_LOG_FILE") 2>&1 & - else - NO_COLOR=1 setsid bash -c "exec $ROUTER_CMD" > >(sed -u -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' >"$ROUTER_LOG_FILE") 2>&1 & - fi - set +x - proxy_pid=$! - proxy_pgid=$(ps -o pgid= -p "$proxy_pid" 2>/dev/null | tr -d ' ') - : "${proxy_pgid:=$proxy_pid}" - - # Wait for router to be ready via health endpoint - HEALTH_BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-health \ - --health-endpoint /readiness \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $HEALTH_BARRIER_CMD" - else - wait_or_die "$prefill0_pid" bash -c "$HEALTH_BARRIER_CMD" || exit 1 - fi - - # ---- End-to-end router readiness canary (run 28696443568) ---- - # The /readiness barrier above only proves the router PROCESS is up; it does - # NOT prove the router can reach a prefill worker and complete a generation. - # In that run the eval fired the instant /readiness passed and EVERY request - # 503'd ("No available prefill workers (all circuits open or unhealthy)") from - # request #1 -> lm_eval gave up -> 0 result files -> "Verify eval scores" failed. - # Gate the benchmark on ONE successful generation THROUGH the router so the eval - # never starts against a router whose prefill path is not yet actually serving. - # - # Run the poll loop under wait_or_die (like the barriers above) rather than - # inline: the canary is the FIRST real generation through prefill, so a - # prefill crash right after /readiness passes is plausible. Without - # wait_or_die watching prefill0_pid, that just looks like repeated 503s and - # the loop burns the full ROUTER_CANARY_TIMEOUT (default 600s) instead of - # detecting the dead pid and aborting in ~5s. - run_router_canary() { - local canary_url="http://${NODE0_ADDR}:30000/v1/chat/completions" - local canary_model="${MODEL_DIR}/${MODEL_NAME}" - local canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT:-600} )) - local canary_code - while [ "$(date +%s)" -lt "$canary_deadline" ]; do - canary_code=$(curl -s -o /tmp/router_canary.out -w '%{http_code}' \ - -m "${ROUTER_CANARY_REQ_TIMEOUT:-120}" \ - -X POST "$canary_url" -H 'Content-Type: application/json' \ - -d "{\"model\":\"${canary_model}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null) - if [ "$canary_code" = "200" ] && \ - ! grep -qE "circuits open|server_selection_failed|No available" /tmp/router_canary.out 2>/dev/null; then - echo "Router readiness canary passed (end-to-end generation OK)" - return 0 - fi - echo "Router readiness canary not ready yet (http=${canary_code}); retrying in 5s . . ." - sleep 5 - done - echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT:-600}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router." - head -c 800 /tmp/router_canary.out 2>/dev/null - return 1 - } - if [[ "${ROUTER_READINESS_CANARY:-1}" == "1" ]]; then - wait_or_die "$prefill0_pid" run_router_canary || exit 1 - fi - - echo "Router is ready for benchmarking" - fi - - - echo "Ready for benchmarking on ${host_name}:${host_ip}" - - echo "Benchmarking on ${host_name}:${host_ip}" - cd $SGLANG_WS_PATH - - # Export IS_MTP based on whether MTP is enabled - if [ "$DECODE_MTP_SIZE" -gt 0 ]; then - export IS_MTP=true - else - export IS_MTP=false - fi - - # Select the benchmark runner. - # IS_AGENTIC=1/true → agentic trace replay (trace_replay.sh) - # IS_AGENTIC unset/0 → fixed-seq-len throughput benchmark (bench.sh) - if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - # Point aiperf's server-metrics scrape at the per-worker Prometheus - # /metrics endpoints. The router (:30000) that aiperf auto-detects from - # --url does not expose Prometheus, so without this the scrape finds no - # reachable endpoint and all server-side cache/KV fields come out null. - # Only set it when the workers were actually started with --enable-metrics. - if [[ "${ENABLE_METRICS:-0}" == "1" && "${#SERVER_METRICS_URLS[@]}" -gt 0 ]]; then - AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${SERVER_METRICS_URLS[*]}") - export AIPERF_SERVER_METRICS_URLS - echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" - fi - # Per-worker base URLs for cache flushing between concurrency points. - # trace_replay.sh consults these when CLEAR_CACHE_BETWEEN_CONC=1. - if [[ "${#SERVER_FLUSH_URLS[@]}" -gt 0 ]]; then - SERVER_FLUSH_URLS_CSV=$(IFS=,; echo "${SERVER_FLUSH_URLS[*]}") - export SERVER_FLUSH_URLS_CSV - echo "SERVER_FLUSH_URLS_CSV=${SERVER_FLUSH_URLS_CSV}" - fi - # trace_replay.sh signature: model_path model_name concurrency_list log_path - BENCH_CMD="bash $SGLANG_WS_PATH/trace_replay.sh \ - $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" - echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING:-none}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" - else - # bench.sh signature: - # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path - # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier - BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - echo "Benchmark runner: bench.sh (fixed-seq-len)" - fi - - IS_AGENTIC_RUN=0 - if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - IS_AGENTIC_RUN=1 - fi - - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - elif [[ -n "${CLIENT_IMAGE:-}" && "$IS_AGENTIC_RUN" == "1" ]]; then - # Separate client image (node-0 sibling container): run the aiperf trace - # replay in its own sibling container built from CLIENT_IMAGE (which ships - # a pre-baked aiperf + deps) instead of rebuilding the aiperf venv inside - # this server container. The server/router stay up in this container while - # the client container drives the benchmark against the router on - # localhost (--network host). job.slurm mounts the host docker socket + CLI - # into this container and forwards HOST_REPO_DIR / HOST_MODEL_DIR / - # HOST_BENCH_LOGS / CLIENT_CONT_NAME so the sibling can be launched here. - CLIENT_ENV_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/client.env" - mkdir -p "/run_logs/slurm_job-${SLURM_JOB_ID}" - # Forward the benchmark-relevant env (incl. runtime-computed metrics/flush - # URLs) to the client container; override the few paths/flags that differ - # inside the pre-baked image. Unset vars are skipped, so the client keeps - # its own defaults for anything not exported here. - { - for _v in ENGINE MODEL_NAME MODEL_PREFIX PRECISION FRAMEWORK SPEC_DECODING \ - DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME RUNNER_TYPE IMAGE \ - AIPERF_SERVER_METRICS_URLS SERVER_FLUSH_URLS_CSV \ - ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ - DISAGG IS_MULTINODE \ - TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ - PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_ENABLE_DP PREFILL_HARDWARE \ - DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_ENABLE_DP DECODE_HARDWARE \ - KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ - WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ - AIPERF_WARMUP_REQUESTS_PER_LANE AIPERF_TRACE_IDLE_GAP_CAP_SECONDS \ - AIPERF_EXPERIMENTAL_FAST AIPERF_UNSAFE_OVERRIDE \ - AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES ROUTER_PORT TQDM_MININTERVAL; do - if [[ -n "${!_v+x}" ]]; then - _val="${!_v}" - # docker --env-file requires one KEY=VALUE per line with no - # embedded newlines; KV_OFFLOAD_BACKEND_METADATA carries - # pretty-printed multi-line JSON, which otherwise splits - # into unparseable lines (e.g. '"name": "hicache",') and - # aborts the client container launch. Re-serialize it to - # compact single-line JSON (round-tripping through - # json.loads/json.dumps) instead of naively stripping - # newlines, so this stays correct even if a value ever - # contained a literal newline inside a string. Empty/ - # "none"/"null" is the normal case when KV offloading is - # disabled (job.slurm always sets this var, even to ""), - # and must pass through untouched -- matching how - # optional_kv_offload_backend_metadata() in - # process_agentic_result.py treats those as "no metadata" - # rather than invalid JSON. - if [[ "$_v" == "KV_OFFLOAD_BACKEND_METADATA" && -n "$_val" && "$_val" != "null" ]]; then - _val="$(python3 -c 'import json, sys -print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { - echo "KV_OFFLOAD_BACKEND_METADATA must contain valid JSON" >&2 - exit 1 - } - fi - printf '%s=%s\n' "$_v" "$_val" - fi - done - echo "INFMAX_CONTAINER_WORKSPACE=/workspace" - # Do NOT pin AGENTIC_OUTPUT_DIR: it must default to /workspace (the - # host repo mount == GITHUB_WORKSPACE) so the aggregated - # ${RESULT_FILENAME}_conc.json lands where the workflow guard globs - # it. /workspace is bind-mounted writable, same as the co-located path. - echo "HF_HOME=/run_logs/hf_cache" - echo "MODEL_DIR=/models" - # A pre-baked client image ships aiperf at CLIENT_AIPERF_VENV; when - # unset (e.g. reusing the server image, which carries no pre-baked - # venv), trace_replay builds aiperf on the fly from - # /workspace/utils/aiperf — same as the co-located path. - if [[ -n "${CLIENT_AIPERF_VENV:-}" ]]; then - echo "AIPERF_USE_PREBUILT=1" - echo "AIPERF_VENV=${CLIENT_AIPERF_VENV}" - fi - } > "$CLIENT_ENV_FILE" - - echo "Launching agentic benchmark in separate client container: ${CLIENT_IMAGE}" - docker rm -f "${CLIENT_CONT_NAME}" 2>/dev/null || true - set -x - docker run --rm --network host \ - --name "${CLIENT_CONT_NAME}" \ - --shm-size 32G \ - -v "${HOST_REPO_DIR}:/workspace" \ - -v "${HOST_MODEL_DIR}:/models" \ - -v /tmp:/run_logs \ - -v "${HOST_BENCH_LOGS}:/benchmark_logs" \ - --env-file "${CLIENT_ENV_FILE}" \ - --entrypoint "" \ - "${CLIENT_IMAGE}" \ - bash -lc "cd /workspace/benchmarks/multi_node/amd_utils && bash trace_replay.sh /models ${MODEL_NAME} \"${BENCH_MAX_CONCURRENCY}\" /run_logs/slurm_job-${SLURM_JOB_ID}" - set +x - else - set -x - eval "$BENCH_CMD" - set +x - fi - - # Run evaluation if requested (before killing router) - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "Running lm-eval (GSM8K) evaluation on Node 0..." - - # Health check: verify the router is still serving before running eval. - # The throughput benchmark may have crashed/exhausted decode workers. - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:30000/readiness" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - # Must run from repo root so utils/evals/gsm8k.yaml resolves - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - # Use EVAL_CONC from workflow if set, otherwise fall back to max of conc list. - # Export CONC before run_eval so meta_env.json matches validate_scores.py. - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - export CONC="${EVAL_CONCURRENT_REQUESTS}" - - # Override eval context length with model's configured context_length - if [[ -n "$prefill_context_length" ]]; then - export EVAL_MAX_MODEL_LEN="$prefill_context_length" - fi - - export ISL="${BENCH_INPUT_LEN:-0}" - export OSL="${BENCH_OUTPUT_LEN:-0}" - bridge_disagg_eval_metadata - # IS_MULTINODE, FRAMEWORK, PRECISION, MODEL_PREFIX, RUNNER_TYPE, - # RESULT_FILENAME are already set via Docker -e flags from job.slurm - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --framework lm-eval --port 30000 (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" - else - run_eval --framework lm-eval --port 30000 - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 - EVAL_FAILED=1 - else - # Always rewrite meta_env.json so EP/DPA match the workflow - # topology even when run_eval() staged artifacts internally. - rewrite_lm_eval_meta_env - - # Fixed-seq-len post-bench eval still needs append to move - # results out of the temp EVAL_RESULT_DIR. - if [[ "${EVAL_ONLY:-false}" != "true" || "$IS_AGENTIC_RUN" != "1" ]]; then - append_lm_eval_summary - fi - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - mkdir -p "$EVAL_COPY_DIR" - for f in meta_env.json; do - [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" - done - find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - - echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" - fi - fi - - popd - fi - fi - - # Copy benchmark results to BENCHMARK_LOGS_DIR (mounted from host) - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR:-/run_logs}/logs" - mkdir -p "$LOGS_OUTPUT" - - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Killing the proxy server and prefill server" - - if [[ "$DRY_RUN" -eq 0 ]]; then - # Kill the router's entire process group (isolated via setsid at launch). - # The python launcher (proxy_pid) has usually already exited after - # spawning the detached Rust worker; the worker reparents to init but - # stays in this process group, so a group-kill reliably closes :30000. - # `kill $proxy_pid` alone misses the worker and hangs decode/prefill. - kill -TERM -"${proxy_pgid:-$proxy_pid}" 2>/dev/null || true - # Group-kill the prefill server tree (setsid at launch) so its - # TP-scheduler children die too and release the process-sub tee -> - # the container's outer `| tee` gets EOF and the container can exit. - kill -TERM -"${prefill0_pgid:-$prefill0_pid}" 2>/dev/null || true - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then - echo "${host_name}:${host_ip} is Prefill Node (Model: ${MODEL_NAME:-'default'})" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - - PREFILL_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then - PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" - fi - set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path $MODEL_DIR/${MODEL_NAME} \ - --disaggregation-mode prefill \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${PREFILL_SERVER_CONFIG} " - - if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then - rank=$((NODE_RANK % PREFILL_NODES_PER_WORKER)) - prefill_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) - PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[$prefill_idx]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank $rank" - fi - - dump_cmd "PREFILL (rank ${NODE_RANK})" "$PREFILL_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - # setsid isolates the server tree in its own process group so teardown - # can group-kill it (python + TP-scheduler children); otherwise the - # children hold the process-sub tee's pipe and the container never exits. - setsid bash -c "$PREFILL_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log >/dev/null) 2>&1 & - set +x - prefill_pid=$! - prefill_pgid=$(ps -o pgid= -p "$prefill_pid" 2>/dev/null | tr -d ' ') - : "${prefill_pgid:=$prefill_pid}" - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$prefill_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $SGLANG_WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port 30000" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - wait_or_die "$prefill_pid" bash -c "$WAIT_CMD" || exit 1 - fi - - echo "Killing the rank $NODE_RANK prefill server" - - if [[ "$DRY_RUN" -eq 0 ]]; then - # Group-kill the whole server tree (setsid at launch) so TP-scheduler - # children die and the process-sub tee gets EOF -> container can exit. - kill -TERM -"${prefill_pgid:-$prefill_pid}" 2>/dev/null || true - fi - -else - RANK=$((NODE_RANK - xP * PREFILL_NODES_PER_WORKER)) - echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME:-'default'})" - echo "Using decode config: $DECODE_SERVER_CONFIG" - echo "Decode node rank: $RANK" - echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - - DECODE_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_DECODE" ]]; then - DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" - fi - set +x - - # Agentic trace replay doesn't reproduce real token-by-token traffic, so - # measured MTP/EAGLE acceptance there isn't representative (PR #2309 - # review: https://github.com/SemiAnalysisAI/InferenceX/pull/2309#pullrequestreview-4778348624). - # Per the AgentX fairness guidelines (golden_al_distribution/README.md), - # agentic throughput benchmarks simulate acceptance at the model's - # committed golden AL instead of measuring real (non-representative) - # acceptance. Eval runs (RUN_EVAL / EVAL_ONLY) need real acceptance so - # GSM8K scores reflect actual MTP behavior. Golden curve source: - # golden_al_distribution/dsv4_mtp.yaml (thinking_on). - DECODE_SIM_ACC_ENV="" - if [[ "$DECODE_MTP_SIZE" -gt 0 ]] && { [[ "${IS_AGENTIC:-0}" == "1" ]] || [[ "${IS_AGENTIC:-}" == "true" ]]; }; then - if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "[INFO] Eval mode: synthetic MTP disabled (using real acceptance)" - else - DSV4_GOLDEN_AL="" - case "${MODEL_NAME}:${DECODE_MTP_SIZE}" in - *DeepSeek-V4*:1) DSV4_GOLDEN_AL=1.79 ;; - *DeepSeek-V4*:2) DSV4_GOLDEN_AL=2.27 ;; - *DeepSeek-V4*:3) DSV4_GOLDEN_AL=2.49 ;; - esac - if [[ -n "$DSV4_GOLDEN_AL" ]]; then - DECODE_SIM_ACC_ENV="SGLANG_SIMULATE_ACC_LEN=${DSV4_GOLDEN_AL} SGLANG_SIMULATE_ACC_METHOD=match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token" - else - echo "WARNING: agentic MTP run (model=${MODEL_NAME}, DECODE_MTP_SIZE=${DECODE_MTP_SIZE}) has no golden AL wired in server_sglang.sh -- falling back to real (unsimulated, non-representative) acceptance. Add a case in server_sglang.sh and golden_al_distribution/ before shipping this arm. See golden_al_distribution/README.md." >&2 - fi - fi - fi - - DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${DECODE_SIM_ACC_ENV} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path ${MODEL_DIR}/${MODEL_NAME} \ - --disaggregation-mode decode \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${DECODE_SERVER_CONFIG} " - - if [ "$DECODE_NODES_PER_WORKER" -gt 1 ]; then - rank=$((RANK % DECODE_NODES_PER_WORKER)) - decode_idx=$((RANK / DECODE_NODES_PER_WORKER)) - DECODE_CMD="$DECODE_CMD --dist-init-addr ${DECODE_HEADNODE_URLS[$decode_idx]} --nnodes ${DECODE_NODES_PER_WORKER} --node-rank $rank" - fi - - dump_cmd "DECODE (rank ${NODE_RANK})" "$DECODE_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - set -x - # setsid isolates the server tree in its own process group so teardown - # can group-kill it (python + TP-scheduler children); otherwise the - # children hold the process-sub tee's pipe and the container never exits. - setsid bash -c "$DECODE_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log >/dev/null) 2>&1 & - - set +x - decode_pid=$! - decode_pgid=$(ps -o pgid= -p "$decode_pid" 2>/dev/null | tr -d ' ') - : "${decode_pgid:=$decode_pid}" - fi - - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$decode_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $SGLANG_WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port 30000" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - wait_or_die "$decode_pid" bash -c "$WAIT_CMD" || exit 1 - fi - - echo "Killing the rank $RANK decode server" - if [[ "$DRY_RUN" -eq 0 ]]; then - # Group-kill the whole server tree (setsid at launch) so TP-scheduler - # children die and the process-sub tee gets EOF -> container can exit. - kill -TERM -"${decode_pgid:-$decode_pid}" 2>/dev/null || true - fi - -fi - -echo "Script completed successfully" -exit 0 diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh deleted file mode 100755 index 55154cd015..0000000000 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ /dev/null @@ -1,542 +0,0 @@ -#!/bin/bash -# vLLM Disaggregated Server Launcher with Model-Specific Configurations -# ============================================================================= -# -# Node role assignment (by NODE_RANK): -# 0 -> Proxy/Router + first Prefill node (kv_producer) -# 1..xP-1 -> Additional Prefill nodes (kv_producer) -# xP..xP+yD-1 -> Decode nodes (kv_consumer) -# -# Total nodes = xP + yD (router co-located with first prefill, like SGLang). - -# ============================================================================= -# Dependency Setup (idempotent; required when using base vLLM image) -# ============================================================================= -source "$(dirname "${BASH_SOURCE[0]}")/setup_deps.sh" - -# ============================================================================= -# Environment Configuration -# ============================================================================= - -NODE0_ADDR="${NODE0_ADDR:-localhost}" -NODE_RANK="${NODE_RANK:-0}" -MODEL_DIR="${MODEL_DIR:-}" -MODEL_NAME="${MODEL_NAME:-}" - -xP="${xP:-1}" -yD="${yD:-1}" - -IPADDRS="${IPADDRS:-localhost}" - -# Benchmark Configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" - -DRY_RUN="${DRY_RUN:-0}" -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-$GPUS_PER_NODE}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-$GPUS_PER_NODE}" - -ROUTER_PORT="${ROUTER_PORT:-30000}" -SERVER_PORT="${SERVER_PORT:-2584}" -ENGINE_ID="${ENGINE_ID:-${MODEL_NAME}-pd-run}" - -# Prefer MODEL_PATH from job.slurm (handles HF cache snapshot resolution) -MODEL_PATH="${MODEL_PATH:-${MODEL_DIR}/${MODEL_NAME}}" - -# ============================================================================= -# Dependencies and Environment Setup -# ============================================================================= -source $WS_PATH/env.sh - -host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}') -# RDMA IP for Nixl KV transfer (prefer 192.168.x.x subnet if available) -rdma_ip=$(hostname -I | tr ' ' '\n' | grep '^192\.168\.' | head -1) -rdma_ip="${rdma_ip:-$host_ip}" -host_name=$(hostname) - -echo "[INFO] Management IP (barriers/proxy): $host_ip" -echo "[INFO] RDMA IP (Nixl KV transfer): $rdma_ip" - -# ============================================================================= -# RDMA / Nixl Workarounds -# ============================================================================= - -setup_rdma_env() { - # Pensando ionic (RoCEv2) point-to-point /31 route fix. - # Each benic interface has a /31 to the TOR switch. Without explicit routes, - # traffic to other nodes' RDMA IPs falls through to the management network. - if [[ "$rdma_ip" =~ ^192\.168\.([0-9]+)\.([0-9]+)$ ]]; then - local rdma_subnet="${BASH_REMATCH[1]}" - local rdma_host="${BASH_REMATCH[2]}" - local rdma_gw="192.168.${rdma_subnet}.$(( rdma_host | 1 ))" - local rdma_iface - rdma_iface=$(ip -o addr show | awk -v ip="$rdma_ip" '$4 ~ ip {print $2}' | head -1) - if [[ -n "$rdma_iface" ]]; then - ip route replace "192.168.${rdma_subnet}.0/24" via "$rdma_gw" dev "$rdma_iface" 2>/dev/null && \ - echo "[RDMA-ROUTE] Added 192.168.${rdma_subnet}.0/24 via $rdma_gw dev $rdma_iface" || \ - echo "[RDMA-ROUTE] Route add failed for 192.168.${rdma_subnet}.0/24" - fi - fi - - # Patch Nixl UCX backend: set ucx_error_handling_mode=none. - # Required for ALL NIC types under high concurrency (C512+). Without this, - # UCX's default UCP_ERR_HANDLING_MODE_PEER triggers transport-level error - # recovery on ibv_post_send failures, preventing RIXL RDMA READ retries from - # recovering gracefully. This causes the prefill KV cache to fill to 100% - # and deadlock the pipeline. On ionic NICs this was already applied (rdmacm - # incompatibility); on mlx5 NICs it was incorrectly skipped. - local nixl_api - nixl_api=$(python3 -c "import rixl._api; print(rixl._api.__file__)" 2>/dev/null) - if [[ -n "$nixl_api" ]]; then - if ! grep -q 'ucx_error_handling_mode' "$nixl_api"; then - sed -i '/self\.create_backend(bknd, init)/i\ init["ucx_error_handling_mode"] = "none"' "$nixl_api" - echo "[PATCH] Added ucx_error_handling_mode=none to $nixl_api (IBDEVICES=${IBDEVICES:-unset})" - else - echo "[PATCH] ucx_error_handling_mode already set in $nixl_api" - fi - fi -} - -setup_rdma_env - -if [[ -z "$UCX_NET_DEVICES" ]]; then - echo "Error: UCX_NET_DEVICES is empty after env.sh detection" >&2 - exit 1 -fi - -# ============================================================================= -# Model-Specific Configuration from YAML -# ============================================================================= -MODELS_YAML="${WS_PATH}/models_vllm.yaml" - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "ERROR: models.yaml not found at $MODELS_YAML" - exit 1 -fi - -if [[ -z "$MODEL_NAME" ]]; then - echo "ERROR: MODEL_NAME is not set"; exit 1 -fi - -eval "$(python3 -c " -import yaml, sys - -with open('${MODELS_YAML}') as f: - models = yaml.safe_load(f) - -model_name = '${MODEL_NAME}' -if model_name not in models: - print(f'echo \"ERROR: Model {model_name} not in models.yaml\"; exit 1') - sys.exit(0) - -m = models[model_name] - -def bash_escape(s): - \"\"\"Escape a value for safe embedding in a bash double-quoted assignment.\"\"\" - return s.replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"').replace('\$', '\\\\\$').replace('\`', '\\\\\`') - -pf = bash_escape(m.get('prefill_flags', '--tensor-parallel-size 8')) -df = bash_escape(m.get('decode_flags', '--tensor-parallel-size 8')) -ev = bash_escape(m.get('env', '')) -dev = bash_escape(m.get('decode_env', '')) -pev = bash_escape(m.get('prefill_env', '')) -print(f'PREFILL_SERVER_CONFIG=\"{pf}\"') -print(f'DECODE_SERVER_CONFIG=\"{df}\"') -print(f'MODEL_ENVS=\"{ev}\"') -print(f'DECODE_MODEL_ENVS=\"{dev}\"') -print(f'PREFILL_MODEL_ENVS=\"{pev}\"') -")" - -echo "Loaded model configuration for: $MODEL_NAME" - -# Apply tensor-parallel size and EP/DP flags from submit pipeline. -if [[ -n "${PREFILL_TP_SIZE:-}" ]]; then - if echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--tensor-parallel-size'; then - PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed -E "s/--tensor-parallel-size[[:space:]]+[0-9]+/--tensor-parallel-size ${PREFILL_TP_SIZE}/g") - else - PREFILL_SERVER_CONFIG+=" --tensor-parallel-size ${PREFILL_TP_SIZE}" - fi -fi -if [[ -n "${DECODE_TP_SIZE:-}" ]]; then - if echo "$DECODE_SERVER_CONFIG" | grep -q -- '--tensor-parallel-size'; then - DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed -E "s/--tensor-parallel-size[[:space:]]+[0-9]+/--tensor-parallel-size ${DECODE_TP_SIZE}/g") - else - DECODE_SERVER_CONFIG+=" --tensor-parallel-size ${DECODE_TP_SIZE}" - fi -fi -if [[ "${PREFILL_ENABLE_EP:-false}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then - PREFILL_SERVER_CONFIG+=" --enable-expert-parallel" -fi -if [[ "${PREFILL_ENABLE_DP:-false}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-dp-attention'; then - PREFILL_SERVER_CONFIG+=" --enable-dp-attention" -fi -if [[ "${DECODE_ENABLE_EP:-false}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then - DECODE_SERVER_CONFIG+=" --enable-expert-parallel" -fi -if [[ "${DECODE_ENABLE_DP:-false}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG" | grep -q -- '--enable-dp-attention'; then - DECODE_SERVER_CONFIG+=" --enable-dp-attention" -fi - -echo "PREFILL_SERVER_CONFIG (after TP/EP/DP): $PREFILL_SERVER_CONFIG" -echo "DECODE_SERVER_CONFIG (after TP/EP/DP): $DECODE_SERVER_CONFIG" - -# ============================================================================= -# Container Synchronization -# ============================================================================= - -echo "Waiting at the container creation barrier on $host_name" -python3 $WS_PATH/sync.py barrier \ - --local-ip ${host_ip} \ - --local-port 5000 \ - --enable-port \ - --node-ips ${IPADDRS} \ - --node-ports 5000 \ - --wait-for-all-ports \ - --timeout 600 - -# ============================================================================= -# Cluster Topology Configuration -# ============================================================================= -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_ARGS="" -DECODE_ARGS="" - -for ((i=0; i "$PREFILL_LOG_FILE" 2>&1 & - set +x - prefill_pid=$! - fi - - echo "Waiting for all prefill and decode servers to be up . . ." - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: skipping barrier (wait-for-all-ports)" - else - python3 $WS_PATH/sync.py barrier \ - --node-ips ${IPADDRS} \ - --node-ports $SERVER_PORT \ - --wait-for-all-ports \ - --timeout 1800 - fi - - echo "Congratulations!!! All prefill and decode servers are up . . ." - - # Wait for proxy /health to confirm it is accepting requests - HEALTH_BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-health \ - --health-endpoint /health \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $HEALTH_BARRIER_CMD" - else - eval "$HEALTH_BARRIER_CMD" - echo "MoRI-IO proxy is ready for benchmarking" - fi - - echo "Ready for benchmarking on ${host_name}:${host_ip}" - echo "Benchmarking on ${host_name}:${host_ip}" - cd $WS_PATH - - export ROUTER_PORT=$ROUTER_PORT - BENCH_CMD="bash $WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - # Run evaluation if requested (before killing router) - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/health" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --framework lm-eval --port $ROUTER_PORT (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" - else - run_eval --framework lm-eval --port "$ROUTER_PORT" - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 - EVAL_FAILED=1 - else - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export EP_SIZE=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && EP_SIZE="${PREFILL_TP_SIZE}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - [[ "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" - export DECODE_NUM_WORKERS="${yD}" - export DP_ATTENTION="${PREFILL_ENABLE_DP}" - export PREFILL_DP_ATTENTION="${PREFILL_ENABLE_DP}" - export DECODE_DP_ATTENTION="${DECODE_ENABLE_DP}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - - append_lm_eval_summary - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - mkdir -p "$EVAL_COPY_DIR" - for f in meta_env.json; do - [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" - done - find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - - echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" - fi - fi - - popd - fi - fi - - # Copy benchmark/eval results to BENCHMARK_LOGS_DIR (mounted from host) - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR:-/run_logs}/logs" - mkdir -p "$LOGS_OUTPUT" - - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Killing the prefill server" - if [[ "$DRY_RUN" -eq 0 ]]; then - [[ -n "${prefill_pid:-}" ]] && kill $prefill_pid 2>/dev/null || true - sleep 2 - pkill -f "vllm serve" 2>/dev/null || true - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$xP" ]; then - echo "${host_name}:${host_ip} is Additional Prefill Node (Model: ${MODEL_NAME})" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - - setup_vllm_env - - for env_pair in ${PREFILL_MODEL_ENVS}; do - export "$env_pair" - echo "[PREFILL_ENV] $env_pair" - done - - SERVED_MODEL="${MODEL_NAME}" - PREFILL_CMD="vllm serve ${MODEL_PATH} \ - --served-model-name ${SERVED_MODEL} \ - --port $SERVER_PORT \ - --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ - ${PREFILL_SERVER_CONFIG}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - PREFILL_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log" - set -x - eval "$PREFILL_CMD" > "$PREFILL_LOG_FILE" 2>&1 & - set +x - prefill_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port ${ROUTER_PORT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the prefill server" - [[ "$DRY_RUN" -eq 0 ]] && kill $prefill_pid 2>/dev/null || true - -else - echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME})" - echo "Using decode config: $DECODE_SERVER_CONFIG" - - setup_vllm_env - - for env_pair in ${DECODE_MODEL_ENVS}; do - export "$env_pair" - echo "[DECODE_ENV] $env_pair" - done - - SERVED_MODEL="${MODEL_NAME}" - DECODE_CMD="vllm serve ${MODEL_PATH} \ - --served-model-name ${SERVED_MODEL} \ - --port $SERVER_PORT \ - --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_consumer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ - ${DECODE_SERVER_CONFIG}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - DECODE_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log" - set -x - eval "$DECODE_CMD" > "$DECODE_LOG_FILE" 2>&1 & - set +x - decode_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port ${ROUTER_PORT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the decode server" - [[ "$DRY_RUN" -eq 0 ]] && kill $decode_pid 2>/dev/null || true -fi - -# echo "Killing the etcd server" -# kill $etcd_pid 2>/dev/null || true -# pkill -f etcd 2>/dev/null || true - -echo "Script completed successfully" -exit 0 diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh deleted file mode 100644 index 8fa49dfb27..0000000000 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# ============================================================================= -# setup_deps.sh — Install missing disagg dependencies at container start. -# -# Dispatched by $ENGINE (set by server.sh dispatcher): -# vllm-disagg -> recipe deps + amd-quark + UCX/RIXL path exports -# (base image: vllm/vllm-openai-rocm:nightly) -# sglang-disagg -> SGLang aiter gluon patch + per-model installs -# (base image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-*) -# -# Sourced by server_vllm.sh and server_sglang.sh so PATH / LD_LIBRARY_PATH -# exports persist. Each patch is idempotent: skipped if already applied. -# -# Build steps run in subshells to avoid CWD pollution between installers. -# ============================================================================= - -ROCM_PATH="${ROCM_PATH:-/opt/rocm}" -UCX_HOME="${UCX_HOME:-/usr/local/ucx}" -RIXL_HOME="${RIXL_HOME:-/usr/local/rixl}" - -_SETUP_START=$(date +%s) -_SETUP_INSTALLED=() - -git_clone_retry() { - local url="$1" dest="$2" max_tries=3 try=1 - while (( try <= max_tries )); do - if git clone --quiet "$url" "$dest" 2>/dev/null; then return 0; fi - echo "[SETUP] git clone attempt $try/$max_tries failed for $url, retrying in 10s..." - rm -rf "$dest" - sleep 10 - (( try++ )) - done - echo "[SETUP] git clone failed after $max_tries attempts: $url" - return 1 -} - -# --------------------------------------------------------------------------- -# 5. Container RDMA/net tools -# - ibv_devinfo comes from ibverbs-utils -# - iproute2 provides the `ip` command -# Used for in-container NIC/RDMA validation and routing checks. -# --------------------------------------------------------------------------- -install_recipe_deps() { - if command -v ibv_devinfo >/dev/null 2>&1 && command -v ip >/dev/null 2>&1; then - echo "[SETUP] Container RDMA/net tools already present" - return 0 - fi - - echo "[SETUP] Installing ibv_devinfo + iproute2 in container..." - apt-get update -q -y && apt-get install -q -y \ - ibverbs-utils iproute2 \ - && rm -rf /var/lib/apt/lists/* - - if ! command -v ibv_devinfo >/dev/null 2>&1 || ! command -v ip >/dev/null 2>&1; then - echo "[SETUP] ERROR: Failed to install ibv_devinfo/iproute2"; exit 1 - fi - _SETUP_INSTALLED+=("ibverbs-utils+iproute2") -} - -# --------------------------------------------------------------------------- -# 6b. amd-quark (MXFP4 quantization support for Kimi-K2.5-MXFP4 and similar) -# Required due to ROCm vLLM missing the quark dependency: -# https://github.com/vllm-project/vllm/issues/35633 -# --------------------------------------------------------------------------- -install_amd_quark() { - if python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] amd-quark already present" - return 0 - fi - - echo "[SETUP] Installing amd-quark for MXFP4 quantization support..." - pip install --quiet amd-quark - - if ! python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] WARN: amd-quark install failed (non-fatal for non-MXFP4 models)" - return 0 - fi - _SETUP_INSTALLED+=("amd-quark") -} - -# --------------------------------------------------------------------------- -# SGLang: Install latest transformers for GLM model type support. -# -# GLM-5 (zai-org/GLM-5-FP8) requires a transformers build that includes -# the glm_moe_dsa model type. The mori images do not ship it. Gated on any -# GLM model name (not just GLM-5-FP8) so other GLM variants pick up the same -# fix; only installs when a GLM model is active (avoid overhead otherwise). -# --------------------------------------------------------------------------- -install_transformers_glm5() { - if [[ "$MODEL_NAME" != *GLM* ]]; then - return 0 - fi - - if python3 -c "from transformers import AutoConfig; AutoConfig.from_pretrained('zai-org/GLM-5-FP8', trust_remote_code=True)" 2>/dev/null; then - echo "[SETUP] transformers already supports GLM-5 model type" - return 0 - fi - - echo "[SETUP] Installing transformers with GLM-5 (glm_moe_dsa) support..." - pip install --quiet -U --no-cache-dir \ - "git+https://github.com/huggingface/transformers.git@6ed9ee36f608fd145168377345bfc4a5de12e1e2" - _SETUP_INSTALLED+=("transformers-glm5") -} - -# ============================================================================= -# Run installers (engine-gated) -# ============================================================================= - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - install_recipe_deps - install_amd_quark - - # ========================================================================= - # vLLM: Export UCX/RIXL paths (persists since this file is sourced) - # ========================================================================= - export ROCM_PATH="${ROCM_PATH}" - export UCX_HOME="${UCX_HOME}" - export RIXL_HOME="${RIXL_HOME}" - export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" - export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" -else - install_transformers_glm5 -fi - -_SETUP_END=$(date +%s) -if [[ ${#_SETUP_INSTALLED[@]} -eq 0 ]]; then - echo "[SETUP] All dependencies already present ($(( _SETUP_END - _SETUP_START ))s wallclock)" -else - echo "[SETUP] Installed: ${_SETUP_INSTALLED[*]} in $(( _SETUP_END - _SETUP_START ))s" -fi diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh deleted file mode 100755 index e7ecc8d913..0000000000 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ /dev/null @@ -1,334 +0,0 @@ -#!/bin/bash -# -# Cluster Configuration Template for Multi-Node Disaggregated Serving -# -# This script submits a multi-node disaggregated benchmark job to SLURM. -# It must be configured for your specific cluster before use. -# -# ENGINE=sglang (default): SGLang disaggregated serving -# ENGINE=vllm: vLLM disaggregated serving -# -# Router is co-located with the first prefill node (same for both engines), -# so NUM_NODES = PREFILL_NODES + DECODE_NODES. - -usage() { - cat << 'USAGE' -Usage: - bash submit.sh \ - \ - \ - \ - \ - [NODE_LIST] - -Arguments: - PREFILL_NODES Number of prefill nodes - PREFILL_WORKERS Number of prefill workers (usually 1) - DECODE_NODES Number of decode nodes - DECODE_WORKERS Number of decode workers (usually 1) - ISL Input sequence length - OSL Output sequence length - CONCURRENCIES Concurrency levels, delimited by 'x' (e.g., "8x16x32") - REQUEST_RATE Request rate ("inf" for max throughput) - PREFILL_ENABLE_EP true/false or 1/0 (expert parallelism on prefill) - PREFILL_ENABLE_DP true/false or 1/0 (data-parallel attention on prefill) - DECODE_ENABLE_EP true/false or 1/0 (expert parallelism on decode) - DECODE_ENABLE_DP true/false or 1/0 (data-parallel attention on decode) - PREFILL_TP Tensor parallel size per prefill node - DECODE_TP Tensor parallel size per decode node - RANDOM_RANGE_RATIO Random range ratio for benchmark client - NODE_LIST Optional: comma-separated hostnames (must match NUM_NODES) - -Required environment variables: - SLURM_ACCOUNT SLURM account name - SLURM_PARTITION SLURM partition - TIME_LIMIT Job time limit (e.g., "08:00:00") - MODEL_PATH Path to model directory (e.g., /nfsdata) - MODEL_NAME Model name directory - CONTAINER_IMAGE Docker image name (e.g., vllm_disagg_pd:latest) - RUNNER_NAME Runner identifier (for job name) - -Optional environment variables: - DRY_RUN 1 = echo composed server/router launch commands instead of - running them (preview a recipe against a real allocation). -USAGE -} - -check_env() { - local name="$1" - if [[ -z "${!name:-}" ]]; then - echo "Error: ${name} not specified" >&2 - usage >&2 - exit 1 - fi -} - -check_env SLURM_ACCOUNT -check_env SLURM_PARTITION -check_env TIME_LIMIT - -check_env MODEL_PATH -check_env MODEL_NAME -check_env CONTAINER_IMAGE -check_env RUNNER_NAME -check_env FRAMEWORK - -# GPUS_PER_NODE defaults to 8 (MI355X). Set to 4 for MI325X if needed. -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -# COMMAND_LINE ARGS -PREFILL_NODES=$1 -PREFILL_WORKERS=${2:-1} -DECODE_NODES=$3 -DECODE_WORKERS=${4:-1} -ISL=$5 -OSL=$6 -CONCURRENCIES=$7 -REQUEST_RATE=$8 -PREFILL_ENABLE_EP=${9:-true} -PREFILL_ENABLE_DP=${10:-true} -DECODE_ENABLE_EP=${11:-true} -DECODE_ENABLE_DP=${12:-true} -PREFILL_TP=${13:-8} -DECODE_TP=${14:-8} -RANDOM_RANGE_RATIO=${15:-0.8} -NODE_LIST=${16} - -NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) -profiler_args="${ISL} ${OSL} ${CONCURRENCIES} ${REQUEST_RATE}" - -# Export variables for the SLURM job -export ENGINE="${FRAMEWORK:-sglang}" -export MODEL_DIR=$MODEL_PATH -export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE -export PROFILER_ARGS=$profiler_args - -# Engine-specific xP/yD semantics and TP exports -if [[ "$ENGINE" == "vllm-disagg" ]]; then - export PROXY_STREAM_IDLE_TIMEOUT=${PROXY_STREAM_IDLE_TIMEOUT:-300} -fi -# xP = prefill workers, yD = decode workers (may span multiple nodes) -export xP=$PREFILL_WORKERS -export yD=$DECODE_WORKERS -export PREFILL_TP_SIZE=$(( $PREFILL_NODES * $PREFILL_TP / $PREFILL_WORKERS )) -export PREFILL_ENABLE_EP=${PREFILL_ENABLE_EP} -export PREFILL_ENABLE_DP=${PREFILL_ENABLE_DP} -export PREFILL_TP -export PREFILL_EP=${PREFILL_EP:-1} -export PREFILL_DP_ATTN=${PREFILL_DP_ATTN:-false} -export PREFILL_NUM_WORKERS=${PREFILL_NUM_WORKERS:-$PREFILL_WORKERS} -export PREFILL_PP_SIZE=${PREFILL_PP_SIZE:-1} -export PREFILL_DCP_SIZE=${PREFILL_DCP_SIZE:-1} -export PREFILL_PCP_SIZE=${PREFILL_PCP_SIZE:-1} -export DECODE_TP_SIZE=$(( $DECODE_NODES * $DECODE_TP / $DECODE_WORKERS )) -export DECODE_ENABLE_EP=${DECODE_ENABLE_EP} -export DECODE_ENABLE_DP=${DECODE_ENABLE_DP} -export DECODE_TP -export DECODE_EP=${DECODE_EP:-1} -export DECODE_DP_ATTN=${DECODE_DP_ATTN:-false} -export DECODE_NUM_WORKERS=${DECODE_NUM_WORKERS:-$DECODE_WORKERS} -export DECODE_PP_SIZE=${DECODE_PP_SIZE:-1} -export DECODE_DCP_SIZE=${DECODE_DCP_SIZE:-1} -export DECODE_PCP_SIZE=${DECODE_PCP_SIZE:-1} -export DECODE_MTP_SIZE=${DECODE_MTP_SIZE} - -export NUM_NODES=$NUM_NODES -export GPUS_PER_NODE=$GPUS_PER_NODE -export MODEL_NAME=$MODEL_NAME -export BENCH_INPUT_LEN=${ISL} -export BENCH_OUTPUT_LEN=${OSL} -export BENCH_NUM_PROMPTS_MULTIPLIER=${BENCH_NUM_PROMPTS_MULTIPLIER:-10} -export BENCH_MAX_CONCURRENCY=${CONCURRENCIES} -export BENCH_REQUEST_RATE=${REQUEST_RATE} -export BENCH_RANDOM_RANGE_RATIO=${RANDOM_RANGE_RATIO:-0.8} - -# DRY_RUN=1 makes server_sglang.sh echo the composed prefill/decode/router launch -# commands instead of executing them (useful for previewing a recipe against a real -# allocation). Threaded here → job.slurm → Docker (-e DRY_RUN) → server_sglang.sh. -# sbatch defaults to --export=ALL, so exporting it is what carries it into the job. -export DRY_RUN="${DRY_RUN:-0}" - -# Eval-related env vars (threaded from workflow → runner → here → job.slurm → Docker) -export RUN_EVAL="${RUN_EVAL:-false}" -export EVAL_ONLY="${EVAL_ONLY:-false}" -export EVAL_CONC="${EVAL_CONC:-}" -export FRAMEWORK="${FRAMEWORK:-}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-}" -export IS_MULTINODE="${IS_MULTINODE:-false}" -export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" -export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" -export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" -export HF_TOKEN="${HF_TOKEN:-}" -export SCENARIO_TYPE="${SCENARIO_TYPE:-}" -export EVAL_LIMIT="${EVAL_LIMIT:-}" - -# Log directory: must be on NFS (shared filesystem) so the submit host can read SLURM output. -export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" -mkdir -p "$BENCHMARK_LOGS_DIR" - -# Optional: pass an explicit node list to sbatch. -NODELIST_OPT=() -if [[ -n "${NODE_LIST//[[:space:]]/}" ]]; then - IFS=',' read -r -a NODE_ARR <<< "$NODE_LIST" - if [[ "${#NODE_ARR[@]}" -ne "$NUM_NODES" ]]; then - echo "Error: NODE_LIST has ${#NODE_ARR[@]} nodes but NUM_NODES=${NUM_NODES}" >&2 - echo "Error: NODE_LIST='${NODE_LIST}'" >&2 - exit 1 - fi - NODELIST_CSV="$(IFS=,; echo "${NODE_ARR[*]}")" - NODELIST_OPT=(--nodelist "$NODELIST_CSV") -fi - -# Optional: exclude specific nodes for known-bad (FRAMEWORK, MODEL_NAME) -# combos (e.g. nodes with broken Docker sockets), looked up from -# node_excludes.yaml. Set SLURM_EXCLUDE_NODES to override with an explicit -# comma-separated hostname list (takes precedence over the file). -# -# Resolution must fail loudly (not silently yield an empty exclude list) if -# it can't be trusted: a submit host missing python3/PyYAML, or a genuine -# parse error, must not silently reintroduce the known-bad-node issue this -# exclusion mechanism exists to prevent. -EXCLUDE_OPT=() -NODE_EXCLUDES_YAML="$(dirname "$0")/node_excludes.yaml" -if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then - RESOLVED_EXCLUDE_NODES="$SLURM_EXCLUDE_NODES" -elif [[ -f "$NODE_EXCLUDES_YAML" ]]; then - if command -v python3 >/dev/null 2>&1 && python3 -c "import yaml" >/dev/null 2>&1; then - RESOLVED_EXCLUDE_NODES=$(python3 -c " -import yaml - -with open('${NODE_EXCLUDES_YAML}') as f: - cfg = yaml.safe_load(f) or {} - -framework = '${FRAMEWORK}' -model = '${MODEL_NAME}' -for rule in cfg.get('rules', []): - if rule.get('framework') == framework and model in (rule.get('models') or []): - print(rule.get('exclude_nodes', '')) - break -") - PYTHON_EXCLUDE_RC=$? - if [[ $PYTHON_EXCLUDE_RC -ne 0 ]]; then - echo "Error: python3 failed (exit ${PYTHON_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 - echo "Error: fix the YAML, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 - exit 1 - fi - else - # Fall back to an awk parser (mirrors job.slurm's awk-based models.yaml - # parsing) matched to node_excludes.yaml's fixed rule/models/exclude_nodes - # shape. Only exercised when python3 or its yaml module is unavailable. - echo "Warning: python3/PyYAML unavailable on submit host; falling back to awk parsing of ${NODE_EXCLUDES_YAML}" >&2 - RESOLVED_EXCLUDE_NODES=$(awk -v fw="$FRAMEWORK" -v model="$MODEL_NAME" ' - /^ - framework:/ { - line = $0 - sub(/^ - framework: */, "", line) - fw_match = (line == fw) - model_match = 0 - next - } - fw_match && /^ - / { - m = $0 - sub(/^ - */, "", m) - gsub(/^"|"$/, "", m) - if (m == model) model_match = 1 - next - } - fw_match && model_match && /^ exclude_nodes:/ { - val = $0 - sub(/^ *exclude_nodes: */, "", val) - gsub(/^"|"$/, "", val) - print val - exit - } - ' "$NODE_EXCLUDES_YAML") - AWK_EXCLUDE_RC=$? - if [[ $AWK_EXCLUDE_RC -ne 0 ]]; then - echo "Error: awk fallback failed (exit ${AWK_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 - echo "Error: fix the YAML/parser, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 - exit 1 - fi - fi -else - RESOLVED_EXCLUDE_NODES="" -fi -if [[ -n "$RESOLVED_EXCLUDE_NODES" ]]; then - EXCLUDE_OPT=(--exclude "$RESOLVED_EXCLUDE_NODES") -fi - -# ============================================================================= -# Reuse existing allocation (skip sbatch) -# ============================================================================= -# When SLURM_REUSE_JOBID is set, run job.slurm directly in the current shell, -# attaching to the existing allocation. Inner `srun` calls pick up the -# allocation via SLURM_JOB_ID; SLURM_OVERLAP=1 lets them share task slots with -# the interactive shell already holding the allocation. -if [[ -n "${SLURM_REUSE_JOBID:-}" ]]; then - REUSE_JID="$SLURM_REUSE_JOBID" - echo "Reusing existing Slurm allocation ${REUSE_JID} (skipping sbatch)" >&2 - - # Resolve allocation's nodelist if not already provided. - ALLOC_NODELIST="${SLURM_JOB_NODELIST:-$(squeue -h -j "$REUSE_JID" -o '%N' 2>/dev/null)}" - if [[ -z "$ALLOC_NODELIST" ]]; then - echo "Error: could not resolve nodelist for job ${REUSE_JID}" >&2 - exit 1 - fi - ALLOC_NNODES=$(scontrol show hostnames "$ALLOC_NODELIST" | wc -l) - if [[ "$ALLOC_NNODES" -lt "$NUM_NODES" ]]; then - echo "Error: allocation ${REUSE_JID} has ${ALLOC_NNODES} nodes, need ${NUM_NODES}" >&2 - exit 1 - fi - - export SLURM_JOB_ID="$REUSE_JID" - export SLURM_JOBID="$REUSE_JID" - export SLURM_JOB_NODELIST="$ALLOC_NODELIST" - export SLURM_NODELIST="$ALLOC_NODELIST" - export SLURM_NNODES="$ALLOC_NNODES" - export SLURM_JOB_NUM_NODES="$ALLOC_NNODES" - export SLURM_NTASKS="$ALLOC_NNODES" - export SLURM_NPROCS="$ALLOC_NNODES" - export SLURM_NTASKS_PER_NODE=1 - export SLURM_TASKS_PER_NODE="1(x${ALLOC_NNODES})" - export SLURM_OVERLAP=1 - export SLURM_SUBMIT_DIR="$(pwd)" - - STDOUT_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.out" - STDERR_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.err" - rm -f "$STDOUT_LOG" "$STDERR_LOG" - - nohup bash "$(dirname "$0")/job.slurm" >"$STDOUT_LOG" 2>"$STDERR_LOG" & - INLINE_PID=$! - echo "$INLINE_PID" > "${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.pid" - echo "Started job.slurm (pid=${INLINE_PID}); logs: ${STDOUT_LOG}" >&2 - - echo "$REUSE_JID" - exit 0 -fi - -# Construct the sbatch command -sbatch_cmd=( - sbatch - --parsable - --exclusive - -N "$NUM_NODES" - -n "$NUM_NODES" - "${NODELIST_OPT[@]}" - "${EXCLUDE_OPT[@]}" - --time "$TIME_LIMIT" - --partition "$SLURM_PARTITION" - --account "$SLURM_ACCOUNT" - --job-name "$RUNNER_NAME" - --output "${BENCHMARK_LOGS_DIR}/slurm_job-%j.out" - --error "${BENCHMARK_LOGS_DIR}/slurm_job-%j.err" - "$(dirname "$0")/job.slurm" -) - -JOB_ID=$("${sbatch_cmd[@]}") -if [[ $? -ne 0 ]]; then - echo "Error: Failed to submit job with sbatch" >&2 - exit 1 -fi -echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/sync.py b/benchmarks/multi_node/amd_utils/sync.py deleted file mode 100755 index 96e94c1b08..0000000000 --- a/benchmarks/multi_node/amd_utils/sync.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python3 -""" -Multi-node synchronization utilities for disaggregated inference. - -Subcommands: - barrier - Wait until all specified nodes have opened their ports (TCP barrier) - Optionally wait for HTTP health endpoints to return 200 - wait - Block until a remote port closes (shutdown coordination) -""" - -import socket -import time -import threading -import argparse -import sys -import urllib.request -import urllib.error - - -def is_port_open(ip, port, timeout=2): - """Check if a given IP and port are accessible.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(timeout) - return s.connect_ex((ip, port)) == 0 - - -def check_health(ip, port, path="/health", timeout=2): - """Return True if http://ip:port/path returns HTTP 200.""" - try: - url = f"http://{ip}:{port}{path}" - req = urllib.request.Request(url) - with urllib.request.urlopen(req, timeout=timeout) as resp: - return getattr(resp, "status", 200) == 200 - except (urllib.error.URLError, urllib.error.HTTPError, OSError): - return False - - -# ============================================================================= -# barrier subcommand -# ============================================================================= - -def cmd_barrier(args): - """Wait until all nodes have opened the specified ports.""" - NODE_IPS = [ip.strip() for ip in args.node_ips.split(",") if ip.strip()] - NODE_PORTS = [int(p.strip()) for p in args.node_ports.split(",") if p.strip()] - - if not NODE_IPS: - print("Error: NODE_IPS argument is empty or not set.") - sys.exit(1) - - if len(NODE_PORTS) == 1: - NODE_PORTS *= len(NODE_IPS) - elif len(NODE_PORTS) != len(NODE_IPS): - print("Error: Number of ports must match number of node IPs or only one port should be given for all.") - sys.exit(1) - - server_socket = None - - def open_port(): - nonlocal server_socket - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind((args.local_ip, args.local_port)) - server_socket.listen(5) - print(f"Port {args.local_port} is now open on {args.local_ip}.") - while True: - conn, addr = server_socket.accept() - conn.close() - - def close_port(): - nonlocal server_socket - if server_socket: - server_socket.close() - print(f"Port {args.local_port} has been closed on {args.local_ip}.") - - if args.enable_port: - threading.Thread(target=open_port, daemon=True).start() - - # Wait for all ports (TCP check) - if args.wait_for_all_ports: - start_time = time.time() - timeout = args.timeout - seen_open = set() - - while True: - status = {(ip, port): is_port_open(ip, port) - for ip, port in zip(NODE_IPS, NODE_PORTS)} - - # Fail fast: a port that was open and is now closed means that - # server died/was killed; don't wait out the full timeout. - regressed = [t for t in seen_open if not status[t]] - if regressed: - print("ERROR: the following ports were open then went down (server died):", flush=True) - for ip, port in regressed: - print(f" - {ip}:{port}", flush=True) - sys.exit(1) - seen_open.update(t for t, ok in status.items() if ok) - - if all(status.values()): - break - - if timeout > 0: - elapsed = time.time() - start_time - if elapsed >= timeout: - not_open = [t for t, ok in status.items() if not ok] - print(f"ERROR: Timeout after {timeout} seconds waiting for ports to open.", flush=True) - print("The following nodes/ports are still not responding:", flush=True) - for ip, port in not_open: - print(f" - {ip}:{port}", flush=True) - sys.exit(1) - remaining = timeout - (time.time() - start_time) - print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . . ({remaining:.0f}s remaining)", flush=True) - else: - print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . .", flush=True) - time.sleep(5) - - # Wait for all health endpoints (HTTP check) - if args.wait_for_all_health: - health_path = args.health_endpoint - start_time = time.time() - timeout = args.timeout - seen_ready = set() - - while True: - status = { - (ip, port): check_health(ip, port, health_path) - for ip, port in zip(NODE_IPS, NODE_PORTS) - } - - # Fail fast: an endpoint that was healthy and is now down means the - # server died/was killed; don't wait out the full timeout. - regressed = [t for t in seen_ready if not status[t]] - if regressed: - print(f"ERROR: the following ({health_path}) were healthy then went down (server died):", flush=True) - for ip, port in regressed: - print(f" - http://{ip}:{port}{health_path}", flush=True) - sys.exit(1) - seen_ready.update(t for t, ok in status.items() if ok) - - if all(status.values()): - break - - if timeout > 0: - elapsed = time.time() - start_time - if elapsed >= timeout: - not_ready = [t for t, ok in status.items() if not ok] - print(f"ERROR: Timeout after {timeout} seconds waiting for health endpoints.", flush=True) - print(f"The following (http://ip:port{health_path}) are still not responding:", flush=True) - for ip, port in not_ready: - print(f" - http://{ip}:{port}{health_path}", flush=True) - sys.exit(1) - remaining = timeout - (time.time() - start_time) - print( - f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) .. ({remaining:.0f}s remaining)", - flush=True, - ) - else: - print(f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) ..", flush=True) - time.sleep(30) - - if args.enable_port: - # Keep the port open long enough for slow nodes to pass their barrier. - # The previous 30s was too short when setup times vary by minutes. - grace = max(60, args.timeout // 2) if args.timeout > 0 else 300 - time.sleep(grace) - close_port() - - -# ============================================================================= -# wait subcommand -# ============================================================================= - -def cmd_wait(args): - """Wait while a remote port remains open, exit when it closes.""" - print( - f"Waiting while port {args.remote_port} on {args.remote_ip} is open...", - flush=True, - ) - while is_port_open(args.remote_ip, args.remote_port): - time.sleep(5) - print(f"Port {args.remote_port} on {args.remote_ip} is now closed.", flush=True) - - -# ============================================================================= -# CLI -# ============================================================================= - -def main(): - parser = argparse.ArgumentParser(description="Multi-node synchronization utilities.") - subparsers = parser.add_subparsers(dest="command", required=True) - - # barrier subcommand - bp = subparsers.add_parser("barrier", help="Wait for all nodes to open specified ports.") - bp.add_argument("--local-ip", required=False, help="Local IP address to bind the server.") - bp.add_argument("--local-port", type=int, required=False, help="Port number to bind the server.") - bp.add_argument("--enable-port", action="store_true", help="Enable opening and closing of local port.") - bp.add_argument("--node-ips", required=True, help="Comma-separated list of node IPs.") - bp.add_argument("--node-ports", required=True, help="Comma-separated list of ports to check.") - bp.add_argument("--timeout", type=int, default=600, - help="Timeout in seconds (default: 600). Set to 0 for no timeout.") - bp.add_argument("--wait-for-all-ports", action="store_true", - help="Wait until all node ports are open (TCP).") - bp.add_argument("--wait-for-all-health", action="store_true", - help="Wait until http://ip:port/health returns 200 for all nodes.") - bp.add_argument("--health-endpoint", default="/health", - help="Path for health check (default: /health).") - bp.set_defaults(func=cmd_barrier) - - # wait subcommand - wp = subparsers.add_parser("wait", help="Wait while a remote port remains open.") - wp.add_argument("--remote-ip", required=True, help="Remote server IP address.") - wp.add_argument("--remote-port", type=int, required=True, help="Remote port number.") - wp.set_defaults(func=cmd_wait) - - args = parser.parse_args() - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh deleted file mode 100644 index f4e9f84c23..0000000000 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/bin/bash -# Dual-Engine Disaggregated Benchmark Runner -# -# ENGINE=sglang (default): SGLang benchmark -# ENGINE=vllm: vLLM benchmark -# -# Produces JSON result files via benchmark_serving.py so that the CI pipeline -# can collect and process results. -# -# Usage: bash bench.sh \ -# \ -# - -ENGINE="${ENGINE:-sglang-disagg}" - -model_path=$1 -model_name=$2 -concurrency_list=${3:-"1"} -MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODEL="${MODEL_NAME:-${MODEL_PATH}}" -else - MODEL="${MODEL_PATH}" -fi -log_path=${4:-/run_logs} - -# Split BENCH_MAX_CONCURRENCY (x-delimited, e.g. "8x16x32") into an array. -# Falls back to 1 if unset so the loop always runs at least once. -IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" - - -ROUTER_PORT="${ROUTER_PORT:-30000}" - -export TRANSFORMERS_VERBOSITY=error -export TOKENIZERS_PARALLELISM=false - -# echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" - -RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" -mkdir -p "$RESULT_DIR" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# clear_kv_caches — wipe all KV cache tiers on every backend worker before a -# concurrency point, so each conc is measured cold (no prefix reuse bleeding in -# from the previous conc). Mirrors mori-scheduler/scripts/benchmark/lib/ -# clear_caches.sh, but the worker base URLs are already resolved by -# server_sglang.sh (SERVER_FLUSH_URLS_CSV) so no SSH/IP lookup is needed. -# -# Tiers (SGLang server APIs), hit on EACH worker directly (the router does not -# fan /flush_cache out): -# L1 (GPU radix) + L2 (host hicache): POST /flush_cache — NO-OP while any -# request is in flight, so we drain-retry until "Cache flushed" or -# FLUSH_DRAIN_TIMEOUT (default 120s) elapses. -# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear -# — HTTP != 200 when L3 is off, tolerated. -# Best-effort: logs WARN, never hard-fails the sweep. -clear_kv_caches() { - local drain_tmo="${FLUSH_DRAIN_TIMEOUT:-120}" - local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" - if [[ -z "$urls_csv" ]]; then - echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 - return 0 - fi - local -a urls - IFS=',' read -r -a urls <<< "$urls_csv" - local url start ok resp code - for url in "${urls[@]}"; do - [[ -n "$url" ]] || continue - # L1 + L2: drain-retry until flushed (no-op while requests in flight). - start=$(date +%s); ok=0; resp="" - while :; do - resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) - echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } - (( $(date +%s) - start >= drain_tmo )) && break - sleep 3 - done - if [[ "$ok" == 1 ]]; then - echo "[clear_caches] ${url}: L1+L2 flushed" - else - echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 - fi - # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. - code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) - if [[ "$code" == 200 ]]; then - echo "[clear_caches] ${url}: L3 store cleared" - else - echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" - fi - done -} - -# REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -PORT="${ROUTER_PORT}" -MODEL="${MODEL:-${BENCH_MODEL}}" -DURATION="${DURATION:-1800}" -export MODEL DURATION MAX_MODEL_LEN -RESULT_DIR="${RESULT_DIR:-${profile_folder}}" -# Base name for the per-conc aggregate written by the existing -# utils.agentic.aggregation.process_agentic_result module. -# The workflow guard / upload steps expect a "${RESULT_FILENAME}_conc.json" -# file per concurrency, so each concurrency below is always suffixed with -# _conc (matching agentic_srt.sh on the gb200 path). -RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" - -mkdir -p "$RESULT_DIR" - -if [ "$PREFILL_ENABLE_DP" = "true" ]; then - set -x - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - set +x -fi - -resolve_trace_source -install_agentic_deps - -ANY_FAILED=0 -for max_concurrency in "${chosen_concurrencies[@]}"; do - - echo "==========================================" - echo "Agentic trace replay: conc=$max_concurrency" - echo "==========================================" - - # Clear all KV cache tiers on every backend before this conc point so it is - # measured cold (no prefix reuse from the previous conc). Default on; set - # CLEAR_CACHE_BETWEEN_CONC=0 to disable. Best-effort — never fails the run. - if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then - echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" - clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 - fi - - # Mirror agentic_srt.sh (the srtctl/gb200 path): every concurrency writes - # its artifacts into a conc_/ subdir of RESULT_DIR. The CI matrix explodes - # agentic runs to one concurrency per job, but benchmark-multinode-tmpl.yml - # still expects the per-conc nesting (LOGS/agentic/conc_*/...) and the - # _conc result-file suffix, so we always nest to keep the layout identical - # across runners and avoid overwriting earlier runs in local multi-conc sweeps. - CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" - mkdir -p "$CONC_RESULT_DIR" - - CONC="$max_concurrency" - USERS="$max_concurrency" - export CONC USERS - build_replay_cmd "$CONC_RESULT_DIR" - - # Per-conc result name consumed by write_agentic_result_json. Always suffix - # with _conc so the file matches - # the workflow guard's "${RESULT_FILENAME}_conc*.json" glob (and the agg / - # checkpoint upload steps) for both single-conc CI runs and multi-conc sweeps. - export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" - if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then - echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 - ANY_FAILED=1 - fi - - echo "-----------------------------------------" - -done - -export RESULT_FILENAME="$RESULT_FILENAME_BASE" - -if [ "$ANY_FAILED" -ne 0 ]; then - echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 -fi diff --git a/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index d92dad7503..0000000000 --- a/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index 1505b905de..0000000000 --- a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh deleted file mode 100755 index d7995fb250..0000000000 --- a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# Same EP/DP booleans as dsr1_fp8_mi355x_sglang-disagg.sh → amd_utils/submit.sh -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -# Parameter order matches SGLang disagg submit.sh; arg 16 is optional NODELIST. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh deleted file mode 100644 index a9a28d889b..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index e86e044a45..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh deleted file mode 100755 index ac0f0a27d7..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODE_LIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh deleted file mode 100644 index e86e044a45..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh deleted file mode 100644 index 91628d632f..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -# MiniMax-M3 MXFP8 (~414 GB) is pre-staged in this cluster's shared HF cache -# (/it-share/hf-hub-cache/models--MiniMaxAI--MiniMax-M3-MXFP8), not the default -# /it-share/data the launcher sets. Point the disagg model dir there for M3 only; -# submit.sh exports MODEL_DIR=$MODEL_PATH and job.slurm resolves the snapshot under -# it and bind-mounts MODEL_DIR into the prefill/decode serving containers. -export MODEL_PATH=/it-share/hf-hub-cache -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh deleted file mode 100644 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh deleted file mode 100644 index a8c0d2743b..0000000000 --- a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" \ No newline at end of file diff --git a/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/llm-d/README.md b/benchmarks/multi_node/llm-d/README.md index 81dbd51995..d6acbc30d2 100644 --- a/benchmarks/multi_node/llm-d/README.md +++ b/benchmarks/multi_node/llm-d/README.md @@ -2,7 +2,7 @@ This directory holds the SLURM-side orchestration for the `llmd-vllm` benchmark framework. It mirrors the AMD `sglang-disagg` pattern under -`benchmarks/multi_node/amd_utils/` (NOT the Dynamo / srt-slurm pattern): +the standalone llm-d Slurm path (not the srt-slurm pattern): InferenceX itself owns the SLURM job, no vendor multi-node tool involved. | File | Role | diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 7e189767bc..227bcf0478 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -556,7 +556,7 @@ PY IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" # GPU counts embedded in the result filename as _gpus_/_ctx_/_gen_ tokens so the # CI "Process result" step (benchmark-multinode-tmpl.yml) can parse them and run - # process_result.py for llm-d -- same filename convention as amd_utils/bench.sh. + # process_result.py for llm-d using the standard InferenceX filename convention. # ctx = prefill GPUs, gen = decode GPUs; nodes*GPUS_PER_NODE is correct for any # PREFILL_WORKERS/DECODE_WORKERS split (e.g. high-tpt 2P -> 16 prefill GPUs). _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh index 11c34736f8..49a873fe4a 100755 --- a/benchmarks/multi_node/llm-d/submit.sh +++ b/benchmarks/multi_node/llm-d/submit.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash # # Submit a multi-node llmd-vllm wide-EP P/D disagg benchmark job to SLURM. -# Modeled after benchmarks/multi_node/amd_utils/submit.sh; prints JOB_ID on -# stdout so the runner can poll for completion. +# Prints JOB_ID on stdout so the runner can poll for completion. # # Topology (matches the llm-d wide-EP guide reference): # 1 prefill instance with DP=PREFILL_NODES * GPUS_PER_NODE diff --git a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c7..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c7..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml new file mode 100644 index 0000000000..e1b878bc2b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml @@ -0,0 +1,35 @@ +# srt-slurm cluster profile for the MI355X AMD Slurm cluster. The login and +# compute nodes share /it-share, so source, output, image, and result paths do +# not need node-local transport. + +cluster: mi355x-amds +default_partition: compute +default_time_limit: "01:00:00" +output_dir: /it-share/gharunners2/srt-slurm/outputs + +gpus_per_node: 8 +accelerator_vendor: amd +network_interface: eno0 + +gpu_sbatch_directive: gres +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: false +runtime_config_transport: shared-filesystem + +default_sbatch_directives: + cpus-per-task: "32" + mem: "128G" + +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + /it-share/hf-hub-cache: /hf_hub_cache + # Legacy production checkpoints live here. The launcher may expose one as a + # zero-copy seed under the canonical /hf_hub_cache/hub layout. + /it-share/hf_home: /it-share/hf_home + # The launcher appends the exact shared workspace and result roots. + +containers: + sglang-rocm-v0.5.17-mi35x: /it-share/gharunners2/srt-slurm/containers/sglang-rocm-v0.5.17-mi35x-20260809.sqsh + +nginx_raise_ulimit: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-dep8-mtp-hicache-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-dep8-mtp-hicache-c128.yaml new file mode 100644 index 0000000000..2e19549b56 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-dep8-mtp-hicache-c128.yaml @@ -0,0 +1,210 @@ +name: "mi355x-dsv4-agentic-disagg-1p1d-dep8-mtp-hicache-c128" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "sglang-rocm-dsv4-mi355x" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817" + frameworks: + sglang: "0.5.17.dev20260817+gd97b796c16" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260817+gd97b796c16" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: consistent_hashing + dp-aware: true + cache-threshold: 0.3 + balance-abs-threshold: 2 + balance-rel-threshold: 1.1 + disable-circuit-breaker: true + health-failure-threshold: 100 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + PYTHONPATH: "/sgl-workspace/aiter" + # Validated DeepSeek V4 ROCm kernel routing. These settings keep FP4 MoE, + # indexer, MLA, and fused-compress dispatch on the supported MI355X paths. + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + NCCL_IB_HCA: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + AITER_LOG_LEVEL: "ERROR" + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "high" + SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false" + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_OPT_USE_FUSED_COMPRESS: "true" + SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton" + SGLANG_OPT_FP8_WO_A_GEMM: "false" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false" + SGLANG_OPT_USE_TOPK_V2: "false" + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_USE_TILELANG_INDEXER: "false" + SGLANG_OPT_USE_TILELANG_MHC_PRE: "false" + SGLANG_OPT_USE_TILELANG_MHC_POST: "false" + SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1" + SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true" + SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false" + SGLANG_ROCM_USE_MULTI_STREAM: "false" + AITER_BF16_FP8_MOE_BOUND: "0" + SGLANG_EAGER_INPUT_NO_COPY: "true" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_IO_TC_DISABLE: "0" + MORI_SHMEM_MODE: "ISOLATION" + MORI_EP_LAUNCH_CONFIG_MODE: "AUTO" + MORI_APP_LOG_LEVEL: "WARNING" + SGLANG_MORI_COMBINE_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + MORI_MAX_DISPATCH_TOKENS_PREFILL: "8192" + MORI_MAX_DISPATCH_TOKENS_DECODE: "64" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0" + SGLANG_LOG_MS: "true" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_ROUTER_STDOUT_LOGS: "0" + decode_environment: + <<: *common_environment + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "128" + SGLANG_SIMULATE_ACC_LEN: "2.49" + SGLANG_SIMULATE_ACC_METHOD: "match-expected" + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + sglang_config: + prefill: &common_server + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + tp-size: 8 + ep-size: 8 + dp-size: 8 + watchdog-timeout: 3600 + load-balance-method: round_robin + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + disable-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + log-level: info + log-level-http: error + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-algorithm: EAGLE + speculative-eagle-topk: 1 + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + enable-dp-attention: true + enable-prefill-delayer: true + context-length: 1048576 + enable-metrics: true + mem-fraction-static: 0.85 + max-running-requests: 192 + chunked-prefill-size: 65536 + disable-cuda-graph: true + enable-hierarchical-cache: true + hicache-ratio: 3 + hicache-io-backend: direct + hicache-mem-layout: page_first + hicache-write-policy: write_through + hicache-storage-prefetch-policy: best_effort + decode: + <<: *common_server + mem-fraction-static: 0.85 + chunked-prefill-size: null + disable-cuda-graph: false + enable-hierarchical-cache: false + hicache-ratio: null + hicache-io-backend: null + hicache-mem-layout: null + hicache-write-policy: null + hicache-storage-prefetch-policy: null + prefill-round-robin-balance: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + +sbatch_directives: + # SGLang maps the eight TP ranks across the MI355X host CPU topology. Give + # each backend task the complete node cpuset so every placement stays within + # its Slurm allocation. + cpus-per-task: "128" + mem: "0" + +srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + mkdir -p /infmax-workspace/LOGS/agentic + export RESULT_DIR=/infmax-workspace/LOGS/agentic + export AGENTIC_OUTPUT_DIR=/infmax-workspace + exec bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: "error" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-dep8-mtp-hicache-c96.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-dep8-mtp-hicache-c96.yaml new file mode 100644 index 0000000000..194ec1878a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-dep8-mtp-hicache-c96.yaml @@ -0,0 +1,210 @@ +name: "mi355x-dsv4-agentic-disagg-1p1d-dep8-mtp-hicache-c96" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "sglang-rocm-dsv4-mi355x" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817" + frameworks: + sglang: "0.5.17.dev20260817+gd97b796c16" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260817+gd97b796c16" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: consistent_hashing + dp-aware: true + cache-threshold: 0.3 + balance-abs-threshold: 2 + balance-rel-threshold: 1.1 + disable-circuit-breaker: true + health-failure-threshold: 100 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + PYTHONPATH: "/sgl-workspace/aiter" + # Validated DeepSeek V4 ROCm kernel routing. These settings keep FP4 MoE, + # indexer, MLA, and fused-compress dispatch on the supported MI355X paths. + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + NCCL_IB_HCA: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + AITER_LOG_LEVEL: "ERROR" + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "high" + SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false" + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_OPT_USE_FUSED_COMPRESS: "true" + SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton" + SGLANG_OPT_FP8_WO_A_GEMM: "false" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false" + SGLANG_OPT_USE_TOPK_V2: "false" + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_USE_TILELANG_INDEXER: "false" + SGLANG_OPT_USE_TILELANG_MHC_PRE: "false" + SGLANG_OPT_USE_TILELANG_MHC_POST: "false" + SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1" + SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true" + SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false" + SGLANG_ROCM_USE_MULTI_STREAM: "false" + AITER_BF16_FP8_MOE_BOUND: "0" + SGLANG_EAGER_INPUT_NO_COPY: "true" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_IO_TC_DISABLE: "0" + MORI_SHMEM_MODE: "ISOLATION" + MORI_EP_LAUNCH_CONFIG_MODE: "AUTO" + MORI_APP_LOG_LEVEL: "WARNING" + SGLANG_MORI_COMBINE_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + MORI_MAX_DISPATCH_TOKENS_PREFILL: "8192" + MORI_MAX_DISPATCH_TOKENS_DECODE: "64" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0" + SGLANG_LOG_MS: "true" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_ROUTER_STDOUT_LOGS: "0" + decode_environment: + <<: *common_environment + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "128" + SGLANG_SIMULATE_ACC_LEN: "2.49" + SGLANG_SIMULATE_ACC_METHOD: "match-expected" + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + sglang_config: + prefill: &common_server + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + tp-size: 8 + ep-size: 8 + dp-size: 8 + watchdog-timeout: 3600 + load-balance-method: round_robin + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + disable-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + log-level: info + log-level-http: error + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-algorithm: EAGLE + speculative-eagle-topk: 1 + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + enable-dp-attention: true + enable-prefill-delayer: true + context-length: 1048576 + enable-metrics: true + mem-fraction-static: 0.85 + max-running-requests: 96 + chunked-prefill-size: 65536 + disable-cuda-graph: true + enable-hierarchical-cache: true + hicache-ratio: 3 + hicache-io-backend: direct + hicache-mem-layout: page_first + hicache-write-policy: write_through + hicache-storage-prefetch-policy: best_effort + decode: + <<: *common_server + mem-fraction-static: 0.85 + chunked-prefill-size: null + disable-cuda-graph: false + enable-hierarchical-cache: false + hicache-ratio: null + hicache-io-backend: null + hicache-mem-layout: null + hicache-write-policy: null + hicache-storage-prefetch-policy: null + prefill-round-robin-balance: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + +sbatch_directives: + # SGLang maps the eight TP ranks across the MI355X host CPU topology. Give + # each backend task the complete node cpuset so every placement stays within + # its Slurm allocation. + cpus-per-task: "128" + mem: "0" + +srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + mkdir -p /infmax-workspace/LOGS/agentic + export RESULT_DIR=/infmax-workspace/LOGS/agentic + export AGENTIC_OUTPUT_DIR=/infmax-workspace + exec bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: "error" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-tp8-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-tp8-mtp-hicache.yaml new file mode 100644 index 0000000000..c3ed7f2416 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-tp8-mtp-hicache.yaml @@ -0,0 +1,205 @@ +name: "mi355x-dsv4-agentic-disagg-1p1d-tp8-mtp-hicache" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "sglang-rocm-dsv4-mi355x" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817" + frameworks: + sglang: "0.5.17.dev20260817+gd97b796c16" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260817+gd97b796c16" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: consistent_hashing + dp-aware: true + cache-threshold: 0.3 + balance-abs-threshold: 2 + balance-rel-threshold: 1.1 + disable-circuit-breaker: true + health-failure-threshold: 100 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + PYTHONPATH: "/sgl-workspace/aiter" + # Validated DeepSeek V4 ROCm kernel routing. These settings keep FP4 MoE, + # indexer, MLA, and fused-compress dispatch on the supported MI355X paths. + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + NCCL_IB_HCA: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + AITER_LOG_LEVEL: "ERROR" + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "high" + SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false" + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_OPT_USE_FUSED_COMPRESS: "true" + SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton" + SGLANG_OPT_FP8_WO_A_GEMM: "false" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false" + SGLANG_OPT_USE_TOPK_V2: "false" + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_USE_TILELANG_INDEXER: "false" + SGLANG_OPT_USE_TILELANG_MHC_PRE: "false" + SGLANG_OPT_USE_TILELANG_MHC_POST: "false" + SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1" + SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true" + SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false" + SGLANG_ROCM_USE_MULTI_STREAM: "false" + AITER_BF16_FP8_MOE_BOUND: "0" + SGLANG_EAGER_INPUT_NO_COPY: "true" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_IO_TC_DISABLE: "0" + MORI_SHMEM_MODE: "ISOLATION" + MORI_EP_LAUNCH_CONFIG_MODE: "AUTO" + MORI_APP_LOG_LEVEL: "WARNING" + SGLANG_MORI_COMBINE_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + MORI_MAX_DISPATCH_TOKENS_PREFILL: "8192" + MORI_MAX_DISPATCH_TOKENS_DECODE: "64" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0" + SGLANG_LOG_MS: "true" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_ROUTER_STDOUT_LOGS: "0" + decode_environment: + <<: *common_environment + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "128" + SGLANG_SIMULATE_ACC_LEN: "2.49" + SGLANG_SIMULATE_ACC_METHOD: "match-expected" + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + sglang_config: + prefill: &common_server + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + tp-size: 8 + watchdog-timeout: 3600 + load-balance-method: round_robin + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + disable-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + log-level: info + log-level-http: error + speculative-algorithm: EAGLE + speculative-eagle-topk: 1 + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + context-length: 1048576 + enable-metrics: true + mem-fraction-static: 0.85 + max-running-requests: 64 + chunked-prefill-size: 8192 + disable-cuda-graph: true + enable-hierarchical-cache: true + hicache-ratio: 3 + hicache-io-backend: direct + hicache-mem-layout: page_first + hicache-write-policy: write_through + hicache-storage-prefetch-policy: best_effort + decode: + <<: *common_server + mem-fraction-static: 0.85 + max-running-requests: 128 + chunked-prefill-size: null + disable-cuda-graph: false + enable-hierarchical-cache: false + hicache-ratio: null + hicache-io-backend: null + hicache-mem-layout: null + hicache-write-policy: null + hicache-storage-prefetch-policy: null + prefill-round-robin-balance: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + +sbatch_directives: + # SGLang maps the eight TP ranks across the MI355X host CPU topology. Give + # each backend task the complete node cpuset so every placement stays within + # its Slurm allocation. + cpus-per-task: "128" + mem: "0" + +srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + mkdir -p /infmax-workspace/LOGS/agentic + export RESULT_DIR=/infmax-workspace/LOGS/agentic + export AGENTIC_OUTPUT_DIR=/infmax-workspace + exec bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: "error" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml new file mode 100644 index 0000000000..beb4777979 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml @@ -0,0 +1,94 @@ +# Minimal MI355X aggregate validation. It exercises native SGLang Router and +# one aggregate SGLang worker; this is orchestration validation, not tuning. + +name: "mi355x-sglang-qwen3-0.6b-agg-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + +slurm: + time_limit: "00:30:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 1 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 1 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + +backend: + type: sglang + aggregated_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + sglang_config: + aggregated: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 180 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-agg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml new file mode 100644 index 0000000000..b630a0ef65 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml @@ -0,0 +1,107 @@ +# Minimal two-node MI355X P/D validation. SGLang Router provides the request +# plane and the bundled AMD MoRI transport moves KV between prefill and decode. + +name: "mi355x-sglang-qwen3-0.6b-disagg-1p1d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + +slurm: + time_limit: "00:45:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 1 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 1 + gpus_per_decode: 1 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +backend: + type: sglang + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + decode_environment: *worker_environment + sglang_config: + prefill: &worker_config + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + disaggregation-transfer-backend: mori + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + decode: *worker_config + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 240 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-disagg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ab36d1ce24..baacad9b8c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -268,50 +268,6 @@ qwen3.5-fp8-mi355x-atom-mtp: search-space: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang:v0.5.16-rocm720-mi35x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP4P+TP8D/EP1 baseline (no speculative decoding). - # TP4 prefill saves 4 GPUs vs TP8P while delivering identical decode - # interactivity and 24-31% better throughput/GPU (12 vs 16 GPUs). - # dp-attn intentionally false: with --enable-dp-attention + - # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size, - # but is_deepep_class_backend() excludes MoRI, so - # num_shared_slots stays at the global value (1) and the - # (num_experts - num_shared_slots) % moe_ep_size assertion in - # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 model: amd/Qwen3.5-397B-A17B-MXFP4 @@ -376,41 +332,6 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40, 48, 56, 64] } - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32] } - -qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -473,157 +394,6 @@ dsr1-fp8-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -dsr1-fp8-mi355x-sglang-disagg: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - - spec-decoding: "none" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp8-mi355x-sglang-disagg-mtp: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4, 2 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - kimik3-fp4-mi355x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 model: moonshotai/Kimi-K3 @@ -640,508 +410,50 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev60+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark -# speculative decoding. Acceptance is pinned to the committed golden curve in -# golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml -# at both draft lengths used here: 7 draft tokens -> AL 3.84 at concurrency 1-4, -# 3 draft tokens -> AL 3.00 from concurrency 8 up. -# Companion to kimik3-fp4-mi355x-vllm-agentic-mtp: same checkpoint, same runner. +# speculative decoding (2 draft tokens -> golden AL 2.51, +# golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml, +# the same golden the vLLM arm feeds to synthetic_acceptance_length). +# Companion to kimik3-fp4-mi355x-vllm-agentic-mtp: same checkpoint, same +# runner, same concurrency points, so the two engines are directly comparable. # TP8 only -- the 1.56 TB MXFP4 checkpoint is ~195 GB/GPU and TP4 cannot load, # which is also why there is no DP-attention arm. # Concurrency 1-4 is the latency floor and stays GPU-resident. From concurrency # 8 up, decode is KV-bandwidth-bound over 100k+ token contexts, so dcp-size 8 # shards the KV read across all 8 GPUs and the LMCache DRAM tier backs the # paged KV. -# Two utilization blocks because the per-rank CPU split differs: -# 0.268 -> 803 GB aggregate -> 100 GB/rank, all paged KV (conc 8, 12). -# 0.343 -> 1028 GB aggregate -> 128 GB/rank, split 96 GB paged KV + 32 GB -# for ATOM's CPU state tier (conc 16 and up). K3 is a hybrid, so a -# resumed agentic turn needs the KDA recurrent state back and the -# paged KV tier alone cannot restore one. -# The paged-KV half of that budget is deliberately smaller than the recipe was -# measured with (200 and 192 GB/rank). Every rank pins its own pool, and at the -# measured sizes the eight ranks finished pinning more than 600 s apart, which -# is exactly PyTorch's hardcoded NCCL process-group timeout: the ranks that -# arrived first at the barrier ending allocate_kv_cache() timed out waiting for -# rank 0 and the ModelRunner processes died before the server ever served a -# request. Halving the paged-KV pool halves the pinning work. It costs little: -# the CPU tier's measured hit rate was 0.0% at concurrency 8 and 12 and 0.3-2.4% -# above that, and 96 GB/rank still holds ~53M tokens, roughly 530 full 100k-token -# contexts. The state tier is left at 32 GB/rank. -kimik3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0821 - model: moonshotai/Kimi-K3 - model-prefix: kimik3 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.268 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } - - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [8, 12], spec-decoding: mtp } - - dram-utilization: 0.343 - search-space: - - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [32, 40, 56], spec-decoding: mtp } - -dsr1-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D pure TP8 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 32, 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 640, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - -dsv4-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260701 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D pure TP8 (mori KV transfer) - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - - spec-decoding: "none" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# MTP variant of dsv4-fp4-mi355x-sglang-disagg (base run 29821778206). Mirrors -# the base disagg search space and topologies exactly (same 20260701 image, -# mori KV transfer + MoE a2a, sglang-router --pd-disaggregation) and enables -# DeepSeek-V4 built-in MTP on the decode side via spec-decoding: mtp — the same -# enablement path as dsr1-fp8-mi355x-sglang-disagg-mtp. DECODE_MTP_SIZE=1 (draft -# depth 1) on every arm. Depth 2 was tried on the pure-TP8 (bottom-of-curve) arm, -# but this 20260701 build's 2-step EAGLE verification is lossy: greedy gsm8k fell -# to ~0.85 (below the 0.91 dsv4 floor), while depth 1 is lossless (~0.96, matching -# the spec-none base). So all arms use depth 1. The dsv4 ROCm MTP -# CUDA-graph fix (sgl-project/sglang#26383, merged 2026-05-27) is already -# present in the pinned 20260701 nightly, so no image bump is needed. -dsv4-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 +# Two utilization blocks because the per-rank CPU split differs: +# 0.268 -> 803 GB aggregate -> 100 GB/rank, all paged KV (conc 8, 12). +# 0.343 -> 1028 GB aggregate -> 128 GB/rank, split 96 GB paged KV + 32 GB +# for ATOM's CPU state tier (conc 16 and up). K3 is a hybrid, so a +# resumed agentic turn needs the KDA recurrent state back and the +# paged KV tier alone cannot restore one. +# The paged-KV half of that budget is deliberately smaller than the recipe was +# measured with (200 and 192 GB/rank). Every rank pins its own pool, and at the +# measured sizes the eight ranks finished pinning more than 600 s apart, which +# is exactly PyTorch's hardcoded NCCL process-group timeout: the ranks that +# arrived first at the barrier ending allocate_kv_cache() timed out waiting for +# rank 0 and the ModelRunner processes died before the server ever served a +# request. Halving the paged-KV pool halves the pinning work. It costs little: +# the CPU tier's measured hit rate was 0.0% at concurrency 8 and 12 and 0.3-2.4% +# above that, and 96 GB/rank still holds ~53M tokens, roughly 530 full 100k-token +# contexts. The state tier is left at 32 GB/rank. +kimik3-fp4-mi355x-atom-agentic-mtp: + image: rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0821 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 runner: cluster:mi355x-amds precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true + framework: atom + multinode: false scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 + agentic-coding: + - dram-utilization: 0.268 search-space: - # MTP configurations - # 1P1D pure TP8 (mori KV transfer) - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - - spec-decoding: "mtp" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - - spec-decoding: "mtp" - conc-list: [ 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [8, 12], spec-decoding: mtp } + - dram-utilization: 0.343 + search-space: + - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [32, 40, 56], spec-decoding: mtp } dsv4-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 model: deepseek-ai/DeepSeek-V4-Pro @@ -1499,7 +811,7 @@ dsv4-fp4-mi355x-atom-disagg: - isl: 8192 osl: 1024 search-space: - # 2P1D DPA+TP8 + # 2P1D DPA+TP8 - conc-list: [ 256, 512, 768, 1024, 2048 ] prefill: num-worker: 2 @@ -1515,7 +827,7 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: true additional-settings: - "DECODE_NODES=1" - # 1P1D TP8 + # 1P1D TP8 - conc-list: [ 4, 8, 16, 32, 64, 128 ] prefill: num-worker: 1 @@ -1577,70 +889,6 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: - dram-utilization: 0.20 search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } - -dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: "mtp" - conc-list: [ 1,2,4,8,16,32,48,64 ] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - spec-decoding: "mtp" - conc-list: [ 64, 96, 128, 192 ] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - -# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. -# EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no -# gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels, so vLLM runs the -# checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), -# the H200 dsv4 vLLM path. Config mirrors the same-model dsv4-fp4-mi355x-vllm -# (TP8, conc 4-512); 8x256GB (2TB) has ample headroom for the ~1.05TB FP8 -# footprint. Launch script dsv4_fp8_mi325x.sh carries the deepseek_v4 + gfx942 -# AITER flags. dsv4-fp8-mi325x-vllm: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: deepseek-ai/DeepSeek-V4-Pro @@ -1751,3 +999,140 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } + +qwen3-0.6b-fp16-mi355x-sglang-srt-agg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: sglang + multinode: true + disagg: false + router: { name: sglang-router, version: "0.3.2" } + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + num-nodes: 1 + worker: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml" + +qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: sglang-disagg + multinode: true + disagg: true + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + +# Production-scale migration of the existing Qwen3.5 FP8 MI355X P/D lane to +# srt-slurm. One full TP8 node serves prefill, one TP8 node serves decode, and +# the request/KV planes remain native SGLang Router + AMD MoRI. + +dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: "mtp" + conc-list: [ 1,2,4,8,16,32,48,64 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-tp8-mtp-hicache.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_MTP_SIZE=3" + - spec-decoding: "mtp" + conc-list: [ 64, 96 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-dep8-mtp-hicache-c96.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_MTP_SIZE=3" + - spec-decoding: "mtp" + conc-list: [ 128, 192 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/mi355x/agentic/disagg-1p1d-dep8-mtp-hicache-c128.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_MTP_SIZE=3" + +# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. +# EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no +# gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels, so vLLM runs the +# checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), +# the H200 dsv4 vLLM path. Config mirrors the same-model dsv4-fp4-mi355x-vllm +# (TP8, conc 4-512); 8x256GB (2TB) has ample headroom for the ~1.05TB FP8 +# footprint. Launch script dsv4_fp8_mi325x.sh carries the deepseek_v4 + gfx942 +# AITER flags. diff --git a/configs/deprecated/amd-1k1k-master.yaml b/configs/deprecated/amd-1k1k-master.yaml index 6a64e965ae..fb6ef21300 100644 --- a/configs/deprecated/amd-1k1k-master.yaml +++ b/configs/deprecated/amd-1k1k-master.yaml @@ -274,41 +274,6 @@ qwen3.5-fp8-mi355x-atom-mtp: search-space: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } -qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang:v0.5.14-rocm720-mi35x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Matches qwen3.5-fp8-mi355x-sglang TP8/EP1 low-concurrency sweep - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 model: amd/Qwen3.5-397B-A17B-MXFP4 @@ -354,41 +319,6 @@ qwen3.5-fp4-mi355x-sglang-mtp: search-space: - { tp: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } -qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP8/EP1, dp-attn false; MoRI conn.py overlay via job.slurm. - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -432,41 +362,6 @@ glm5-fp8-mi355x-sglang-mtp: search-space: - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } -glm5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep (aligned with glm5-fp8-mi355x-sglang conc range) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - glm5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: zai-org/GLM-5-FP8 @@ -612,452 +507,6 @@ dsr1-fp8-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } -dsr1-fp8-mi355x-sglang-disagg: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # non-MTP configurations - # "Top of curve" (1 prefill workers each at DEP8 and 1 decode workers at DEP16) - - spec-decoding: "none" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # "Middle of curve" (1 prefill workers each at TP8 and 2 decode workers at DEP8) - - spec-decoding: "none" - conc-list: [ 1536, 1024, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8) - - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp8-mi355x-sglang-disagg-mtp: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # "Top of curve" (1 prefill worker at DEP8 and 1 decode worker at DEP16) - - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=1" - - # "Middle of curve" (1 prefill worker at TP8 and 2 decode workers each at DEP8) - - spec-decoding: "mtp" - conc-list: [ 1536, 1024, 512, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8) - - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - -kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:v0.24.0 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -dsr1-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D TP8 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp4-mi355x-sglang-disagg-1k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # 1P1D TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1P2D TP4 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - dsv4-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 model: deepseek-ai/DeepSeek-V4-Pro @@ -1219,153 +668,6 @@ glm5-fp8-mi325x-sglang-mtp: osl: 1024 search-space: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } -dsr1-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # 1P1D TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1P2D TP4 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - -dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [ 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - -# MiniMax-M3 MXFP8 MI355X recipe: -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. minimaxm3-fp8-mi355x-vllm: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -1487,82 +789,6 @@ minimaxm3-fp8-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } -minimaxm3-fp8-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -minimaxm3-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# MiniMax-M3 MXFP8 MI300X day-zero recipe. Reuse the dedicated ROCm image and -# MI355X serving shape, but retain the default BF16 KV cache because this -# checkpoint lacks calibrated ROCm FP8 attention scales. TP8-only, plain -# (non-expert-parallel) search space across the full conc range: EP8 -# (--enable-expert-parallel) produces garbage/incoherent output on this -# MXFP8+gfx942 combination (confirmed locally: TP8/EP8 returns garbled tokens -# even on trivial prompts, TP8/EP1 answers correctly), and was already the -# lower-throughput topology where measured. minimaxm3-fp8-mi300x-vllm: image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -1628,62 +854,6 @@ minimaxm3-fp8-mi325x-vllm-mtp: - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } -minimaxm3-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Balanced 1P TP4 + 1D TP4 (2 nodes) across the full curve. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Decode-heavy 1P TP4 + 2D TP4 (3 nodes): double the decode engines to - # absorb the decode-bound 1k1k tail at high concurrency. - - spec-decoding: "none" - conc-list: [ 256, 512, 1024, 2048 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - # 8k1k is prefill-bound (8192-token prompts vs 1024 decode steps): pair the - # balanced layout with a prefill-heavy 2P/1D layout. Concurrency is capped at - # 512 so the multi-node eval policy (8k1k + conc >= 16, highest eligible conc) - # marks lm-eval at conc 512 — matching the range NVIDIA's aggregated 8k1k - # sweep tops out at and keeping the lm-eval async client stable. dsv4-fp8-mi325x-vllm: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: deepseek-ai/DeepSeek-V4-Pro diff --git a/configs/deprecated/amd-glm5-glm5.1-master.yaml b/configs/deprecated/amd-glm5-glm5.1-master.yaml index 8906a0a89b..7a7cf845eb 100644 --- a/configs/deprecated/amd-glm5-glm5.1-master.yaml +++ b/configs/deprecated/amd-glm5-glm5.1-master.yaml @@ -32,44 +32,6 @@ glm5-fp8-mi355x-sglang-mtp: search-space: - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } - - -glm5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep; dp-attn false (NSA / MoRI path) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - glm5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: zai-org/GLM-5-FP8 diff --git a/configs/deprecated/amd-kimik2.5-8k1k-master.yaml b/configs/deprecated/amd-kimik2.5-8k1k-master.yaml index 252848fc69..73aed4fa27 100644 --- a/configs/deprecated/amd-kimik2.5-8k1k-master.yaml +++ b/configs/deprecated/amd-kimik2.5-8k1k-master.yaml @@ -81,110 +81,3 @@ kimik2.5-fp4-mi355x-atom: osl: 1024 search-space: - { tp: 4, conc-start: 4, conc-end: 128 } - -kimik2.5-fp4-mi355x-atom-disagg: - image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atomesh_202607121715 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # 1P(tp4)1D(tp4) - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P(tp4)2D(tp8) - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # All workers TP4 (real-weight sweep: TP8 decode is no better than TP4). - # Split across P/D topologies: 1D is stable at low conc; the high-conc tail - # runs on 2D so decode load is spread across two engines. - - isl: 8192 - osl: 1024 - search-space: - # 1P(TP4) 1D(TP4) = 2 nodes. Low concurrency only. - - spec-decoding: "none" - conc-list: [ 1, 2, 4 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P(TP4) 2D(TP4) = 3 nodes. High concurrency (decode KV/load spread over 2D). - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" diff --git a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml index ed6b003afc..5fbe55925b 100644 --- a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml +++ b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml @@ -143,63 +143,3 @@ minimaxm2.5-fp8-mi325x-vllm: search-space: - { tp: 2, conc-start: 4, conc-end: 64 } - { tp: 8, ep: 8, conc-start: 4, conc-end: 256 } - -# Diverged from minimaxm2.5-fp8-mi325x-vllm (agentic-coding sibling). Reasons below; -# the original minimaxm2.5-fp8-mi325x-vllm entry is left identical to origin/main so -# its fixed-seq-len sweep is unaffected. -# - image: 'vllm/vllm-openai-rocm:v0.18.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf' -minimaxm2.5-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-a6682d1d259cca69a9ae737ea5608fbbe7520031 - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total - # Prefill also needs EP=8: MiniMax M2.5 expert intermediate_size=1536, - # TP8 shards to 192 which is not divisible by FP8 block_n=128. - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" diff --git a/configs/deprecated/amd-minimaxm3-8k1k-master.yaml b/configs/deprecated/amd-minimaxm3-8k1k-master.yaml index ad509a518b..c8f5386738 100644 --- a/configs/deprecated/amd-minimaxm3-8k1k-master.yaml +++ b/configs/deprecated/amd-minimaxm3-8k1k-master.yaml @@ -42,45 +42,6 @@ minimaxm3-fp8-mi355x-vllm-mtp: osl: 1024 search-space: - { tp: 4, conc-start: 1, conc-end: 512, spec-decoding: mtp } - -# MiniMax-M3 MXFP4 MI355X vLLM disaggregated (prefill/decode) config. -minimaxm3-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2dfaae752b4db0d43cfc0715c780e33be030d0f1 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P TP4 + 1D TP4 (2 nodes total), conc sweep 1..256 (single job, looped) - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" -# MiniMax-M3 MXFP4 MI355X vLLM recipe. The pinned nightly includes upstream -# MiniMax-M3 Quark MXFP4 support (vllm-project/vllm#45794). Use the text-only -# language-model path and mirror the MXFP8 MI355X search space for a direct -# precision comparison. minimaxm3-fp4-mi355x-vllm: image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1 model: amd/MiniMax-M3-MXFP4 @@ -181,109 +142,6 @@ minimaxm3-fp8-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } - -minimaxm3-fp8-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 2P1D, DPA TP4 - - conc-list: [ 256, 512, 768, 1024 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D TP4 -minimaxm3-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 2P1D, DPA TP4 - - conc-list: [ 256, 512, 768, 1024 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D TP4 minimaxm3-fp8-mi300x-vllm: image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -370,68 +228,3 @@ minimaxm3-fp8-mi325x-vllm-mtp: - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 256, spec-decoding: mtp } - -# MiniMax-M3 MXFP8 MI355X vLLM disaggregated (prefill/decode) sweep on the -# day-zero ROCm image, over the MoRI-IO KV-transfer pipeline (MoRI-patch-removal -# infra #1585). All workers are TP4, no EP: the single-node M3 MXFP8 recipe -# (minimaxm3-fp8-mi355x-vllm, PR #2003) found plain TP4 beats both TP8 and -# TP4/EP4 on tok/s/GPU for this model on gfx950, so prefill and decode both use -# TP4 and we tune the prefill:decode worker ratio (xP:yD) instead of TP. The -# mi355x-disagg pool has 3 nodes and the launcher places one worker per node -# (NUM_NODES = xP + yD), so every layout keeps xP + yD <= 3: -# - 1P-TP4 / 1D-TP4 (2 nodes): balanced, full concurrency curve. -# - 1P-TP4 / 2D-TP4 (3 nodes): decode-heavy, for the decode-bound 1k1k tail. -# - 2P-TP4 / 1D-TP4 (3 nodes): prefill-heavy, for the prefill-bound 8k1k tail. -# Per-worker serve flags live in -# benchmarks/multi_node/amd_utils/models_vllm.yaml (MiniMax-M3-MXFP8). -minimaxm3-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Balanced 1P TP4 + 1D TP4 (2 nodes) across the full curve. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Prefill-heavy 2P TP4 + 1D TP4 (3 nodes): two half-node TP4 prefill workers - # keep the single TP4 decode engine fed for the prefill-bound 8k1k tail. - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a619ab7c7d..941c3f8369 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6606,7 +6606,6 @@ description: - "Update the vLLM B200 Kimi-K3 AgentX configs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2776 - - config-keys: - kimik3-fp4-mi355x-vllm-agentic-mtp scenario-type: @@ -6674,3 +6673,30 @@ - "Bump image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260828" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2763 +- config-keys: + - qwen3-0.6b-fp16-mi355x-sglang-srt-agg + - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg + scenario-type: + - fixed-seq-len + description: + - "Add small-model aggregate and 1P1D disaggregated correctness lanes for the shared MI355X srt-slurm launcher." + - "Use the native SGLang router for request routing and AMD MoRI for disaggregated KV transfer, without Dynamo, NIXL, or the deleted amd_utils control plane." + - "Remove the active and deprecated MI355X matrix entries that depended on amd_utils; configuration-specific child PRs reintroduce the active production lanes as native srt-slurm recipes." + - "Preserve the retired AgentX launch contract in the shared srt path: route DP-attention sessions by correlation ID, flush every advertised worker cache before each concurrency point, and run the MI355X RDMA device/QoS/DCQCN plus bare-process GPU drain preflights on every allocated host." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + description: + - "Port the merged MI355X DeepSeek V4 AgentX MTP 1P1D search space from the legacy AMD multinode launcher to srt-slurm." + - "Use srt-slurm's native SGLang Router frontend and AMD MoRI transport without Dynamo, NATS, etcd, or NIXL." + - "Preserve the current TP8/EP1 and TP8/EP8/DPA concurrency grids, EAGLE MTP, the golden simulated average length, HiCache settings, and logical prefill/decode server-metrics collection through the custom AgentX benchmark." + - "Remove the superseded DSV4-specific amd_utils benchmark wrapper, model stanza, HiCache override plumbing, and hard-coded synthetic-acceptance injection; the srt-slurm recipes now own those settings directly." + - "Allocate all 128 logical CPUs per MI355X backend node so SGLang TP8 affinity follows the complete host topology, and pin srt-slurm diagnostics that preserve failure-log tails containing non-UTF-8 ROCm compiler output." + - "Move the known-green DeepSeek V4 ROCm kernel-routing contract into each self-contained srt-slurm recipe; this avoids the unsupported AITer fused-MoE heuristic dispatch path while keeping legacy harness-only environment indirection removed." + - "Declare the exact SGLang and AMD MoRI runtime fingerprint reported by the pinned ROCm image so srt-slurm identity verification is reproducible." + - "Rebase the AMD runtime on the active native-router branch and wait for every advertised SGLang HTTP worker to become healthy before launching Model Gateway, preventing slow ROCm startup from permanently dropping a static prefill worker after the router's one-shot registration timeout." + - "Use the shared MI355X launcher pinned to srt-slurm PR #7 commit 02ce6d6ec7ccc7cba7a97328a156b2380d885e46, including the AMD runtime, native routers, SGLang worker-readiness gate, DP-attention rank registration, writable job-local benchmark and evaluation runtimes, and explicit eval result artifact staging." + - "Preserve the legacy DeepSeek V4 MoRI contract explicitly: 16384 prefill and 128 decode dispatch tokens per rank, 3600-second bootstrap/waiting timeouts, socket/RDMA settings, speculative-v2 and preallocation controls, correlation-ID DP routing, per-point backend cache flushing, and the per-node RDMA QoS/DCQCN preflight." + - "Preserve the proven serving parameters exactly across the launcher migration: mem-fraction-static 0.85, HiCache ratio 3, and round-robin SGLang worker load balancing for both TP8 and DP-attention paths." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2555 diff --git a/runners/check_mi355x_rdma.sh b/runners/check_mi355x_rdma.sh new file mode 100755 index 0000000000..f8aa662f54 --- /dev/null +++ b/runners/check_mi355x_rdma.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Fast, per-node fabric preflight for MI355X srt-slurm allocations. This keeps +# the meaningful QoS/DCQCN gate from the retired amd_utils launcher without its +# Docker or job-control plumbing. + +log() { printf '[%s] %s\n' "$(hostname -s)" "$*"; } +fail() { log "RDMA preflight failed: $*" >&2; exit 1; } + +expected_devices="${IBDEVICES:-rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7}" +IFS=',' read -r -a devices <<< "$expected_devices" +for device in "${devices[@]}"; do + [[ -d "/sys/class/infiniband/${device}" ]] || fail "missing device ${device}" +done +log "found all ${#devices[@]} expected RDMA devices: ${expected_devices}" + +if ! command -v nicctl >/dev/null 2>&1; then + log "nicctl is unavailable; device presence passed, QoS/DCQCN checks skipped" + exit 0 +fi + +probe=$(sudo -n nicctl show version firmware 2>&1 || true) +if grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC' <<< "$probe"; then + fail "nicctl cannot access the AMD NICs" +fi + +qos=$(sudo -n nicctl show qos 2>/dev/null) || fail "nicctl show qos failed" +classification=$(awk '/Classification type/ {print $NF; exit}' <<< "$qos") +[[ "$classification" == "DSCP" ]] || fail "classification is ${classification:-unset}, expected DSCP" + +priorities=$(awk '/PFC no-drop priorities/ {print $NF; exit}' <<< "$qos") +bitmap=$(awk '/PFC priority bitmap/ {print $NF; exit}' <<< "$qos") +[[ -n "$priorities" ]] || fail "PFC no-drop priorities are missing" +[[ -n "$bitmap" && "$bitmap" != "0x0" ]] || fail "PFC is disabled" +IFS=',' read -r -a priority_values <<< "$priorities" +for priority in "${priority_values[@]}"; do + priority="${priority//[^0-9]/}" + [[ -n "$priority" ]] || fail "invalid PFC priority list: ${priorities}" + (( bitmap & (1 << priority) )) || fail "PFC bitmap ${bitmap} does not cover priority ${priority}" +done + +dcqcn=$(sudo -n nicctl show dcqcn 2>/dev/null) || fail "nicctl show dcqcn failed" +device_count=$(grep -c 'ROCE device' <<< "$dcqcn" || true) +(( device_count > 0 )) || fail "no RoCE devices reported by nicctl" +if grep 'Status' <<< "$dcqcn" | grep -qv 'Enabled'; then + fail "DCQCN is disabled on at least one RoCE device" +fi +cnp_count=$(awk '/DSCP value used for CNP/ {print $NF}' <<< "$dcqcn" | sort -u | grep -c . || true) +(( cnp_count == 1 )) || fail "CNP DSCP is inconsistent across NICs" + +log "RDMA QoS/DCQCN preflight passed" diff --git a/runners/launch_mi355x-amds-srt.sh b/runners/launch_mi355x-amds-srt.sh new file mode 100755 index 0000000000..5535c23636 --- /dev/null +++ b/runners/launch_mi355x-amds-srt.sh @@ -0,0 +1,440 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Shared MI355X entry point for AMD-capable srt-slurm recipes. Matrix rows opt +# in explicitly with CONFIG_FILE; recipe files own model- and topology-specific +# behavior while this launcher owns staging, submission, logs, and results. +# shellcheck source=runners/slurm_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" + +SRT_SLURM_REPOSITORY="https://github.com/SemiAnalysisAI/srt-slurm.git" +SRT_SLURM_COMMIT="f1537b3810ebe4fc287dbc1ad572b370e15fc4d6" +SLURM_PARTITION="compute" +SHARED_BASE="/it-share/gharunners2/srt-slurm" +SHARED_HF_CACHE="/it-share/hf-hub-cache" +LEGACY_HF_CACHE="/it-share/hf_home" +SHARED_AIPERF_CACHE="/it-share/aiperf-cache" +SHARED_RESULTS="${SHARED_BASE}/results" + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set by Actions}" +: "${RESULT_FILENAME:?RESULT_FILENAME must be set by the benchmark workflow}" +: "${CONFIG_FILE:?CONFIG_FILE must name an srt-slurm recipe}" +: "${IMAGE:?IMAGE must identify the SGLang container image}" +: "${MODEL:?MODEL must identify the Hugging Face model}" + +SGLANG_IMAGE="$IMAGE" +IMAGE_KEY="${SGLANG_IMAGE//\//_}" +IMAGE_KEY="${IMAGE_KEY//:/_}" +SHARED_IMAGE="${SHARED_BASE}/containers/${IMAGE_KEY}.sqsh" +LOCAL_IMAGE="/var/lib/squash/${IMAGE_KEY}.sqsh" +SRT_MODEL_LOCAL_PATH="${SRT_MODEL_LOCAL_PATH:-}" +SRT_DRAFT_MODEL="${SRT_DRAFT_MODEL:-}" + +CONFIG_PATH="${CONFIG_FILE%%:*}" +LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" +CLUSTER_PROFILE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml" +[[ -f "$LOCAL_RECIPE" ]] || { echo "Missing recipe: $LOCAL_RECIPE" >&2; exit 1; } +[[ -f "$CLUSTER_PROFILE" ]] || { echo "Missing cluster profile: $CLUSTER_PROFILE" >&2; exit 1; } + +RUN_KEY="${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUNNER_NAME:-runner}" +WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" +SRT_REPO_DIR="${WORK_DIR}/srt-slurm" +mkdir -p "$WORK_DIR" "$SHARED_RESULTS" "$SHARED_AIPERF_CACHE" + +# Materialize one immutable shared squashfs and the requested model inputs. +# Production checkpoints may be Hugging Face repositories or an existing +# shared local directory. Legacy cache layouts are linked into the canonical +# HF_HOME/hub layout so large checkpoints are reused without copies. +STAGE_SCRIPT="${WORK_DIR}/stage-mi355x-runtime.sbatch" +cat > "$STAGE_SCRIPT" <"${SHARED_IMAGE}.lock" +flock -w 2400 9 +if ! unsquashfs -s "$SHARED_IMAGE" >/dev/null 2>&1; then + tmp="${SHARED_IMAGE}.tmp.\${SLURM_JOB_ID}" + rm -f "\$tmp" + local_image="${LOCAL_IMAGE}" + if unsquashfs -s "\$local_image" >/dev/null 2>&1; then + cp --sparse=always "\$local_image" "\$tmp" + else + enroot import -o "\$tmp" "docker://${SGLANG_IMAGE}" + fi + unsquashfs -s "\$tmp" >/dev/null + mv "\$tmp" "$SHARED_IMAGE" +fi +flock -u 9 +mkdir -p "$SHARED_HF_CACHE/hub" +seed_legacy_cache() { + local repo="\$1" + local cache_key="models--\${repo//\//--}" + local legacy_model_dir="$SHARED_HF_CACHE/\${cache_key}" + local canonical_model_dir="$SHARED_HF_CACHE/hub/\${cache_key}" + exec 8>"$SHARED_HF_CACHE/.\${cache_key}.stage.lock" + flock -w 2400 8 + if [[ ! -e "\$canonical_model_dir" ]]; then + if [[ -f "\$legacy_model_dir/refs/main" && -d "\$legacy_model_dir/snapshots" ]]; then + ln -s "../\${cache_key}" "\$canonical_model_dir" + elif [[ -f "$LEGACY_HF_CACHE/\${cache_key}/refs/main" && -d "$LEGACY_HF_CACHE/\${cache_key}/snapshots" ]]; then + ln -s "$LEGACY_HF_CACHE/\${cache_key}" "\$canonical_model_dir" + fi + fi + flock -u 8 +} + +model_repos=() +if [[ -n "$SRT_MODEL_LOCAL_PATH" ]]; then + python3 - "$SRT_MODEL_LOCAL_PATH" <<'PYMODEL' +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +if not root.is_dir(): + raise SystemExit(f"local model directory does not exist: {root}") +for required in ("config.json", "tokenizer_config.json", "model.safetensors.index.json"): + if not (root / required).is_file(): + raise SystemExit(f"local model is missing {required}: {root}") +index = json.loads((root / "model.safetensors.index.json").read_text()) +shards = sorted(set(index.get("weight_map", {}).values())) +if not shards: + raise SystemExit(f"local model index has no shards: {root}") +missing = [shard for shard in shards if not (root / shard).is_file()] +if missing: + raise SystemExit(f"local model is missing {len(missing)} indexed shards: {missing[:5]}") +print(f"validated local model {root}: {len(shards)} indexed shards") +PYMODEL +else + model_repos+=("$MODEL") +fi +if [[ -n "$SRT_DRAFT_MODEL" ]]; then + model_repos+=("$SRT_DRAFT_MODEL") +fi +for repo in "\${model_repos[@]}"; do + seed_legacy_cache "\$repo" +done + +if (( \${#model_repos[@]} == 0 )); then + exit 0 +fi +model_repo_list="\$(IFS=,; echo "\${model_repos[*]}")" +srun --nodes=1 --ntasks=1 \ + --container-image="$SHARED_IMAGE" \ + --container-mounts="$SHARED_HF_CACHE:/hf_hub_cache,$LEGACY_HF_CACHE:$LEGACY_HF_CACHE" \ + --container-writable --container-remap-root --no-container-entrypoint \ + --export=ALL,HF_HOME=/hf_hub_cache,HF_HUB_CACHE=/hf_hub_cache/hub,HUGGINGFACE_HUB_CACHE=/hf_hub_cache/hub,MODEL_REPOS="\$model_repo_list" \ + python3 -c 'import os; from huggingface_hub import snapshot_download; [snapshot_download(repo) for repo in os.environ["MODEL_REPOS"].split(",") if repo]' +EOF +STAGE_JOB_ID=$(sbatch --wait --parsable "$STAGE_SCRIPT") +STAGE_JOB_ID="${STAGE_JOB_ID%%;*}" +echo "MI355X runtime prerequisites verified with Slurm job ${STAGE_JOB_ID}" + +git clone "$SRT_SLURM_REPOSITORY" "$SRT_REPO_DIR" +git -C "$SRT_REPO_DIR" checkout "$SRT_SLURM_COMMIT" +ACTUAL_SRT_COMMIT=$(git -C "$SRT_REPO_DIR" rev-parse HEAD) +[[ "$ACTUAL_SRT_COMMIT" == "$SRT_SLURM_COMMIT" ]] || { + echo "srt-slurm checkout mismatch: $ACTUAL_SRT_COMMIT" >&2 + exit 1 +} + +mkdir -p "${SRT_REPO_DIR}/$(dirname "$CONFIG_PATH")" +cp "$LOCAL_RECIPE" "${SRT_REPO_DIR}/${CONFIG_PATH}" +cp "$CLUSTER_PROFILE" "${WORK_DIR}/srtslurm.yaml" +python3 - "${WORK_DIR}/srtslurm.yaml" "${SRT_REPO_DIR}/${CONFIG_PATH}" \ + "$GITHUB_WORKSPACE" "$SHARED_RESULTS" "$SHARED_AIPERF_CACHE" "$SHARED_IMAGE" <<'PY' +import os +import sys +from pathlib import Path + +import yaml + +profile_path = Path(sys.argv[1]) +recipe_path = Path(sys.argv[2]) +workspace, results, aiperf_cache, image_path = sys.argv[3:] +needle = " /it-share/hf-hub-cache: /hf_hub_cache\n" +text = profile_path.read_text() +if text.count(needle) != 1: + raise SystemExit("expected exactly one Hugging Face cache mount") +profile_path.write_text( + text.replace( + needle, + needle + + f" {aiperf_cache}: /aiperf_mmap_cache\n" + + f" {workspace}: /infmax-workspace\n" + + f" {results}: /results\n", + ) +) + +recipe = yaml.safe_load(recipe_path.read_text()) +host_setup_script = Path(workspace) / "runners" / "prepare_mi355x_srt_host.sh" +if not host_setup_script.is_file(): + raise SystemExit(f"missing MI355X host setup script: {host_setup_script}") +recipe["host_setup_script"] = str(host_setup_script) +container_alias = recipe["model"]["container"] +profile = yaml.safe_load(profile_path.read_text()) +profile.setdefault("containers", {})[container_alias] = image_path +profile_path.write_text(yaml.safe_dump(profile, sort_keys=False)) + +benchmark_env = recipe.setdefault("benchmark", {}).setdefault("env", {}) +forwarded = ( + "AIPERF_EXPERIMENTAL_FAST", + "CONC", + "CONC_LIST", + "DECODE_DP_ATTN", + "DECODE_EP", + "DECODE_NUM_WORKERS", + "DECODE_PCP_SIZE", + "DECODE_PP_SIZE", + "DECODE_TP", + "DURATION", + "EVAL_CONC", + "EVAL_LIMIT", + "EVAL_ONLY", + "FRAMEWORK", + "IS_AGENTIC", + "ISL", + "KV_OFFLOADING", + "MAX_MODEL_LEN", + "MODEL", + "MODEL_PREFIX", + "PREFILL_DP_ATTN", + "PREFILL_EP", + "PREFILL_NUM_WORKERS", + "PREFILL_PCP_SIZE", + "PREFILL_PP_SIZE", + "PREFILL_TP", + "PRECISION", + "RESULT_FILENAME", + "RUN_EVAL", + "RUNNER_TYPE", + "OSL", + "SPEC_DECODING", + "TOTAL_CPU_DRAM_GB", +) +for key in forwarded: + value = os.environ.get(key) + if value: + benchmark_env[key] = value + +# The legacy MI355X launcher sized DP+EP admission from the largest +# concurrency exercised by a recipe. It also honored a model-specific MoRI +# dispatch pin when present; only the inter-kernel switch threshold was +# derived per topology. Preserve those semantics rather than treating MTP +# draft tokens as additional independent requests. +if ( + os.environ.get("PREFILL_DP_ATTN", "false").lower() == "true" + and int(os.environ.get("PREFILL_EP", "1")) > 1 +): + concurrency_text = os.environ.get("CONC_LIST") or os.environ.get("CONC") + if not concurrency_text: + raise SystemExit("DP+EP recipe requires CONC_LIST or CONC") + concurrency_values = concurrency_text.split() + concurrency = max(int(value) for value in concurrency_values) + prefill = recipe["backend"]["sglang_config"]["prefill"] + decode = recipe["backend"]["sglang_config"]["decode"] + prefill["max-running-requests"] = concurrency + decode["max-running-requests"] = concurrency + + decode_tp = int(os.environ["DECODE_TP"]) + decode_environment = recipe["backend"]["decode_environment"] + dispatch_tokens = max(1, concurrency // decode_tp) + decode_environment.setdefault( + "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK", str(dispatch_tokens) + ) + # The retired launcher also exposed its harness-level dispatch budget to + # the server environment after scaling it by the MTP draft width. Keep that + # auxiliary value distinct from the model-specific per-rank SGLang pin. + if "MORI_MAX_DISPATCH_TOKENS_DECODE" in decode_environment: + mtp_size = int(os.environ.get("DECODE_MTP_SIZE", "0")) + decode_environment["MORI_MAX_DISPATCH_TOKENS_DECODE"] = str( + dispatch_tokens * (mtp_size + 1) + ) + decode_environment["SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD"] = str( + 2 * dispatch_tokens + ) + +eval_only = os.environ.get("EVAL_ONLY", "false").lower() == "true" +run_eval = os.environ.get("RUN_EVAL", "false").lower() == "true" +if eval_only or run_eval: + benchmark_env["SRTCTL_LM_EVAL_RESULT_DIR"] = "/results/{job_id}/eval" + decode_env = recipe.get("backend", {}).get("decode_environment", {}) + for key in ( + "SGLANG_SIMULATE_ACC_LEN", + "SGLANG_SIMULATE_ACC_METHOD", + "SGLANG_SIMULATE_ACC_TOKEN_MODE", + ): + decode_env.pop(key, None) + server_config = recipe.get("backend", {}).get("sglang_config", {}) + for mode in ("prefill", "decode"): + server_config.get(mode, {}).pop("ep-dispatch-algorithm", None) + + resources = recipe.get("resources", {}) + prefill = server_config.get("prefill", server_config.get("aggregated", {})) + decode = server_config.get("decode", prefill) + + def topology_value(config, *keys, default=1): + for key in keys: + if key in config: + return int(config[key]) + return default + + topology_defaults = { + "IS_MULTINODE": "true", + "MODEL_NAME": os.environ["MODEL"], + "EVAL_MAX_MODEL_LEN": str( + prefill.get( + "context-length", os.environ.get("MAX_MODEL_LEN", "16384") + ) + ), + "PREFILL_TP": str( + topology_value(prefill, "tp-size", "tensor-parallel-size") + ), + "PREFILL_EP": str( + topology_value(prefill, "ep-size", "expert-parallel-size") + ), + "PREFILL_NUM_WORKERS": str( + resources.get("prefill_workers", resources.get("agg_workers", 1)) + ), + "DECODE_TP": str( + topology_value(decode, "tp-size", "tensor-parallel-size") + ), + "DECODE_EP": str( + topology_value(decode, "ep-size", "expert-parallel-size") + ), + "DECODE_NUM_WORKERS": str( + resources.get("decode_workers", resources.get("agg_workers", 1)) + ), + "PREFILL_DP_ATTN": str(prefill.get("enable-dp-attention", False)).lower(), + "DECODE_DP_ATTN": str(decode.get("enable-dp-attention", False)).lower(), + } + for key, value in topology_defaults.items(): + benchmark_env.setdefault(key, value) + + eval_command = r''' +set -euo pipefail +eval_root="/results/${SLURM_JOB_ID}/eval" +mkdir -p "${eval_root}" +cd "${eval_root}" +export SRTCTL_LM_EVAL_RESULT_DIR="${eval_root}" +source /infmax-workspace/benchmarks/benchmark_lib.sh +export EVAL_SERVER_HOST="${SRT_FRONTEND_HOST}" +if [[ -n "${EVAL_CONC:-}" ]]; then + export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" +else + export EVAL_CONCURRENT_REQUESTS="$(printf '%s\n' "${CONC_LIST:-${CONC:-1}}" | tr ' ' '\n' | sort -n | tail -1)" +fi +export CONC="${EVAL_CONCURRENT_REQUESTS}" +bridge_disagg_eval_metadata +run_eval --framework lm-eval --port "${SRT_FRONTEND_PORT}" +append_lm_eval_summary +'''.strip() + if eval_only: + recipe["benchmark"]["command"] = eval_command + else: + recipe["benchmark"]["command"] = ( + recipe["benchmark"]["command"].rstrip() + "\n" + eval_command + ) +recipe_path.write_text(yaml.safe_dump(recipe, sort_keys=False)) +PY + +export PATH="$HOME/.local/bin:$PATH" +cd "$SRT_REPO_DIR" +uv venv --python 3.12 +uv pip install -e . +make setup-compute ARCH=x86_64 +source .venv/bin/activate +export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" +export SRTCTL_RUNTIME_SOURCE_DIR="$SRT_REPO_DIR" + +echo "Submitting ${CONFIG_PATH} with srt-slurm ${SRT_SLURM_COMMIT}" +set +e +SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" \ + --tags "mi355x,inferencex,github-actions,${RUN_KEY}" 2>&1) +SRTCTL_RC=$? +set -e +echo "$SRTCTL_OUTPUT" +if [[ $SRTCTL_RC -ne 0 ]]; then + echo "srtctl apply failed with exit code ${SRTCTL_RC}" >&2 + exit "$SRTCTL_RC" +fi +JOB_ID=$(grep -oE 'Job [0-9]+' <<< "$SRTCTL_OUTPUT" | awk '{print $2}' | tail -1) +[[ -n "$JOB_ID" ]] || { echo "Unable to parse srt-slurm job ID" >&2; exit 1; } +echo "SRT_SLURM_JOB_ID=$JOB_ID" + +OUTPUT_LOG_DIR="${SHARED_BASE}/outputs/${JOB_ID}/logs" +LOG_FILE="${OUTPUT_LOG_DIR}/sweep_${JOB_ID}.log" +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + +read -r JOB_STATE JOB_EXIT JOB_NODELIST < <( + sacct -X --noheader --parsable2 --jobs "$JOB_ID" \ + --format=State,ExitCode,NodeList | head -1 | tr '|' ' ' +) +echo "srt-slurm job ${JOB_ID}: state=${JOB_STATE} exit=${JOB_EXIT} nodes=${JOB_NODELIST}" + +RESULT_DIR="${SHARED_RESULTS}/${JOB_ID}" +mkdir -p "$GITHUB_WORKSPACE/LOGS" +if [[ -d "$OUTPUT_LOG_DIR" ]]; then + tar -C "$OUTPUT_LOG_DIR" -czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" . +fi +if [[ -d "$RESULT_DIR" ]]; then + cp -R "$RESULT_DIR/." "$GITHUB_WORKSPACE/LOGS/" +fi + +if [[ "${DISAGG:-false}" == "true" ]]; then + PREFILL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + DECODE_GPUS=$((DECODE_NUM_WORKERS * DECODE_TP)) + TOTAL_GPUS=$((PREFILL_GPUS + DECODE_GPUS)) +else + TOTAL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP * ${PREFILL_PP_SIZE:-1} * ${PREFILL_PCP_SIZE:-1})) +fi + +if [[ "${EVAL_ONLY:-false}" != "true" && "${IS_AGENTIC:-0}" == "1" ]]; then + shopt -s nullglob + RESULTS=("$GITHUB_WORKSPACE/${RESULT_FILENAME}"_conc*.json) + shopt -u nullglob + [[ ${#RESULTS[@]} -gt 0 ]] || { + echo "No AgentX aggregate results found for ${RESULT_FILENAME}" >&2 + exit 1 + } + printf 'Collected %s\n' "${RESULTS[@]}" +elif [[ "${EVAL_ONLY:-false}" != "true" ]]; then + shopt -s nullglob + RESULTS=("$RESULT_DIR"/fixed-seq/*.json) + shopt -u nullglob + [[ ${#RESULTS[@]} -gt 0 ]] || { echo "No fixed-sequence results found in $RESULT_DIR" >&2; exit 1; } + for result in "${RESULTS[@]}"; do + concurrency=$(basename "$result" | sed -n 's/.*-c\([0-9][0-9]*\)\.json/\1/p') + [[ -n "$concurrency" ]] || { echo "Cannot parse concurrency from $result" >&2; exit 1; } + if [[ "${DISAGG:-false}" == "true" ]]; then + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}_ctx_${PREFILL_GPUS}_gen_${DECODE_GPUS}.json" + else + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}.json" + fi + cp "$result" "$output" + echo "Collected $output" + done +fi + +if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then + if [[ "${EVAL_ONLY:-false}" == "true" && ! -f "$RESULT_DIR/eval/meta_env.json" ]]; then + echo "No eval metadata found in $RESULT_DIR/eval" >&2 + exit 1 + fi + copy_eval_artifacts "$RESULT_DIR/eval" "$GITHUB_WORKSPACE" || exit 1 +fi + +if [[ "$JOB_STATE" != COMPLETED || "$JOB_EXIT" != 0:0 ]]; then + echo "srt-slurm validation failed: ${JOB_STATE} (${JOB_EXIT})" >&2 + exit 1 +fi + +printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" +echo "MI355X srt-slurm validation completed successfully" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 49c3af0cf5..a111f19637 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +if [[ -n "${CONFIG_FILE:-}" ]]; then + exec bash "$(dirname "${BASH_SOURCE[0]}")/launch_mi355x-amds-srt.sh" +fi + scancel_sync() { local jobid=$1 local timeout=${2:-600} diff --git a/runners/prepare_mi355x_srt_host.sh b/runners/prepare_mi355x_srt_host.sh new file mode 100755 index 0000000000..e5ac5e7be3 --- /dev/null +++ b/runners/prepare_mi355x_srt_host.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +"$(dirname "${BASH_SOURCE[0]}")/check_mi355x_rdma.sh" + +# Preserve the legacy bare-process GPU drain gate. Slurm owns the node, but a +# process left outside the prior job's container can still retain VRAM and make +# the next model load fail much later with a misleading OOM. +( + unset IS_AGENTIC SCENARIO_TYPE + # shellcheck source=benchmarks/benchmark_lib.sh + source "$(dirname "${BASH_SOURCE[0]}")/../benchmarks/benchmark_lib.sh" + wait_for_amd_gpu_clean +) + +# Some MI355X experiments reserve large 2 MiB HugeTLB pools and leave the +# reservation behind after their Slurm allocation exits. Those free hugepages +# remain unavailable to ordinary host allocations, which can make a later +# unchanged SGLang HiCache recipe fail even on a 3 TiB node. Reclaim only free +# pages; pages currently used or reserved by host services are preserved. +meminfo=/proc/meminfo +nr_hugepages=/proc/sys/vm/nr_hugepages + +read_hugepage_value() { + local key="$1" + awk -v key="${key}:" '$1 == key {print $2}' "$meminfo" +} + +total=$(read_hugepage_value HugePages_Total) +free=$(read_hugepage_value HugePages_Free) +reserved=$(read_hugepage_value HugePages_Rsvd) +used=$((total - free)) +target=$((used + reserved)) + +echo "MI355X host memory before preparation:" +grep -E '^(MemAvailable|HugePages_Total|HugePages_Free|HugePages_Rsvd|HugePages_Surp|Hugetlb):' "$meminfo" + +if (( target < total )); then + printf '%s\n' "$target" | sudo -n tee "$nr_hugepages" >/dev/null +fi + +after_total=$(read_hugepage_value HugePages_Total) +after_free=$(read_hugepage_value HugePages_Free) +echo "MI355X host memory after preparation:" +grep -E '^(MemAvailable|HugePages_Total|HugePages_Free|HugePages_Rsvd|HugePages_Surp|Hugetlb):' "$meminfo" + +if (( after_total - after_free < used )); then + echo "Host preparation released hugepages that were in use" >&2 + exit 1 +fi +if (( after_free > reserved )); then + echo "Host preparation could not reclaim all unused hugepages" >&2 + exit 1 +fi diff --git a/utils/evals/test_batched_eval.py b/utils/evals/test_batched_eval.py index a5d6df0085..bcb39fd26b 100644 --- a/utils/evals/test_batched_eval.py +++ b/utils/evals/test_batched_eval.py @@ -228,17 +228,22 @@ def test_validate_scores_checks_threshold_for_every_concurrency( def test_amd_multinode_container_forwards_eval_concurrency_list() -> None: - job_slurm = ( + launcher = ( Path(__file__).resolve().parents[2] - / "benchmarks" - / "multi_node" - / "amd_utils" - / "job.slurm" + / "runners" + / "launch_mi355x-amds-srt.sh" ) - contents = job_slurm.read_text() + contents = launcher.read_text() - assert r'-e \"EVAL_CONC=\$EVAL_CONC\"' in contents - assert "-e EVAL_CONC\n" not in contents + assert ' "EVAL_CONC",' in contents + assert 'benchmark_env[key] = value' in contents + assert ( + 'benchmark_env["SRTCTL_LM_EVAL_RESULT_DIR"] = "/results/{job_id}/eval"' + in contents + ) + assert 'export EVAL_SERVER_HOST="${SRT_FRONTEND_HOST}"' in contents + assert 'export SRTCTL_LM_EVAL_RESULT_DIR="${eval_root}"' in contents + assert 'copy_eval_artifacts "$RESULT_DIR/eval"' in contents workflow = ( Path(__file__).resolve().parents[2] @@ -248,3 +253,29 @@ def test_amd_multinode_container_forwards_eval_concurrency_list() -> None: ).read_text() assert 'expected_concs="${EVAL_CONC}"' in workflow assert 'validate_scores.py --expected-concs "${expected_concs}"' in workflow + + +def test_amd_srt_launcher_streams_canonical_slurm_stdout() -> None: + launcher = ( + Path(__file__).resolve().parents[2] + / "runners" + / "launch_mi355x-amds-srt.sh" + ).read_text() + + assert 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"' in launcher + assert 'LOG_FILE="${OUTPUT_LOG_DIR}/sweep_${JOB_ID}.log"' in launcher + assert 'stream_slurm_job_log "$JOB_ID" "$LOG_FILE"' in launcher + + +def test_amd_srt_launcher_preserves_mori_dispatch_pin() -> None: + launcher = ( + Path(__file__).resolve().parents[2] + / "runners" + / "launch_mi355x-amds-srt.sh" + ).read_text() + + assert 'decode_environment.setdefault(' in launcher + assert '"SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK", str(dispatch_tokens)' in launcher + assert '"SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD"' in launcher + assert 'if "MORI_MAX_DISPATCH_TOKENS_DECODE" in decode_environment:' in launcher + assert "dispatch_tokens * (mtp_size + 1)" in launcher diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 45be3a6e4d..076fcafdd3 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -143,15 +143,18 @@ def test_lm_patch_copy_resolves_outside_repo(tmp_path): ''' -def _run_lm_eval_cmdline(*, eval_limit=None) -> str: +def _run_lm_eval_cmdline(*, eval_limit=None, eval_server_host=None) -> str: env = { **os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), "KV_OFFLOADING": "none", } env.pop("EVAL_LIMIT", None) + env.pop("EVAL_SERVER_HOST", None) if eval_limit is not None: env["EVAL_LIMIT"] = str(eval_limit) + if eval_server_host is not None: + env["EVAL_SERVER_HOST"] = eval_server_host res = subprocess.run( ["bash", "-c", _EVAL_LIMIT_SCRIPT], env=env, @@ -177,6 +180,11 @@ def test_lm_eval_defaults_to_gsm8k(): assert "utils/evals/gsm8k.yaml" in out +def test_lm_eval_uses_routed_server_host_when_set(): + out = _run_lm_eval_cmdline(eval_server_host="10.0.0.42") + assert "base_url=http://10.0.0.42:9999/v1/chat/completions" in out + + _MODAL_CREDS_SCRIPT = r''' source "$BENCHMARK_LIB"