diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index c973df94e6..d90f3fc543 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -261,6 +261,7 @@ jobs: MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval) SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) + EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval) { echo "agentic-config=$AGENTIC" diff --git a/benchmarks/multi_node/srt-slurm-recipes/AMD_BRINGUP.md b/benchmarks/multi_node/srt-slurm-recipes/AMD_BRINGUP.md new file mode 100644 index 0000000000..f8548d3686 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/AMD_BRINGUP.md @@ -0,0 +1,111 @@ +# AMD srt-slurm bring-up + +This document tracks the work-in-progress integration of +[`SemiAnalysisAI/srt-slurm`](https://github.com/SemiAnalysisAI/srt-slurm) with +InferenceX AMD Slurm clusters. The project is a functional orchestration +bring-up, not a performance-tuning exercise. + +Current development pin for both AMD launchers: + +- repository: `SemiAnalysisAI/srt-slurm` +- branch: `agent/amd-multinode-runtime` +- commit: `315e4b06a7e0806194a646ea21832e750e896a46` + +The MI300X launcher uses srt-slurm's supported `--no-preflight` submission mode +because the immutable squashfs files live on compute-node-local RAID rather +than the login node. Before submission, the launcher stages the benchmark +runtime across the same eligible node pool. Missing engine and router images +are imported atomically under per-image locks from the pinned public +`vllm/vllm-openai-rocm:v0.26.0` and +`vllm/vllm-router:nightly-20260809-d2ba586` images. + +The MI300X login and compute nodes also do not share the Actions checkout. The +staging allocation checks out the exact pinned srt-slurm commit on every +eligible compute node, installs its compute-only runtime, and injects that +node-local path through srt-slurm's `SRTCTL_RUNTIME_SOURCE_DIR` transport +override. The submitter continues to validate against its local pinned checkout. + +## Scope + +1. Prove a single-node aggregate vLLM deployment on MI300X. +2. Prove a multi-node vLLM Router prefill/decode deployment on MI300X using + AMD's supported MoRI-IO KV connector. +3. Exercise both paths with fixed input/output sequence lengths and lightweight + models before introducing production-size models. +4. Validate the same paths through the upstream InferenceX GitHub Actions + runner infrastructure. +5. Port a representative existing MI355X disaggregated configuration after the + MI300X runtime contract is stable. + +## MI300X cluster contract under validation + +- eight AMD GPUs per healthy compute node; +- Slurm GPU allocation through `--gres=gpu:` rather than the current + srt-slurm `--gpus-per-node` default; +- no site-specific `--segment` directive; +- ROCm device access through `/dev/kfd` and `/dev/dri`; +- Pyxis/Enroot writable, remap-root, and mount-home behavior matching the + established MI300X launcher; +- the shared Hugging Face cache and runner workspace remain user-owned; +- fixed-sequence validation runs InferenceX's existing + `utils/bench_serving/benchmark_serving.py` through srt-slurm's `custom` + benchmark hook rather than maintaining a second benchmark copy in + srt-slurm; +- the routable inter-node network interface is selected from live cluster + evidence rather than copied from an NVIDIA recipe. + +The launcher will continue to exclude compute nodes already documented as +unsuitable. It must not resume down nodes, cancel or preempt existing jobs, or +alter unrelated shared software. + +## Acceptance criteria + +### Aggregate + +- one srt-slurm allocation starts one aggregate vLLM service; +- every requested GPU is visible to ROCm and vLLM exactly once; +- the OpenAI-compatible health/model endpoint becomes ready; +- a fixed-sequence request completes successfully; +- srt-slurm tears down all owned processes and exits successfully. + +### Disaggregated + +- one allocation places distinct prefill and decode roles across multiple + MI300X nodes; +- vLLM Router and direct vLLM workers become healthy without Dynamo, NATS, + etcd, or bespoke per-recipe orchestration; +- role endpoints use routable node addresses and unique ports; +- KV transfer completes across AMD nodes and a fixed-sequence request succeeds; +- teardown removes only processes owned by the allocation. + +### Regression safety + +- NVIDIA remains the default accelerator runtime in srt-slurm; +- existing NVIDIA recipes and device binding tests remain green; +- AMD-specific mounts, Slurm directives, and environment variables live in a + reusable cluster profile rather than duplicated recipe shell fragments. + +## Current status + +The srt-slurm branch now contains the first accelerator-aware runtime slice: +cluster configuration accepts `accelerator_vendor: amd`, partial-GPU workers +use Linux ROCm's `ROCR_VISIBLE_DEVICES`, and legacy NVIDIA/CUDA behavior remains +the default. It also supports `gpu_sbatch_directive: gres` without changing the +legacy NVIDIA `--gpus-per-node` default. The initial MI300X cluster profile and +small-model aggregate recipe are checked in alongside this document. The first +aggregate path uses a direct private `vllm serve` endpoint. The two-node +1-prefill/1-decode path uses the official vLLM Router and vLLM's ROCm-only +`MoRIIOConnector`: srt-slurm owns the router discovery port and generates +role-aware worker registration config from the realized Slurm topology. The +control-plane endpoints use automatic RFC1918-preferring discovery because +private NIC names vary across MI300X node generations. The earlier +Dynamo/NIXL experiment reached KV-cache initialization but failed ROCm memory +registration; that NVIDIA-oriented data plane is now explicitly out of scope +rather than patched into the AMD implementation. + +The aggregate recipe has completed end to end on MI300X with both fixed-length +concurrency points. The disaggregated recipe pins ROCm's supported AITER Flash +Attention backend for both roles. This is required by the released MoRI-IO +connector's registered-memory contract: AITER exposes a contiguous logical KV +cache tensor, whereas the default Triton NHD view is strided and cannot be +registered by `mori.io` without copying or patching vLLM. diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi300x/agg-2w-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi300x/agg-2w-fixed-seq.yaml new file mode 100644 index 0000000000..0f557e4992 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi300x/agg-2w-fixed-seq.yaml @@ -0,0 +1,99 @@ +# Small two-worker aggregate correctness lane for the srt-slurm ATOM backend +# and Infera's dynamic KV-aware router. This validates orchestration, not tuning. + +name: "mi300x-atom-qwen3-0.6b-agg-2w-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "infera-atom-v0.1.1" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "rocm/infera:atom-v0.1.1" + frameworks: + atom: "0.1.4.dev113+g5837907f3" + infera: "0.0.0" + +slurm: + time_limit: "00:45:00" + +resources: + gpu_type: "mi300x" + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 2 + gpus_per_agg: 1 + +frontend: + type: infera + enable_multiple_frontends: false + env: + PYTHONPATH: "/atom-source:/infera-source" + args: + router-policy: kv-aware + +backend: + type: atom + enable_kv_events: true + aggregated_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + PYTHONPATH: "/atom-source:/infera-source" + PYTHONUNBUFFERED: "1" + OMP_NUM_THREADS: "1" + atom_config: + aggregated: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.50 + max-model-len: 2048 + max-num-seqs: 8 + block-size: 16 + enforce-eager: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 240 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-prefix-len 96 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-atom-agg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi300x/disagg-1p1d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi300x/disagg-1p1d-fixed-seq.yaml new file mode 100644 index 0000000000..87a1e9e528 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi300x/disagg-1p1d-fixed-seq.yaml @@ -0,0 +1,110 @@ +# Small two-node ATOM P/D correctness lane. Infera discovers both workers, +# routes the completions API, and ATOM transfers KV through Mooncake. + +name: "mi300x-atom-qwen3-0.6b-disagg-1p1d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "infera-atom-v0.1.1" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "rocm/infera:atom-v0.1.1" + frameworks: + atom: "0.1.4.dev113+g5837907f3" + infera: "0.0.0" + +slurm: + time_limit: "00:45:00" + +resources: + gpu_type: "mi300x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 1 + gpus_per_decode: 1 + +frontend: + type: infera + enable_multiple_frontends: false + env: + PYTHONPATH: "/atom-source:/infera-source" + args: + router-policy: kv-aware + +backend: + type: atom + connector: mooncake + # The stable atom-v0.1.1 image bundles Mooncake before ROCm DMA-BUF memory + # registration support. Use its supported TCP transport for this correctness + # lane; connection pooling is supplied by the srt-slurm ATOM adapter. + mooncake_protocol: tcp + enable_kv_events: true + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + PYTHONPATH: "/atom-source:/infera-source" + PYTHONUNBUFFERED: "1" + OMP_NUM_THREADS: "1" + decode_environment: *worker_environment + atom_config: + prefill: &worker_config + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.50 + max-model-len: 2048 + max-num-seqs: 8 + block-size: 16 + enforce-eager: true + no-enable_prefix_caching: true + decode: *worker_config + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 240 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-prefix-len 96 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-atom-disagg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi355x/agg-2w-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi355x/agg-2w-fixed-seq.yaml new file mode 100644 index 0000000000..54fc657e3c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi355x/agg-2w-fixed-seq.yaml @@ -0,0 +1,99 @@ +# Small two-worker aggregate correctness lane for ATOM and Infera on MI355X. +# It validates orchestration and KV-aware routing rather than performance. + +name: "mi355x-atom-qwen3-0.6b-agg-2w-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "infera-atom-v0.1.1" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "rocm/infera:atom-v0.1.1" + frameworks: + atom: "0.1.4.dev113+g5837907f3" + infera: "0.0.0" + +slurm: + time_limit: "00:30:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 2 + gpus_per_agg: 1 + +frontend: + type: infera + enable_multiple_frontends: false + env: + PYTHONPATH: "/atom-source:/infera-source" + args: + router-policy: kv-aware + +backend: + type: atom + enable_kv_events: true + aggregated_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONPATH: "/atom-source:/infera-source" + PYTHONUNBUFFERED: "1" + OMP_NUM_THREADS: "1" + atom_config: + aggregated: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.50 + max-model-len: 2048 + max-num-seqs: 8 + block-size: 16 + enforce-eager: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 180 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-prefix-len 96 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-atom-mi355x-agg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml new file mode 100644 index 0000000000..a5cad0d691 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml @@ -0,0 +1,110 @@ +# Small two-node ATOM P/D correctness lane on MI355X. Infera discovers the +# role-specific workers and ATOM transfers KV through Mooncake. + +name: "mi355x-atom-qwen3-0.6b-disagg-1p1d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "infera-atom-v0.1.1" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "rocm/infera:atom-v0.1.1" + frameworks: + atom: "0.1.4.dev113+g5837907f3" + infera: "0.0.0" + +slurm: + time_limit: "00:45:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 1 + gpus_per_decode: 1 + +frontend: + type: infera + enable_multiple_frontends: false + env: + PYTHONPATH: "/atom-source:/infera-source" + args: + router-policy: kv-aware + +backend: + type: atom + connector: mooncake + # The stable atom-v0.1.1 image predates Mooncake's ROCm DMA-BUF RDMA fix. + # Explicit TCP still exercises the real P/D KV path without false RDMA + # success; srt-slurm enables Mooncake connection pooling for this mode. + mooncake_protocol: tcp + enable_kv_events: true + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONPATH: "/atom-source:/infera-source" + PYTHONUNBUFFERED: "1" + OMP_NUM_THREADS: "1" + decode_environment: *worker_environment + atom_config: + prefill: &worker_config + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.50 + max-model-len: 2048 + max-num-seqs: 8 + block-size: 16 + enforce-eager: true + no-enable_prefix_caching: true + decode: *worker_config + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 240 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-prefix-len 96 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-atom-mi355x-disagg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amds.yaml b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amds.yaml new file mode 100644 index 0000000000..d661afc42a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amds.yaml @@ -0,0 +1,44 @@ +# srt-slurm cluster profile for the MI300X AMD Slurm cluster. +# Runtime-specific source/output paths are supplied by the launcher. + +cluster: mi300x-amds +default_partition: compute +default_time_limit: "01:00:00" + +gpus_per_node: 8 +accelerator_vendor: amd +# Private NIC names vary by node generation (ens51f1np1 vs ens61f1np1), so +# use srt-slurm's RFC1918-preferring automatic resolver. +network_interface: null + +# This cluster allocates accelerators through GRES and does not implement the +# NVIDIA cluster's topology-segment directive. +gpu_sbatch_directive: gres +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: false + +# The login node and compute nodes do not share this output path. Carry the +# exact resolved recipe in the batch script and materialize it on the allocated +# head node instead of depending on a submitter-side copy. +runtime_config_transport: embedded + +default_sbatch_directives: + cpus-per-task: "32" + mem: "128G" + exclude: "chi-mi300x-049,chi-mi300x-121" + +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + /raid/hf-hub-cache: /hf_hub_cache + # The launcher appends per-run /infmax-workspace and /results mounts here. + +# Pyxis otherwise re-imports the large ROCm image for every srun step. The +# validation launcher materializes this immutable squashfs on each healthy +# node before submitting a sweep. +containers: + vllm-rocm-v0.26.0: /raid/hf-hub-cache/inferencex/srt-slurm/containers/vllm-openai-rocm-v0.26.0.sqsh + vllm-router-20260809: /raid/hf-hub-cache/inferencex/srt-slurm/containers/vllm-router-nightly-20260809-d2ba586.sqsh + infera-atom-v0.1.1: /raid/hf-hub-cache/inferencex/srt-slurm/containers/infera-atom-v0.1.1.sqsh + +nginx_raise_ulimit: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml new file mode 100644 index 0000000000..3b4cd881c1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml @@ -0,0 +1,33 @@ +# srt-slurm cluster profile for the MI355X AMD Slurm cluster. The login and +# compute nodes share /it-share, so source, output, image, and result paths do +# not need node-local transport. + +cluster: mi355x-amds +default_partition: compute +default_time_limit: "01:00:00" +output_dir: /it-share/gharunners2/srt-slurm/outputs + +gpus_per_node: 8 +accelerator_vendor: amd +network_interface: eno0 + +gpu_sbatch_directive: gres +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: false +runtime_config_transport: shared-filesystem + +default_sbatch_directives: + cpus-per-task: "32" + mem: "128G" + +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + /it-share/hf-hub-cache: /hf_hub_cache + # The launcher appends the exact shared workspace and result roots. + +containers: + sglang-rocm-v0.5.17-mi35x: /it-share/gharunners2/srt-slurm/containers/sglang-rocm-v0.5.17-mi35x-20260809.sqsh + infera-atom-v0.1.1: /it-share/gharunners2/srt-slurm/containers/infera-atom-v0.1.1.sqsh + +nginx_raise_ulimit: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml new file mode 100644 index 0000000000..beb4777979 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml @@ -0,0 +1,94 @@ +# Minimal MI355X aggregate validation. It exercises native SGLang Router and +# one aggregate SGLang worker; this is orchestration validation, not tuning. + +name: "mi355x-sglang-qwen3-0.6b-agg-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + +slurm: + time_limit: "00:30:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 1 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 1 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + +backend: + type: sglang + aggregated_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + sglang_config: + aggregated: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 180 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-agg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml new file mode 100644 index 0000000000..b630a0ef65 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml @@ -0,0 +1,107 @@ +# Minimal two-node MI355X P/D validation. SGLang Router provides the request +# plane and the bundled AMD MoRI transport moves KV between prefill and decode. + +name: "mi355x-sglang-qwen3-0.6b-disagg-1p1d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + +slurm: + time_limit: "00:45:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 1 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 1 + gpus_per_decode: 1 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +backend: + type: sglang + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + decode_environment: *worker_environment + sglang_config: + prefill: &worker_config + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + disaggregation-transfer-backend: mori + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + decode: *worker_config + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 240 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-disagg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml new file mode 100644 index 0000000000..7af3198531 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml @@ -0,0 +1,145 @@ +# Production-scale MI355X port of the existing Qwen3.5 FP8 1P1D recipe. +# Each role owns one full 8-GPU node; the native SGLang Router provides the +# request plane and AMD MoRI moves KV directly between the P/D workers. + +name: "mi355x-sglang-qwen3.5-fp8-disagg-1p1d-tp8-fixed-seq" + +model: + path: "hf:Qwen/Qwen3.5-397B-A17B-FP8" + container: "sglang-rocm-v0.5.17-mi35x" + precision: "fp8" + +identity: + model: + repo: "Qwen/Qwen3.5-397B-A17B-FP8" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + +slurm: + time_limit: "02:00:00" + +# This topology owns every GPU on each MI355X node. Give SGLang the complete +# 128-core cpuset as well: its ROCm image enables per-GPU CPU affinity, which +# maps the eight TP ranks across the full dual-socket CPU topology. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + # Hugging Face stores hub snapshots under $HF_HOME/hub. Keep the explicit + # cache variables on that same path so srt-slurm's prefetch and every + # backend process resolve the identical, current snapshot. + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: "ISOLATION" + SGLANG_USE_AITER: "1" + AITER_LOG_LEVEL: "ERROR" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + decode_environment: *common_environment + sglang_config: + prefill: &common_config + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + tensor-parallel-size: 8 + disaggregation-transfer-backend: mori + disaggregation-ib-device: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + kv-cache-dtype: "fp8_e4m3" + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 262144 + context-length: 16384 + disable-radix-cache: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8] + enable-metrics: true + decode: + <<: *common_config + mem-fraction-static: 0.85 + prefill-round-robin-balance: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + concurrency=8 + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3.5-397B-A17B-FP8 \ + --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 2))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3.5-fp8-disagg-isl8192-osl1024-c${concurrency}.json" + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/qwen3-0.6b/mi300x/agg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/qwen3-0.6b/mi300x/agg-fixed-seq.yaml new file mode 100644 index 0000000000..2f201560be --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/qwen3-0.6b/mi300x/agg-fixed-seq.yaml @@ -0,0 +1,93 @@ +# Minimal MI300X aggregate validation. This recipe deliberately uses one GPU, +# a small public model, and a short fixed-sequence workload. It validates +# srt-slurm orchestration rather than performance. + +name: "mi300x-vllm-qwen3-0.6b-agg-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "vllm-rocm-v0.26.0" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "vllm/vllm-openai-rocm:v0.26.0" + frameworks: + vllm: "0.26.0+rocm723" + +slurm: + time_limit: "00:30:00" + +resources: + gpu_type: "mi300x" + gpus_per_node: 1 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 1 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + set_visible_devices: true + aggregated_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + PYTHONUNBUFFERED: "1" + vllm_config: + aggregated: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.50 + max-model-len: 2048 + max-num-seqs: 8 + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 180 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/qwen3-0.6b/mi300x/disagg-1p1d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/qwen3-0.6b/mi300x/disagg-1p1d-fixed-seq.yaml new file mode 100644 index 0000000000..239e2b6cc4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/qwen3-0.6b/mi300x/disagg-1p1d-fixed-seq.yaml @@ -0,0 +1,121 @@ +# Minimal two-node MI300X prefill/decode validation. This recipe exercises +# vLLM Router discovery and AMD MoRI-IO KV transfer without performance tuning. + +name: "mi300x-vllm-qwen3-0.6b-disagg-1p1d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "vllm-rocm-v0.26.0" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "vllm/vllm-openai-rocm:v0.26.0" + frameworks: + vllm: "0.26.0+rocm723" + vllm-router: "nightly-20260809-d2ba586" + +slurm: + time_limit: "00:45:00" + +resources: + gpu_type: "mi300x" + gpus_per_node: 1 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 1 + gpus_per_decode: 1 + +frontend: + type: vllm-router + enable_multiple_frontends: false + container_image: "vllm-router-20260809" + args: + policy: consistent_hash + prefill-policy: consistent_hash + decode-policy: consistent_hash + +backend: + type: vllm + connector: moriio + set_visible_devices: true + prefill_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + PYTHONUNBUFFERED: "1" + VLLM_ROCM_USE_AITER: "1" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + decode_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + PYTHONUNBUFFERED: "1" + VLLM_ROCM_USE_AITER: "1" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + vllm_config: + prefill: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.50 + max-model-len: 2048 + max-num-seqs: 8 + enforce-eager: true + attention-backend: "ROCM_AITER_FA" + decode: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.50 + max-model-len: 2048 + max-num-seqs: 8 + enforce-eager: true + attention-backend: "ROCM_AITER_FA" + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 240 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 1 4; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-disagg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d26796a52f..15ec613e4c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1497,6 +1497,292 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } +# Small-model orchestration validation for SemiAnalysisAI/srt-slurm on AMD. +# This is intentionally a correctness lane, not a performance submission. +qwen3-0.6b-fp16-mi300x-vllm-srt-agg: + image: vllm/vllm-openai-rocm:v0.26.0 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: mi300x-disagg + precision: fp16 + framework: vllm + # Route through the multinode workflow even though the recipe itself owns a + # single aggregate node. That workflow is the srt-slurm orchestration entry + # point; disagg remains false and the decode worker count stays zero so the + # aggregate GPU is counted exactly once. + multinode: true + disagg: false + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + # The srt-slurm custom benchmark exercises both c1 and c4 in one + # allocation; keep one matrix point so the server is launched once. + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/qwen3-0.6b/mi300x/agg-fixed-seq.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 1 + dp-attn: false + +qwen3-0.6b-fp16-mi300x-vllm-srt-disagg: + image: vllm/vllm-openai-rocm:v0.26.0 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: mi300x-disagg + precision: fp16 + framework: vllm + multinode: true + disagg: true + router: { name: vllm-router, version: "nightly-20260809-d2ba586" } + kv-p2p-transfer: moriio + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/qwen3-0.6b/mi300x/disagg-1p1d-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + +# Small-model correctness lanes for the native ATOM backend and the current +# open-source Infera router (the successor to ATOMesh). The aggregate lane +# runs two independent ATOM workers so discovery and routing are exercised; +# the P/D lane validates Mooncake transfer across two MI300X nodes. +qwen3-0.6b-fp16-mi300x-atom-infera-srt-agg: + image: rocm/infera:atom-v0.1.1 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: mi300x-disagg + precision: fp16 + framework: atom + router: { name: infera, version: "0.1.1" } + multinode: true + disagg: false + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + prefill: + num-worker: 2 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/atom/qwen3-0.6b/mi300x/agg-2w-fixed-seq.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 1 + dp-attn: false + +qwen3-0.6b-fp16-mi300x-atom-infera-srt-disagg: + image: rocm/infera:atom-v0.1.1 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: mi300x-disagg + precision: fp16 + framework: atom-disagg + router: { name: infera, version: "0.1.1" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/atom/qwen3-0.6b/mi300x/disagg-1p1d-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + +# Small-model correctness lanes for ATOM and Infera orchestration on MI355X. +# Aggregate uses two workers so KV-event-aware routing is exercised; P/D uses +# one prefill and one decode worker with Mooncake transfer. +qwen3-0.6b-fp16-mi355x-atom-infera-srt-agg: + image: rocm/infera:atom-v0.1.1 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: atom + multinode: true + disagg: false + router: { name: infera, version: "0.1.1" } + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + prefill: + num-worker: 2 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/atom/qwen3-0.6b/mi355x/agg-2w-fixed-seq.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 1 + dp-attn: false + +qwen3-0.6b-fp16-mi355x-atom-infera-srt-disagg: + image: rocm/infera:atom-v0.1.1 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: atom-disagg + multinode: true + disagg: true + router: { name: infera, version: "0.1.1" } + kv-p2p-transfer: mooncake + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/atom/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + +# Small-model correctness lanes for native SGLang Router orchestration on +# MI355X. Both use the unchanged InferenceX fixed-sequence benchmark; the P/D +# lane uses AMD MoRI and neither lane starts Dynamo, NATS, etcd, or NIXL. +qwen3-0.6b-fp16-mi355x-sglang-srt-agg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: sglang + multinode: true + disagg: false + router: { name: sglang-router, version: "0.3.2" } + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 1 + dp-attn: false + +qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: sglang-disagg + multinode: true + disagg: true + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + +# Production-scale migration of the existing Qwen3.5 FP8 MI355X P/D lane to +# srt-slurm. One full TP8 node serves prefill, one TP8 node serves decode, and +# the request/KV planes remain native SGLang Router + AMD MoRI. +qwen3.5-fp8-mi355x-sglang-srt-disagg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi355x-amds + precision: fp8 + framework: sglang-disagg + multinode: true + disagg: true + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [8] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + minimaxm3-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf model: amd/MiniMax-M3-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b89b34e5ff..635670c080 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5726,6 +5726,24 @@ - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2520 +- config-keys: + - qwen3-0.6b-fp16-mi300x-vllm-srt-agg + - qwen3-0.6b-fp16-mi300x-vllm-srt-disagg + - qwen3-0.6b-fp16-mi355x-sglang-srt-agg + - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg + - qwen3.5-fp8-mi355x-sglang-srt-disagg + description: + - "Add native AMD orchestration to srt-slurm and validate aggregate plus disaggregated serving on MI300X and MI355X without Dynamo, NIXL, NATS, or etcd." + - "Use the native vLLM Router with MoRIIO for MI300X disaggregation and the native SGLang Router with AMD MoRI for MI355X disaggregation; aggregate paths use the corresponding direct backend." + - "Run the repository's unchanged utils/bench_serving/benchmark_serving.py through srt-slurm's custom benchmark contract for fixed-sequence validation." + - "Port the production Qwen3.5-397B-A17B-FP8 MI355X topology as two full TP8 nodes, one prefill and one decode, using lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809, SGLang Router, MoRI, AITer attention, FP8 KV cache, and an 8192-input/1024-output concurrency-8 workload." + - "Use the current steady ROCm image because its MoRI IO path contains the merged automatic dma-buf fallback for large GPU memory registrations; the prior 20260728 image used plain ibv_reg_mr and failed reproducibly above 3.75 GiB." + - "Propagate runtime Slurm options to native static routers so the router inherits the same writable/remapped container contract as workers and benchmarks." + - "Pin the InferenceX launcher to the exact srt-slurm AMD runtime commit and validate model, container, backend, router, and MoRI package identity before serving." + - "Stage every MI355X model into the standard $HF_HOME/hub layout used by all aggregate, prefill, decode, and benchmark processes, avoiding a second flat cache tree on cold clusters." + - "Recreate missing MI300X engine and router squashfs files from their pinned public vLLM images under per-node locks, validate each temporary squashfs, and publish it atomically before submission." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + - config-keys: - dsv4-fp4-gb300-dynamo-sglang-agentic-agg - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg @@ -5776,4 +5794,16 @@ description: - "Update the B300 DSV4 AgentX MTP image and SimpleCPU sweep through c576." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 - + +- config-keys: + - qwen3-0.6b-fp16-mi300x-atom-infera-srt-agg + - qwen3-0.6b-fp16-mi300x-atom-infera-srt-disagg + - qwen3-0.6b-fp16-mi355x-atom-infera-srt-agg + - qwen3-0.6b-fp16-mi355x-atom-infera-srt-disagg + description: + - "Add srt-slurm correctness lanes for the ROCm ATOM engine and the open-source Infera successor to ATOMesh." + - "Validate two aggregate ATOM workers behind Infera KV-aware routing and a two-node 1P/1D ATOM deployment using Mooncake TCP with connection pooling." + - "Use rocm/infera:atom-v0.1.1 with Qwen3-0.6B fixed-sequence requests on MI300X and MI355X, verify the installed ATOM and Infera distributions, and pin the combined AMD, vLLM Router, ATOM, and Infera srt-slurm branch." + - "Overlay the exact upstream Infera KV-event decoder fix from AMD-AGI/Infera#114 so ATOM's tagged-map events populate the router cache view during pre-release hardware validation." + - "Serialize shared Infera and ATOM source staging per checkout and publish completed clones atomically so concurrent aggregate and disaggregated validation jobs cannot corrupt one another." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2558 diff --git a/runners/launch_mi300x-amds-srt.sh b/runners/launch_mi300x-amds-srt.sh new file mode 100644 index 0000000000..31ab31d0bf --- /dev/null +++ b/runners/launch_mi300x-amds-srt.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash +set -euo pipefail + +# MI300X srt-slurm validation path. The existing launcher remains the default; +# matrix rows opt in by exporting CONFIG_FILE through additional-settings. +SRT_SLURM_REPOSITORY="https://github.com/SemiAnalysisAI/srt-slurm.git" +SRT_SLURM_COMMIT="5ecfb13d1ba0960045482f1ef006312d8729d37a" +INFERA_REPOSITORY="https://github.com/cquil11/Infera.git" +INFERA_COMMIT="8ed8f1728c745d4e91ba9eaa09ed81159aa57e41" +ATOM_REPOSITORY="https://github.com/cquil11/ATOM.git" +ATOM_COMMIT="2ab42bc2c64d1ad04f698c396da48473e71a6dbb" +SLURM_PARTITION="compute" +EXCLUDED_NODES="chi-mi300x-049,chi-mi300x-121" +REMOTE_BASE="/raid/hf-hub-cache/inferencex/srt-slurm" +VLLM_IMAGE="vllm/vllm-openai-rocm:v0.26.0" +VLLM_ROUTER_IMAGE="vllm/vllm-router:nightly-20260809-d2ba586" +ATOM_IMAGE="rocm/infera:atom-v0.1.1" +VLLM_SQSH="${REMOTE_BASE}/containers/vllm-openai-rocm-v0.26.0.sqsh" +VLLM_ROUTER_SQSH="${REMOTE_BASE}/containers/vllm-router-nightly-20260809-d2ba586.sqsh" +ATOM_SQSH="${REMOTE_BASE}/containers/infera-atom-v0.1.1.sqsh" + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set by Actions}" +: "${RESULT_FILENAME:?RESULT_FILENAME must be set by the benchmark workflow}" + +: "${CONFIG_FILE:?CONFIG_FILE must name an srt-slurm recipe}" + +case "${IMAGE:?IMAGE must identify the recipe container}" in + "$VLLM_IMAGE") + ENGINE_IMAGE="$VLLM_IMAGE" + ENGINE_SQSH="$VLLM_SQSH" + AUX_IMAGE="$VLLM_ROUTER_IMAGE" + AUX_SQSH="$VLLM_ROUTER_SQSH" + ;; + "$ATOM_IMAGE") + ENGINE_IMAGE="$ATOM_IMAGE" + ENGINE_SQSH="$ATOM_SQSH" + AUX_IMAGE="" + AUX_SQSH="" + ;; + *) + echo "Unsupported MI300X srt-slurm image: $IMAGE" >&2 + exit 1 + ;; +esac + +CONFIG_PATH="${CONFIG_FILE%%:*}" +LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" +CLUSTER_PROFILE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amds.yaml" +[[ -f "$LOCAL_RECIPE" ]] || { echo "Missing recipe: $LOCAL_RECIPE" >&2; exit 1; } +[[ -f "$CLUSTER_PROFILE" ]] || { echo "Missing cluster profile: $CLUSTER_PROFILE" >&2; exit 1; } + +RUN_KEY="${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUNNER_NAME:-runner}" +REMOTE_RUNTIME="${REMOTE_BASE}/runtime/inferencex-${RUN_KEY}" +REMOTE_SRT_RUNTIME="${REMOTE_BASE}/runtime/srt-slurm-${SRT_SLURM_COMMIT}" +REMOTE_INFERA_RUNTIME="${REMOTE_BASE}/runtime/infera-${INFERA_COMMIT}" +REMOTE_ATOM_RUNTIME="${REMOTE_BASE}/runtime/atom-${ATOM_COMMIT}" +REMOTE_RESULTS="${REMOTE_BASE}/results" +WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" +SRT_REPO_DIR="${WORK_DIR}/srt-slurm" +mkdir -p "$WORK_DIR" + +# The login and compute nodes do not share a filesystem. Stage only the +# unchanged InferenceX benchmark client and immutable public container images +# onto every eligible node. The cluster has nine nodes and this validation +# excludes two, so the staging allocation must cover all seven remaining nodes. +# The batch job exits normally; it does not cancel or preempt any allocation. +RUNTIME_ARCHIVE="${WORK_DIR}/inferencex-benchmark.tar.gz" +tar -C "$GITHUB_WORKSPACE" -czf "$RUNTIME_ARCHIVE" utils/bench_serving +RUNTIME_PAYLOAD=$(base64 -w0 "$RUNTIME_ARCHIVE") +STAGE_SCRIPT="${WORK_DIR}/stage-runtime.sbatch" +cat > "$STAGE_SCRIPT" < "\$source_archive" +sbcast --force "\$source_archive" "\$node_archive" +srun --ntasks-per-node=1 bash -c ' + set -euo pipefail + runtime="${REMOTE_RUNTIME}" + srt_runtime="${REMOTE_SRT_RUNTIME}" + infera_runtime="${REMOTE_INFERA_RUNTIME}" + atom_runtime="${REMOTE_ATOM_RUNTIME}" + export ENROOT_RUNTIME_PATH="\${TMPDIR:-/tmp}/enroot-runtime-\${UID}" + mkdir -p "\$ENROOT_RUNTIME_PATH" "\$runtime" "${REMOTE_RESULTS}" "${REMOTE_BASE}/containers" + chmod 700 "\$ENROOT_RUNTIME_PATH" + ensure_container_image() { + local target="\$1" + local image="\$2" + local lock_fd + local tmp + if unsquashfs -s "\$target" >/dev/null 2>&1; then + return + fi + exec {lock_fd}>"\${target}.lock" + flock -w 2400 "\$lock_fd" + if ! unsquashfs -s "\$target" >/dev/null 2>&1; then + tmp="\${target}.tmp.\${SLURM_JOB_ID}" + for attempt in 1 2 3; do + rm -f "\$tmp" + if enroot import -o "\$tmp" "docker://\${image}"; then + break + fi + if [[ "\$attempt" -eq 3 ]]; then + echo "Failed to import \${image} after \${attempt} attempts" >&2 + return 1 + fi + echo "Retrying \${image} import after attempt \${attempt}" >&2 + sleep "\$((attempt * 10))" + done + unsquashfs -s "\$tmp" >/dev/null + mv "\$tmp" "\$target" + fi + flock -u "\$lock_fd" + exec {lock_fd}>&- + } + ensure_git_checkout() { + local target="\$1" + local repository="\$2" + local commit="\$3" + local lock_fd + local temporary="\${target}.tmp.\${SLURM_JOB_ID}.\${BASHPID}" + local quarantine="\${target}.incomplete.\${SLURM_JOB_ID}.\${BASHPID}" + + mkdir -p "\$(dirname "\$target")" + exec {lock_fd}>"\${target}.lock" + flock -w 2400 "\$lock_fd" + if [[ ! -d "\$target/.git" ]]; then + if [[ -e "\$target" ]]; then + mv -T "\$target" "\$quarantine" + fi + git clone --quiet "\$repository" "\$temporary" + git -C "\$temporary" fetch --quiet origin "\$commit" + git -C "\$temporary" checkout --quiet --detach "\$commit" + test "\$(git -C "\$temporary" rev-parse HEAD)" = "\$commit" + mv -T "\$temporary" "\$target" + else + git -C "\$target" fetch --quiet origin "\$commit" + git -C "\$target" checkout --quiet --detach "\$commit" + test "\$(git -C "\$target" rev-parse HEAD)" = "\$commit" + fi + flock -u "\$lock_fd" + exec {lock_fd}>&- + } + ensure_container_image "${ENGINE_SQSH}" "${ENGINE_IMAGE}" + if [[ -n "${AUX_IMAGE}" ]]; then + ensure_container_image "${AUX_SQSH}" "${AUX_IMAGE}" + fi + ensure_git_checkout "\$srt_runtime" "${SRT_SLURM_REPOSITORY}" "${SRT_SLURM_COMMIT}" + make -C "\$srt_runtime" --no-print-directory setup ARCH=x86_64 + ensure_git_checkout "\$infera_runtime" "${INFERA_REPOSITORY}" "${INFERA_COMMIT}" + ensure_git_checkout "\$atom_runtime" "${ATOM_REPOSITORY}" "${ATOM_COMMIT}" + tar -xzf "/tmp/inferencex-benchmark-\${SLURM_JOB_ID}.tar.gz" -C "\$runtime" + printf "%s\\n" "${GITHUB_SHA:-unknown}" > "\$runtime/.inferencex-source-head" +' +EOF +STAGE_JOB_ID=$(sbatch --wait --parsable "$STAGE_SCRIPT") +echo "Staged InferenceX benchmark client with Slurm job ${STAGE_JOB_ID}" + +git clone "$SRT_SLURM_REPOSITORY" "$SRT_REPO_DIR" +git -C "$SRT_REPO_DIR" checkout "$SRT_SLURM_COMMIT" +ACTUAL_SRT_COMMIT=$(git -C "$SRT_REPO_DIR" rev-parse HEAD) +[[ "$ACTUAL_SRT_COMMIT" == "$SRT_SLURM_COMMIT" ]] || { + echo "srt-slurm checkout mismatch: $ACTUAL_SRT_COMMIT" >&2 + exit 1 +} + +mkdir -p "${SRT_REPO_DIR}/$(dirname "$CONFIG_PATH")" +cp "$LOCAL_RECIPE" "${SRT_REPO_DIR}/${CONFIG_PATH}" +cp "$CLUSTER_PROFILE" "${WORK_DIR}/srtslurm.yaml" +python3 - "${WORK_DIR}/srtslurm.yaml" "$REMOTE_RUNTIME" "$REMOTE_RESULTS" "$REMOTE_INFERA_RUNTIME" "$REMOTE_ATOM_RUNTIME" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +runtime, results, infera_runtime, atom_runtime = sys.argv[2:] +needle = " /raid/hf-hub-cache: /hf_hub_cache\n" +text = path.read_text() +if text.count(needle) != 1: + raise SystemExit("expected exactly one Hugging Face cache mount") +path.write_text( + text.replace( + needle, + needle + + f" {runtime}: /infmax-workspace\n" + + f" {results}: /results\n" + + f" {infera_runtime}: /infera-source\n" + + f" {atom_runtime}: /atom-source\n", + ) +) +PY + +export PATH="$HOME/.local/bin:$PATH" +cd "$SRT_REPO_DIR" +uv venv --python 3.12 +uv pip install -e . +make setup ARCH=x86_64 +source .venv/bin/activate +export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" +export SRTCTL_RUNTIME_SOURCE_DIR="$REMOTE_SRT_RUNTIME" +export INFMAX_WORKSPACE="$REMOTE_RUNTIME" + +echo "Submitting ${CONFIG_PATH} with srt-slurm ${SRT_SLURM_COMMIT}" +set +e +SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" \ + --no-preflight \ + --tags "mi300x,inferencex,github-actions,${RUN_KEY}" 2>&1) +SRTCTL_RC=$? +set -e +echo "$SRTCTL_OUTPUT" +if [[ $SRTCTL_RC -ne 0 ]]; then + echo "srtctl apply failed with exit code ${SRTCTL_RC}" >&2 + exit "$SRTCTL_RC" +fi +JOB_ID=$(grep -oE 'Job [0-9]+' <<< "$SRTCTL_OUTPUT" | awk '{print $2}' | tail -1) +[[ -n "$JOB_ID" ]] || { echo "Unable to parse srt-slurm job ID" >&2; exit 1; } +echo "SRT_SLURM_JOB_ID=$JOB_ID" + +while squeue --noheader --jobs "$JOB_ID" | grep -q .; do + squeue --noheader --jobs "$JOB_ID" --format='srt-slurm %i %T %M %R' + sleep 15 +done + +read -r JOB_STATE JOB_EXIT JOB_NODELIST < <( + sacct -X --noheader --parsable2 --jobs "$JOB_ID" \ + --format=State,ExitCode,NodeList | head -1 | tr '|' ' ' +) +JOB_BATCH_HOST=$(scontrol show job "$JOB_ID" -dd | sed -n 's/.*BatchHost=\([^ ]*\).*/\1/p' | head -1) +[[ -n "$JOB_BATCH_HOST" ]] || { + echo "Unable to resolve BatchHost for srt-slurm job ${JOB_ID}" >&2 + exit 1 +} +echo "srt-slurm job ${JOB_ID}: state=${JOB_STATE} exit=${JOB_EXIT} nodes=${JOB_NODELIST} batch_host=${JOB_BATCH_HOST}" + +# Results live on the allocation's node-local RAID. Retrieve the small result +# bundle with a separate completed Slurm job on the batch node. +RETRIEVE_DIR="${WORK_DIR}/retrieved" +mkdir -p "$RETRIEVE_DIR" +RESULT_PAYLOAD=$(srun --partition="$SLURM_PARTITION" --nodes=1 --ntasks=1 \ + --cpus-per-task=1 --time=00:05:00 --nodelist="$JOB_BATCH_HOST" \ + bash -c "tar -C '${REMOTE_RESULTS}/${JOB_ID}' -czf - . | base64 -w0") +printf '%s' "$RESULT_PAYLOAD" | base64 -d | tar -xzf - -C "$RETRIEVE_DIR" + +mkdir -p "$GITHUB_WORKSPACE/LOGS" +if [[ -f "$RETRIEVE_DIR/runtime-logs.tar.gz" ]]; then + cp "$RETRIEVE_DIR/runtime-logs.tar.gz" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" +fi +cp -R "$RETRIEVE_DIR/." "$GITHUB_WORKSPACE/LOGS/" + +if [[ "${DISAGG:-false}" == "true" ]]; then + PREFILL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + DECODE_GPUS=$((DECODE_NUM_WORKERS * DECODE_TP)) + TOTAL_GPUS=$((PREFILL_GPUS + DECODE_GPUS)) +else + # Aggregate srt-slurm rows intentionally use the multinode workflow so + # this launcher owns orchestration. The aggregate worker is represented + # by the prefill-shaped matrix fields; decode workers are zero. + TOTAL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP * ${PREFILL_PP_SIZE:-1} * ${PREFILL_PCP_SIZE:-1})) +fi +shopt -s nullglob +RESULTS=("$RETRIEVE_DIR"/fixed-seq/*.json) +shopt -u nullglob +[[ ${#RESULTS[@]} -gt 0 ]] || { echo "No fixed-sequence results retrieved" >&2; exit 1; } +for result in "${RESULTS[@]}"; do + concurrency=$(basename "$result" | sed -n 's/.*-c\([0-9][0-9]*\)\.json/\1/p') + [[ -n "$concurrency" ]] || { echo "Cannot parse concurrency from $result" >&2; exit 1; } + if [[ "${DISAGG:-false}" == "true" ]]; then + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}_ctx_${PREFILL_GPUS}_gen_${DECODE_GPUS}.json" + else + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}.json" + fi + cp "$result" "$output" + echo "Collected $output" +done + +if [[ "$JOB_STATE" != COMPLETED || "$JOB_EXIT" != 0:0 ]]; then + echo "srt-slurm validation failed: ${JOB_STATE} (${JOB_EXIT})" >&2 + exit 1 +fi + +printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" +echo "MI300X srt-slurm validation completed successfully" diff --git a/runners/launch_mi300x-amds.sh b/runners/launch_mi300x-amds.sh index fdd03889a0..a69e22edbe 100644 --- a/runners/launch_mi300x-amds.sh +++ b/runners/launch_mi300x-amds.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash set -eo pipefail +if [[ -n "${CONFIG_FILE:-}" ]]; then + exec bash "$(dirname "${BASH_SOURCE[0]}")/launch_mi300x-amds-srt.sh" +fi + export HF_HUB_CACHE_MOUNT="/raid/hf-hub-cache/" export PORT=8888 diff --git a/runners/launch_mi355x-amds-srt.sh b/runners/launch_mi355x-amds-srt.sh new file mode 100755 index 0000000000..72ab623ae9 --- /dev/null +++ b/runners/launch_mi355x-amds-srt.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash +set -euo pipefail + +# MI355X validation path for the AMD-capable srt-slurm branch. Matrix rows opt +# in explicitly with CONFIG_FILE; all existing MI355X launch behavior remains +# unchanged for every other row. +SRT_SLURM_REPOSITORY="https://github.com/SemiAnalysisAI/srt-slurm.git" +SRT_SLURM_COMMIT="5ecfb13d1ba0960045482f1ef006312d8729d37a" +INFERA_REPOSITORY="https://github.com/cquil11/Infera.git" +INFERA_COMMIT="8ed8f1728c745d4e91ba9eaa09ed81159aa57e41" +ATOM_REPOSITORY="https://github.com/cquil11/ATOM.git" +ATOM_COMMIT="2ab42bc2c64d1ad04f698c396da48473e71a6dbb" +SLURM_PARTITION="compute" +SGLANG_IMAGE="lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" +ATOM_IMAGE="rocm/infera:atom-v0.1.1" +SHARED_BASE="/it-share/gharunners2/srt-slurm" +SHARED_HF_CACHE="/it-share/hf-hub-cache" +SHARED_RESULTS="${SHARED_BASE}/results" +SHARED_INFERA_RUNTIME="${SHARED_BASE}/runtime/infera-${INFERA_COMMIT}" +SHARED_ATOM_RUNTIME="${SHARED_BASE}/runtime/atom-${ATOM_COMMIT}" + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set by Actions}" +: "${RESULT_FILENAME:?RESULT_FILENAME must be set by the benchmark workflow}" +: "${CONFIG_FILE:?CONFIG_FILE must name an srt-slurm recipe}" +: "${MODEL:?MODEL must identify the Hugging Face model}" + +case "${IMAGE:?IMAGE must identify the recipe container}" in + "$SGLANG_IMAGE") + RUNTIME_IMAGE="$SGLANG_IMAGE" + SHARED_IMAGE="${SHARED_BASE}/containers/sglang-rocm-v0.5.17-mi35x-20260809.sqsh" + LOCAL_FALLBACK_IMAGE="/var/lib/squash/lmsysorg_sglang-rocm_v0.5.17-rocm720-mi35x-20260809.sqsh" + ;; + "$ATOM_IMAGE") + RUNTIME_IMAGE="$ATOM_IMAGE" + SHARED_IMAGE="${SHARED_BASE}/containers/infera-atom-v0.1.1.sqsh" + LOCAL_FALLBACK_IMAGE="" + ;; + *) + echo "Unsupported MI355X srt-slurm image: $IMAGE" >&2 + exit 1 + ;; +esac + +CONFIG_PATH="${CONFIG_FILE%%:*}" +LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" +CLUSTER_PROFILE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml" +[[ -f "$LOCAL_RECIPE" ]] || { echo "Missing recipe: $LOCAL_RECIPE" >&2; exit 1; } +[[ -f "$CLUSTER_PROFILE" ]] || { echo "Missing cluster profile: $CLUSTER_PROFILE" >&2; exit 1; } + +RUN_KEY="${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUNNER_NAME:-runner}" +WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" +SRT_REPO_DIR="${WORK_DIR}/srt-slurm" +mkdir -p "$WORK_DIR" "$SHARED_RESULTS" + +# Materialize one immutable, shared squashfs and the small public validation +# model. This job exits normally and never cancels or preempts another job. +# The shared lock makes concurrent aggregate/disaggregated validations safe. +STAGE_SCRIPT="${WORK_DIR}/stage-mi355x-runtime.sbatch" +cat > "$STAGE_SCRIPT" <"\${target}.lock" + flock -w 2400 "\$lock_fd" + if [[ ! -d "\$target/.git" ]]; then + if [[ -e "\$target" ]]; then + mv -T "\$target" "\$quarantine" + fi + git clone --quiet "\$repository" "\$temporary" + git -C "\$temporary" fetch --quiet origin "\$commit" + git -C "\$temporary" checkout --quiet --detach "\$commit" + test "\$(git -C "\$temporary" rev-parse HEAD)" = "\$commit" + mv -T "\$temporary" "\$target" + else + git -C "\$target" fetch --quiet origin "\$commit" + git -C "\$target" checkout --quiet --detach "\$commit" + test "\$(git -C "\$target" rev-parse HEAD)" = "\$commit" + fi + flock -u "\$lock_fd" + exec {lock_fd}>&- +} +ensure_git_checkout "${SHARED_INFERA_RUNTIME}" "${INFERA_REPOSITORY}" "${INFERA_COMMIT}" +ensure_git_checkout "${SHARED_ATOM_RUNTIME}" "${ATOM_REPOSITORY}" "${ATOM_COMMIT}" +exec 9>"${SHARED_IMAGE}.lock" +flock -w 2400 9 +if ! unsquashfs -s "$SHARED_IMAGE" >/dev/null 2>&1; then + tmp="${SHARED_IMAGE}.tmp.\${SLURM_JOB_ID}" + rm -f "\$tmp" + if [[ -n "${LOCAL_FALLBACK_IMAGE}" ]] && unsquashfs -s "${LOCAL_FALLBACK_IMAGE}" >/dev/null 2>&1; then + cp --sparse=always "${LOCAL_FALLBACK_IMAGE}" "\$tmp" + else + for attempt in 1 2 3; do + rm -f "\$tmp" + if enroot import -o "\$tmp" "docker://${RUNTIME_IMAGE}"; then + break + fi + if [[ "\$attempt" -eq 3 ]]; then + echo "Failed to import ${RUNTIME_IMAGE} after \${attempt} attempts" >&2 + exit 1 + fi + echo "Retrying ${RUNTIME_IMAGE} import after attempt \${attempt}" >&2 + sleep "\$((attempt * 10))" + done + fi + unsquashfs -s "\$tmp" >/dev/null + mv "\$tmp" "$SHARED_IMAGE" +fi +flock -u 9 +srun --nodes=1 --ntasks=1 \ + --container-image="$SHARED_IMAGE" \ + --container-mounts="$SHARED_HF_CACHE:/hf_hub_cache" \ + --container-writable --container-remap-root --no-container-entrypoint \ + --export=ALL,HF_HOME=/hf_hub_cache,HF_HUB_CACHE=/hf_hub_cache/hub,HUGGINGFACE_HUB_CACHE=/hf_hub_cache/hub,MODEL_REPO=${MODEL} \ + python3 -c 'import os; from huggingface_hub import snapshot_download; snapshot_download(os.environ["MODEL_REPO"])' +EOF +STAGE_JOB_ID=$(sbatch --wait --parsable "$STAGE_SCRIPT") +STAGE_JOB_ID="${STAGE_JOB_ID%%;*}" +echo "MI355X runtime prerequisites verified with Slurm job ${STAGE_JOB_ID}" + +git clone "$SRT_SLURM_REPOSITORY" "$SRT_REPO_DIR" +git -C "$SRT_REPO_DIR" checkout "$SRT_SLURM_COMMIT" +ACTUAL_SRT_COMMIT=$(git -C "$SRT_REPO_DIR" rev-parse HEAD) +[[ "$ACTUAL_SRT_COMMIT" == "$SRT_SLURM_COMMIT" ]] || { + echo "srt-slurm checkout mismatch: $ACTUAL_SRT_COMMIT" >&2 + exit 1 +} + +mkdir -p "${SRT_REPO_DIR}/$(dirname "$CONFIG_PATH")" +cp "$LOCAL_RECIPE" "${SRT_REPO_DIR}/${CONFIG_PATH}" +cp "$CLUSTER_PROFILE" "${WORK_DIR}/srtslurm.yaml" +python3 - "${WORK_DIR}/srtslurm.yaml" "$GITHUB_WORKSPACE" "$SHARED_RESULTS" "$SHARED_INFERA_RUNTIME" "$SHARED_ATOM_RUNTIME" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +workspace, results, infera_runtime, atom_runtime = sys.argv[2:] +needle = " /it-share/hf-hub-cache: /hf_hub_cache\n" +text = path.read_text() +if text.count(needle) != 1: + raise SystemExit("expected exactly one Hugging Face cache mount") +path.write_text( + text.replace( + needle, + needle + + f" {workspace}: /infmax-workspace\n" + + f" {results}: /results\n" + + f" {infera_runtime}: /infera-source\n" + + f" {atom_runtime}: /atom-source\n", + ) +) +PY + +export PATH="$HOME/.local/bin:$PATH" +cd "$SRT_REPO_DIR" +uv venv --python 3.12 +uv pip install -e . +make setup ARCH=x86_64 +source .venv/bin/activate +export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" +export SRTCTL_RUNTIME_SOURCE_DIR="$SRT_REPO_DIR" + +echo "Submitting ${CONFIG_PATH} with srt-slurm ${SRT_SLURM_COMMIT}" +set +e +SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" \ + --tags "mi355x,inferencex,github-actions,${RUN_KEY}" 2>&1) +SRTCTL_RC=$? +set -e +echo "$SRTCTL_OUTPUT" +if [[ $SRTCTL_RC -ne 0 ]]; then + echo "srtctl apply failed with exit code ${SRTCTL_RC}" >&2 + exit "$SRTCTL_RC" +fi +JOB_ID=$(grep -oE 'Job [0-9]+' <<< "$SRTCTL_OUTPUT" | awk '{print $2}' | tail -1) +[[ -n "$JOB_ID" ]] || { echo "Unable to parse srt-slurm job ID" >&2; exit 1; } +echo "SRT_SLURM_JOB_ID=$JOB_ID" + +while squeue --noheader --jobs "$JOB_ID" | grep -q .; do + squeue --noheader --jobs "$JOB_ID" --format='srt-slurm %i %T %M %R' + sleep 15 +done + +read -r JOB_STATE JOB_EXIT JOB_NODELIST < <( + sacct -X --noheader --parsable2 --jobs "$JOB_ID" \ + --format=State,ExitCode,NodeList | head -1 | tr '|' ' ' +) +echo "srt-slurm job ${JOB_ID}: state=${JOB_STATE} exit=${JOB_EXIT} nodes=${JOB_NODELIST}" + +RESULT_DIR="${SHARED_RESULTS}/${JOB_ID}" +mkdir -p "$GITHUB_WORKSPACE/LOGS" +if [[ -f "$RESULT_DIR/runtime-logs.tar.gz" ]]; then + cp "$RESULT_DIR/runtime-logs.tar.gz" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" +fi +cp -R "$RESULT_DIR/." "$GITHUB_WORKSPACE/LOGS/" + +if [[ "${DISAGG:-false}" == "true" ]]; then + PREFILL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + DECODE_GPUS=$((DECODE_NUM_WORKERS * DECODE_TP)) + TOTAL_GPUS=$((PREFILL_GPUS + DECODE_GPUS)) +else + TOTAL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP * ${PREFILL_PP_SIZE:-1} * ${PREFILL_PCP_SIZE:-1})) +fi + +shopt -s nullglob +RESULTS=("$RESULT_DIR"/fixed-seq/*.json) +shopt -u nullglob +[[ ${#RESULTS[@]} -gt 0 ]] || { echo "No fixed-sequence results found in $RESULT_DIR" >&2; exit 1; } +for result in "${RESULTS[@]}"; do + concurrency=$(basename "$result" | sed -n 's/.*-c\([0-9][0-9]*\)\.json/\1/p') + [[ -n "$concurrency" ]] || { echo "Cannot parse concurrency from $result" >&2; exit 1; } + if [[ "${DISAGG:-false}" == "true" ]]; then + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}_ctx_${PREFILL_GPUS}_gen_${DECODE_GPUS}.json" + else + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}.json" + fi + cp "$result" "$output" + echo "Collected $output" +done + +if [[ "$JOB_STATE" != COMPLETED || "$JOB_EXIT" != 0:0 ]]; then + echo "srt-slurm validation failed: ${JOB_STATE} (${JOB_EXIT})" >&2 + exit 1 +fi + +printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" +echo "MI355X srt-slurm validation completed successfully" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index ab838d11f3..e772e8b19a 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +if [[ -n "${CONFIG_FILE:-}" ]]; then + exec bash "$(dirname "${BASH_SOURCE[0]}")/launch_mi355x-amds-srt.sh" +fi + scancel_sync() { local jobid=$1 local timeout=${2:-600} diff --git a/utils/changelog_gate_tests/test_run_sweep_gating.py b/utils/changelog_gate_tests/test_run_sweep_gating.py index ed30ec5ad9..a2ecad93d4 100644 --- a/utils/changelog_gate_tests/test_run_sweep_gating.py +++ b/utils/changelog_gate_tests/test_run_sweep_gating.py @@ -417,6 +417,14 @@ def test_e2e_workflow_cannot_dispatch_database_ingest() -> None: assert "INFX_FRONTEND_PAT" not in workflow +def test_e2e_workflow_defines_every_emitted_matrix_variable() -> None: + workflow = (REPO_ROOT / ".github/workflows/e2e-tests.yml").read_text() + assigned = set(re.findall(r"^\s+([A-Z_]+)=", workflow, re.MULTILINE)) + emitted = set(re.findall(r'echo "[a-z-]+=\$([A-Z_]+)"', workflow)) + + assert emitted <= assigned + + def test_priority_classifier_runs_for_enabled_actions() -> None: scenario = { **_PR, diff --git a/utils/test_mi300x_srt_slurm_contract.py b/utils/test_mi300x_srt_slurm_contract.py new file mode 100644 index 0000000000..e27bac84a5 --- /dev/null +++ b/utils/test_mi300x_srt_slurm_contract.py @@ -0,0 +1,290 @@ +"""High-signal contract checks for the MI300X srt-slurm bring-up lane.""" + +import os +import subprocess +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[1] +RECIPE_PATH = ( + REPO_ROOT + / "benchmarks/multi_node/srt-slurm-recipes/vllm/qwen3-0.6b/mi300x/agg-fixed-seq.yaml" +) +DISAGG_RECIPE_PATH = RECIPE_PATH.with_name("disagg-1p1d-fixed-seq.yaml") +ATOM_RECIPE_PATH = ( + REPO_ROOT + / "benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi300x/agg-2w-fixed-seq.yaml" +) +ATOM_DISAGG_RECIPE_PATH = ATOM_RECIPE_PATH.with_name("disagg-1p1d-fixed-seq.yaml") +CLUSTER_PATH = ( + REPO_ROOT + / "benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amds.yaml" +) +MASTER_CONFIG_PATH = REPO_ROOT / "configs/amd-master.yaml" +SRT_LAUNCHER_PATH = REPO_ROOT / "runners/launch_mi300x-amds-srt.sh" + + +def test_mi300x_cluster_uses_the_rocm_slurm_contract(): + cluster = yaml.safe_load(CLUSTER_PATH.read_text()) + + assert cluster["accelerator_vendor"] == "amd" + assert cluster["network_interface"] is None + assert cluster["gpu_sbatch_directive"] == "gres" + assert cluster["use_segment_sbatch_directive"] is False + assert cluster["runtime_config_transport"] == "embedded" + assert cluster["default_sbatch_directives"]["exclude"] == ( + "chi-mi300x-049,chi-mi300x-121" + ) + assert cluster["default_mounts"]["/dev/kfd"] == "/dev/kfd" + assert cluster["default_mounts"]["/dev/dri"] == "/dev/dri" + image_path = cluster["containers"]["vllm-rocm-v0.26.0"] + assert image_path.endswith("/vllm-openai-rocm-v0.26.0.sqsh") + router_path = cluster["containers"]["vllm-router-20260809"] + assert router_path.endswith("/vllm-router-nightly-20260809-d2ba586.sqsh") + + for recipe_path in (RECIPE_PATH, DISAGG_RECIPE_PATH): + recipe = yaml.safe_load(recipe_path.read_text()) + assert recipe["model"]["container"] == "vllm-rocm-v0.26.0" + assert recipe["identity"]["container"]["image"] == ( + "vllm/vllm-openai-rocm:v0.26.0" + ) + + +def test_disaggregated_recipe_uses_native_router_and_moriio(): + recipe = yaml.safe_load(DISAGG_RECIPE_PATH.read_text()) + + assert recipe["frontend"] == { + "type": "vllm-router", + "enable_multiple_frontends": False, + "container_image": "vllm-router-20260809", + "args": { + "policy": "consistent_hash", + "prefill-policy": "consistent_hash", + "decode-policy": "consistent_hash", + }, + } + assert recipe["backend"]["connector"] == "moriio" + for role in ("prefill", "decode"): + assert recipe["backend"][f"{role}_environment"][ + "VLLM_ROCM_USE_AITER" + ] == "1" + assert recipe["backend"]["vllm_config"][role][ + "attention-backend" + ] == "ROCM_AITER_FA" + assert "dynamo" not in recipe + serialized = DISAGG_RECIPE_PATH.read_text().lower() + assert "nixl" not in serialized + assert "nats" not in serialized + assert "etcd" not in serialized + + +def test_official_matrix_routes_disagg_through_the_pinned_srt_launcher(): + config = yaml.safe_load(MASTER_CONFIG_PATH.read_text())[ + "qwen3-0.6b-fp16-mi300x-vllm-srt-disagg" + ] + search = config["scenarios"]["fixed-seq-len"][0]["search-space"][0] + launcher = SRT_LAUNCHER_PATH.read_text() + + assert config["runner"] == "mi300x-disagg" + assert config["router"] == { + "name": "vllm-router", + "version": "nightly-20260809-d2ba586", + } + assert config["kv-p2p-transfer"] == "moriio" + assert search["prefill"]["additional-settings"] == [ + "CONFIG_FILE=recipes/vllm/qwen3-0.6b/mi300x/" + "disagg-1p1d-fixed-seq.yaml" + ] + assert "5ecfb13d1ba0960045482f1ef006312d8729d37a" in launcher + assert launcher.count("setup ARCH=x86_64") == 2 + assert "--no-preflight" in launcher + assert 'ENROOT_RUNTIME_PATH="\\${TMPDIR:-/tmp}/enroot-runtime-\\${UID}"' in launcher + assert "for attempt in 1 2 3" in launcher + assert 'VLLM_IMAGE="vllm/vllm-openai-rocm:v0.26.0"' in launcher + assert ( + 'VLLM_ROUTER_IMAGE="vllm/vllm-router:nightly-20260809-d2ba586"' + in launcher + ) + assert 'enroot import -o "\\$tmp" "docker://\\${image}"' in launcher + assert 'exec {lock_fd}>"\\${target}.lock"' in launcher + assert 'flock -w 2400 "\\$lock_fd"' in launcher + assert 'unsquashfs -s "\\$tmp"' in launcher + assert 'mv "\\$tmp" "\\$target"' in launcher + assert 'REMOTE_SRT_RUNTIME="${REMOTE_BASE}/runtime/srt-slurm-${SRT_SLURM_COMMIT}"' in launcher + assert ( + 'ensure_git_checkout "\\$srt_runtime" "${SRT_SLURM_REPOSITORY}" ' + '"${SRT_SLURM_COMMIT}"' + in launcher + ) + assert 'make -C "\\$srt_runtime" --no-print-directory setup ARCH=x86_64' in launcher + assert 'export SRTCTL_RUNTIME_SOURCE_DIR="$REMOTE_SRT_RUNTIME"' in launcher + assert '#SBATCH --nodes=7' in launcher + assert 'ensure_git_checkout()' in launcher + assert 'mv -T "\\$target" "\\$quarantine"' in launcher + assert 'mv -T "\\$temporary" "\\$target"' in launcher + assert "scancel" not in launcher + + +def test_official_matrix_routes_aggregate_through_the_pinned_srt_launcher(): + config = yaml.safe_load(MASTER_CONFIG_PATH.read_text())[ + "qwen3-0.6b-fp16-mi300x-vllm-srt-agg" + ] + search = config["scenarios"]["fixed-seq-len"][0]["search-space"] + launcher = SRT_LAUNCHER_PATH.read_text() + + assert config["runner"] == "mi300x-disagg" + assert config["multinode"] is True + assert config["disagg"] is False + assert search == [ + { + "conc-list": [1], + "prefill": { + "num-worker": 1, + "tp": 1, + "ep": 1, + "dp-attn": False, + "additional-settings": [ + "CONFIG_FILE=recipes/vllm/qwen3-0.6b/mi300x/" + "agg-fixed-seq.yaml" + ], + }, + "decode": { + "num-worker": 0, + "tp": 1, + "ep": 1, + "dp-attn": False, + }, + } + ] + assert ': "${CONFIG_FILE:?CONFIG_FILE must name an srt-slurm recipe}"' in launcher + assert 'JOB_BATCH_HOST=$(scontrol show job "$JOB_ID" -dd' in launcher + assert '--nodelist="$JOB_BATCH_HOST"' in launcher + assert "TOTAL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP" in launcher + + +def test_aggregate_recipe_uses_direct_vllm_without_dynamo_or_a_router(): + recipe = yaml.safe_load(RECIPE_PATH.read_text()) + + assert recipe["resources"] == { + "gpu_type": "mi300x", + "gpus_per_node": 1, + "agg_nodes": 1, + "agg_workers": 1, + "gpus_per_agg": 1, + } + assert recipe["frontend"] == { + "type": "vllm", + "enable_multiple_frontends": False, + } + assert recipe["backend"]["connector"] is None + serialized = RECIPE_PATH.read_text().lower() + assert "dynamo" not in serialized + assert "nixl" not in serialized + assert "moriio" not in serialized + + +def test_fixed_sequence_recipe_uses_inferencex_custom_benchmark(): + recipe = yaml.safe_load(RECIPE_PATH.read_text()) + benchmark = recipe["benchmark"] + command = benchmark["command"] + + assert benchmark["type"] == "custom" + assert "/infmax-workspace/utils/bench_serving/benchmark_serving.py" in command + assert 'result_root="/results/${SLURM_JOB_ID}"' in command + assert "--backend openai-chat" in command + assert "--endpoint /v1/chat/completions" in command + assert "--random-input-len 128" in command + assert "--random-output-len 32" in command + assert "--random-range-ratio 1.0" in command + assert "best-of" not in command + assert "sa-bench" not in command + + +def test_atom_recipes_use_infera_and_keep_worker_metrics_honest(): + cluster = yaml.safe_load(CLUSTER_PATH.read_text()) + aggregate = yaml.safe_load(ATOM_RECIPE_PATH.read_text()) + disaggregate = yaml.safe_load(ATOM_DISAGG_RECIPE_PATH.read_text()) + launcher = SRT_LAUNCHER_PATH.read_text() + + assert cluster["containers"]["infera-atom-v0.1.1"].endswith( + "/infera-atom-v0.1.1.sqsh" + ) + assert aggregate["resources"]["agg_workers"] == 2 + for recipe in (aggregate, disaggregate): + assert recipe["model"]["container"] == "infera-atom-v0.1.1" + assert recipe["identity"]["container"]["image"] == ( + "rocm/infera:atom-v0.1.1" + ) + assert recipe["frontend"]["type"] == "infera" + assert recipe["frontend"]["args"]["router-policy"] == "kv-aware" + assert recipe["backend"]["type"] == "atom" + assert recipe["backend"]["enable_kv_events"] is True + command = recipe["benchmark"]["command"] + assert "--backend openai" in command + assert "--endpoint /v1/completions" in command + assert disaggregate["backend"]["connector"] == "mooncake" + assert disaggregate["backend"]["mooncake_protocol"] == "tcp" + assert 'ATOM_IMAGE="rocm/infera:atom-v0.1.1"' in launcher + assert 'INFERA_COMMIT="8ed8f1728c745d4e91ba9eaa09ed81159aa57e41"' in launcher + assert 'ATOM_COMMIT="2ab42bc2c64d1ad04f698c396da48473e71a6dbb"' in launcher + assert 'REMOTE_INFERA_RUNTIME="${REMOTE_BASE}/runtime/infera-${INFERA_COMMIT}"' in launcher + for recipe in (aggregate, disaggregate): + assert recipe["frontend"]["env"]["PYTHONPATH"] == ( + "/atom-source:/infera-source" + ) + role_environments = [ + value + for key, value in recipe["backend"].items() + if key.endswith("_environment") + ] + assert role_environments + assert all( + env["PYTHONPATH"] == "/atom-source:/infera-source" + for env in role_environments + ) + + +def test_fixed_sequence_commands_keep_all_arguments_attached(tmp_path): + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_python = fake_bin / "python3" + fake_python.write_text( + "#!/bin/bash\n" + 'printf "%s\\n" "$@" >> "$FAKE_ARGS_LOG"\n' + 'printf "%s\\n" --CALL-END-- >> "$FAKE_ARGS_LOG"\n' + ) + fake_python.chmod(0o755) + + recipe_paths = ( + RECIPE_PATH, + DISAGG_RECIPE_PATH, + ATOM_RECIPE_PATH, + ATOM_DISAGG_RECIPE_PATH, + ) + for index, recipe_path in enumerate(recipe_paths): + command = yaml.safe_load(recipe_path.read_text())["benchmark"]["command"] + result_dir = tmp_path / f"recipe-{index}" + command = command.replace( + 'result_root="/results/${SLURM_JOB_ID}"', + f'result_root="{result_dir}"', + ) + args_log = tmp_path / f"recipe-{index}.args" + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "FAKE_ARGS_LOG": str(args_log), + "SRT_FRONTEND_HOST": "127.0.0.1", + "SRT_FRONTEND_PORT": "8000", + "SLURM_JOB_ID": "123", + } + + subprocess.run(["bash", "-n"], input=command, text=True, check=True) + subprocess.run(["bash", "-c", command], env=env, check=True) + + calls = args_log.read_text().split("--CALL-END--\n") + calls = [[arg for arg in call.splitlines() if arg] for call in calls if call] + assert len(calls) == 2 + assert all("--model" in call and "Qwen/Qwen3-0.6B" in call for call in calls) + assert [call[call.index("--num-prompts") + 1] for call in calls] == ["4", "16"] diff --git a/utils/test_mi355x_atom_srt_contract.py b/utils/test_mi355x_atom_srt_contract.py new file mode 100644 index 0000000000..50a9e7f449 --- /dev/null +++ b/utils/test_mi355x_atom_srt_contract.py @@ -0,0 +1,84 @@ +"""High-signal contracts for ATOM/Infera validation on MI355X.""" + +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[1] +RECIPE_ROOT = ( + REPO_ROOT + / "benchmarks/multi_node/srt-slurm-recipes/atom/qwen3-0.6b/mi355x" +) +AGG_RECIPE = RECIPE_ROOT / "agg-2w-fixed-seq.yaml" +DISAGG_RECIPE = RECIPE_ROOT / "disagg-1p1d-fixed-seq.yaml" +MASTER_CONFIG = REPO_ROOT / "configs/amd-master.yaml" +LAUNCHER = REPO_ROOT / "runners/launch_mi355x-amds-srt.sh" + + +def test_aggregate_exercises_two_atom_workers_and_kv_aware_infera(): + recipe = yaml.safe_load(AGG_RECIPE.read_text()) + + assert recipe["resources"] == { + "gpu_type": "mi355x", + "gpus_per_node": 8, + "agg_nodes": 1, + "agg_workers": 2, + "gpus_per_agg": 1, + } + assert recipe["frontend"]["type"] == "infera" + assert recipe["frontend"]["args"]["router-policy"] == "kv-aware" + assert recipe["backend"]["type"] == "atom" + assert recipe["backend"]["enable_kv_events"] is True + assert recipe["backend"]["aggregated_environment"]["PYTHONPATH"] == ( + "/atom-source:/infera-source" + ) + command = recipe["benchmark"]["command"] + assert "--random-prefix-len 96" in command + assert "--endpoint /v1/completions" in command + + +def test_disaggregate_uses_one_prefill_one_decode_and_mooncake(): + recipe = yaml.safe_load(DISAGG_RECIPE.read_text()) + + assert recipe["resources"]["prefill_workers"] == 1 + assert recipe["resources"]["decode_workers"] == 1 + assert recipe["resources"]["prefill_nodes"] == 1 + assert recipe["resources"]["decode_nodes"] == 1 + assert recipe["backend"]["type"] == "atom" + assert recipe["backend"]["connector"] == "mooncake" + assert recipe["backend"]["mooncake_protocol"] == "tcp" + assert recipe["backend"]["enable_kv_events"] is True + assert recipe["backend"]["prefill_environment"]["PYTHONPATH"] == ( + "/atom-source:/infera-source" + ) + + +def test_matrix_rows_route_only_through_the_atom_srt_launcher(): + configs = yaml.safe_load(MASTER_CONFIG.read_text()) + launcher = LAUNCHER.read_text() + + agg = configs["qwen3-0.6b-fp16-mi355x-atom-infera-srt-agg"] + disagg = configs["qwen3-0.6b-fp16-mi355x-atom-infera-srt-disagg"] + assert agg["framework"] == "atom" + assert agg["router"] == {"name": "infera", "version": "0.1.1"} + assert agg["scenarios"]["fixed-seq-len"][0]["search-space"][0][ + "prefill" + ]["additional-settings"] == [ + "CONFIG_FILE=recipes/atom/qwen3-0.6b/mi355x/agg-2w-fixed-seq.yaml" + ] + assert disagg["framework"] == "atom-disagg" + assert disagg["kv-p2p-transfer"] == "mooncake" + assert disagg["scenarios"]["fixed-seq-len"][0]["search-space"][0][ + "prefill" + ]["additional-settings"] == [ + "CONFIG_FILE=recipes/atom/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml" + ] + assert "5ecfb13d1ba0960045482f1ef006312d8729d37a" in launcher + assert "8ed8f1728c745d4e91ba9eaa09ed81159aa57e41" in launcher + assert "2ab42bc2c64d1ad04f698c396da48473e71a6dbb" in launcher + assert "ensure_git_checkout()" in launcher + assert 'exec {lock_fd}>"\\${target}.lock"' in launcher + assert 'flock -w 2400 "\\$lock_fd"' in launcher + assert 'mv -T "\\$temporary" "\\$target"' in launcher + assert "scancel" not in launcher