diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml new file mode 100644 index 000000000..f3cd36eb0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml @@ -0,0 +1,136 @@ +# DeepSeek-V4-Pro 1P1D ATOM P/D serving on MI355X. Native AToMesh +# routes prefill/decode traffic while ATOM transfers KV state through +# Mooncake's default RDMA transport. The custom benchmark preserves the +# legacy 8k/1k search and request-length distribution. + +name: "mi355x-atom-dsv4-disagg-1p1d-tp8-8k1k" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "rocm/atom-dev:nightly_202609031453" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "rocm/atom-dev:nightly_202609031453" + frameworks: + atomesh: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: atomesh + enable_multiple_frontends: false + args: + policy: random + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + +environment: &runtime_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + # Use the official image's ROCm/Mooncake library paths (Python 3.12). + PYTHONUNBUFFERED: "1" + PYTHONDONTWRITEBYTECODE: "1" + SAFETENSORS_FAST_GPU: "1" + VLLM_LOG_LEVEL: "WARNING" + ATOM_LOG_LEVEL: "WARNING" + AITER_LOG_LEVEL: "WARNING" + LOG_LEVEL: "WARNING" + LOGLEVEL: "WARNING" + +backend: + type: atom + connector: mooncake + prefill_environment: &worker_environment + ATOM_MOE_GU_ITLV: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + decode_environment: *worker_environment + atom_config: + prefill: + # Upstream workflow MAX_MODEL_LEN = ISL + OSL + 256. + max-model-len: 9472 + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 256 + block-size: 16 + no-enable_prefix_caching: true + trust-remote-code: true + decode: + max-model-len: 9472 + cudagraph-capture-sizes: "[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]" + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 128 + block-size: 16 + no-enable_prefix_caching: true + trust-remote-code: true + +# The legacy exclusive allocation exposed the full 128-core, unlimited-memory +# node; do not inherit the shared small-model profile's 32-core / 128G limits. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + exclusive: "" + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio "${RANDOM_RANGE_RATIO:-0.8}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "dsv4-atom-mi355x-disagg-1p1d-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + <<: *runtime_environment diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml new file mode 100644 index 000000000..22c7ad775 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml @@ -0,0 +1,140 @@ +# DeepSeek-V4-Pro 2P1D ATOM P/D serving on MI355X. Each logical endpoint +# occupies one TP8 node; the two prefill endpoints and decode endpoint use +# data-parallel attention/TBO as in the legacy production configuration. + +name: "mi355x-atom-dsv4-disagg-2p1d-dpa-tp8-8k1k" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "rocm/atom-dev:nightly_202609031453" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "rocm/atom-dev:nightly_202609031453" + frameworks: + atomesh: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: atomesh + enable_multiple_frontends: false + args: + policy: random + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + +environment: &runtime_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + # Use the official image's ROCm/Mooncake library paths (Python 3.12). + PYTHONUNBUFFERED: "1" + PYTHONDONTWRITEBYTECODE: "1" + SAFETENSORS_FAST_GPU: "1" + VLLM_LOG_LEVEL: "WARNING" + ATOM_LOG_LEVEL: "WARNING" + AITER_LOG_LEVEL: "WARNING" + LOG_LEVEL: "WARNING" + LOGLEVEL: "WARNING" + +backend: + type: atom + connector: mooncake + prefill_environment: &worker_environment + ATOM_MOE_GU_ITLV: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + GPU_MAX_HW_QUEUES: "5" + ATOM_CPU_AFFINITY: "1" + decode_environment: *worker_environment + atom_config: + prefill: + # Upstream workflow MAX_MODEL_LEN = ISL + OSL + 256. + max-model-len: 9472 + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 256 + block-size: 16 + no-enable_prefix_caching: true + enable-dp-attention: true + enable-tbo: true + trust-remote-code: true + decode: + max-model-len: 9472 + cudagraph-capture-sizes: "[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]" + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 2048 + block-size: 16 + no-enable_prefix_caching: true + enable-dp-attention: true + enable-tbo: true + trust-remote-code: true + +# The legacy exclusive allocation exposed the full 128-core, unlimited-memory +# node; do not inherit the shared small-model profile's 32-core / 128G limits. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + exclusive: "" + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio "${RANDOM_RANGE_RATIO:-0.8}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "dsv4-atom-mi355x-disagg-2p1d-dpa-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + <<: *runtime_environment diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d5dbbec0..3937a8f00 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -796,13 +796,13 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 + image: rocm/atom-dev:nightly_202609031453 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: mi355x + runner: cluster:mi355x-amds precision: fp4 framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } + router: { name: atomesh, version: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -820,6 +820,7 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: true additional-settings: - "PREFILL_NODES=2" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml" decode: num-worker: 1 tp: 8 @@ -836,6 +837,7 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: false additional-settings: - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a0f56d8a7..756c5f761 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6885,3 +6885,23 @@ - "Remove the mandatory host-repair hook, RDMA/GPU-drain helper scripts, and separate image/model staging allocation; use provisioned caches and native Pyxis container lifecycle." - "Keep InferenceX-specific benchmark/eval metadata and artifact collection in a portable adapter while preserving the existing serving and workload settings." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg + description: + - "Port the DeepSeek-V4-Pro FP4 MI355X ATOM disaggregated 8k/1k submission from the configuration-specific AMD launcher to srt-slurm ATOM orchestration and native AToMesh routing." + - "Preserve all original search points: 1P1D TP8 at concurrency 4, 8, 16, 32, 64, and 128; 2P1D DPA+TP8 at concurrency 256, 512, 768, 1024, and 2048." + - "Use the unchanged InferenceX benchmark_serving.py through the custom benchmark contract and remove the exact legacy wrapper plus its obsolete deprecated 1k/1k entry." + - "Keep the original image, Mooncake default RDMA transport, random length ratio 0.8, decode graph capture sizes, router options, and TP8 / DP-attention / TBO settings." + - "Use the ATOM image's bundled native AToMesh binary directly, without Infera source overlays or Infera routing, and archive the complete runtime logs from the custom benchmark." + - "Keep rocm/atom-dev:nightly_202606101403 and the original AToMesh revision 087b82d9c1f630e79149ba37e6213257ec9a76f8; use the shared launcher and srt-slurm PR #7 from base PR #2542." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg + description: + - "Use the unmodified official rocm/atom-dev:nightly_202609031453 image and its native AToMesh revision f95ef3ec30a16876a676e0af96d60f8e0605c7c5; no source patches, overlays, or custom image." + - "The June image ignored max_completion_tokens after AToMesh normalized max_tokens, causing the 2P1D GSM8K decode worker to exceed its context capacity. The upstream ATOM API fix is included in this nightly." + - "Use the image-provided ROCm/Mooncake library paths instead of the retired Python 3.10 path; keep all explicit engine options, topologies, concurrency points, benchmark inputs, and eval thresholds unchanged." + - "Validate the shared single-entrypoint launcher and native Pyxis lifecycle without host-repair, RDMA-preflight, or GPU-drain scripts." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627