From 893d0f0062b808a0746ab471c594888bf3d4f8d1 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 17 Sep 2026 14:12:44 -0700 Subject: [PATCH 1/2] feat: add GB300 MiniMax-M3 AgentX recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 GB300 MiniMax-M3 Dynamo-vLLM AgentX 聚合与分离式部署配置，固定运行依赖，并更新 srt-slurm 以支持各工作进程独立配置 Mooncake 存储。 --- .../multi_node/srt-slurm-recipes/RECIPES.md | 2 +- .../srt-slurm-recipes/RECIPES_zh.md | 2 +- .../minimaxm3-mooncake/setup-disagg.sh | 5 + .../gb300-fp4/agentx/agg-tp2-c20-mtp.yaml | 136 +++++++++++ .../gb300-fp4/agentx/agg-tp2-c24-mtp.yaml | 136 +++++++++++ .../vllm/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml | 138 +++++++++++ .../agentx/disagg-1p4d-dep4-tep4-c64-mtp.yaml | 227 ++++++++++++++++++ .../agentx/disagg-1p5d-dep4-tp2-c100-mtp.yaml | 226 +++++++++++++++++ .../agentx/disagg-1p5d-dep4-tp2-c110-mtp.yaml | 226 +++++++++++++++++ .../agentx/disagg-1p5d-dep4-tp2-c90-mtp.yaml | 226 +++++++++++++++++ configs/nvidia-master.yaml | 160 ++++++++++++ perf-changelog.yaml | 9 + utils/srt-slurm | 2 +- 13 files changed, 1492 insertions(+), 3 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-mooncake/setup-disagg.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c20-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c24-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep4-c64-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c100-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c110-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c90-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index a731508e05..40f93ee455 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -4,7 +4,7 @@ InferenceX owns the recipes in this directory. Every NVIDIA srt-slurm launcher uses `setup_srt_slurm()` in [`runners/slurm_utils.sh`](../../../runners/slurm_utils.sh), makes a job-local Git clone of the pinned submodule, and copies this entire tree into `recipes/`. The shared helper records the actual revision in `srt-slurm-sha.txt`; power lanes copy that revision into `power-producer-sha.txt` for result validation. -The shared version is the Git submodule pointer at [`utils/srt-slurm`](../../../utils/srt-slurm), currently [v2.2.1](https://github.com/NVIDIA/srt-slurm/releases/tag/v2.2.1) (`984180e5b8755aef85e9995048b5a16cb5336bce`). Update that submodule pointer when upgrading, then run the recipe and integration checks. Do not add model-specific checkout branches to launchers. +The shared version is the Git submodule pointer at [`utils/srt-slurm`](../../../utils/srt-slurm), currently [v2.11.0](https://github.com/NVIDIA/srt-slurm/releases/tag/v2.11.0) (`8939ac513c2646a301dfef46971ff8bb48a9d6ff`). Update that submodule pointer when upgrading, then run the recipe and integration checks. Do not add model-specific checkout branches to launchers. InferenceX requires srt-slurm 2.0 or newer and `schema: 2` recipes. Legacy recipe layouts are unsupported; migrate them before adding them to this tree. diff --git a/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md b/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md index ef3c0ed034..864387aa06 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md +++ b/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md @@ -4,7 +4,7 @@ InferenceX 负责维护本目录中的配置。所有 NVIDIA srt-slurm 启动器均调用 [`runners/slurm_utils.sh`](../../../runners/slurm_utils.sh) 中的 `setup_srt_slurm()`，为作业创建固定版本子模块的本地 Git 克隆，并将整个目录复制到 `recipes/`。共享函数将实际提交记录到 `srt-slurm-sha.txt`；功耗测试路径还会将其复制到 `power-producer-sha.txt`，供结果校验使用。 -统一版本由 [`utils/srt-slurm`](../../../utils/srt-slurm) 的 Git 子模块指针指定，目前为 [v2.2.1](https://github.com/NVIDIA/srt-slurm/releases/tag/v2.2.1)（`984180e5b8755aef85e9995048b5a16cb5336bce`）。升级时更新该子模块指针，然后运行配置和集成检查。不要在启动器中新增按模型选择检出版本的分支。 +统一版本由 [`utils/srt-slurm`](../../../utils/srt-slurm) 的 Git 子模块指针指定，目前为 [v2.11.0](https://github.com/NVIDIA/srt-slurm/releases/tag/v2.11.0)（`8939ac513c2646a301dfef46971ff8bb48a9d6ff`）。升级时更新该子模块指针，然后运行配置和集成检查。不要在启动器中新增按模型选择检出版本的分支。 InferenceX 要求 srt-slurm 2.0 或更新版本，且配置必须声明 `schema: 2`。不支持旧版配置结构；加入本目录前必须先完成迁移。 diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-mooncake/setup-disagg.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-mooncake/setup-disagg.sh new file mode 100755 index 0000000000..711bdf1567 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-mooncake/setup-disagg.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -eo pipefail +# Download, verify, and install the published wheel inside each backend container. +uv pip install --system --no-deps --reinstall --require-hashes \ + 'https://test-files.pythonhosted.org/packages/f3/00/fb2847f5564864be132d0f70384dde5688689d9f91b2c81d2609632f36e4/mooncake_transfer_engine_cuda13-0.3.14.dev20260910-cp312-cp312-manylinux_2_28_aarch64.whl#sha256=c55fcc42cf189fcdecdcdf9d776ef84b348a9970a9bd3aad2fad64cb2d9e6f82' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c20-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c20-mtp.yaml new file mode 100644 index 0000000000..7b71903d12 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c20-mtp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: minimax-m3-gb300-agg-tp2-c20 +dynamo: + install: true + source: + wheel: 1.5.0.dev20260914 + request_plane: tcp +services: +- name: etcd + type: etcd + placement: + node: infra +- name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 14400 + env: + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + DYN_TCP_CONNECT_TIMEOUT: '120' +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + precision: fp4 +identity: + model: + repo: nvidia/MiniMax-M3-NVFP4 + container: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + frameworks: + dynamo: 1.5.0.dev20260914 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: vllm + connector: null + dp_launch_mode: per_gpu +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + no-enable-flashinfer-autotune: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":429496729600,"cpu_bytes_to_use_per_rank":214748364800,"lazy_offload":true}}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + ENABLE_AGENTX_POWER: '0' + AIPERF_EXTRA_INPUTS: thinking:true + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + AIPERF_WARMUP_REQUESTS_PER_LANE: '10' + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + AIPERF_DRAIN_TIMEOUT_SECONDS: '1800' + AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + TP: '2' + EP_SIZE: '1' + IS_MULTINODE: 'false' + DISAGG: 'false' + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +observability: + tachometer: + enabled: false +telemetry: + enabled: false +health_check: + max_attempts: 2160 + interval_seconds: 10 +sbatch_directives: + cpus-per-task: '144' + mem: '0' +srun_options: + container-remap-root: '' + cpu-bind: cores +slurm: + time_limit: '8:00:00' +environment: + PYTHONHASHSEED: '0' + WITH_NVIDIA_PEERMEM: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c24-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c24-mtp.yaml new file mode 100644 index 0000000000..a7eca6e3cf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c24-mtp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: minimax-m3-gb300-agg-tp2-c24 +dynamo: + install: true + source: + wheel: 1.5.0.dev20260914 + request_plane: tcp +services: +- name: etcd + type: etcd + placement: + node: infra +- name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 14400 + env: + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + DYN_TCP_CONNECT_TIMEOUT: '120' +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + precision: fp4 +identity: + model: + repo: nvidia/MiniMax-M3-NVFP4 + container: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + frameworks: + dynamo: 1.5.0.dev20260914 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: vllm + connector: null + dp_launch_mode: per_gpu +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + no-enable-flashinfer-autotune: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":429496729600,"cpu_bytes_to_use_per_rank":214748364800,"lazy_offload":true}}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + ENABLE_AGENTX_POWER: '0' + AIPERF_EXTRA_INPUTS: thinking:true + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + AIPERF_WARMUP_REQUESTS_PER_LANE: '10' + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + AIPERF_DRAIN_TIMEOUT_SECONDS: '1800' + AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + TP: '2' + EP_SIZE: '1' + IS_MULTINODE: 'false' + DISAGG: 'false' + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +observability: + tachometer: + enabled: false +telemetry: + enabled: false +health_check: + max_attempts: 2160 + interval_seconds: 10 +sbatch_directives: + cpus-per-task: '144' + mem: '0' +srun_options: + container-remap-root: '' + cpu-bind: cores +slurm: + time_limit: '8:00:00' +environment: + PYTHONHASHSEED: '0' + WITH_NVIDIA_PEERMEM: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml new file mode 100644 index 0000000000..5537cf8414 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: minimax-m3-gb300-agg-tp8-c1 +dynamo: + install: true + source: + wheel: 1.5.0.dev20260914 + request_plane: tcp +services: +- name: etcd + type: etcd + placement: + node: infra +- name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: least-loaded + router-session-affinity-ttl-secs: 14400 + env: + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + DYN_TCP_CONNECT_TIMEOUT: '120' +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + precision: fp4 +identity: + model: + repo: nvidia/MiniMax-M3-NVFP4 + container: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + frameworks: + dynamo: 1.5.0.dev20260914 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: vllm + connector: null + dp_launch_mode: per_gpu +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: mnnvl + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_LOG_STATS_INTERVAL: '1' + UCX_MEMTYPE_CACHE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + UCX_TLS: rc,cuda_copy + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + no-enable-flashinfer-autotune: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + ENABLE_AGENTX_POWER: '0' + AIPERF_EXTRA_INPUTS: thinking:true + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + AIPERF_WARMUP_REQUESTS_PER_LANE: '10' + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + AIPERF_DRAIN_TIMEOUT_SECONDS: '1800' + AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + TP: '8' + EP_SIZE: '1' + IS_MULTINODE: 'false' + DISAGG: 'false' + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +observability: + tachometer: + enabled: false +telemetry: + enabled: false +health_check: + max_attempts: 2160 + interval_seconds: 10 +sbatch_directives: + cpus-per-task: '144' + mem: '0' +srun_options: + container-remap-root: '' + cpu-bind: cores +slurm: + time_limit: '8:00:00' +environment: + PYTHONHASHSEED: '0' + WITH_NVIDIA_PEERMEM: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep4-c64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep4-c64-mtp.yaml new file mode 100644 index 0000000000..fc1f52909e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep4-c64-mtp.yaml @@ -0,0 +1,227 @@ +schema: 2 +name: minimax-m3-gb300-1p-dep4-4d-tep4-c64 +setup_script: minimaxm3-mooncake/setup-disagg.sh +host_setup: + commands: + - numactl --hardware + - test -n "${SLURM_JOB_ID:-}" && test ! -L "/tmp/srt-fdx-${SLURM_JOB_ID}" && install -d -m 700 "/tmp/srt-fdx-${SLURM_JOB_ID}" + nodes: all +container_mounts: + /tmp/srt-fdx-{job_id}: /srt-ipc +dynamo: + install: true + source: + wheel: 1.5.0.dev20260914 + request_plane: tcp +services: +- name: etcd + type: etcd + placement: + node: infra +- name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + options: + device_names_by_gpu: + - mlx5_0 + - mlx5_1 + - mlx5_2 + - mlx5_3 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 150GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + mode: embedded + enable_offload: true +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + args: + router-mode: least-loaded + env: + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + DYN_TCP_CONNECT_TIMEOUT: '120' +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + precision: fp4 +identity: + model: + repo: nvidia/MiniMax-M3-NVFP4 + container: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + frameworks: + dynamo: 1.5.0.dev20260914 +resources: + gpu_type: gb300 + gpus_per_node: 4 + spread_workers: false +engine: + type: vllm + connector: null + dp_launch_mode: per_gpu + set_cuda_visible_devices: true +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TMPDIR: /srt-ipc + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '20' + UCX_MEMTYPE_CACHE: n + UCX_CUDA_IPC_ENABLE_MNNVL: y + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + UCX_TLS: ^cuda_ipc,gdr_copy + UCX_RNDV_PIPELINE_ERROR_HANDLING: y + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '1' + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + numa-bind: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: flashinfer_nvlink_one_sided + kv-cache-dtype: fp8 + block-size: 512 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false}}]}}' + enable-cumem-allocator: true + decode: + nodes: 4 + workers: 4 + gpus: 4 + env: + TMPDIR: /srt-ipc + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '20' + UCX_MEMTYPE_CACHE: n + UCX_CUDA_IPC_ENABLE_MNNVL: y + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + UCX_TLS: ^cuda_ipc,gdr_copy + UCX_RNDV_PIPELINE_ERROR_HANDLING: y + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '1' + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + numa-bind: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: flashinfer_nvlink_one_sided + max-num-seqs: 2 + kv-cache-dtype: fp8 + block-size: 512 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + ENABLE_AGENTX_POWER: '0' + AIPERF_EXTRA_INPUTS: thinking:true + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + AIPERF_WARMUP_REQUESTS_PER_LANE: '10' + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + AIPERF_DRAIN_TIMEOUT_SECONDS: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + DISAGG: 'true' + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +observability: + tachometer: + enabled: false +telemetry: + enabled: false +health_check: + max_attempts: 2160 + interval_seconds: 10 +sbatch_directives: + cpus-per-task: '144' + mem: '0' +srun_options: + container-remap-root: '' + cpu-bind: cores +slurm: + time_limit: '8:00:00' +environment: + PYTHONHASHSEED: '0' + WITH_NVIDIA_PEERMEM: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c100-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c100-mtp.yaml new file mode 100644 index 0000000000..1ccafe18eb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c100-mtp.yaml @@ -0,0 +1,226 @@ +schema: 2 +name: minimax-m3-gb300-1p-dep4-5d-tp2-c100 +setup_script: minimaxm3-mooncake/setup-disagg.sh +host_setup: + commands: + - numactl --hardware + - test -n "${SLURM_JOB_ID:-}" && test ! -L "/tmp/srt-fdx-${SLURM_JOB_ID}" && install -d -m 700 "/tmp/srt-fdx-${SLURM_JOB_ID}" + nodes: all +container_mounts: + /tmp/srt-fdx-{job_id}: /srt-ipc +dynamo: + install: true + source: + wheel: 1.5.0.dev20260914 + request_plane: tcp +services: +- name: etcd + type: etcd + placement: + node: infra +- name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + options: + device_names_by_gpu: + - mlx5_0 + - mlx5_1 + - mlx5_2 + - mlx5_3 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 170GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + mode: embedded + enable_offload: true +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + args: + router-mode: least-loaded + env: + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + DYN_TCP_CONNECT_TIMEOUT: '120' +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + precision: fp4 +identity: + model: + repo: nvidia/MiniMax-M3-NVFP4 + container: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + frameworks: + dynamo: 1.5.0.dev20260914 +resources: + gpu_type: gb300 + gpus_per_node: 4 + spread_workers: false +engine: + type: vllm + connector: null + dp_launch_mode: per_gpu + set_cuda_visible_devices: true +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '20' + UCX_MEMTYPE_CACHE: n + UCX_CUDA_IPC_ENABLE_MNNVL: y + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + UCX_TLS: ^cuda_ipc,gdr_copy + UCX_RNDV_PIPELINE_ERROR_HANDLING: y + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '1' + TMPDIR: /srt-ipc + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + numa-bind: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: flashinfer_nvlink_one_sided + kv-cache-dtype: fp8 + block-size: 512 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false}}]}}' + enable-cumem-allocator: true + decode: + nodes: 3 + workers: 5 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '20' + UCX_MEMTYPE_CACHE: n + UCX_CUDA_IPC_ENABLE_MNNVL: y + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + UCX_TLS: ^cuda_ipc,gdr_copy + UCX_RNDV_PIPELINE_ERROR_HANDLING: y + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '1' + TMPDIR: /srt-ipc + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + max-num-seqs: 4 + numa-bind: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 2 + enable-expert-parallel: false + kv-cache-dtype: fp8 + block-size: 512 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + ENABLE_AGENTX_POWER: '0' + AIPERF_EXTRA_INPUTS: thinking:true + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + AIPERF_WARMUP_REQUESTS_PER_LANE: '10' + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + AIPERF_DRAIN_TIMEOUT_SECONDS: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + DISAGG: 'true' + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +observability: + tachometer: + enabled: false +telemetry: + enabled: false +health_check: + max_attempts: 2160 + interval_seconds: 10 +sbatch_directives: + cpus-per-task: '144' + mem: '0' +srun_options: + container-remap-root: '' + cpu-bind: cores +slurm: + time_limit: '8:00:00' +environment: + PYTHONHASHSEED: '0' + WITH_NVIDIA_PEERMEM: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c110-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c110-mtp.yaml new file mode 100644 index 0000000000..1009e4ba31 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c110-mtp.yaml @@ -0,0 +1,226 @@ +schema: 2 +name: minimax-m3-gb300-1p-dep4-5d-tp2-c110 +setup_script: minimaxm3-mooncake/setup-disagg.sh +host_setup: + commands: + - numactl --hardware + - test -n "${SLURM_JOB_ID:-}" && test ! -L "/tmp/srt-fdx-${SLURM_JOB_ID}" && install -d -m 700 "/tmp/srt-fdx-${SLURM_JOB_ID}" + nodes: all +container_mounts: + /tmp/srt-fdx-{job_id}: /srt-ipc +dynamo: + install: true + source: + wheel: 1.5.0.dev20260914 + request_plane: tcp +services: +- name: etcd + type: etcd + placement: + node: infra +- name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + options: + device_names_by_gpu: + - mlx5_0 + - mlx5_1 + - mlx5_2 + - mlx5_3 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 170GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + mode: embedded + enable_offload: true +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + args: + router-mode: least-loaded + env: + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + DYN_TCP_CONNECT_TIMEOUT: '120' +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + precision: fp4 +identity: + model: + repo: nvidia/MiniMax-M3-NVFP4 + container: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + frameworks: + dynamo: 1.5.0.dev20260914 +resources: + gpu_type: gb300 + gpus_per_node: 4 + spread_workers: false +engine: + type: vllm + connector: null + dp_launch_mode: per_gpu + set_cuda_visible_devices: true +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '20' + UCX_MEMTYPE_CACHE: n + UCX_CUDA_IPC_ENABLE_MNNVL: y + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + UCX_TLS: ^cuda_ipc,gdr_copy + UCX_RNDV_PIPELINE_ERROR_HANDLING: y + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '1' + TMPDIR: /srt-ipc + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + numa-bind: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: flashinfer_nvlink_one_sided + kv-cache-dtype: fp8 + block-size: 512 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false}}]}}' + enable-cumem-allocator: true + decode: + nodes: 3 + workers: 5 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '20' + UCX_MEMTYPE_CACHE: n + UCX_CUDA_IPC_ENABLE_MNNVL: y + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + UCX_TLS: ^cuda_ipc,gdr_copy + UCX_RNDV_PIPELINE_ERROR_HANDLING: y + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '1' + TMPDIR: /srt-ipc + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + max-num-seqs: 8 + numa-bind: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 2 + enable-expert-parallel: false + kv-cache-dtype: fp8 + block-size: 512 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + ENABLE_AGENTX_POWER: '0' + AIPERF_EXTRA_INPUTS: thinking:true + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + AIPERF_WARMUP_REQUESTS_PER_LANE: '10' + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + AIPERF_DRAIN_TIMEOUT_SECONDS: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + DISAGG: 'true' + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +observability: + tachometer: + enabled: false +telemetry: + enabled: false +health_check: + max_attempts: 2160 + interval_seconds: 10 +sbatch_directives: + cpus-per-task: '144' + mem: '0' +srun_options: + container-remap-root: '' + cpu-bind: cores +slurm: + time_limit: '8:00:00' +environment: + PYTHONHASHSEED: '0' + WITH_NVIDIA_PEERMEM: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c90-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c90-mtp.yaml new file mode 100644 index 0000000000..d2bef3637b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c90-mtp.yaml @@ -0,0 +1,226 @@ +schema: 2 +name: minimax-m3-gb300-1p-dep4-5d-tp2-c90 +setup_script: minimaxm3-mooncake/setup-disagg.sh +host_setup: + commands: + - numactl --hardware + - test -n "${SLURM_JOB_ID:-}" && test ! -L "/tmp/srt-fdx-${SLURM_JOB_ID}" && install -d -m 700 "/tmp/srt-fdx-${SLURM_JOB_ID}" + nodes: all +container_mounts: + /tmp/srt-fdx-{job_id}: /srt-ipc +dynamo: + install: true + source: + wheel: 1.5.0.dev20260914 + request_plane: tcp +services: +- name: etcd + type: etcd + placement: + node: infra +- name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + options: + device_names_by_gpu: + - mlx5_0 + - mlx5_1 + - mlx5_2 + - mlx5_3 + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 170GB + local_buffer_size: 4GB + protocol: rdma + device_name: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + mode: embedded + enable_offload: true +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + args: + router-mode: least-loaded + env: + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE_BYTES: '8589934592' + DYN_TCP_CONNECT_TIMEOUT: '120' +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + precision: fp4 +identity: + model: + repo: nvidia/MiniMax-M3-NVFP4 + container: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + frameworks: + dynamo: 1.5.0.dev20260914 +resources: + gpu_type: gb300 + gpus_per_node: 4 + spread_workers: false +engine: + type: vllm + connector: null + dp_launch_mode: per_gpu + set_cuda_visible_devices: true +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '20' + UCX_MEMTYPE_CACHE: n + UCX_CUDA_IPC_ENABLE_MNNVL: y + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + UCX_TLS: ^cuda_ipc,gdr_copy + UCX_RNDV_PIPELINE_ERROR_HANDLING: y + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '1' + TMPDIR: /srt-ipc + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + numa-bind: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: flashinfer_nvlink_one_sided + kv-cache-dtype: fp8 + block-size: 512 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false}}]}}' + enable-cumem-allocator: true + decode: + nodes: 3 + workers: 5 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + DYN_TCP_CONNECT_TIMEOUT: '120' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '20' + UCX_MEMTYPE_CACHE: n + UCX_CUDA_IPC_ENABLE_MNNVL: y + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + UCX_TLS: ^cuda_ipc,gdr_copy + UCX_RNDV_PIPELINE_ERROR_HANDLING: y + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 + NCCL_IB_HCA: mlx5_0,mlx5_1,mlx5_2,mlx5_3 + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '1' + TMPDIR: /srt-ipc + HF_HUB_CACHE: /hf_hub_cache + HUGGINGFACE_HUB_CACHE: /hf_hub_cache + args: + max-num-seqs: 2 + numa-bind: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + tensor-parallel-size: 2 + enable-expert-parallel: false + kv-cache-dtype: fp8 + block-size: 512 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + dyn-default-thinking-mode: enabled + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +benchmark: + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + ENABLE_AGENTX_POWER: '0' + AIPERF_EXTRA_INPUTS: thinking:true + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + AIPERF_WARMUP_REQUESTS_PER_LANE: '10' + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: '300' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + AIPERF_DRAIN_TIMEOUT_SECONDS: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + DISAGG: 'true' + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +observability: + tachometer: + enabled: false +telemetry: + enabled: false +health_check: + max_attempts: 2160 + interval_seconds: 10 +sbatch_directives: + cpus-per-task: '144' + mem: '0' +srun_options: + container-remap-root: '' + cpu-bind: cores +slurm: + time_limit: '8:00:00' +environment: + PYTHONHASHSEED: '0' + WITH_NVIDIA_PEERMEM: '0' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 83b1fb860d..19db166db5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8167,3 +8167,163 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: # 8x141 GB holds the 511 GB checkpoint minus the UVA-offloaded Engram # tables, so the KV cache stays GPU-resident across the full range. - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + +minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-agg: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.5.0.dev20260914 + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: + - 1 + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml + - dram-utilization: 0.9523809523809523 + search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 20 + num-nodes: 1 + kv-offload-backend: + name: vllm-simple + worker: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c20-mtp.yaml + - dram-utilization: 0.9523809523809523 + search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 24 + num-nodes: 1 + kv-offload-backend: + name: vllm-simple + worker: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/agg-tp2-c24-mtp.yaml +minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg-mooncake: + image: vllm/vllm-openai:nightly-0bfc7a15d095fe83ecc82b50561a93c177fece2d@sha256:ac75d722e093a1d6b1515fdb0d9cff692225e47e8bcbaf614a1a220d302fa116 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.5.0.dev20260914 + multinode: true + disagg: true + kv-p2p-transfer: nixl + scenarios: + agentic-coding: + - dram-utilization: 0.7142857142857143 + search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 64 + kv-offload-backend: + name: mooncake + version: 0.3.14.dev20260910 + prefill: + num-worker: 4 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep4-c64-mtp.yaml + decode: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: false + - dram-utilization: 0.8095238095238095 + search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 90 + kv-offload-backend: + name: mooncake + version: 0.3.14.dev20260910 + prefill: + num-worker: 4 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c90-mtp.yaml + decode: + num-worker: 5 + tp: 2 + ep: 1 + dp-attn: false + - dram-utilization: 0.8095238095238095 + search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 100 + kv-offload-backend: + name: mooncake + version: 0.3.14.dev20260910 + prefill: + num-worker: 4 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c100-mtp.yaml + decode: + num-worker: 5 + tp: 2 + ep: 1 + dp-attn: false + - dram-utilization: 0.8095238095238095 + search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 110 + kv-offload-backend: + name: mooncake + version: 0.3.14.dev20260910 + prefill: + num-worker: 4 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p5d-dep4-tp2-c110-mtp.yaml + decode: + num-worker: 5 + tp: 2 + ep: 1 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9d51cba8b5..aa54cb1cf1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8032,3 +8032,12 @@ - "配置键由 dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp 改名为 dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark，以反映 DSpark 算法与 UMBP-linker 分支；该配方此前的条目仍记录在旧键名下。改名不伴随任何其他字段变化。" - "Collect completed logs from every Slurm node so both prefill and decode artifacts are uploaded; forward workflow-owned AgentX fast-mode and power inputs, and supply the existing router port to evaluation." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3188 + +- config-keys: + - minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg-mooncake + description: + - "Add GB300 MiniMax-M3 Dynamo-vLLM AgentX aggregate and disaggregated recipes with EAGLE3-GQA" + - "Use srt-slurm v2.11.0 for process-local Mooncake store configuration" + - "新增采用 EAGLE3-GQA 的 GB300 MiniMax-M3 Dynamo-vLLM AgentX 聚合与分离式部署配置；采用 srt-slurm v2.11.0 为各工作进程配置 Mooncake 存储" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/utils/srt-slurm b/utils/srt-slurm index 984180e5b8..8939ac513c 160000 --- a/utils/srt-slurm +++ b/utils/srt-slurm @@ -1 +1 @@ -Subproject commit 984180e5b8755aef85e9995048b5a16cb5336bce +Subproject commit 8939ac513c2646a301dfef46971ff8bb48a9d6ff From 53ba150d7098835b594622e2a815e8586a5d467d Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 17 Sep 2026 14:14:57 -0700 Subject: [PATCH 2/2] chore: link GB300 MiniMax-M3 recipes changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 GB300 MiniMax-M3 配置的变更日志关联到 PR #3232。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index aa54cb1cf1..85b9fbe11a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8040,4 +8040,4 @@ - "Add GB300 MiniMax-M3 Dynamo-vLLM AgentX aggregate and disaggregated recipes with EAGLE3-GQA" - "Use srt-slurm v2.11.0 for process-local Mooncake store configuration" - "新增采用 EAGLE3-GQA 的 GB300 MiniMax-M3 Dynamo-vLLM AgentX 聚合与分离式部署配置；采用 srt-slurm v2.11.0 为各工作进程配置 Mooncake 存储" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3232