diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml new file mode 100644 index 000000000..a3f8bb681 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml @@ -0,0 +1,175 @@ +# Experimental; retain only measured improvements over the current Pareto frontier. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 16 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 16 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '16' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml new file mode 100644 index 000000000..a9af6f125 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml @@ -0,0 +1,175 @@ +# Experimental; retain only measured improvements over the current Pareto frontier. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 48 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 48 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 24 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 24 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '24' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml new file mode 100644 index 000000000..5dab4917a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml @@ -0,0 +1,175 @@ +# Fast replay improves the published frontier; canonical sweep qualification is pending. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 32 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 32 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml new file mode 100644 index 000000000..5197d3be5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml @@ -0,0 +1,176 @@ +# Experimental; retain only measured improvements over the current Pareto frontier. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 96 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + # Combined KV/Mamba budget: 860 GB per TP4 prefill worker. + hicache-size: 215 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.88 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 96 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 48 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 48 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml new file mode 100644 index 000000000..ac6a00a1d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml @@ -0,0 +1,176 @@ +# Short replay improves the published frontier; full official qualification remains required. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 128 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + # Combined KV/Mamba budget: 860 GB per TP4 prefill worker. + hicache-size: 215 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.88 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 128 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 64 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 64 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '64' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml new file mode 100644 index 000000000..2492edcae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml @@ -0,0 +1,175 @@ +# Experimental; retain only measured improvements over the current Pareto frontier. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 8 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 8 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '8' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36e171688..06e8c65fc 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8249,3 +8249,129 @@ dsv41flash-fp4-h200-sglang-agentic-dspark: - dram-utilization: 0.80 search-space: - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16] } + +qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:b200-nscale + precision: fp8 + framework: dynamo-sglang + router: + name: dynamo-router + version: 805a77f053d807b0d8def5d27f674a6df0ed839e + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.8 + search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 8 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 16 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 24 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 32 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 48 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 64 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6b5e011b1..258691d3d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8423,3 +8423,15 @@ - "Enable required measured power for the complete H200 Kimi-K3 balanced recipe through the shared recipe-gated launcher." - "为 H200 Kimi-K3 balanced 完整配方启用必需实测功耗。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3053 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Add B200 Qwen3.5-397B-A17B FP8 AgentX disaggregation with colocated TP4 prefill + and TP4 decode on eight GPUs at concurrency 8, 16, 24, 32, 48 and 64. + - Use Dynamo/SGLang nightly 20260918-20518d85 with NEXTN MTP and Mooncake NVLink transfers. + - Enable 104 GB per-rank prefill HiCache at C8-C32; use 215 GB with first-use + write-through and decode static memory 0.88 at C48/C64. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index cc24c382a..0c97d696a 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -545,6 +545,7 @@ run_multinode_srt() { fi USES_DCGM_POWER=0 + USES_AGENTX_POWER=0 _POWER_CONFIG_FILE="${CONFIG_FILE:-}" if [[ "${EVAL_ONLY}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then _POWER_CONFIG_FILE="$EVAL_CONFIG_FILE" @@ -560,13 +561,16 @@ run_multinode_srt() { ' "$_RECIPE_SRC"; then USES_DCGM_POWER=1 fi - if [[ "$USES_DCGM_POWER" == "1" && ( + if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + USES_AGENTX_POWER=1 + elif [[ "$USES_DCGM_POWER" == "1" && ( "${IS_AGENTIC}" == "1" || "$MODEL_PREFIX" != "dsv4" || "$PRECISION" != "fp4" || "$FRAMEWORK" != "dynamo-vllm" ) ]]; then - echo "Error: B200 Nscale dcgm-power is limited to fixed-sequence DSV4 FP4 dynamo-vllm" >&2 + echo "Error: B200 Nscale dcgm-power requires fixed-sequence DSV4 FP4 dynamo-vllm or Qwen3.5 FP8 AgentX dynamo-sglang" >&2 exit 1 fi @@ -678,8 +682,10 @@ run_multinode_srt() { sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" SRTCTL_PREFLIGHT_ARGS=() - # Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node. - if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + # These weights are staged on the Slurm compute nodes, not the login node. + # SRT still checks the resolved model path when the worker starts. + if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]] || + [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi @@ -700,29 +706,11 @@ run_multinode_srt() { LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" - while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 - done - - ( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 - done - ) & - POLL_PID=$! - - echo "Tailing LOG_FILE: $LOG_FILE" - - # -F follows by name and polls; inotify does not work on NFS. - tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null - - wait $POLL_PID + local srt_job_rc=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || srt_job_rc=$? + if [[ "$srt_job_rc" -eq 0 ]]; then + verify_slurm_job_status "$JOB_ID" || srt_job_rc=$? + fi set -x @@ -736,6 +724,15 @@ run_multinode_srt() { echo "Found logs directory: $LOGS_DIR" + if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY}" != "true" ]]; then + check_env_vars CONC_LIST + local -a power_concurrencies + read -r -a power_concurrencies <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" \ + "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" \ + "$SRT_SLURM_COMMIT" "${power_concurrencies[@]}" || srt_job_rc=$? + fi + if [[ "$USES_DCGM_POWER" == "1" ]]; then mkdir -p "$LOGS_DIR/power" cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" @@ -776,6 +773,8 @@ run_multinode_srt() { sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + # Failed runs still provide the diagnostics and eval outputs above. + return "$srt_job_rc" } # --------------------------------------------------------------------------- diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index edb6d2a66..67a6eac78 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -157,6 +157,32 @@ stream_slurm_job_log() { wait "$poll_pid" } +verify_slurm_job_status() { + local job_id="$1" + # Disappearing from squeue means terminal, not successful. Accounting can + # lag briefly; inspect only the allocation, never successful service steps. + local attempt accounting state exit_code + for attempt in {1..10}; do + accounting=$(sacct -X -n -P -j "$job_id" --format=State,ExitCode 2>/dev/null) || accounting="" + IFS='|' read -r state exit_code <<< "$accounting" + case "$state" in + COMPLETED) + if [[ "$exit_code" == "0:0" ]]; then + return 0 + fi + ;; + ""|PENDING|RUNNING|CONFIGURING|COMPLETING) + sleep 1 + continue + ;; + esac + echo "ERROR: Slurm job $job_id ended with state=$state exit_code=$exit_code" >&2 + return 1 + done + echo "ERROR: could not verify terminal Slurm status for job $job_id" >&2 + return 1 +} + copy_to_workspace() { local source_file="$1" local destination_file="$2"