From 10f1a60c1c0c2860f08679b30c4618c081b0f97c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 12:03:26 -0500 Subject: [PATCH 01/28] feat: add B200 Qwen3.5 FP8 disaggregated AgentX draft --- .../agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml | 137 ++++++++++++++++++ configs/nvidia-master.yaml | 37 +++++ perf-changelog.yaml | 9 ++ 3 files changed, 183 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml new file mode 100644 index 0000000000..1b50d13941 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml @@ -0,0 +1,137 @@ +# SGLang recipe: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# Adapt the existing FP8 TP4 PD recipe and B200 AgentX serving flags. +# GPU qualification pending; shared apply_srt_recipe selects golden AL for throughput only. +schema: 2 +name: qwen35-b200-fp8-agentx-1p1d-tp4-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 16 + stream-interval: 50 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 1 + - 4 + - 8 + - 16 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f28e74de34..0b4925b18b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8150,3 +8150,40 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: # 8x141 GB holds the 511 GB checkpoint minus the UVA-offloaded Engram # tables, so the KV cache stays GPU-resident across the full range. - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + +qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:b200-nscale + precision: fp8 + framework: dynamo-sglang + router: + name: dynamo-router + version: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.8 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: + - 1 + - 4 + - 8 + - 16 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a112ba56db..bba838ab3e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8082,3 +8082,12 @@ - "Capture only full decode CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=0 with cudagraph_mode FULL_DECODE_ONLY, as the MiniMax-M3 gfx942 arm does) and restore --moe-backend aiter: on gfx942 every worker segfaulted during piecewise graph capture with both the Triton W4A16 MoE kernel (run 35305045778) and the auto-selected unfused Triton kernel (run 35306398350), so the capture mode rather than the MoE kernel is the failing piece; prefill runs eagerly" - "仅捕获完整的 decode CUDA graph（VLLM_USE_BREAKABLE_CUDAGRAPH=0 并设置 cudagraph_mode FULL_DECODE_ONLY，与 MiniMax-M3 gfx942 配方一致）并恢复 --moe-backend aiter：在 gfx942 上，无论使用 Triton W4A16 MoE 内核（运行 35305045778）还是自动选择的未融合 Triton 内核（运行 35306398350），所有 worker 都在 piecewise graph 捕获期间段错误，说明问题在于捕获模式而非 MoE 内核；prefill 以 eager 方式运行" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3247 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - 'Add Qwen3.5 397B FP8 B200 Dynamo + SGLang disaggregated AgentX: one TP4 prefill + worker and one TP4 decode worker with NEXTN MTP, concurrency 1/4/8/16.' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From aca85d866a27fef926795a7925be77a6151b6147 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 12:04:04 -0500 Subject: [PATCH 02/28] docs: link B200 AgentX performance changelog to PR --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bba838ab3e..d3be26293d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8090,4 +8090,4 @@ description: - 'Add Qwen3.5 397B FP8 B200 Dynamo + SGLang disaggregated AgentX: one TP4 prefill worker and one TP4 decode worker with NEXTN MTP, concurrency 1/4/8/16.' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 33371d2c731df37c427c400941c4f4f3f988b096 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 12:55:34 -0500 Subject: [PATCH 03/28] feat: align B200 disaggregated AgentX with aggregate baseline --- ...agg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml | 153 ++++++++++++++++++ ...g-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml} | 32 +++- ...gg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml | 153 ++++++++++++++++++ ...agg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml | 153 ++++++++++++++++++ ...agg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml | 153 ++++++++++++++++++ configs/nvidia-master.yaml | 63 +++++++- perf-changelog.yaml | 10 ++ 7 files changed, 708 insertions(+), 9 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/{disagg-1p1d-p-tp4-d-tp4-mtp.yaml => disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml} (76%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml new file mode 100644 index 0000000000..c042bd170f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml @@ -0,0 +1,153 @@ +# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. +# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. +# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. +# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. +schema: 2 +name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c1-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 2 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 2 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 1 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 1 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml similarity index 76% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml index 1b50d13941..ae4ea825ae 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml @@ -1,8 +1,10 @@ -# SGLang recipe: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# Adapt the existing FP8 TP4 PD recipe and B200 AgentX serving flags. -# GPU qualification pending; shared apply_srt_recipe selects golden AL for throughput only. +# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. +# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. +# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. +# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. schema: 2 -name: qwen35-b200-fp8-agentx-1p1d-tp4-mtp +name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c16-mtp model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 @@ -40,6 +42,10 @@ roles: SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -65,8 +71,13 @@ roles: disaggregation-mode: prefill disaggregation-transfer-backend: mooncake disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 decode: - nodes: 1 + nodes: colocate workers: 1 gpus: 4 env: @@ -75,6 +86,10 @@ roles: SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -106,6 +121,10 @@ roles: speculative-num-draft-tokens: 4 cuda-graph-max-bs-decode: 16 stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 health_check: max_attempts: 720 interval_seconds: 10 @@ -118,9 +137,6 @@ benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh concurrencies: - - 1 - - 4 - - 8 - 16 env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml new file mode 100644 index 0000000000..75b9662322 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml @@ -0,0 +1,153 @@ +# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. +# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. +# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. +# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. +schema: 2 +name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c32-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 32 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 32 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml new file mode 100644 index 0000000000..b3128a7d9a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml @@ -0,0 +1,153 @@ +# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. +# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. +# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. +# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. +schema: 2 +name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c4-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 8 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 8 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 4 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 4 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml new file mode 100644 index 0000000000..df5a2e4051 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml @@ -0,0 +1,153 @@ +# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. +# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. +# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. +# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. +schema: 2 +name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c8-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 16 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 16 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 8 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 8 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0b4925b18b..40ea150a41 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8172,8 +8172,53 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: kv-offloading: none conc-list: - 1 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: - 4 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: - 8 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: - 16 prefill: num-worker: 1 @@ -8181,7 +8226,23 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-mtp.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: + - 32 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml decode: num-worker: 1 tp: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d3be26293d..3da1b9695b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8091,3 +8091,13 @@ - 'Add Qwen3.5 397B FP8 B200 Dynamo + SGLang disaggregated AgentX: one TP4 prefill worker and one TP4 decode worker with NEXTN MTP, concurrency 1/4/8/16.' pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Align B200 Qwen3.5 FP8 1P1D AgentX with the aggregate serving baseline; colocate + TP4 prefill and TP4 decode on one eight-GPU node, match per-concurrency request/graph + limits, and extend the initial study to concurrency 32. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From e64b9c1e5b5d786b09a17b6462ef56b647aef97c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 13:21:23 -0500 Subject: [PATCH 04/28] perf: use latest SGLang nightly for B200 Qwen3.5 FP8 --- .../disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml | 6 +++--- .../disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml | 6 +++--- .../disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml | 6 +++--- .../disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml | 6 +++--- .../disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml | 6 +++--- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 10 ++++++++++ 7 files changed, 26 insertions(+), 16 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml index c042bd170f..ea1d448ae2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml @@ -7,7 +7,7 @@ schema: 2 name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c1-mtp model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 engine: sglang dynamo: @@ -74,7 +74,7 @@ roles: enable-symm-mem: true scheduler-recv-interval: 10 stream-interval: 50 - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 decode: nodes: colocate @@ -123,7 +123,7 @@ roles: stream-interval: 50 enable-symm-mem: true scheduler-recv-interval: 10 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-track-interval: 128 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml index ae4ea825ae..bf3cc39f78 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml @@ -7,7 +7,7 @@ schema: 2 name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c16-mtp model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 engine: sglang dynamo: @@ -74,7 +74,7 @@ roles: enable-symm-mem: true scheduler-recv-interval: 10 stream-interval: 50 - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 decode: nodes: colocate @@ -123,7 +123,7 @@ roles: stream-interval: 50 enable-symm-mem: true scheduler-recv-interval: 10 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-track-interval: 128 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml index 75b9662322..13f26160e5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml @@ -7,7 +7,7 @@ schema: 2 name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c32-mtp model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 engine: sglang dynamo: @@ -74,7 +74,7 @@ roles: enable-symm-mem: true scheduler-recv-interval: 10 stream-interval: 50 - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 decode: nodes: colocate @@ -123,7 +123,7 @@ roles: stream-interval: 50 enable-symm-mem: true scheduler-recv-interval: 10 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-track-interval: 128 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml index b3128a7d9a..f8b7267188 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml @@ -7,7 +7,7 @@ schema: 2 name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c4-mtp model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 engine: sglang dynamo: @@ -74,7 +74,7 @@ roles: enable-symm-mem: true scheduler-recv-interval: 10 stream-interval: 50 - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 decode: nodes: colocate @@ -123,7 +123,7 @@ roles: stream-interval: 50 enable-symm-mem: true scheduler-recv-interval: 10 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-track-interval: 128 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml index df5a2e4051..b1263a9994 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml @@ -7,7 +7,7 @@ schema: 2 name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c8-mtp model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 engine: sglang dynamo: @@ -74,7 +74,7 @@ roles: enable-symm-mem: true scheduler-recv-interval: 10 stream-interval: 50 - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 decode: nodes: colocate @@ -123,7 +123,7 @@ roles: stream-interval: 50 enable-symm-mem: true scheduler-recv-interval: 10 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-track-interval: 128 health_check: max_attempts: 720 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 40ea150a41..c328f89b88 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8152,7 +8152,7 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: cluster:b200-nscale diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3da1b9695b..9d2ef85dbb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8101,3 +8101,13 @@ TP4 prefill and TP4 decode on one eight-GPU node, match per-concurrency request/graph limits, and extend the initial study to concurrency 32. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Pin B200 Qwen3.5 FP8 disaggregated AgentX to the latest CUDA 13 amd64 nightly, + nightly-dev-cu13-20260918-20518d85, and use the current mamba-radix-cache-strategy + argument for prefill and decode. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From d1368f7f00c647775be6d3d27f9777ed29bb79f9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 13:34:44 -0500 Subject: [PATCH 05/28] fix: align Dynamo with latest SGLang module layout --- .../disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml | 2 +- .../disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml | 2 +- .../disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml | 2 +- .../disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml | 2 +- .../disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml | 2 +- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 10 ++++++++++ 7 files changed, 16 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml index ea1d448ae2..f3b059d142 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml @@ -13,7 +13,7 @@ engine: sglang dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e resources: gpu_type: b200 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml index bf3cc39f78..c311acda82 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml @@ -13,7 +13,7 @@ engine: sglang dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e resources: gpu_type: b200 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml index 13f26160e5..527726e53e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml @@ -13,7 +13,7 @@ engine: sglang dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e resources: gpu_type: b200 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml index f8b7267188..781cff326d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml @@ -13,7 +13,7 @@ engine: sglang dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e resources: gpu_type: b200 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml index b1263a9994..c1609d780b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml @@ -13,7 +13,7 @@ engine: sglang dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e resources: gpu_type: b200 gpus_per_node: 8 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c328f89b88..32c715bc98 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8160,7 +8160,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: framework: dynamo-sglang router: name: dynamo-router - version: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + version: 805a77f053d807b0d8def5d27f674a6df0ed839e kv-p2p-transfer: mooncake multinode: true disagg: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 670ad12318..6bab173019 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8120,3 +8120,13 @@ nightly-dev-cu13-20260918-20518d85, and use the current mamba-radix-cache-strategy argument for prefill and decode. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Update B200 Qwen3.5 FP8 AgentX to Dynamo 805a77f053d807b0d8def5d27f674a6df0ed839e + for compatibility with the latest SGLang nightly's relocated argument parser + and encoder modules. Keep recipe and result router version pins aligned. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 683c6c8f17b22a72fb1a35cafddbacda4b1cd7e9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:03:13 -0500 Subject: [PATCH 06/28] fix: handle B200 Qwen3.5 node-local weights in SRT preflight --- perf-changelog.yaml | 10 ++++++++++ runners/launch_b200-nscale-compat.sh | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6bab173019..17f7d60943 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8130,3 +8130,13 @@ for compatibility with the latest SGLang nightly's relocated argument parser and encoder modules. Keep recipe and result router version pins aligned. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Apply the existing B200 node-local-weight preflight handling to Qwen3.5 FP8 + Dynamo SGLang jobs. The checkpoint is present on compute-node NVMe but absent + from the login node; retain worker-side model loading checks. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 39d78ca99b..8f1830fa5d 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -326,8 +326,10 @@ EOF sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" SRTCTL_PREFLIGHT_ARGS=() - # Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node. - if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + # These weights are staged on the Slurm compute nodes, not the login node. + # SRT still checks the resolved model path when the worker starts. + if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]] || + [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi From 64234cd22533b9085e5cec86d2b4094632ecb8c6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:12:42 -0500 Subject: [PATCH 07/28] fix: use intra-node NVLink for colocated B200 Qwen3.5 FP8 --- .../disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml | 4 ++++ .../disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml | 4 ++++ .../disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml | 4 ++++ .../disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml | 4 ++++ .../disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml | 4 ++++ perf-changelog.yaml | 11 +++++++++++ 6 files changed, 31 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml index f3b059d142..963c586fe7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml @@ -46,6 +46,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -90,6 +92,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml index c311acda82..18043afc51 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml @@ -46,6 +46,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -90,6 +92,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml index 527726e53e..de126efca3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml @@ -46,6 +46,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -90,6 +92,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml index 781cff326d..96cf22b1ee 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml @@ -46,6 +46,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -90,6 +92,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml index c1609d780b..51e7bb76eb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml @@ -46,6 +46,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -90,6 +92,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 17f7d60943..3b925d597e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8140,3 +8140,14 @@ Dynamo SGLang jobs. The checkpoint is present on compute-node NVMe but absent from the login node; retain worker-side model loading checks. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Use Mooncake intra-node NVLink transport and compatible device allocations + for the colocated B200 Qwen3.5 FP8 1P1D AgentX recipes. This transport passed + four real-verification short/long-input probes on B300; B200 validation and + normalized performance qualification remain pending. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From f0219646545f9c3a971297a26aebcfa8835eb5fd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:41:59 -0500 Subject: [PATCH 08/28] perf: add isolated latest B200 Qwen3.5 FP8 aggregate candidates --- .../agentic/qwen3.5_fp8_b200_sglang_mtp.sh | 29 ++++++++++++++----- configs/nvidia-master.yaml | 16 ++++++++++ perf-changelog.yaml | 11 +++++++ runners/launch_b200-nscale-slurm.sh | 9 ++++++ 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh index 2a2d4b62e2..a558bb0dbb 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh @@ -14,7 +14,14 @@ export EVAL_FRAMEWORK="lm-eval" check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY +check_env_vars EVAL_ONLY QWEN35_HICACHE_BUDGET_MODE +case "$QWEN35_HICACHE_BUDGET_MODE" in + legacy|combined) ;; + *) + echo "Error: QWEN35_HICACHE_BUDGET_MODE must be legacy or combined" >&2 + exit 1 + ;; +esac SCHEDULER_RECV_INTERVAL=10 @@ -41,27 +48,33 @@ mkdir -p "$RESULT_DIR" CACHE_ARGS=() if require_agentic_kv_offload_backend hicache; then - # SGLang applies --hicache-size independently to Qwen's target KV and - # Mamba pools. Native NEXTN also creates a draft KV pool with the same - # slot count; its one attention layer adds 1/15 of the target KV bytes. - # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. + # Preserve the existing budget unless the launcher selects the pinned + # nightly's combined KV/Mamba pool. Reserve 1 GB/rank for page alignment. HICACHE_ALIGNMENT_RESERVE_GB=$TP HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 exit 1 fi - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) + case "$QWEN35_HICACHE_BUDGET_MODE" in + legacy) + HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) + PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) + ;; + combined) + HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB / TP)) + PROJECTED_HICACHE_TOTAL_GB=$((HICACHE_SIZE_GB * TP + HICACHE_ALIGNMENT_RESERVE_GB)) + ;; + esac if [ "$HICACHE_SIZE_GB" -lt 1 ]; then echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 exit 1 fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 exit 1 fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" + echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB per rank with ${QWEN35_HICACHE_BUDGET_MODE} budgeting across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" CACHE_ARGS=( --page-size 64 --enable-hierarchical-cache diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 32c715bc98..19d943d39b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8248,3 +8248,19 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: tp: 4 ep: 1 dp-attn: false + +# Aggregate candidates pending canonical sweep and performance qualification. +qwen3.5-fp8-b200-sglang-agentic-mtp-latest: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:b200-nscale + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.8 + search-space: + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32] } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3b925d597e..0f9e6000fd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8151,3 +8151,14 @@ four real-verification short/long-input probes on B300; B200 validation and normalized performance qualification remain pending. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-sglang-agentic-mtp-latest + scenario-type: + - agentic-coding + description: + - Add latest-nightly B200 Qwen3.5 FP8 aggregate AgentX candidates at TP4 C4 + without offloading and TP4 C32 with HiCache, pending official sweep and + normalized performance qualification. Use the combined KV/Mamba host-pool + budget only for the pinned latest image, preserving legacy aggregate sizing. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b53e667696..91286f2627 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -839,6 +839,15 @@ run_single_node() { # submitted to the Nscale batch_1 partition. check_env_vars SALLOC_TIME_LIMIT GPU_COUNT + if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && + "$PRECISION" == "fp8" && "$FRAMEWORK" == "sglang" && "$SPEC_DECODING" == "mtp" ]]; then + check_env_vars IMAGE + export QWEN35_HICACHE_BUDGET_MODE=legacy + if [[ "$IMAGE" == "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85" ]]; then + export QWEN35_HICACHE_BUDGET_MODE=combined + fi + fi + SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" || "$SPEC_DECODING" == "draft_model" ]] && printf '_mtp' || printf '') From b3d72d15df8756e015831bdfc0a3ad8bbacd76d4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:45:56 -0500 Subject: [PATCH 09/28] fix: preserve standalone B200 aggregate budget defaults --- benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh index a558bb0dbb..e7098cce4e 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh @@ -14,6 +14,8 @@ export EVAL_FRAMEWORK="lm-eval" check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION +# Standalone callers retain the historical budget unless they opt in. +QWEN35_HICACHE_BUDGET_MODE="${QWEN35_HICACHE_BUDGET_MODE:-legacy}" check_env_vars EVAL_ONLY QWEN35_HICACHE_BUDGET_MODE case "$QWEN35_HICACHE_BUDGET_MODE" in legacy|combined) ;; From e82ee036f72dcc6f5e84ca10a6969855fa4f63b0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:50:09 -0500 Subject: [PATCH 10/28] fix: propagate failed B200 Slurm sweeps after artifact collection --- perf-changelog.yaml | 10 ++ runners/launch_b200-nscale-slurm.sh | 27 +---- runners/slurm_utils.sh | 23 ++++ runners/test_b200_srt_status.py | 179 ++++++++++++++++++++++++++++ 4 files changed, 216 insertions(+), 23 deletions(-) create mode 100644 runners/test_b200_srt_status.py diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0f9e6000fd..50b1a7e05e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8162,3 +8162,13 @@ normalized performance qualification. Use the combined KV/Mamba host-pool budget only for the pinned latest image, preserving legacy aggregate sizing. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Verify terminal Slurm allocation status for B200 Qwen3.5 FP8 disaggregated + runs and preserve diagnostic artifacts before returning benchmark failure. + Prevent aborted replay jobs from appearing successful after leaving squeue. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 91286f2627..a8356c02a9 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -752,29 +752,8 @@ EOF LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" - while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 - done - - ( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 - done - ) & - POLL_PID=$! - - echo "Tailing LOG_FILE: $LOG_FILE" - - # -F follows by name and polls; inotify does not work on NFS. - tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null - - wait $POLL_PID + local srt_job_rc=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || srt_job_rc=$? set -x @@ -828,6 +807,8 @@ EOF sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + # Failed runs still provide the diagnostics and eval outputs above. + return "$srt_job_rc" } # --------------------------------------------------------------------------- diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 79f9175915..a15b94aa40 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -132,6 +132,29 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" + + # Disappearing from squeue means terminal, not successful. Accounting can + # lag briefly; inspect only the allocation, never successful service steps. + local attempt accounting state exit_code + for attempt in {1..10}; do + accounting=$(sacct -X -n -P -j "$job_id" --format=State,ExitCode 2>/dev/null) || accounting="" + IFS='|' read -r state exit_code <<< "$accounting" + case "$state" in + COMPLETED) + if [[ "$exit_code" == "0:0" ]]; then + return 0 + fi + ;; + ""|PENDING|RUNNING|CONFIGURING|COMPLETING) + sleep 1 + continue + ;; + esac + echo "ERROR: Slurm job $job_id ended with state=$state exit_code=$exit_code" >&2 + return 1 + done + echo "ERROR: could not verify terminal Slurm status for job $job_id" >&2 + return 1 } copy_to_workspace() { diff --git a/runners/test_b200_srt_status.py b/runners/test_b200_srt_status.py new file mode 100644 index 0000000000..007d600169 --- /dev/null +++ b/runners/test_b200_srt_status.py @@ -0,0 +1,179 @@ +"""A completed Slurm allocation must not hide a failed benchmark.""" + +import os +import shutil +import subprocess +import tarfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def _stub(directory: Path, name: str, body: str) -> None: + path = directory / name + path.write_text("#!/bin/bash\n" + body + "\n") + path.chmod(0o755) + + +@pytest.mark.parametrize( + "accounting,expected_status", + [ + ("COMPLETED|0:0", 0), + ("FAILED|1:0", 1), + ("COMPLETED|0:9", 1), + ("CANCELLED by 123|0:15", 1), + ("DELAYED", 0), + ("", 1), + ], +) +def test_b200_collects_artifacts_before_returning_slurm_status( + tmp_path: Path, accounting: str, expected_status: int +) -> None: + """Run the actual Qwen launcher path with only external services stubbed.""" + binaries = tmp_path / "bin" + binaries.mkdir() + for name in ( + "curl", + "uv", + "make", + "srtctl", + "flock", + "unsquashfs", + "squeue", + "sleep", + ): + _stub(binaries, name, "exit 0") + _stub( + binaries, + "sacct", + r""" +[[ " $* " == *" -X "* ]] || exit 2 +count=0 +[[ -f "$MOCK_SACCT_COUNT" ]] && read -r count < "$MOCK_SACCT_COUNT" +count=$((count + 1)) +printf '%s\n' "$count" > "$MOCK_SACCT_COUNT" +if [[ "$MOCK_ACCOUNTING" == DELAYED ]]; then + case "$count" in + 1) exit 0 ;; + 2) printf 'RUNNING|0:0\n' ;; + *) printf 'COMPLETED|0:0\n' ;; + esac +else + printf '%s\n' "$MOCK_ACCOUNTING" +fi +""", + ) + # Tail's long-running process is a clock collaborator; logs remain real. + _stub( + binaries, + "tail", + 'for arg in "$@"; do [[ -f "$arg" ]] && cat "$arg"; done; exit 0', + ) + workspace = tmp_path / "workspace" + workspace.mkdir() + for relative in ( + "runners/launch_b200-nscale-slurm.sh", + "runners/slurm_utils.sh", + "benchmarks/benchmark_lib.sh", + ): + destination = workspace / relative + destination.parent.mkdir(exist_ok=True) + shutil.copyfile(ROOT / relative, destination) + # Stub remote checkout and submission after loading the real shared helpers. + with (workspace / "runners/slurm_utils.sh").open("a") as helpers: + helpers.write(r""" +setup_srt_slurm() { + mkdir -p "$1/recipes" + cd "$1" || return 1 + printf 'name: fixture\n' > recipes/test.yaml +} +apply_srt_recipe() { + mkdir -p outputs/42/logs + cp -R "$MOCK_FIXTURE/." outputs/42/logs/ + printf '{"diagnostic":"retained"}\n' > "$GITHUB_WORKSPACE/aggregate_conc1.json" + printf '✅ Job 42\n' +} +""") + activation = workspace / ".venv/bin/activate" + activation.parent.mkdir(parents=True) + activation.write_text(":\n") + fixture = tmp_path / "fixture" + (fixture / "agentic/conc_1/aiperf_artifacts").mkdir(parents=True) + (fixture / "eval_results").mkdir() + (fixture / "sweep_42.log").write_text("benchmark diagnostics\n") + (fixture / "agentic/conc_1/aiperf_artifacts/profile_export_aiperf.json").write_text( + '{"metadata":{"submission_valid":false}}\n' + ) + (fixture / "eval_results/results_eval.json").write_text('{"eval":"retained"}\n') + env = { + **os.environ, + "PATH": f"{binaries}:{os.environ['PATH']}", + "MOCK_ACCOUNTING": accounting, + "MOCK_SACCT_COUNT": str(tmp_path / "sacct-count"), + "MOCK_FIXTURE": str(fixture), + "GITHUB_WORKSPACE": str(workspace), + "EVAL_ONLY": "false", + "IS_AGENTIC": "0", + "IS_MULTINODE": "true", + "RUN_EVAL": "true", + "SLURM_PARTITION": "batch_1", + "SLURM_ACCOUNT": "benchmark", + "FRAMEWORK": "dynamo-sglang", + "MODEL_PREFIX": "qwen3.5", + "PRECISION": "fp8", + "MODEL": "Qwen/Qwen3.5-397B-A17B-FP8", + "SPEC_DECODING": "mtp", + "CONFIG_FILE": "recipes/test.yaml", + "RUNNER_NAME": "b200-nscale-slurm_08", + "IMAGE": "fixture-image", + "B200_SQUASH_DIR": str(tmp_path / "containers"), + "B200_SQUASH_LOCK_TIMEOUT": "1", + "RESULT_FILENAME": "aggregate", + "ISL": "1", + "OSL": "1", + } + result = subprocess.run( + [ + "bash", + "-c", + r""" +# GNU-only command options in this Linux launcher are irrelevant to the +# completion contract; keep the test runnable on macOS too. +grep() { + if [[ "$1" == -oP ]]; then cat >/dev/null; printf '42\n'; else command grep "$@"; fi +} +sed() { + if [[ "$1" == -i && "$(uname)" == Darwin ]]; then + shift; command sed -i '' "$@" + else + command sed "$@" + fi +} +builtin source "$1/runners/launch_b200-nscale-slurm.sh" +""", + "bash", + str(workspace), + ], + cwd=workspace, + env=env, + text=True, + capture_output=True, + check=False, + timeout=15, + ) + assert result.returncode == expected_status, result.stdout + result.stderr + assert ( + workspace / "aggregate_conc1.json" + ).read_text() == '{"diagnostic":"retained"}\n' + assert (workspace / "results_eval.json").read_text() == '{"eval":"retained"}\n' + assert (workspace / "LOGS/sweep_42.log").read_text() == "benchmark diagnostics\n" + with tarfile.open(workspace / "multinode_server_logs.tar.gz") as archive: + assert ( + "./agentic/conc_1/aiperf_artifacts/profile_export_aiperf.json" + in archive.getnames() + ) + assert not (workspace / "srt-slurm/outputs").exists() + expected_queries = 3 if accounting == "DELAYED" else 10 if accounting == "" else 1 + assert int((tmp_path / "sacct-count").read_text()) == expected_queries From b03becba9925325e374e04ec7d09c4e67b9bc78c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:51:56 -0500 Subject: [PATCH 11/28] fix: use verified NVIDIA wheels for B200 Dynamo dependencies --- .../disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml | 3 +++ .../disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml | 3 +++ .../disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml | 3 +++ .../disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml | 3 +++ .../disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml | 3 +++ perf-changelog.yaml | 10 ++++++++++ 6 files changed, 25 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml index 963c586fe7..f640795dcc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml @@ -28,6 +28,7 @@ frontend: num_additional_frontends: 1 env: PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com args: router-mode: kv router-session-affinity-ttl-secs: '3600' @@ -39,6 +40,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' @@ -85,6 +87,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml index 18043afc51..b4bb996b24 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml @@ -28,6 +28,7 @@ frontend: num_additional_frontends: 1 env: PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com args: router-mode: kv router-session-affinity-ttl-secs: '3600' @@ -39,6 +40,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' @@ -85,6 +87,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml index de126efca3..d25955b6c6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml @@ -28,6 +28,7 @@ frontend: num_additional_frontends: 1 env: PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com args: router-mode: kv router-session-affinity-ttl-secs: '3600' @@ -39,6 +40,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' @@ -85,6 +87,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml index 96cf22b1ee..ef2dbeb95f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml @@ -28,6 +28,7 @@ frontend: num_additional_frontends: 1 env: PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com args: router-mode: kv router-session-affinity-ttl-secs: '3600' @@ -39,6 +40,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' @@ -85,6 +87,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml index 51e7bb76eb..9936a8958a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml @@ -28,6 +28,7 @@ frontend: num_additional_frontends: 1 env: PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com args: router-mode: kv router-session-affinity-ttl-secs: '3600' @@ -39,6 +40,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' @@ -85,6 +87,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com SGLANG_ENABLE_SPEC_V2: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 50b1a7e05e..f4ee304037 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8172,3 +8172,13 @@ runs and preserve diagnostic artifacts before returning benchmark failure. Prevent aborted replay jobs from appearing successful after leaving squeue. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Resolve NVIDIA Dynamo dependencies directly from the official NVIDIA + wheel index. Retain pip integrity verification while avoiding the PyPI + placeholder downloader that repeatedly truncated aisimulate wheels. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 2560380de20028a000599bd21af2ed317cc40a22 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 15:02:34 -0500 Subject: [PATCH 12/28] feat: select measured B200 FP8 HiCache disaggregation candidate --- ...agg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml | 160 ------------------ ...gg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml | 160 ------------------ ...agg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml | 160 ------------------ ...agg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml | 160 ------------------ ...agg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml} | 13 +- configs/nvidia-master.yaml | 70 +------- perf-changelog.yaml | 13 ++ 7 files changed, 24 insertions(+), 712 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/{disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml => disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml} (90%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml deleted file mode 100644 index f640795dcc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml +++ /dev/null @@ -1,160 +0,0 @@ -# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. -# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. -# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. -# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. -schema: 2 -name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c1-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -engine: sglang -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -resources: - gpu_type: b200 - gpus_per_node: 8 -slurm: - time_limit: '4:00:00' -frontend: - type: dynamo - nginx_container: nginx-sqsh - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - SGLANG_ENABLE_SPEC_V2: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - PYTHONNOUSERSITE: '1' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - trust-remote-code: true - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - page-size: 64 - mem-fraction-static: 0.8 - context-length: 262144 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 2 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - enable-metrics: true - enable-cache-report: true - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - disable-cuda-graph: true - enable-symm-mem: true - scheduler-recv-interval: 10 - stream-interval: 50 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - SGLANG_ENABLE_SPEC_V2: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - PYTHONNOUSERSITE: '1' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - trust-remote-code: true - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - page-size: 64 - mem-fraction-static: 0.8 - context-length: 262144 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 2 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - enable-metrics: true - enable-cache-report: true - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs-decode: 1 - stream-interval: 50 - enable-symm-mem: true - scheduler-recv-interval: 10 - mamba-radix-cache-strategy: no_buffer - mamba-track-interval: 128 -health_check: - max_attempts: 720 - interval_seconds: 10 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - concurrencies: - - 1 - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml deleted file mode 100644 index b4bb996b24..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml +++ /dev/null @@ -1,160 +0,0 @@ -# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. -# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. -# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. -# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. -schema: 2 -name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c16-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -engine: sglang -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -resources: - gpu_type: b200 - gpus_per_node: 8 -slurm: - time_limit: '4:00:00' -frontend: - type: dynamo - nginx_container: nginx-sqsh - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - SGLANG_ENABLE_SPEC_V2: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - PYTHONNOUSERSITE: '1' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - trust-remote-code: true - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - page-size: 64 - mem-fraction-static: 0.8 - context-length: 262144 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 32 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - enable-metrics: true - enable-cache-report: true - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - disable-cuda-graph: true - enable-symm-mem: true - scheduler-recv-interval: 10 - stream-interval: 50 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - SGLANG_ENABLE_SPEC_V2: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - PYTHONNOUSERSITE: '1' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - trust-remote-code: true - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - page-size: 64 - mem-fraction-static: 0.8 - context-length: 262144 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 32 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - enable-metrics: true - enable-cache-report: true - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs-decode: 16 - stream-interval: 50 - enable-symm-mem: true - scheduler-recv-interval: 10 - mamba-radix-cache-strategy: no_buffer - mamba-track-interval: 128 -health_check: - max_attempts: 720 - interval_seconds: 10 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - concurrencies: - - 16 - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml deleted file mode 100644 index ef2dbeb95f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml +++ /dev/null @@ -1,160 +0,0 @@ -# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. -# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. -# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. -# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. -schema: 2 -name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c4-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -engine: sglang -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -resources: - gpu_type: b200 - gpus_per_node: 8 -slurm: - time_limit: '4:00:00' -frontend: - type: dynamo - nginx_container: nginx-sqsh - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - SGLANG_ENABLE_SPEC_V2: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - PYTHONNOUSERSITE: '1' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - trust-remote-code: true - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - page-size: 64 - mem-fraction-static: 0.8 - context-length: 262144 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 8 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - enable-metrics: true - enable-cache-report: true - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - disable-cuda-graph: true - enable-symm-mem: true - scheduler-recv-interval: 10 - stream-interval: 50 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - SGLANG_ENABLE_SPEC_V2: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - PYTHONNOUSERSITE: '1' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - trust-remote-code: true - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - page-size: 64 - mem-fraction-static: 0.8 - context-length: 262144 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 8 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - enable-metrics: true - enable-cache-report: true - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs-decode: 4 - stream-interval: 50 - enable-symm-mem: true - scheduler-recv-interval: 10 - mamba-radix-cache-strategy: no_buffer - mamba-track-interval: 128 -health_check: - max_attempts: 720 - interval_seconds: 10 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - concurrencies: - - 4 - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml deleted file mode 100644 index 9936a8958a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml +++ /dev/null @@ -1,160 +0,0 @@ -# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. -# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. -# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. -# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. -schema: 2 -name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c8-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -engine: sglang -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -resources: - gpu_type: b200 - gpus_per_node: 8 -slurm: - time_limit: '4:00:00' -frontend: - type: dynamo - nginx_container: nginx-sqsh - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - SGLANG_ENABLE_SPEC_V2: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - PYTHONNOUSERSITE: '1' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - trust-remote-code: true - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - page-size: 64 - mem-fraction-static: 0.8 - context-length: 262144 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 16 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - enable-metrics: true - enable-cache-report: true - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - disable-cuda-graph: true - enable-symm-mem: true - scheduler-recv-interval: 10 - stream-interval: 50 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - SGLANG_ENABLE_SPEC_V2: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - PYTHONNOUSERSITE: '1' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - trust-remote-code: true - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - page-size: 64 - mem-fraction-static: 0.8 - context-length: 262144 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 16 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - enable-metrics: true - enable-cache-report: true - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs-decode: 8 - stream-interval: 50 - enable-symm-mem: true - scheduler-recv-interval: 10 - mamba-radix-cache-strategy: no_buffer - mamba-track-interval: 128 -health_check: - max_attempts: 720 - interval_seconds: 10 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - concurrencies: - - 8 - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml similarity index 90% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml index d25955b6c6..c4ec375e2b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml @@ -1,10 +1,6 @@ -# Based on the B200 FP8 aggregate AgentX recipe and SGLang Qwen3.5 PD recipe. -# https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# 8 active GPUs on one 8-GPU node: prefill GPU0-3, decode GPU4-7. -# Request/graph limits match aggregate C; Dynamo owns frontend tokenization. -# Shared apply_srt_recipe selects golden AL3.39 for throughput, real verification for evals. +# Fast replay improves the published frontier; canonical sweep qualification is pending. schema: 2 -name: qwen35-b200-fp8-agentx-1p1d-tp4-colocated-c32-mtp +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -80,6 +76,11 @@ roles: stream-interval: 50 mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective decode: nodes: colocate workers: 1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 19d943d39b..13f1945735 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8169,71 +8169,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: - dram-utilization: 0.8 search-space: - spec-decoding: mtp - kv-offloading: none - conc-list: - - 1 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c1-mtp.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none - conc-list: - - 4 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c4-mtp.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none - conc-list: - - 8 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c8-mtp.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none - conc-list: - - 16 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c16-mtp.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none + kv-offloading: dram conc-list: - 32 prefill: @@ -8242,12 +8178,14 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-colocated-c32-mtp.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false + kv-offload-backend: + name: hicache # Aggregate candidates pending canonical sweep and performance qualification. qwen3.5-fp8-b200-sglang-agentic-mtp-latest: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f4ee304037..035aa2d95a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8182,3 +8182,16 @@ wheel index. Retain pip integrity verification while avoiding the PyPI placeholder downloader that repeatedly truncated aisimulate wheels. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Select the measured B200 Qwen3.5 FP8 colocated TP4 prefill plus TP4 decode + C32 candidate with prefill HiCache at 104 GB per rank. The short replay + reaches 20645.73 logical tokens/s/GPU at 211.219 p90 interactivity with + zero request errors, improving the published frontier; canonical sweep + and real-verification evals remain required. Remove unqualified starter + points from the PR selection. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 58fe416f28a352666d6cc5e778b9d512d0d26dee Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 15:05:08 -0500 Subject: [PATCH 13/28] feat: qualify B200 FP8 aggregate high-concurrency frontier candidate --- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 13f1945735..c8409be140 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8201,4 +8201,4 @@ qwen3.5-fp8-b200-sglang-agentic-mtp-latest: - dram-utilization: 0.8 search-space: - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 64] } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 035aa2d95a..6bb93f1d9a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8195,3 +8195,14 @@ and real-verification evals remain required. Remove unqualified starter points from the PR selection. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-sglang-agentic-mtp-latest + scenario-type: + - agentic-coding + description: + - Add measured B200 Qwen3.5 FP8 TP4 aggregate C64 HiCache for canonical + qualification. Its short replay reaches 41896.93 logical tokens/s/GPU + at 50.042 p90 interactivity with zero errors, extending the published + throughput frontier at the cost of lower interactivity and higher TTFT. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 5fd1bedd9df6fa84ecb07b7a719fb517c26bdcee Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 15:23:33 -0500 Subject: [PATCH 14/28] test: include runtime inputs in isolated B200 launcher fixture --- runners/test_b200_srt_status.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/runners/test_b200_srt_status.py b/runners/test_b200_srt_status.py index 007d600169..7e1b291a28 100644 --- a/runners/test_b200_srt_status.py +++ b/runners/test_b200_srt_status.py @@ -73,14 +73,14 @@ def test_b200_collects_artifacts_before_returning_slurm_status( ) workspace = tmp_path / "workspace" workspace.mkdir() - for relative in ( - "runners/launch_b200-nscale-slurm.sh", - "runners/slurm_utils.sh", - "benchmarks/benchmark_lib.sh", - ): - destination = workspace / relative - destination.parent.mkdir(exist_ok=True) - shutil.copyfile(ROOT / relative, destination) + # Keep launcher-owned runtime inputs with the real launcher and helpers. + shutil.copytree(ROOT / "runners", workspace / "runners") + (workspace / "benchmarks").mkdir() + shutil.copyfile( + ROOT / "benchmarks/benchmark_lib.sh", workspace / "benchmarks/benchmark_lib.sh" + ) + # Run the real configuration renderer regardless of editable-install state. + (workspace / "infx").symlink_to(ROOT / "infx", target_is_directory=True) # Stub remote checkout and submission after loading the real shared helpers. with (workspace / "runners/slurm_utils.sh").open("a") as helpers: helpers.write(r""" From 3dcbfda929c46b375dc55f894bfd84a738d359cb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 15:34:31 -0500 Subject: [PATCH 15/28] feat: stage B200 FP8 2P1D frontier candidate for qualification --- ...sagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml | 161 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 13 ++ 3 files changed, 192 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml new file mode 100644 index 0000000000..efefc1867b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml @@ -0,0 +1,161 @@ +# Provisional fast frontier extension; compare neighboring 1P1D points and qualify officially. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 4 + env: + # Use the eight GPU-associated 400 Gb/s NDR rails on this pool. + MC_TE_FILTERS: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + # Exclude secondary 100 Gb/s HDR functions and the Ethernet bond. + MC_TE_FILTERS: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 32 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 32 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c8409be140..7af6a2a0e9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8186,6 +8186,24 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: dp-attn: false kv-offload-backend: name: hicache + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 32 + prefill: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache # Aggregate candidates pending canonical sweep and performance qualification. qwen3.5-fp8-b200-sglang-agentic-mtp-latest: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6bb93f1d9a..9620a3b570 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8206,3 +8206,16 @@ at 50.042 p90 interactivity with zero errors, extending the published throughput frontier at the cost of lower interactivity and higher TTFT. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Stage B200 Qwen3.5 FP8 2P1D C32 HiCache with verified 400 Gb/s RDMA + interfaces for expanded qualification. The short replay reaches 14511.73 + logical tokens/s per all 12 active GPUs at 228.969 p90 interactivity and + 0.851s p90 TTFT with zero errors, extending the published frontier. + Compare neighboring 1P1D candidates and require official sweep success + before final retention; the allocation contains 16 GPUs. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 27b44dc9599be7d9d8b2ec67407229b52c4e2623 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 15:41:23 -0500 Subject: [PATCH 16/28] fix: require explicit aggregate HiCache budget mode from caller --- .../agentic/qwen3.5_fp8_b200_sglang_mtp.sh | 2 -- perf-changelog.yaml | 11 +++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh index e7098cce4e..a558bb0dbb 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh @@ -14,8 +14,6 @@ export EVAL_FRAMEWORK="lm-eval" check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -# Standalone callers retain the historical budget unless they opt in. -QWEN35_HICACHE_BUDGET_MODE="${QWEN35_HICACHE_BUDGET_MODE:-legacy}" check_env_vars EVAL_ONLY QWEN35_HICACHE_BUDGET_MODE case "$QWEN35_HICACHE_BUDGET_MODE" in legacy|combined) ;; diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9620a3b570..ce58cabfcc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8219,3 +8219,14 @@ Compare neighboring 1P1D candidates and require official sweep success before final retention; the allocation contains 16 GPUs. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-sglang-agentic-mtp-latest + scenario-type: + - agentic-coding + description: + - Require the B200 aggregate caller to supply the HiCache budget mode, + following repository environment-input policy. The unified launcher + already supplies legacy or combined explicitly; generated benchmark + arguments and existing image-specific cache sizes remain unchanged. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 873bf0467fd8f63bc5d080800b08018411e77d3f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 15:57:22 -0500 Subject: [PATCH 17/28] fix: preserve shared log streaming while verifying B200 job status --- perf-changelog.yaml | 11 +++++++++++ runners/launch_b200-nscale-slurm.sh | 3 +++ runners/slurm_utils.sh | 3 +++ 3 files changed, 17 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2a06a90603..02323bd0eb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8253,3 +8253,14 @@ already supplies legacy or combined explicitly; generated benchmark arguments and existing image-specific cache sizes remain unchanged. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Scope terminal Slurm accounting verification to the artifact-preserving + multinode launcher path. Keep the shared log-streaming helper's existing + behavior for other callers so failed native jobs retain their diagnostics. + Benchmark arguments, workload and selected recipes are unchanged. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index c948d8d44d..c376569bf7 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -704,6 +704,9 @@ run_multinode_srt() { local srt_job_rc=0 stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || srt_job_rc=$? + if [[ "$srt_job_rc" -eq 0 ]]; then + verify_slurm_job_status "$JOB_ID" || srt_job_rc=$? + fi set -x diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index b8a98a08fc..67a6eac780 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -155,7 +155,10 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" +} +verify_slurm_job_status() { + local job_id="$1" # Disappearing from squeue means terminal, not successful. Accounting can # lag briefly; inspect only the allocation, never successful service steps. local attempt accounting state exit_code From 6498b907257ac80b14310b948a837ba79796eed0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:18:26 -0500 Subject: [PATCH 18/28] fix: preserve published aggregate curves in B200 qualification scope --- .../agentic/qwen3.5_fp8_b200_sglang_mtp.sh | 29 +++++-------------- configs/nvidia-master.yaml | 16 ---------- perf-changelog.yaml | 21 ++++++++++++-- runners/launch_b200-nscale-slurm.sh | 9 ------ 4 files changed, 26 insertions(+), 49 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh index a558bb0dbb..2a2d4b62e2 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh @@ -14,14 +14,7 @@ export EVAL_FRAMEWORK="lm-eval" check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY QWEN35_HICACHE_BUDGET_MODE -case "$QWEN35_HICACHE_BUDGET_MODE" in - legacy|combined) ;; - *) - echo "Error: QWEN35_HICACHE_BUDGET_MODE must be legacy or combined" >&2 - exit 1 - ;; -esac +check_env_vars EVAL_ONLY SCHEDULER_RECV_INTERVAL=10 @@ -48,33 +41,27 @@ mkdir -p "$RESULT_DIR" CACHE_ARGS=() if require_agentic_kv_offload_backend hicache; then - # Preserve the existing budget unless the launcher selects the pinned - # nightly's combined KV/Mamba pool. Reserve 1 GB/rank for page alignment. + # SGLang applies --hicache-size independently to Qwen's target KV and + # Mamba pools. Native NEXTN also creates a draft KV pool with the same + # slot count; its one attention layer adds 1/15 of the target KV bytes. + # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. HICACHE_ALIGNMENT_RESERVE_GB=$TP HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 exit 1 fi - case "$QWEN35_HICACHE_BUDGET_MODE" in - legacy) - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - ;; - combined) - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB / TP)) - PROJECTED_HICACHE_TOTAL_GB=$((HICACHE_SIZE_GB * TP + HICACHE_ALIGNMENT_RESERVE_GB)) - ;; - esac + HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) if [ "$HICACHE_SIZE_GB" -lt 1 ]; then echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 exit 1 fi + PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 exit 1 fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB per rank with ${QWEN35_HICACHE_BUDGET_MODE} budgeting across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" + echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" CACHE_ARGS=( --page-size 64 --enable-hierarchical-cache diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b1e6cf2671..cbf8649fed 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8204,19 +8204,3 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: dp-attn: false kv-offload-backend: name: hicache - -# Aggregate candidates pending canonical sweep and performance qualification. -qwen3.5-fp8-b200-sglang-agentic-mtp-latest: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:b200-nscale - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.8 - search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 64] } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 02323bd0eb..87ed8fa04b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8176,7 +8176,7 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - config-keys: - - qwen3.5-fp8-b200-sglang-agentic-mtp-latest + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp scenario-type: - agentic-coding description: @@ -8220,7 +8220,7 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - config-keys: - - qwen3.5-fp8-b200-sglang-agentic-mtp-latest + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp scenario-type: - agentic-coding description: @@ -8244,7 +8244,7 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - config-keys: - - qwen3.5-fp8-b200-sglang-agentic-mtp-latest + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp scenario-type: - agentic-coding description: @@ -8264,3 +8264,18 @@ behavior for other callers so failed native jobs retain their diagnostics. Benchmark arguments, workload and selected recipes are unchanged. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Withdraw the provisional sparse aggregate publication scope and restore + the aggregate launcher/script to main. A regular latest-curve snapshot + replaces the prior aggregate scope; publishing only C4/C32/C64 would + discard existing frontier points. Keep aggregate experiments as comparison + evidence while qualifying only winning disaggregated additions alongside + the unchanged published aggregate curves. Earlier draft entries naming + the withdrawn aggregate key are retargeted to this final PR scope only; + all published changelog history remains byte-identical to main. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index c376569bf7..1b18a15dd6 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -773,15 +773,6 @@ run_single_node() { # submitted to the Nscale batch_1 partition. check_env_vars SALLOC_TIME_LIMIT GPU_COUNT - if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && - "$PRECISION" == "fp8" && "$FRAMEWORK" == "sglang" && "$SPEC_DECODING" == "mtp" ]]; then - check_env_vars IMAGE - export QWEN35_HICACHE_BUDGET_MODE=legacy - if [[ "$IMAGE" == "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85" ]]; then - export QWEN35_HICACHE_BUDGET_MODE=combined - fi - fi - SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" || "$SPEC_DECODING" == "draft_model" ]] && printf '_mtp' || printf '') From 1efe0d3c279b5e1a228810cb542da4c85306fc04 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:26:07 -0500 Subject: [PATCH 19/28] feat: expand B200 FP8 frontier qualification with concurrency 16 --- ...sagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml | 161 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 12 ++ 3 files changed, 191 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml new file mode 100644 index 0000000000..90670c32e9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml @@ -0,0 +1,161 @@ +# Experimental; retain only measured improvements over the current Pareto frontier. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 16 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 16 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index cbf8649fed..b8da88b53f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8168,6 +8168,24 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: agentic-coding: - dram-utilization: 0.8 search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 16 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache - spec-decoding: mtp kv-offloading: dram conc-list: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 87ed8fa04b..cedb773617 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8279,3 +8279,15 @@ the withdrawn aggregate key are retargeted to this final PR scope only; all published changelog history remains byte-identical to main. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Add measured B200 Qwen3.5 FP8 1P1D C16 H104 for expanded official + qualification. The short replay reaches 9493.56 logical tokens/s per + all eight active GPUs at 294.095 p90 interactivity and 0.908s p90 TTFT + with zero errors, extending the published frontier. The C8 diagnostic + is excluded because the same-image aggregate C4 control dominates it. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 8170706cd4cc7efc47b17485ef339eb9405c24bb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:27:47 -0500 Subject: [PATCH 20/28] feat: select B200 FP8 concurrency 24 over dominated 2P1D candidate --- ...agg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml} | 24 +++++++++---------- configs/nvidia-master.yaml | 8 +++---- perf-changelog.yaml | 14 +++++++++++ 3 files changed, 30 insertions(+), 16 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/{disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml => disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml} (88%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml similarity index 88% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml index efefc1867b..274f4343b4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml @@ -1,6 +1,6 @@ -# Provisional fast frontier extension; compare neighboring 1P1D points and qualify officially. +# Experimental; retain only measured improvements over the current Pareto frontier. schema: 2 -name: qwen35-b200-fp8-agentx-disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -31,11 +31,9 @@ frontend: roles: prefill: nodes: 1 - workers: 2 + workers: 1 gpus: 4 env: - # Use the eight GPU-associated 400 Gb/s NDR rails on this pool. - MC_TE_FILTERS: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com @@ -46,6 +44,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -63,7 +63,7 @@ roles: context-length: 262144 chunked-prefill-size: 16384 max-prefill-tokens: 16384 - max-running-requests: 64 + max-running-requests: 48 reasoning-parser: qwen3 tool-call-parser: qwen3_coder enable-metrics: true @@ -82,12 +82,10 @@ roles: hicache-mem-layout: page_first hicache-write-policy: write_through_selective decode: - nodes: 1 + nodes: colocate workers: 1 gpus: 4 env: - # Exclude secondary 100 Gb/s HDR functions and the Ethernet bond. - MC_TE_FILTERS: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com @@ -98,6 +96,8 @@ roles: SGL_ENABLE_JIT_DEEPGEMM: 'false' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 trust-remote-code: true @@ -115,7 +115,7 @@ roles: context-length: 262144 chunked-prefill-size: 16384 max-prefill-tokens: 16384 - max-running-requests: 64 + max-running-requests: 48 reasoning-parser: qwen3 tool-call-parser: qwen3_coder enable-metrics: true @@ -127,7 +127,7 @@ roles: speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 - cuda-graph-max-bs-decode: 32 + cuda-graph-max-bs-decode: 24 stream-interval: 50 enable-symm-mem: true scheduler-recv-interval: 10 @@ -145,7 +145,7 @@ benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh concurrencies: - - 32 + - 24 env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b8da88b53f..d3737fab20 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8189,14 +8189,14 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: - spec-decoding: mtp kv-offloading: dram conc-list: - - 32 + - 24 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml decode: num-worker: 1 tp: 4 @@ -8209,12 +8209,12 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: conc-list: - 32 prefill: - num-worker: 2 + num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-2p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml decode: num-worker: 1 tp: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cedb773617..6f6844d6c8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8291,3 +8291,17 @@ with zero errors, extending the published frontier. The C8 diagnostic is excluded because the same-image aggregate C4 control dominates it. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Replace the provisional 2P1D C32 point with measured 1P1D C24 H104. + Its short replay reaches 15784.02 logical tokens/s per eight active GPUs + at 231.403 p90 interactivity with zero errors or drain cancellations, + exceeding 2P1D C32 at 14511.73 per twelve active GPUs and 228.969. + The 2P layout has better TTFT but loses the requested primary frontier; + retain it as experimental evidence, not a final publication point. + Require fresh expanded official qualification for C16/C24/C32. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From a83fe811db5fbc0b4cf60367744387c4e639b883 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:35:49 -0500 Subject: [PATCH 21/28] feat: qualify B200 concurrency 8 against canonical aggregate evidence --- ...isagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml | 161 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 15 ++ 3 files changed, 194 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml new file mode 100644 index 0000000000..b07b5b3aa1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml @@ -0,0 +1,161 @@ +# Experimental; retain only measured improvements over the current Pareto frontier. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 32 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 8 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 8 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d3737fab20..192f7782a5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8168,6 +8168,24 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: agentic-coding: - dram-utilization: 0.8 search-space: + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 8 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache - spec-decoding: mtp kv-offloading: dram conc-list: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6f6844d6c8..2d0fa63167 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8305,3 +8305,18 @@ retain it as experimental evidence, not a final publication point. Require fresh expanded official qualification for C16/C24/C32. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Restore 1P1D C8 H104 as a provisional full-sweep qualification point + after the completed canonical aggregate C4 control measures only + 309.423 p90 interactivity, below its short-run 367.116. The C8 short + replay reaches 4953.76 logical tokens/s/GPU at 362.572 p90 interactivity + with zero errors or cancellations, extending the actual published + endpoint. Its earlier rejection was only against a short-run aggregate + comparator; retain C8 finally only if its own canonical result improves + the measured frontier. No aggregate publication scope is added. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From cbb19c6d7efec2c31bf403a13a0dfb634d351ebc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:54:25 -0500 Subject: [PATCH 22/28] fix: require validated B200 Qwen3.5 AgentX power telemetry --- ...sagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml | 11 +++ ...sagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml | 11 +++ ...sagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml | 11 +++ ...isagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml | 11 +++ perf-changelog.yaml | 12 ++++ runners/launch_b200-nscale-slurm.sh | 17 ++++- runners/test_b200_srt_status.py | 69 ++++++++++++++----- 7 files changed, 124 insertions(+), 18 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml index 90670c32e9..14da507af8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml @@ -141,6 +141,17 @@ sbatch_directives: srun_options: mem: '0' container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml index 274f4343b4..d48476b5d2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml @@ -141,6 +141,17 @@ sbatch_directives: srun_options: mem: '0' container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml index c4ec375e2b..9d862c4011 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml @@ -141,6 +141,17 @@ sbatch_directives: srun_options: mem: '0' container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml index b07b5b3aa1..c8de0c24ef 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml @@ -141,6 +141,17 @@ sbatch_directives: srun_options: mem: '0' container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2d0fa63167..610d5bb40b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8320,3 +8320,15 @@ comparator; retain C8 finally only if its own canonical result improves the measured frontier. No aggregate publication scope is added. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Enable required SRT DCGM power telemetry for the colocated C8/C16/C24/C32 + Qwen3.5 FP8 AgentX recipes and validate each finalized measurement package + before publication. Preserve diagnostic aggregates, power audits and eval + artifacts when collection fails; eval-only jobs skip power collection. + Require fresh official qualification with measured power for these points. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 1b18a15dd6..b58467ccab 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -545,6 +545,7 @@ run_multinode_srt() { fi USES_DCGM_POWER=0 + USES_AGENTX_POWER=0 _POWER_CONFIG_FILE="${CONFIG_FILE:-}" if [[ "${EVAL_ONLY}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then _POWER_CONFIG_FILE="$EVAL_CONFIG_FILE" @@ -560,13 +561,16 @@ run_multinode_srt() { ' "$_RECIPE_SRC"; then USES_DCGM_POWER=1 fi - if [[ "$USES_DCGM_POWER" == "1" && ( + if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + USES_AGENTX_POWER=1 + elif [[ "$USES_DCGM_POWER" == "1" && ( "${IS_AGENTIC}" == "1" || "$MODEL_PREFIX" != "dsv4" || "$PRECISION" != "fp4" || "$FRAMEWORK" != "dynamo-vllm" ) ]]; then - echo "Error: B200 Nscale dcgm-power is limited to fixed-sequence DSV4 FP4 dynamo-vllm" >&2 + echo "Error: B200 Nscale dcgm-power requires fixed-sequence DSV4 FP4 dynamo-vllm or Qwen3.5 FP8 AgentX dynamo-sglang" >&2 exit 1 fi @@ -720,6 +724,15 @@ run_multinode_srt() { echo "Found logs directory: $LOGS_DIR" + if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY}" != "true" ]]; then + check_env_vars CONC_LIST + local -a power_concurrencies + read -r -a power_concurrencies <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" \ + "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" \ + "$SRT_SLURM_COMMIT" "${power_concurrencies[@]}" || srt_job_rc=$? + fi + if [[ "$USES_DCGM_POWER" == "1" ]]; then mkdir -p "$LOGS_DIR/power" cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" diff --git a/runners/test_b200_srt_status.py b/runners/test_b200_srt_status.py index 7e1b291a28..c1881b3588 100644 --- a/runners/test_b200_srt_status.py +++ b/runners/test_b200_srt_status.py @@ -1,5 +1,6 @@ """A completed Slurm allocation must not hide a failed benchmark.""" +import json import os import shutil import subprocess @@ -18,18 +19,20 @@ def _stub(directory: Path, name: str, body: str) -> None: @pytest.mark.parametrize( - "accounting,expected_status", + "accounting,expected_status,power_mode", [ - ("COMPLETED|0:0", 0), - ("FAILED|1:0", 1), - ("COMPLETED|0:9", 1), - ("CANCELLED by 123|0:15", 1), - ("DELAYED", 0), - ("", 1), + ("COMPLETED|0:0", 0, "off"), + ("FAILED|1:0", 1, "off"), + ("COMPLETED|0:9", 1, "off"), + ("CANCELLED by 123|0:15", 1, "off"), + ("DELAYED", 0, "off"), + ("", 1, "off"), + ("COMPLETED|0:0", 1, "missing"), + ("COMPLETED|0:0", 0, "eval"), ], ) def test_b200_collects_artifacts_before_returning_slurm_status( - tmp_path: Path, accounting: str, expected_status: int + tmp_path: Path, accounting: str, expected_status: int, power_mode: str ) -> None: """Run the actual Qwen launcher path with only external services stubbed.""" binaries = tmp_path / "bin" @@ -40,11 +43,11 @@ def test_b200_collects_artifacts_before_returning_slurm_status( "make", "srtctl", "flock", - "unsquashfs", "squeue", "sleep", ): _stub(binaries, name, "exit 0") + _stub(binaries, "unsquashfs", 'touch "$2"; exit 0') _stub( binaries, "sacct", @@ -54,6 +57,10 @@ def test_b200_collects_artifacts_before_returning_slurm_status( [[ -f "$MOCK_SACCT_COUNT" ]] && read -r count < "$MOCK_SACCT_COUNT" count=$((count + 1)) printf '%s\n' "$count" > "$MOCK_SACCT_COUNT" +if [[ "$*" == *JobIDRaw* ]]; then + printf '42|%s\n' "$MOCK_ACCOUNTING" + exit 0 +fi if [[ "$MOCK_ACCOUNTING" == DELAYED ]]; then case "$count" in 1) exit 0 ;; @@ -71,11 +78,23 @@ def test_b200_collects_artifacts_before_returning_slurm_status( "tail", 'for arg in "$@"; do [[ -f "$arg" ]] && cat "$arg"; done; exit 0', ) + # Shared-cluster cache paths are external to this local launcher test. + _stub( + binaries, + "mkdir", + 'for arg in "$@"; do [[ "$arg" == /data/* ]] && exit 0; done; exec /bin/mkdir "$@"', + ) workspace = tmp_path / "workspace" workspace.mkdir() # Keep launcher-owned runtime inputs with the real launcher and helpers. shutil.copytree(ROOT / "runners", workspace / "runners") (workspace / "benchmarks").mkdir() + if power_mode != "off": + recipe = workspace / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text( + "telemetry:\n enabled: true\n dcgm_exporter:\n container_image: dcgm-exporter\n" + ) shutil.copyfile( ROOT / "benchmarks/benchmark_lib.sh", workspace / "benchmarks/benchmark_lib.sh" ) @@ -88,11 +107,13 @@ def test_b200_collects_artifacts_before_returning_slurm_status( mkdir -p "$1/recipes" cd "$1" || return 1 printf 'name: fixture\n' > recipes/test.yaml + SRT_SLURM_COMMIT=1111111111111111111111111111111111111111 + printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/power-producer-sha.txt" } apply_srt_recipe() { mkdir -p outputs/42/logs cp -R "$MOCK_FIXTURE/." outputs/42/logs/ - printf '{"diagnostic":"retained"}\n' > "$GITHUB_WORKSPACE/aggregate_conc1.json" + printf '{"diagnostic":"retained","disagg":true,"num_prefill_gpu":4,"num_decode_gpu":4}\n' > "$GITHUB_WORKSPACE/aggregate_conc1.json" printf '✅ Job 42\n' } """) @@ -114,8 +135,9 @@ def test_b200_collects_artifacts_before_returning_slurm_status( "MOCK_SACCT_COUNT": str(tmp_path / "sacct-count"), "MOCK_FIXTURE": str(fixture), "GITHUB_WORKSPACE": str(workspace), - "EVAL_ONLY": "false", - "IS_AGENTIC": "0", + "EVAL_ONLY": "true" if power_mode == "eval" else "false", + "IS_AGENTIC": "1" if power_mode != "off" else "0", + "CONC_LIST": "1", "IS_MULTINODE": "true", "RUN_EVAL": "true", "SLURM_PARTITION": "batch_1", @@ -164,9 +186,16 @@ def test_b200_collects_artifacts_before_returning_slurm_status( timeout=15, ) assert result.returncode == expected_status, result.stdout + result.stderr - assert ( - workspace / "aggregate_conc1.json" - ).read_text() == '{"diagnostic":"retained"}\n' + aggregate = json.loads((workspace / "aggregate_conc1.json").read_text()) + assert aggregate["diagnostic"] == "retained" + if power_mode == "missing": + validation = json.loads( + (workspace / "LOGS/agentic/conc_1/power_validation.json").read_text() + ) + assert validation["power_valid"] is False + assert validation["reasons"] == ["formal_benchmark_result_missing"] + else: + assert not (workspace / "LOGS/agentic/conc_1/power_validation.json").exists() assert (workspace / "results_eval.json").read_text() == '{"eval":"retained"}\n' assert (workspace / "LOGS/sweep_42.log").read_text() == "benchmark diagnostics\n" with tarfile.open(workspace / "multinode_server_logs.tar.gz") as archive: @@ -175,5 +204,13 @@ def test_b200_collects_artifacts_before_returning_slurm_status( in archive.getnames() ) assert not (workspace / "srt-slurm/outputs").exists() - expected_queries = 3 if accounting == "DELAYED" else 10 if accounting == "" else 1 + expected_queries = ( + 2 + if power_mode == "missing" + else 3 + if accounting == "DELAYED" + else 10 + if accounting == "" + else 1 + ) assert int((tmp_path / "sacct-count").read_text()) == expected_queries From 6a9ecdb3107226fb4149f88c798a7b69154aefb3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:14:25 -0500 Subject: [PATCH 23/28] feat: qualify B200 FP8 concurrency 64 with first-use HiCache backup --- ...4-d-tp4-hicache-c64-write-through-mtp.yaml | 173 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 13 ++ 3 files changed, 204 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml new file mode 100644 index 0000000000..adc312b323 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml @@ -0,0 +1,173 @@ +# Short replay improves the published frontier; full official qualification remains required. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 128 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + # Combined KV/Mamba budget: 860 GB per TP4 prefill worker. + hicache-size: 215 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.88 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 128 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 64 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 64 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 192f7782a5..84c32be6c9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8240,3 +8240,21 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: dp-attn: false kv-offload-backend: name: hicache + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 64 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 610d5bb40b..555cc17d83 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8332,3 +8332,16 @@ artifacts when collection fails; eval-only jobs skip power collection. Require fresh official qualification with measured power for these points. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Add measured B200 Qwen3.5 FP8 concurrency 64 with first-use prefill HiCache write-through + at 215 GB per rank and decode static memory 0.88. The short replay reaches 35684.49 logical + tokens/s/GPU at 132.967 p90 interactivity and 3.861s p90 TTFT, with zero errors or drain + cancellations, dominating published aggregate TP4 C24. Preserve exact serving settings and + require native power telemetry, a full official sweep and real-verification evals before + qualification. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 29d50971da35f69aad3a05fc317f1a8aa3a07736 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:29:22 -0500 Subject: [PATCH 24/28] fix: supply B200 AgentX power window contract --- .../disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml | 3 +++ .../disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml | 3 +++ .../disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml | 3 +++ ...p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml | 3 +++ .../disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml | 3 +++ perf-changelog.yaml | 11 +++++++++++ 6 files changed, 26 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml index 14da507af8..a3f8bb681c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml @@ -160,6 +160,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '16' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' EVAL_FRAMEWORK: lm-eval diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml index d48476b5d2..a9af6f1251 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml @@ -160,6 +160,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '24' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' EVAL_FRAMEWORK: lm-eval diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml index 9d862c4011..5dab4917a0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml @@ -160,6 +160,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' EVAL_FRAMEWORK: lm-eval diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml index adc312b323..ac6a00a1d8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml @@ -161,6 +161,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '64' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' EVAL_FRAMEWORK: lm-eval diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml index c8de0c24ef..2492edcae4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml @@ -160,6 +160,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '8' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' EVAL_FRAMEWORK: lm-eval diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 555cc17d83..9c27c3501a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8345,3 +8345,14 @@ require native power telemetry, a full official sweep and real-verification evals before qualification. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Supply the custom AgentX measurement type, exact point concurrency and result root + explicitly to the native SRT power-window consumer. The pinned producer supplies + the window directory; the benchmark caller supplies the remaining contract fields. + Require fresh power qualification and the complete official sweep. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From beae2374bc40d4ea4a0cf0b3958cddc322c9e882 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:43:54 -0500 Subject: [PATCH 25/28] config: add measured B200 FP8 C48 write-through point --- ...4-d-tp4-hicache-c48-write-through-mtp.yaml | 176 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 13 ++ 3 files changed, 207 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml new file mode 100644 index 0000000000..5197d3be5a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml @@ -0,0 +1,176 @@ +# Experimental; retain only measured improvements over the current Pareto frontier. +schema: 2 +name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +engine: sglang +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +resources: + gpu_type: b200 + gpus_per_node: 8 +slurm: + time_limit: '4:00:00' +frontend: + type: dynamo + nginx_container: nginx-sqsh + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.8 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 96 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + disable-cuda-graph: true + enable-symm-mem: true + scheduler-recv-interval: 10 + stream-interval: 50 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + # Combined KV/Mamba budget: 860 GB per TP4 prefill worker. + hicache-size: 215 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + SGLANG_ENABLE_SPEC_V2: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + PYTHONNOUSERSITE: '1' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + page-size: 64 + mem-fraction-static: 0.88 + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 96 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + enable-metrics: true + enable-cache-report: true + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs-decode: 48 + stream-interval: 50 + enable-symm-mem: true + scheduler-recv-interval: 10 + mamba-radix-cache-strategy: no_buffer + mamba-track-interval: 128 +health_check: + max_attempts: 720 + interval_seconds: 10 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + concurrencies: + - 48 + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + EVAL_FRAMEWORK: lm-eval + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_MAX_CONTEXT_LENGTH: '262144' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index da4b47b7b0..12902fc0f3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8339,6 +8339,24 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: dp-attn: false kv-offload-backend: name: hicache + - spec-decoding: mtp + kv-offloading: dram + conc-list: + - 48 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kv-offload-backend: + name: hicache - spec-decoding: mtp kv-offloading: dram conc-list: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 187a2398c1..866b8e4113 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8539,3 +8539,16 @@ the window directory; the benchmark caller supplies the remaining contract fields. Require fresh power qualification and the complete official sweep. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Add measured colocated 1P1D concurrency 48 with 215 GB per-rank prefill + HiCache write-through and decode static memory 0.88. Its short replay + reaches 28722.10 logical tokens/s per all eight GPUs at 172.553 p90 + interactivity and 2.318s p90 TTFT, with 2643 profile requests, zero errors + and zero drain cancellations. The six-point matrix requires fresh full + official sweep, complete native power artifacts and real-verification evals. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 0b75e138be098de0122959460c765860f79b95fd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:51:30 -0500 Subject: [PATCH 26/28] docs: consolidate B200 PR performance changelog --- perf-changelog.yaml | 272 +++----------------------------------------- 1 file changed, 15 insertions(+), 257 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b7d290782b..17ada77840 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8360,266 +8360,24 @@ - "在该 key 中设置了该参数的 5 个 srt-slurm 配方里，将 decode 角色的 prefill-round-robin-balance: true 替换为 load-balance-method: follow_bootstrap_room：SGLang nightly-dev-cu13-20260918-20518d85 已没有 --prefill-round-robin-balance（运行 35319100186 中 DEP16 分支的所有 decode worker 在参数解析阶段退出），其 --load-balance-method 可选值为 auto、round_robin、follow_bootstrap_room、total_requests 与 total_tokens；prefill 角色保持 load-balance-method: round_robin" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3253 -- config-keys: - - dsv41flash-fp4-h100-vllm-agentic-dspark - scenario-type: - - agentic-coding - description: - - "Add concurrency 20, 24 and 28 to the H100 TP8 AgentX arm to fill in the saturation curve between the existing c16 point and the range where 1M-token trajectories start to preempt; recipe script, image and DSpark settings are unchanged" - - "为 H100 TP8 AgentX 配方新增并发 20、24、28，以填补现有 c16 点与 1M token 轨迹开始抢占区间之间的饱和曲线；配方脚本、镜像与 DSpark 设置不变" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3248 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - 'Add Qwen3.5 397B FP8 B200 Dynamo + SGLang disaggregated AgentX: one TP4 prefill - worker and one TP4 decode worker with NEXTN MTP, concurrency 1/4/8/16.' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Align B200 Qwen3.5 FP8 1P1D AgentX with the aggregate serving baseline; colocate - TP4 prefill and TP4 decode on one eight-GPU node, match per-concurrency request/graph - limits, and extend the initial study to concurrency 32. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Pin B200 Qwen3.5 FP8 disaggregated AgentX to the latest CUDA 13 amd64 nightly, - nightly-dev-cu13-20260918-20518d85, and use the current mamba-radix-cache-strategy - argument for prefill and decode. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Update B200 Qwen3.5 FP8 AgentX to Dynamo 805a77f053d807b0d8def5d27f674a6df0ed839e - for compatibility with the latest SGLang nightly's relocated argument parser - and encoder modules. Keep recipe and result router version pins aligned. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Apply the existing B200 node-local-weight preflight handling to Qwen3.5 FP8 - Dynamo SGLang jobs. The checkpoint is present on compute-node NVMe but absent - from the login node; retain worker-side model loading checks. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Use Mooncake intra-node NVLink transport and compatible device allocations - for the colocated B200 Qwen3.5 FP8 1P1D AgentX recipes. This transport passed - four real-verification short/long-input probes on B300; B200 validation and - normalized performance qualification remain pending. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Add latest-nightly B200 Qwen3.5 FP8 aggregate AgentX candidates at TP4 C4 - without offloading and TP4 C32 with HiCache, pending official sweep and - normalized performance qualification. Use the combined KV/Mamba host-pool - budget only for the pinned latest image, preserving legacy aggregate sizing. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Verify terminal Slurm allocation status for B200 Qwen3.5 FP8 disaggregated - runs and preserve diagnostic artifacts before returning benchmark failure. - Prevent aborted replay jobs from appearing successful after leaving squeue. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Resolve NVIDIA Dynamo dependencies directly from the official NVIDIA - wheel index. Retain pip integrity verification while avoiding the PyPI - placeholder downloader that repeatedly truncated aisimulate wheels. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Select the measured B200 Qwen3.5 FP8 colocated TP4 prefill plus TP4 decode - C32 candidate with prefill HiCache at 104 GB per rank. The short replay - reaches 20645.73 logical tokens/s/GPU at 211.219 p90 interactivity with - zero request errors, improving the published frontier; canonical sweep - and real-verification evals remain required. Remove unqualified starter - points from the PR selection. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Add measured B200 Qwen3.5 FP8 TP4 aggregate C64 HiCache for canonical - qualification. Its short replay reaches 41896.93 logical tokens/s/GPU - at 50.042 p90 interactivity with zero errors, extending the published - throughput frontier at the cost of lower interactivity and higher TTFT. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Stage B200 Qwen3.5 FP8 2P1D C32 HiCache with verified 400 Gb/s RDMA - interfaces for expanded qualification. The short replay reaches 14511.73 - logical tokens/s per all 12 active GPUs at 228.969 p90 interactivity and - 0.851s p90 TTFT with zero errors, extending the published frontier. - Compare neighboring 1P1D candidates and require official sweep success - before final retention; the allocation contains 16 GPUs. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Require the B200 aggregate caller to supply the HiCache budget mode, - following repository environment-input policy. The unified launcher - already supplies legacy or combined explicitly; generated benchmark - arguments and existing image-specific cache sizes remain unchanged. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Scope terminal Slurm accounting verification to the artifact-preserving - multinode launcher path. Keep the shared log-streaming helper's existing - behavior for other callers so failed native jobs retain their diagnostics. - Benchmark arguments, workload and selected recipes are unchanged. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Withdraw the provisional sparse aggregate publication scope and restore - the aggregate launcher/script to main. A regular latest-curve snapshot - replaces the prior aggregate scope; publishing only C4/C32/C64 would - discard existing frontier points. Keep aggregate experiments as comparison - evidence while qualifying only winning disaggregated additions alongside - the unchanged published aggregate curves. Earlier draft entries naming - the withdrawn aggregate key are retargeted to this final PR scope only; - all published changelog history remains byte-identical to main. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Add measured B200 Qwen3.5 FP8 1P1D C16 H104 for expanded official - qualification. The short replay reaches 9493.56 logical tokens/s per - all eight active GPUs at 294.095 p90 interactivity and 0.908s p90 TTFT - with zero errors, extending the published frontier. The C8 diagnostic - is excluded because the same-image aggregate C4 control dominates it. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Replace the provisional 2P1D C32 point with measured 1P1D C24 H104. - Its short replay reaches 15784.02 logical tokens/s per eight active GPUs - at 231.403 p90 interactivity with zero errors or drain cancellations, - exceeding 2P1D C32 at 14511.73 per twelve active GPUs and 228.969. - The 2P layout has better TTFT but loses the requested primary frontier; - retain it as experimental evidence, not a final publication point. - Require fresh expanded official qualification for C16/C24/C32. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Restore 1P1D C8 H104 as a provisional full-sweep qualification point - after the completed canonical aggregate C4 control measures only - 309.423 p90 interactivity, below its short-run 367.116. The C8 short - replay reaches 4953.76 logical tokens/s/GPU at 362.572 p90 interactivity - with zero errors or cancellations, extending the actual published - endpoint. Its earlier rejection was only against a short-run aggregate - comparator; retain C8 finally only if its own canonical result improves - the measured frontier. No aggregate publication scope is added. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Enable required SRT DCGM power telemetry for the colocated C8/C16/C24/C32 - Qwen3.5 FP8 AgentX recipes and validate each finalized measurement package - before publication. Preserve diagnostic aggregates, power audits and eval - artifacts when collection fails; eval-only jobs skip power collection. - Require fresh official qualification with measured power for these points. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Add measured B200 Qwen3.5 FP8 concurrency 64 with first-use prefill HiCache write-through - at 215 GB per rank and decode static memory 0.88. The short replay reaches 35684.49 logical - tokens/s/GPU at 132.967 p90 interactivity and 3.861s p90 TTFT, with zero errors or drain - cancellations, dominating published aggregate TP4 C24. Preserve exact serving settings and - require native power telemetry, a full official sweep and real-verification evals before - qualification. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - -- config-keys: - - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - Supply the custom AgentX measurement type, exact point concurrency and result root - explicitly to the native SRT power-window consumer. The pinned producer supplies - the window directory; the benchmark caller supplies the remaining contract fields. - Require fresh power qualification and the complete official sweep. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 - - config-keys: - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp scenario-type: - agentic-coding description: - - Add measured colocated 1P1D concurrency 48 with 215 GB per-rank prefill - HiCache write-through and decode static memory 0.88. Its short replay - reaches 28722.10 logical tokens/s per all eight GPUs at 172.553 p90 - interactivity and 2.318s p90 TTFT, with 2643 profile requests, zero errors - and zero drain cancellations. The six-point matrix requires fresh full - official sweep, complete native power artifacts and real-verification evals. + - Add Qwen3.5-397B-A17B FP8 AgentX recipes for B200 with one TP4 prefill worker + and one TP4 decode worker colocated on eight GPUs, at concurrency 8, 16, 24, + 32, 48 and 64. Use lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85, + Dynamo source 805a77f053d807b0d8def5d27f674a6df0ed839e and native srt-slurm + 984180e5b8755aef85e9995048b5a16cb5336bce with Mooncake intra-node NVLink. + Use 104 GB per-rank prefill HiCache at C8-C32; C48/C64 use 215 GB per rank + with first-use write-through and decode static memory 0.88. Preserve native + NEXTN MTP, automatic measured acceptance for throughput and real verification + for evals. Require native eight-GPU power telemetry and explicit formal-window + metadata; propagate failed Slurm/power status after preserving artifacts. + Retain compute-node checkpoint loading checks and official NVIDIA wheel integrity. + Short-run candidates improve the published throughput-per-GPU versus p90 + interactivity frontier; require a fresh full official sweep and all applicable + evals before qualification. Aggregate and rejected 2P1D experiments remain + comparison evidence; existing aggregate publication scopes are unchanged. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 From 5ed8f2eb5f39ad35331e9b76811cd616128fac63 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 11:58:18 -0500 Subject: [PATCH 27/28] test: remove B200 SRT status tests --- runners/test_b200_srt_status.py | 216 -------------------------------- 1 file changed, 216 deletions(-) delete mode 100644 runners/test_b200_srt_status.py diff --git a/runners/test_b200_srt_status.py b/runners/test_b200_srt_status.py deleted file mode 100644 index c1881b3588..0000000000 --- a/runners/test_b200_srt_status.py +++ /dev/null @@ -1,216 +0,0 @@ -"""A completed Slurm allocation must not hide a failed benchmark.""" - -import json -import os -import shutil -import subprocess -import tarfile -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[1] - - -def _stub(directory: Path, name: str, body: str) -> None: - path = directory / name - path.write_text("#!/bin/bash\n" + body + "\n") - path.chmod(0o755) - - -@pytest.mark.parametrize( - "accounting,expected_status,power_mode", - [ - ("COMPLETED|0:0", 0, "off"), - ("FAILED|1:0", 1, "off"), - ("COMPLETED|0:9", 1, "off"), - ("CANCELLED by 123|0:15", 1, "off"), - ("DELAYED", 0, "off"), - ("", 1, "off"), - ("COMPLETED|0:0", 1, "missing"), - ("COMPLETED|0:0", 0, "eval"), - ], -) -def test_b200_collects_artifacts_before_returning_slurm_status( - tmp_path: Path, accounting: str, expected_status: int, power_mode: str -) -> None: - """Run the actual Qwen launcher path with only external services stubbed.""" - binaries = tmp_path / "bin" - binaries.mkdir() - for name in ( - "curl", - "uv", - "make", - "srtctl", - "flock", - "squeue", - "sleep", - ): - _stub(binaries, name, "exit 0") - _stub(binaries, "unsquashfs", 'touch "$2"; exit 0') - _stub( - binaries, - "sacct", - r""" -[[ " $* " == *" -X "* ]] || exit 2 -count=0 -[[ -f "$MOCK_SACCT_COUNT" ]] && read -r count < "$MOCK_SACCT_COUNT" -count=$((count + 1)) -printf '%s\n' "$count" > "$MOCK_SACCT_COUNT" -if [[ "$*" == *JobIDRaw* ]]; then - printf '42|%s\n' "$MOCK_ACCOUNTING" - exit 0 -fi -if [[ "$MOCK_ACCOUNTING" == DELAYED ]]; then - case "$count" in - 1) exit 0 ;; - 2) printf 'RUNNING|0:0\n' ;; - *) printf 'COMPLETED|0:0\n' ;; - esac -else - printf '%s\n' "$MOCK_ACCOUNTING" -fi -""", - ) - # Tail's long-running process is a clock collaborator; logs remain real. - _stub( - binaries, - "tail", - 'for arg in "$@"; do [[ -f "$arg" ]] && cat "$arg"; done; exit 0', - ) - # Shared-cluster cache paths are external to this local launcher test. - _stub( - binaries, - "mkdir", - 'for arg in "$@"; do [[ "$arg" == /data/* ]] && exit 0; done; exec /bin/mkdir "$@"', - ) - workspace = tmp_path / "workspace" - workspace.mkdir() - # Keep launcher-owned runtime inputs with the real launcher and helpers. - shutil.copytree(ROOT / "runners", workspace / "runners") - (workspace / "benchmarks").mkdir() - if power_mode != "off": - recipe = workspace / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" - recipe.parent.mkdir(parents=True) - recipe.write_text( - "telemetry:\n enabled: true\n dcgm_exporter:\n container_image: dcgm-exporter\n" - ) - shutil.copyfile( - ROOT / "benchmarks/benchmark_lib.sh", workspace / "benchmarks/benchmark_lib.sh" - ) - # Run the real configuration renderer regardless of editable-install state. - (workspace / "infx").symlink_to(ROOT / "infx", target_is_directory=True) - # Stub remote checkout and submission after loading the real shared helpers. - with (workspace / "runners/slurm_utils.sh").open("a") as helpers: - helpers.write(r""" -setup_srt_slurm() { - mkdir -p "$1/recipes" - cd "$1" || return 1 - printf 'name: fixture\n' > recipes/test.yaml - SRT_SLURM_COMMIT=1111111111111111111111111111111111111111 - printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/power-producer-sha.txt" -} -apply_srt_recipe() { - mkdir -p outputs/42/logs - cp -R "$MOCK_FIXTURE/." outputs/42/logs/ - printf '{"diagnostic":"retained","disagg":true,"num_prefill_gpu":4,"num_decode_gpu":4}\n' > "$GITHUB_WORKSPACE/aggregate_conc1.json" - printf '✅ Job 42\n' -} -""") - activation = workspace / ".venv/bin/activate" - activation.parent.mkdir(parents=True) - activation.write_text(":\n") - fixture = tmp_path / "fixture" - (fixture / "agentic/conc_1/aiperf_artifacts").mkdir(parents=True) - (fixture / "eval_results").mkdir() - (fixture / "sweep_42.log").write_text("benchmark diagnostics\n") - (fixture / "agentic/conc_1/aiperf_artifacts/profile_export_aiperf.json").write_text( - '{"metadata":{"submission_valid":false}}\n' - ) - (fixture / "eval_results/results_eval.json").write_text('{"eval":"retained"}\n') - env = { - **os.environ, - "PATH": f"{binaries}:{os.environ['PATH']}", - "MOCK_ACCOUNTING": accounting, - "MOCK_SACCT_COUNT": str(tmp_path / "sacct-count"), - "MOCK_FIXTURE": str(fixture), - "GITHUB_WORKSPACE": str(workspace), - "EVAL_ONLY": "true" if power_mode == "eval" else "false", - "IS_AGENTIC": "1" if power_mode != "off" else "0", - "CONC_LIST": "1", - "IS_MULTINODE": "true", - "RUN_EVAL": "true", - "SLURM_PARTITION": "batch_1", - "SLURM_ACCOUNT": "benchmark", - "FRAMEWORK": "dynamo-sglang", - "MODEL_PREFIX": "qwen3.5", - "PRECISION": "fp8", - "MODEL": "Qwen/Qwen3.5-397B-A17B-FP8", - "SPEC_DECODING": "mtp", - "CONFIG_FILE": "recipes/test.yaml", - "RUNNER_NAME": "b200-nscale-slurm_08", - "IMAGE": "fixture-image", - "B200_SQUASH_DIR": str(tmp_path / "containers"), - "B200_SQUASH_LOCK_TIMEOUT": "1", - "RESULT_FILENAME": "aggregate", - "ISL": "1", - "OSL": "1", - } - result = subprocess.run( - [ - "bash", - "-c", - r""" -# GNU-only command options in this Linux launcher are irrelevant to the -# completion contract; keep the test runnable on macOS too. -grep() { - if [[ "$1" == -oP ]]; then cat >/dev/null; printf '42\n'; else command grep "$@"; fi -} -sed() { - if [[ "$1" == -i && "$(uname)" == Darwin ]]; then - shift; command sed -i '' "$@" - else - command sed "$@" - fi -} -builtin source "$1/runners/launch_b200-nscale-slurm.sh" -""", - "bash", - str(workspace), - ], - cwd=workspace, - env=env, - text=True, - capture_output=True, - check=False, - timeout=15, - ) - assert result.returncode == expected_status, result.stdout + result.stderr - aggregate = json.loads((workspace / "aggregate_conc1.json").read_text()) - assert aggregate["diagnostic"] == "retained" - if power_mode == "missing": - validation = json.loads( - (workspace / "LOGS/agentic/conc_1/power_validation.json").read_text() - ) - assert validation["power_valid"] is False - assert validation["reasons"] == ["formal_benchmark_result_missing"] - else: - assert not (workspace / "LOGS/agentic/conc_1/power_validation.json").exists() - assert (workspace / "results_eval.json").read_text() == '{"eval":"retained"}\n' - assert (workspace / "LOGS/sweep_42.log").read_text() == "benchmark diagnostics\n" - with tarfile.open(workspace / "multinode_server_logs.tar.gz") as archive: - assert ( - "./agentic/conc_1/aiperf_artifacts/profile_export_aiperf.json" - in archive.getnames() - ) - assert not (workspace / "srt-slurm/outputs").exists() - expected_queries = ( - 2 - if power_mode == "missing" - else 3 - if accounting == "DELAYED" - else 10 - if accounting == "" - else 1 - ) - assert int((tmp_path / "sacct-count").read_text()) == expected_queries From 8ce3db8049e150c9773da712416331de1ce4dbb6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 11:58:57 -0500 Subject: [PATCH 28/28] docs: shorten B200 performance changelog --- perf-changelog.yaml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a8871fa957..7eecb66696 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8411,19 +8411,9 @@ scenario-type: - agentic-coding description: - - Add Qwen3.5-397B-A17B FP8 AgentX recipes for B200 with one TP4 prefill worker - and one TP4 decode worker colocated on eight GPUs, at concurrency 8, 16, 24, - 32, 48 and 64. Use lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85, - Dynamo source 805a77f053d807b0d8def5d27f674a6df0ed839e and native srt-slurm - 984180e5b8755aef85e9995048b5a16cb5336bce with Mooncake intra-node NVLink. - Use 104 GB per-rank prefill HiCache at C8-C32; C48/C64 use 215 GB per rank - with first-use write-through and decode static memory 0.88. Preserve native - NEXTN MTP, automatic measured acceptance for throughput and real verification - for evals. Require native eight-GPU power telemetry and explicit formal-window - metadata; propagate failed Slurm/power status after preserving artifacts. - Retain compute-node checkpoint loading checks and official NVIDIA wheel integrity. - Short-run candidates improve the published throughput-per-GPU versus p90 - interactivity frontier; require a fresh full official sweep and all applicable - evals before qualification. Aggregate and rejected 2P1D experiments remain - comparison evidence; existing aggregate publication scopes are unchanged. + - Add B200 Qwen3.5-397B-A17B FP8 AgentX disaggregation with colocated TP4 prefill + and TP4 decode on eight GPUs at concurrency 8, 16, 24, 32, 48 and 64. + - Use Dynamo/SGLang nightly 20260918-20518d85 with NEXTN MTP and Mooncake NVLink transfers. + - Enable 104 GB per-rank prefill HiCache at C8-C32; use 215 GB with first-use + write-through and decode static memory 0.88 at C48/C64. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269