From ad1e96016ad61d6b76645387700a2852466f157c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 12:02:48 -0500 Subject: [PATCH 01/34] feat: add B300 Qwen3.5 FP8 disaggregated AgentX starter --- .../agentx/disagg-1p1d-tp4-tp4-mtp.yaml | 151 ++++++++++++++++++ configs/nvidia-master.yaml | 37 +++++ perf-changelog.yaml | 10 ++ 3 files changed, 198 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml new file mode 100644 index 0000000000..831a0918c1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml @@ -0,0 +1,151 @@ +# Starter adaptation of the SGLang Qwen3.5 cookbook and existing GB300 AgentX PD recipe. +# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# B300 runtime qualification and tuning are pending. +schema: 2 +name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-mtp +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 64 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + max-running-requests: 64 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-scheduler-strategy: no_buffer + disable-radix-cache: true + cuda-graph-max-bs: 64 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f28e74de34..c8bff8ec3e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8150,3 +8150,40 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: # 8x141 GB holds the 511 GB checkpoint minus the UVA-offloaded Engram # tables, so the KV cache stays GPU-resident across the full range. - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + +qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:b300-dsxe + precision: fp8 + framework: dynamo-sglang + router: + name: dynamo-router + version: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.8 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: + - 1 + - 4 + - 8 + - 16 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a112ba56db..7a5e3f46f2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8082,3 +8082,13 @@ - "Capture only full decode CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=0 with cudagraph_mode FULL_DECODE_ONLY, as the MiniMax-M3 gfx942 arm does) and restore --moe-backend aiter: on gfx942 every worker segfaulted during piecewise graph capture with both the Triton W4A16 MoE kernel (run 35305045778) and the auto-selected unfused Triton kernel (run 35306398350), so the capture mode rather than the MoE kernel is the failing piece; prefill runs eagerly" - "仅捕获完整的 decode CUDA graph（VLLM_USE_BREAKABLE_CUDAGRAPH=0 并设置 cudagraph_mode FULL_DECODE_ONLY，与 MiniMax-M3 gfx942 配方一致）并恢复 --moe-backend aiter：在 gfx942 上，无论使用 Triton W4A16 MoE 内核（运行 35305045778）还是自动选择的未融合 Triton 内核（运行 35306398350），所有 worker 都在 piecewise graph 捕获期间段错误，说明问题在于捕获模式而非 MoE 内核；prefill 以 eager 方式运行" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3247 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - 'Add a starter Qwen3.5-397B FP8 B300 disaggregated AgentX recipe using Dynamo + and SGLang through srt-slurm: 1P1D TP4, NEXTN MTP, and concurrencies 1, 4, 8, + 16.' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 1803e0d796777f6b4992083540647df1c6b37689 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 12:03:23 -0500 Subject: [PATCH 02/34] chore: link B300 starter performance changelog --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7a5e3f46f2..3033ec3965 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8091,4 +8091,4 @@ - 'Add a starter Qwen3.5-397B FP8 B300 disaggregated AgentX recipe using Dynamo and SGLang through srt-slurm: 1P1D TP4, NEXTN MTP, and concurrencies 1, 4, 8, 16.' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From d8cd1b4c80acaf051f3a2e264c78ac58ccdc3c8e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 12:52:40 -0500 Subject: [PATCH 03/34] perf: colocate B300 Qwen3.5 prefill and decode workers --- .../disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml | 162 ++++++++++++++++++ ...disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml | 162 ++++++++++++++++++ ...isagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml} | 25 ++- .../disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml | 162 ++++++++++++++++++ .../disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml | 162 ++++++++++++++++++ configs/nvidia-master.yaml | 63 ++++++- perf-changelog.yaml | 10 ++ 7 files changed, 738 insertions(+), 8 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/{disagg-1p1d-tp4-tp4-mtp.yaml => disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml} (88%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml new file mode 100644 index 0000000000..8bf29c2c89 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml @@ -0,0 +1,162 @@ +# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# Separate TP4 GPU groups share one node. GPU qualification pending. +schema: 2 +name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c1 +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 2 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-scheduler-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 2 + cuda-graph-max-bs-decode: 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml new file mode 100644 index 0000000000..a0d6eb6c3b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml @@ -0,0 +1,162 @@ +# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# Separate TP4 GPU groups share one node. GPU qualification pending. +schema: 2 +name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c16 +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-scheduler-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml similarity index 88% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml index 831a0918c1..aa248be2af 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml @@ -1,8 +1,7 @@ -# Starter adaptation of the SGLang Qwen3.5 cookbook and existing GB300 AgentX PD recipe. # Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# B300 runtime qualification and tuning are pending. +# Separate TP4 GPU groups share one node. GPU qualification pending. schema: 2 -name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-mtp +name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c32 model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 @@ -48,6 +47,10 @@ roles: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 model-path: /model/ @@ -69,7 +72,6 @@ roles: mem-fraction-static: 0.8 chunked-prefill-size: 16384 max-prefill-tokens: 16384 - max-running-requests: 64 page-size: 64 reasoning-parser: qwen3 tool-call-parser: qwen3_coder @@ -79,8 +81,11 @@ roles: disable-cuda-graph: true mamba-scheduler-strategy: extra_buffer mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 decode: - nodes: 1 + nodes: colocate workers: 1 gpus: 4 env: @@ -94,6 +99,10 @@ roles: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' args: served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 model-path: /model/ @@ -115,7 +124,6 @@ roles: mem-fraction-static: 0.8 chunked-prefill-size: 16384 max-prefill-tokens: 16384 - max-running-requests: 64 page-size: 64 reasoning-parser: qwen3 tool-call-parser: qwen3_coder @@ -124,11 +132,14 @@ roles: watchdog-timeout: 3600 mamba-scheduler-strategy: no_buffer disable-radix-cache: true - cuda-graph-max-bs: 64 speculative-algorithm: NEXTN speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 sbatch_directives: mem: '0' cpus-per-task: '144' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml new file mode 100644 index 0000000000..04b2ae93f4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml @@ -0,0 +1,162 @@ +# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# Separate TP4 GPU groups share one node. GPU qualification pending. +schema: 2 +name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c4 +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 8 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-scheduler-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 8 + cuda-graph-max-bs-decode: 4 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml new file mode 100644 index 0000000000..0ccd287fea --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml @@ -0,0 +1,162 @@ +# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +# Separate TP4 GPU groups share one node. GPU qualification pending. +schema: 2 +name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c8 +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 16 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-scheduler-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 16 + cuda-graph-max-bs-decode: 8 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c8bff8ec3e..67e0c101db 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8172,8 +8172,53 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: kv-offloading: none conc-list: - 1 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: - 4 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: - 8 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: - 16 prefill: num-worker: 1 @@ -8181,7 +8226,23 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-mtp.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: + - 32 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml decode: num-worker: 1 tp: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3033ec3965..3ac5649610 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8092,3 +8092,13 @@ and SGLang through srt-slurm: 1P1D TP4, NEXTN MTP, and concurrencies 1, 4, 8, 16.' pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Place the 1P1D TP4/TP4 B300 AgentX starter on one eight-GPU node with disjoint + GPU groups; add concurrency 32 and match aggregate per-concurrency request limits, + decode graph sizing, scheduler interval, and streaming cadence. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From c66edd31f7edf2074e28a780692ecca920db9203 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 13:17:46 -0500 Subject: [PATCH 04/34] fix: match B300 Mamba cache flag to pinned SGLang --- .../agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml | 4 ++-- .../agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml | 4 ++-- .../agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml | 4 ++-- .../agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml | 4 ++-- .../agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml | 4 ++-- perf-changelog.yaml | 8 ++++++++ 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml index 8bf29c2c89..96eb31c277 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml @@ -79,7 +79,7 @@ roles: disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 @@ -130,7 +130,7 @@ roles: disaggregation-mode: decode disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer disable-radix-cache: true speculative-algorithm: NEXTN speculative-num-steps: 3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml index a0d6eb6c3b..4a081251fb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml @@ -79,7 +79,7 @@ roles: disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 @@ -130,7 +130,7 @@ roles: disaggregation-mode: decode disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer disable-radix-cache: true speculative-algorithm: NEXTN speculative-num-steps: 3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml index aa248be2af..4cf6a64a37 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml @@ -79,7 +79,7 @@ roles: disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 @@ -130,7 +130,7 @@ roles: disaggregation-mode: decode disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer disable-radix-cache: true speculative-algorithm: NEXTN speculative-num-steps: 3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml index 04b2ae93f4..b7949422ea 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml @@ -79,7 +79,7 @@ roles: disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 @@ -130,7 +130,7 @@ roles: disaggregation-mode: decode disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer disable-radix-cache: true speculative-algorithm: NEXTN speculative-num-steps: 3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml index 0ccd287fea..6c4aabe2c4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml @@ -79,7 +79,7 @@ roles: disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer + mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 @@ -130,7 +130,7 @@ roles: disaggregation-mode: decode disaggregation-transfer-backend: mooncake watchdog-timeout: 3600 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer disable-radix-cache: true speculative-algorithm: NEXTN speculative-num-steps: 3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a6d29f7a55..bf20d47b6c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8111,3 +8111,11 @@ GPU groups; add concurrency 32 and match aggregate per-concurrency request limits, decode graph sizing, scheduler interval, and streaming cadence. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Use the Mamba radix-cache strategy argument declared by the pinned SGLang nightly for prefill and decode roles. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From 347d6d02803bfac4361f59f26880f532cd385933 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 13:20:04 -0500 Subject: [PATCH 05/34] perf: use latest SGLang nightly for B300 Qwen3.5 FP8 --- .../agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml | 2 +- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 8 ++++++++ 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml index 96eb31c277..598a602fd3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml @@ -4,7 +4,7 @@ schema: 2 name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c1 model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 dynamo: install: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml index 4a081251fb..9af5a1e315 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml @@ -4,7 +4,7 @@ schema: 2 name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c16 model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 dynamo: install: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml index 4cf6a64a37..e4887896d0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml @@ -4,7 +4,7 @@ schema: 2 name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c32 model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 dynamo: install: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml index b7949422ea..0632753a26 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml @@ -4,7 +4,7 @@ schema: 2 name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c4 model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 dynamo: install: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml index 6c4aabe2c4..fa0729f483 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml @@ -4,7 +4,7 @@ schema: 2 name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c8 model: path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 dynamo: install: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 67e0c101db..147f226998 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8152,7 +8152,7 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: cluster:b300-dsxe diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bf20d47b6c..38392cad96 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8119,3 +8119,11 @@ description: - Use the Mamba radix-cache strategy argument declared by the pinned SGLang nightly for prefill and decode roles. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Test B300 Qwen3.5 FP8 disaggregation on the latest amd64 CUDA 13 SGLang nightly, 20260918-20518d85. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From 188f1772a9503802a5c0b9133d0be10fade11350 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 13:35:05 -0500 Subject: [PATCH 06/34] fix: use Dynamo compatibility with latest SGLang --- .../agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml | 2 +- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 8 ++++++++ 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml index 598a602fd3..05cac60fba 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml @@ -9,7 +9,7 @@ model: dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: time_limit: '8:00:00' health_check: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml index 9af5a1e315..58bb7aeb2f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml @@ -9,7 +9,7 @@ model: dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: time_limit: '8:00:00' health_check: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml index e4887896d0..d3ca109dd8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml @@ -9,7 +9,7 @@ model: dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: time_limit: '8:00:00' health_check: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml index 0632753a26..eeab236e2c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml @@ -9,7 +9,7 @@ model: dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: time_limit: '8:00:00' health_check: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml index fa0729f483..4f406960a9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml @@ -9,7 +9,7 @@ model: dynamo: install: true source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: time_limit: '8:00:00' health_check: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 147f226998..77666bfe36 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8160,7 +8160,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: framework: dynamo-sglang router: name: dynamo-router - version: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + version: 805a77f053d807b0d8def5d27f674a6df0ed839e kv-p2p-transfer: mooncake multinode: true disagg: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 38392cad96..57eb4bb78e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8127,3 +8127,11 @@ description: - Test B300 Qwen3.5 FP8 disaggregation on the latest amd64 CUDA 13 SGLang nightly, 20260918-20518d85. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Pin Dynamo 805a77f with compatibility handling for the latest SGLang argument and model configuration APIs. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From a9459107092242bc906aae288c98b7c19b9ed3c4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:11:26 -0500 Subject: [PATCH 07/34] fix: use intra-node NVLink for colocated B300 PD --- .../disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml | 15 +++++++++++++-- .../disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml | 15 +++++++++++++-- .../disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml | 15 +++++++++++++-- .../disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml | 15 +++++++++++++-- .../disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml | 15 +++++++++++++-- perf-changelog.yaml | 8 ++++++++ 6 files changed, 73 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml index 05cac60fba..ef4e798840 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml @@ -6,6 +6,15 @@ model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 dynamo: install: true source: @@ -45,7 +54,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' @@ -97,7 +107,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml index 58bb7aeb2f..21f0971ef2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml @@ -6,6 +6,15 @@ model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 dynamo: install: true source: @@ -45,7 +54,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' @@ -97,7 +107,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml index d3ca109dd8..5af19d8eed 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml @@ -6,6 +6,15 @@ model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 dynamo: install: true source: @@ -45,7 +54,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' @@ -97,7 +107,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml index eeab236e2c..872b5af712 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml @@ -6,6 +6,15 @@ model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 dynamo: install: true source: @@ -45,7 +54,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' @@ -97,7 +107,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml index 4f406960a9..a96d175ac1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml @@ -6,6 +6,15 @@ model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 dynamo: install: true source: @@ -45,7 +54,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' @@ -97,7 +107,8 @@ roles: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 57eb4bb78e..c813bc2836 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8135,3 +8135,11 @@ description: - Pin Dynamo 805a77f with compatibility handling for the latest SGLang argument and model configuration APIs. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Use the supported intra-node NVLink transport and allocator for colocated B300 prefill/decode; the equal-TP real-verification diagnostic passes four short and 16K-input probes after RDMA source-memory registration failed. Record verified model and runtime identities; performance qualification remains pending. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From f8823014d432a814148d212de8091496ed50700e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:51:25 -0500 Subject: [PATCH 08/34] perf: select measured B300 FP8 HiCache PD candidate --- .../disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml | 173 ------------------ ...disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml | 173 ------------------ ...1d-tp4-tp4-colocated-c32-mtp-hicache.yaml} | 15 +- .../disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml | 173 ------------------ .../disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml | 173 ------------------ configs/nvidia-master.yaml | 72 +------- perf-changelog.yaml | 16 ++ 7 files changed, 33 insertions(+), 762 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/{disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml => disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml} (90%) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml deleted file mode 100644 index ef4e798840..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# Separate TP4 GPU groups share one node. GPU qualification pending. -schema: 2 -name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c1 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 2 - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 2 - cuda-graph-max-bs-decode: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml deleted file mode 100644 index 21f0971ef2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# Separate TP4 GPU groups share one node. GPU qualification pending. -schema: 2 -name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c16 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 16 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml similarity index 90% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml index 5af19d8eed..db27c1a56c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml @@ -1,7 +1,6 @@ -# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# Separate TP4 GPU groups share one node. GPU qualification pending. +# Fast AgentX candidate from Slurm 4060; full official sweep and accuracy gates remain required. schema: 2 -name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c32 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -20,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '8:00:00' + time_limit: '2:00:00' health_check: max_attempts: 1440 interval_seconds: 10 @@ -32,6 +31,7 @@ frontend: nginx_session_affinity: true env: PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com args: router-mode: kv router-session-affinity-ttl-secs: '3600' @@ -48,6 +48,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com NCCL_NVLS_ENABLE: '1' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' SGLANG_ENABLE_SPEC_V2: '1' @@ -94,6 +95,11 @@ roles: scheduler-recv-interval: 10 stream-interval: 50 max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 72 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective decode: nodes: colocate workers: 1 @@ -101,6 +107,7 @@ roles: env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com NCCL_NVLS_ENABLE: '1' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' SGLANG_ENABLE_SPEC_V2: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml deleted file mode 100644 index 872b5af712..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# Separate TP4 GPU groups share one node. GPU qualification pending. -schema: 2 -name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c4 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 8 - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 8 - cuda-graph-max-bs-decode: 4 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml deleted file mode 100644 index a96d175ac1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# Source: https://github.com/sgl-project/sglang/blob/191172fa742f8837423ca4e4be4919ebb52f1b51/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx -# Separate TP4 GPU groups share one node. GPU qualification pending. -schema: 2 -name: qwen3.5-b300-fp8-agentx-disagg-1p1d-tp4-tp4-colocated-c8 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 16 - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 16 - cuda-graph-max-bs-decode: 8 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 77666bfe36..8f634d4d79 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8168,72 +8168,12 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.8 search-space: + # Candidate improves the measured published frontier in a 1200-second diagnostic. + # Canonical full sweep and accuracy qualification remain required. - spec-decoding: mtp - kv-offloading: none - conc-list: - - 1 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c1-mtp.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none - conc-list: - - 4 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none - conc-list: - - 8 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none - conc-list: - - 16 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none + kv-offloading: dram + kv-offload-backend: + name: hicache conc-list: - 32 prefill: @@ -8242,7 +8182,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml decode: num-worker: 1 tp: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c813bc2836..d3cd12a15a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8143,3 +8143,19 @@ description: - Use the supported intra-node NVLink transport and allocator for colocated B300 prefill/decode; the equal-TP real-verification diagnostic passes four short and 16K-input probes after RDMA source-memory registration failed. Record verified model and runtime identities; performance qualification remains pending. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Replace unmeasured starter points with the measured concurrency-32 1P1D TP4/TP4 HiCache candidate (72 GB per prefill rank). B300 Slurm 4060 completed 1986 profiled requests without errors at 21258.99 logical tok/s/GPU and 224.89 p90 interactivity, numerically extending the published FP8 frontier. This 1200-second fast diagnostic is not an official result; full sweep, applicable accuracy evals, and optimized same-image aggregate comparisons remain required. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Add the NVIDIA vendor wheel index to the recipe frontend and worker environments so aisimulate can resolve a verified vendor wheel directly instead of its fragile placeholder downloader; retain normal pip hash and package validation. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From fa542049c8e5aa229be403131c95a28dc2d724ae Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 14:54:00 -0500 Subject: [PATCH 09/34] fix: propagate failed B300 Slurm runs after collecting artifacts --- perf-changelog.yaml | 8 ++ runners/launch_b300-dsxe.sh | 27 +---- runners/slurm_utils.sh | 23 ++++ runners/test_b300_srt_status.py | 197 ++++++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 23 deletions(-) create mode 100644 runners/test_b300_srt_status.py diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d3cd12a15a..95efc27a09 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8159,3 +8159,11 @@ description: - Add the NVIDIA vendor wheel index to the recipe frontend and worker environments so aisimulate can resolve a verified vendor wheel directly instead of its fragile placeholder downloader; retain normal pip hash and package validation. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Verify the Slurm allocation terminal state and exit code after log streaming, preserve benchmark and eval artifacts, then fail the B300 launcher when the allocation failed or its final status cannot be verified. Prevent failed disaggregation runs from appearing successful solely because they left the queue. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 665577c171..6c945801a6 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -255,29 +255,8 @@ echo "Extracted JOB_ID: $JOB_ID" LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" -while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 -done - -( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 - done -) & -POLL_PID=$! - -echo "Tailing LOG_FILE: $LOG_FILE" - -# -F follows by name and polls; inotify does not work on NFS. -tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null - -wait $POLL_PID +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x @@ -331,6 +310,8 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true +# Preserve diagnostics and eval outputs before propagating a failed allocation. +exit "$SRT_JOB_RC" else # AgentX trace datasets need a writable persistent cache. Keep the host and diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 79f9175915..a15b94aa40 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -132,6 +132,29 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" + + # Disappearing from squeue means terminal, not successful. Accounting can + # lag briefly; inspect only the allocation, never successful service steps. + local attempt accounting state exit_code + for attempt in {1..10}; do + accounting=$(sacct -X -n -P -j "$job_id" --format=State,ExitCode 2>/dev/null) || accounting="" + IFS='|' read -r state exit_code <<< "$accounting" + case "$state" in + COMPLETED) + if [[ "$exit_code" == "0:0" ]]; then + return 0 + fi + ;; + ""|PENDING|RUNNING|CONFIGURING|COMPLETING) + sleep 1 + continue + ;; + esac + echo "ERROR: Slurm job $job_id ended with state=$state exit_code=$exit_code" >&2 + return 1 + done + echo "ERROR: could not verify terminal Slurm status for job $job_id" >&2 + return 1 } copy_to_workspace() { diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py new file mode 100644 index 0000000000..c43d223960 --- /dev/null +++ b/runners/test_b300_srt_status.py @@ -0,0 +1,197 @@ +"""A completed Slurm allocation must not hide a failed benchmark.""" + +import os +import shutil +import subprocess +import tarfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def _stub(directory: Path, name: str, body: str) -> None: + path = directory / name + path.write_text("#!/bin/bash\n" + body + "\n") + path.chmod(0o755) + + +@pytest.mark.parametrize( + "accounting,expected_status", + [ + ("COMPLETED|0:0", 0), + ("FAILED|1:0", 1), + ("COMPLETED|0:9", 1), + ("CANCELLED by 123|0:15", 1), + ("DELAYED", 0), + ("", 1), + ], +) +def test_b300_collects_artifacts_before_returning_slurm_status( + tmp_path: Path, accounting: str, expected_status: int +) -> None: + """Run the actual Qwen launcher path with only external services stubbed.""" + binaries = tmp_path / "bin" + binaries.mkdir() + for name in ( + "curl", + "uv", + "make", + "srtctl", + "flock", + "unsquashfs", + "squeue", + "sleep", + ): + _stub(binaries, name, "exit 0") + _stub( + binaries, + "sacct", + r""" +[[ " $* " == *" -X "* ]] || exit 2 +count=0 +[[ -f "$MOCK_SACCT_COUNT" ]] && read -r count < "$MOCK_SACCT_COUNT" +count=$((count + 1)) +printf '%s\n' "$count" > "$MOCK_SACCT_COUNT" +if [[ "$MOCK_ACCOUNTING" == DELAYED ]]; then + case "$count" in + 1) exit 0 ;; + 2) printf 'RUNNING|0:0\n' ;; + *) printf 'COMPLETED|0:0\n' ;; + esac +else + printf '%s\n' "$MOCK_ACCOUNTING" +fi +""", + ) + # Tail's long-running process is a clock collaborator; logs remain real. + _stub( + binaries, + "tail", + 'for arg in "$@"; do [[ -f "$arg" ]] && cat "$arg"; done; exit 0', + ) + workspace = tmp_path / "workspace" + workspace.mkdir() + for relative in ( + "runners/launch_b300-dsxe.sh", + "runners/slurm_utils.sh", + "benchmarks/benchmark_lib.sh", + ): + destination = workspace / relative + destination.parent.mkdir(exist_ok=True) + shutil.copyfile(ROOT / relative, destination) + # macOS ships Bash 3, which cannot parse the cluster's associative model + # inventory. It is unrelated to completion/collection; use a fixture alias + # while leaving the executed launcher control flow unchanged. + bash_major = subprocess.check_output( + ["bash", "-c", "printf '%s' \"${BASH_VERSINFO[0]}\""], text=True + ) + if int(bash_major) < 4: + launcher = workspace / "runners/launch_b300-dsxe.sh" + source = launcher.read_text() + start = source.index("declare -A MODEL_ALIASES=(") + end = source.index("\n)", start) + len("\n)") + launcher.write_text(source[:start] + "MODEL_ALIASES=(fixture)" + source[end:]) + # Stub remote checkout and submission after loading the real shared helpers. + with (workspace / "runners/slurm_utils.sh").open("a") as helpers: + helpers.write(r""" +setup_srt_slurm() { + mkdir -p "$1/recipes" + cd "$1" || return 1 + printf 'name: fixture\n' > recipes/test.yaml +} +apply_srt_recipe() { + mkdir -p outputs/42/logs + cp -R "$MOCK_FIXTURE/." outputs/42/logs/ + printf '{"diagnostic":"retained"}\n' > "$GITHUB_WORKSPACE/aggregate_conc1.json" + printf '✅ Job 42\n' +} +""") + activation = workspace / ".venv/bin/activate" + activation.parent.mkdir(parents=True) + activation.write_text(":\n") + fixture = tmp_path / "fixture" + (fixture / "agentic/conc_1/aiperf_artifacts").mkdir(parents=True) + (fixture / "eval_results").mkdir() + (fixture / "sweep_42.log").write_text("benchmark diagnostics\n") + (fixture / "agentic/conc_1/aiperf_artifacts/profile_export_aiperf.json").write_text( + '{"metadata":{"submission_valid":false}}\n' + ) + (fixture / "eval_results/results_eval.json").write_text('{"eval":"retained"}\n') + env = { + **os.environ, + "PATH": f"{binaries}:{os.environ['PATH']}", + "MOCK_ACCOUNTING": accounting, + "MOCK_SACCT_COUNT": str(tmp_path / "sacct-count"), + "MOCK_FIXTURE": str(fixture), + "GITHUB_WORKSPACE": str(workspace), + "EVAL_ONLY": "false", + "IS_AGENTIC": "0", + "IS_MULTINODE": "true", + "RUN_EVAL": "true", + "SLURM_PARTITION": "batch_1", + "SLURM_ACCOUNT": "benchmark", + "FRAMEWORK": "dynamo-sglang", + "MODEL_PREFIX": "qwen3.5", + "PRECISION": "fp8", + "MODEL": "Qwen/Qwen3.5-397B-A17B-FP8", + "SPEC_DECODING": "mtp", + "CONFIG_FILE": "recipes/test.yaml", + "RUNNER_NAME": "b300-dsxe_08", + "IMAGE": "fixture-image", + "ENROOT_IMPORT_TIME_LIMIT": "00:05:00", + "SALLOC_TIME_LIMIT": "00:59:00", + "RESULT_FILENAME": "aggregate", + "ISL": "1", + "OSL": "1", + } + result = subprocess.run( + [ + "bash", + "-c", + r""" +# GNU-only command options in this Linux launcher are irrelevant to the +# completion contract; keep the test runnable on macOS too. +grep() { + if [[ "$1" == -oP ]]; then cat >/dev/null; printf '42\n'; else command grep "$@"; fi +} +sed() { + if [[ "$1" == -i && "$(uname)" == Darwin ]]; then + shift; command sed -i '' "$@" + else + command sed "$@" + fi +} +# The launcher writes the fixed cluster cache directory once; the mock image +# probe does not need it, and the portable test must not touch /data. +mkdir() { + if [[ "$*" == "-p /data/home/sa-gha-runner/squash" ]]; then return 0; fi + command mkdir "$@" +} +builtin source "$1/runners/launch_b300-dsxe.sh" +""", + "bash", + str(workspace), + ], + cwd=workspace, + env=env, + text=True, + capture_output=True, + check=False, + timeout=15, + ) + assert result.returncode == expected_status, result.stdout + result.stderr + assert ( + workspace / "aggregate_conc1.json" + ).read_text() == '{"diagnostic":"retained"}\n' + assert (workspace / "results_eval.json").read_text() == '{"eval":"retained"}\n' + assert (workspace / "LOGS/sweep_42.log").read_text() == "benchmark diagnostics\n" + with tarfile.open(workspace / "multinode_server_logs.tar.gz") as archive: + assert ( + "./agentic/conc_1/aiperf_artifacts/profile_export_aiperf.json" + in archive.getnames() + ) + assert not (workspace / "srt-slurm/outputs").exists() + expected_queries = 3 if accounting == "DELAYED" else 10 if accounting == "" else 1 + assert int((tmp_path / "sacct-count").read_text()) == expected_queries From 08c5597e86b716a5af6e27d91c4c02a37da201d4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:02:14 -0500 Subject: [PATCH 10/34] Prepare expanded measured B300 FP8 qualification curve --- ...p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml | 180 ++++++++++++++++++ ...p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml | 180 ++++++++++++++++++ ...p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml | 180 ++++++++++++++++++ .../agentic/qwen3.5_fp8_b300_sglang_mtp.sh | 38 +++- configs/nvidia-master.yaml | 77 +++++++- docs/configuration-procedures.md | 9 + docs/configuration-procedures_zh.md | 8 + perf-changelog.yaml | 10 + runners/launch_b300-dsxe.sh | 10 + runners/slurm_utils.sh | 3 + runners/test_b300_srt_status.py | 25 +-- 11 files changed, 690 insertions(+), 30 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml new file mode 100644 index 0000000000..9d2f24ed00 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml @@ -0,0 +1,180 @@ +# Measured fast candidate; full official sweep and accuracy gates remain required. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '2:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + enable-hierarchical-cache: true + hicache-size: 72 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml new file mode 100644 index 0000000000..6e71a251c5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml @@ -0,0 +1,180 @@ +# Measured fast candidate; full official sweep and accuracy gates remain required. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '2:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 48 + enable-hierarchical-cache: true + hicache-size: 72 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 48 + cuda-graph-max-bs-decode: 24 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml new file mode 100644 index 0000000000..4c6ddac732 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml @@ -0,0 +1,180 @@ +# Measured fast candidate; full official sweep and accuracy gates remain required. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '2:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 128 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 128 + cuda-graph-max-bs-decode: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh index 3a41757657..6f0a4444e9 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh @@ -14,7 +14,14 @@ export EVAL_FRAMEWORK="lm-eval" check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY +check_env_vars EVAL_ONLY QWEN35_HICACHE_BUDGET_MODE +case "$QWEN35_HICACHE_BUDGET_MODE" in + legacy|combined) ;; + *) + echo "Error: QWEN35_HICACHE_BUDGET_MODE must be legacy or combined" >&2 + exit 1 + ;; +esac SCHEDULER_RECV_INTERVAL=10 @@ -41,27 +48,33 @@ mkdir -p "$RESULT_DIR" CACHE_ARGS=() if require_agentic_kv_offload_backend hicache; then - # SGLang applies --hicache-size independently to Qwen's target KV and - # Mamba pools. Native NEXTN also creates a draft KV pool with the same - # slot count; its one attention layer adds 1/15 of the target KV bytes. - # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. + # Preserve the existing budget unless the launcher selects the pinned + # nightly's combined KV/Mamba pool. Reserve 1 GB/rank for page alignment. HICACHE_ALIGNMENT_RESERVE_GB=$TP HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 exit 1 fi - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) + case "$QWEN35_HICACHE_BUDGET_MODE" in + legacy) + HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) + PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) + ;; + combined) + HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB / TP)) + PROJECTED_HICACHE_TOTAL_GB=$((HICACHE_SIZE_GB * TP + HICACHE_ALIGNMENT_RESERVE_GB)) + ;; + esac if [ "$HICACHE_SIZE_GB" -lt 1 ]; then echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 exit 1 fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 exit 1 fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" + echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB per rank with ${QWEN35_HICACHE_BUDGET_MODE} budgeting across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" CACHE_ARGS=( --page-size 64 --enable-hierarchical-cache @@ -92,6 +105,13 @@ MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS="$CONC" [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +# New nightlies split graph sizing by phase; keep older published images valid. +SGLANG_SERVER_HELP=$(python3 -m sglang.launch_server --help 2>&1) +CUDA_GRAPH_SIZE_FLAG=--cuda-graph-max-bs +if [[ "$SGLANG_SERVER_HELP" == *--cuda-graph-max-bs-decode* ]]; then + CUDA_GRAPH_SIZE_FLAG=--cuda-graph-max-bs-decode +fi + export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 export NCCL_NVLS_ENABLE=1 @@ -121,7 +141,7 @@ SGLANG_CMD=( --mamba-ssm-dtype bfloat16 --attention-backend trtllm_mha --moe-runner-backend flashinfer_trtllm - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + "$CUDA_GRAPH_SIZE_FLAG" "$CUDA_GRAPH_MAX_BS" --max-running-requests "$MAX_RUNNING_REQUESTS" --max-prefill-tokens 16384 --chunked-prefill-size 16384 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0992c0e045..ec97ba24a1 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8168,8 +8168,42 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.8 search-space: - # Candidate improves the measured published frontier in a 1200-second diagnostic. - # Canonical full sweep and accuracy qualification remain required. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 16 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 24 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -8188,3 +8222,42 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 64 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + +qwen3.5-fp8-b300-sglang-agentic-latest: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:b300-dsxe + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.8 + search-space: + - tp: 4 + ep: 1 + spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 32 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 09e7b20064..b6821386f6 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -209,6 +209,15 @@ Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-nego 6. Append a changelog entry selecting all affected keys (wildcards are allowed when intentional), including old/new versions and material runtime changes. 7. Generate each affected family and verify no stale tag survives in its runtime path. +B300 Qwen3.5 FP8 SGLang AgentX uses caller-selected `QWEN35_HICACHE_BUDGET_MODE`. +`launch_b300-dsxe.sh` supplies `combined` for the exact +`nightly-dev-cu13-20260918-20518d85` image and `legacy` for older images; the +benchmark script requires and validates that value. The combined mode divides +`TOTAL_CPU_DRAM_GB` across TP ranks after reserving 1 GB per rank. Legacy mode +retains the target KV, Mamba and draft-pool multiplier. For TP4 with a 1,199 GB +budget, the combined mode supplies `--hicache-size 298` per rank. Qualify a new +image's actual pool allocation before changing this explicit selection. + ## Add or change MTP Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants), [MTP appendix in the model+hardware playbook](../.claude/commands/add-model-hardware.md#appendix--mtp--eagle3-spec-decoding-variant), and current [`*_mtp.sh` siblings](../benchmarks/single_node/fixed_seq_len/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 801a453598..cbfaf02341 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -186,6 +186,14 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 6. 追加选择全部受影响 key 的 changelog 条目（有意覆盖多个 key 时可以使用通配符），并列出旧/新版本及实质运行时变更。 7. 生成每个受影响的配置族，确认其运行时路径中没有残留旧 tag。 +B300 Qwen3.5 FP8 SGLang AgentX 通过调用方选择 `QWEN35_HICACHE_BUDGET_MODE`。 +`launch_b300-dsxe.sh` 对精确镜像 `nightly-dev-cu13-20260918-20518d85` 传入 +`combined`，对较旧镜像传入 `legacy`；benchmark 脚本要求并验证该值。 +combined 模式先为每个 rank 预留 1 GB，再按 TP rank 分配 `TOTAL_CPU_DRAM_GB`。 +legacy 模式保留目标 KV、Mamba 和 draft 池的内存倍率。TP4 的预算为 1,199 GB 时， +combined 模式为每个 rank 传入 `--hicache-size 298`。修改此显式选择前，必须验证 +新镜像的实际内存池分配。 + ## 添加或修改 MTP 来源：[`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants)、[模型+硬件 playbook 的 MTP 附录](../.claude/commands/add-model-hardware.md#appendix--mtp--eagle3-spec-decoding-variant)和现有 [`*_mtp.sh` 同类项](../benchmarks/single_node/fixed_seq_len/)。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0a61d51c25..1ea6750c58 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8190,3 +8190,13 @@ description: - Verify the Slurm allocation terminal state and exit code after log streaming, preserve benchmark and eval artifacts, then fail the B300 launcher when the allocation failed or its final status cannot be verified. Prevent failed disaggregation runs from appearing successful solely because they left the queue. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + - qwen3.5-fp8-b300-sglang-agentic-latest + scenario-type: + - agentic-coding + description: + - "Prepare the expanded B300 FP8 qualification curve from measured fast candidates: colocated 1P1D TP4/TP4 at concurrency 16, 24, 32 and 64, plus TP4 aggregate concurrency 32. Use 72 GB HiCache per prefill rank at C16-C32 and 298 GB at C64; use the latest nightly combined HiCache budget for the aggregate candidate while retaining older-image budgeting. These short measurements require a fresh full official sweep and applicable accuracy evaluations before admission." + - Select the explicit decode CUDA graph flag exposed by new SGLang versions, preserving the older flag for historical images. Scope terminal Slurm status verification to the B300 generic PD path and preserve artifacts before returning failure; run the updated launcher tests through the real runtime-profile renderer. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 5a6f8ada21..d98fdd310f 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -207,6 +207,7 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" SRT_JOB_RC=0 stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? +verify_slurm_job_status "$JOB_ID" || SRT_JOB_RC=$? set -x @@ -264,6 +265,15 @@ find . -name '.nfs*' -delete 2>/dev/null || true exit "$SRT_JOB_RC" else + if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && + "$PRECISION" == "fp8" && "$FRAMEWORK" == "sglang" && "$SPEC_DECODING" == "mtp" ]]; then + check_env_vars IMAGE + export QWEN35_HICACHE_BUDGET_MODE=legacy + if [[ "$IMAGE" == "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85" ]]; then + export QWEN35_HICACHE_BUDGET_MODE=combined + fi + fi + # AgentX trace datasets need a writable persistent cache. Keep the host and # container paths separate so the cache remains valid with # --no-container-mount-home. diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index b8a98a08fc..67a6eac780 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -155,7 +155,10 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" +} +verify_slurm_job_status() { + local job_id="$1" # Disappearing from squeue means terminal, not successful. Accounting can # lag briefly; inspect only the allocation, never successful service steps. local attempt accounting state exit_code diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py index c43d223960..c9b5a836b1 100644 --- a/runners/test_b300_srt_status.py +++ b/runners/test_b300_srt_status.py @@ -73,26 +73,13 @@ def test_b300_collects_artifacts_before_returning_slurm_status( ) workspace = tmp_path / "workspace" workspace.mkdir() - for relative in ( - "runners/launch_b300-dsxe.sh", - "runners/slurm_utils.sh", - "benchmarks/benchmark_lib.sh", - ): - destination = workspace / relative - destination.parent.mkdir(exist_ok=True) - shutil.copyfile(ROOT / relative, destination) - # macOS ships Bash 3, which cannot parse the cluster's associative model - # inventory. It is unrelated to completion/collection; use a fixture alias - # while leaving the executed launcher control flow unchanged. - bash_major = subprocess.check_output( - ["bash", "-c", "printf '%s' \"${BASH_VERSINFO[0]}\""], text=True + # Keep the real launcher-owned profiles and renderer in the fixture checkout. + shutil.copytree(ROOT / "runners", workspace / "runners") + (workspace / "benchmarks").mkdir() + shutil.copyfile( + ROOT / "benchmarks/benchmark_lib.sh", workspace / "benchmarks/benchmark_lib.sh" ) - if int(bash_major) < 4: - launcher = workspace / "runners/launch_b300-dsxe.sh" - source = launcher.read_text() - start = source.index("declare -A MODEL_ALIASES=(") - end = source.index("\n)", start) + len("\n)") - launcher.write_text(source[:start] + "MODEL_ALIASES=(fixture)" + source[end:]) + (workspace / "infx").symlink_to(ROOT / "infx", target_is_directory=True) # Stub remote checkout and submission after loading the real shared helpers. with (workspace / "runners/slurm_utils.sh").open("a") as helpers: helpers.write(r""" From 034155e1c43edb2a881c928fb7cbc53431054459 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:13:45 -0500 Subject: [PATCH 11/34] perf(b300): qualify measured concurrency 48 PD point --- ...p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml | 180 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 8 + 3 files changed, 206 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml new file mode 100644 index 0000000000..4f22f1a950 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml @@ -0,0 +1,180 @@ +# Measured fast candidate; full official sweep and accuracy gates remain required. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '2:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 96 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 96 + cuda-graph-max-bs-decode: 48 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ec97ba24a1..136a328562 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8222,6 +8222,24 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 48 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false - spec-decoding: mtp kv-offloading: dram kv-offload-backend: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1ea6750c58..3fb102e9ae 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8200,3 +8200,11 @@ - "Prepare the expanded B300 FP8 qualification curve from measured fast candidates: colocated 1P1D TP4/TP4 at concurrency 16, 24, 32 and 64, plus TP4 aggregate concurrency 32. Use 72 GB HiCache per prefill rank at C16-C32 and 298 GB at C64; use the latest nightly combined HiCache budget for the aggregate candidate while retaining older-image budgeting. These short measurements require a fresh full official sweep and applicable accuracy evaluations before admission." - Select the explicit decode CUDA graph flag exposed by new SGLang versions, preserving the older flag for historical images. Scope terminal Slurm status verification to the B300 generic PD path and preserve artifacts before returning failure; run the updated launcher tests through the real runtime-profile renderer. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add the measured colocated 1P1D concurrency-48 point with 298 GB HiCache per prefill rank. B300 Slurm 4093 completed 2683 profiled requests with zero errors or grace cancellations at 29720.62 logical tok/s/GPU and 181.40 p90 interactivity, extending the published FP8 frontier between the C32 and C64 candidates. This fast result remains subject to the fresh official full sweep and accuracy evaluations." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From ffa5ca1f00a24c7992eb5f69e3d04efd60cf1d62 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:17:01 -0500 Subject: [PATCH 12/34] perf(b300): preserve existing aggregate publication scopes --- .../agentic/qwen3.5_fp8_b300_sglang_mtp.sh | 38 +++++-------------- configs/nvidia-master.yaml | 20 ---------- docs/configuration-procedures.md | 9 ----- docs/configuration-procedures_zh.md | 8 ---- perf-changelog.yaml | 9 ++++- runners/launch_b300-dsxe.sh | 9 ----- 6 files changed, 17 insertions(+), 76 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh index 6f0a4444e9..3a41757657 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh @@ -14,14 +14,7 @@ export EVAL_FRAMEWORK="lm-eval" check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY QWEN35_HICACHE_BUDGET_MODE -case "$QWEN35_HICACHE_BUDGET_MODE" in - legacy|combined) ;; - *) - echo "Error: QWEN35_HICACHE_BUDGET_MODE must be legacy or combined" >&2 - exit 1 - ;; -esac +check_env_vars EVAL_ONLY SCHEDULER_RECV_INTERVAL=10 @@ -48,33 +41,27 @@ mkdir -p "$RESULT_DIR" CACHE_ARGS=() if require_agentic_kv_offload_backend hicache; then - # Preserve the existing budget unless the launcher selects the pinned - # nightly's combined KV/Mamba pool. Reserve 1 GB/rank for page alignment. + # SGLang applies --hicache-size independently to Qwen's target KV and + # Mamba pools. Native NEXTN also creates a draft KV pool with the same + # slot count; its one attention layer adds 1/15 of the target KV bytes. + # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. HICACHE_ALIGNMENT_RESERVE_GB=$TP HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 exit 1 fi - case "$QWEN35_HICACHE_BUDGET_MODE" in - legacy) - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - ;; - combined) - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB / TP)) - PROJECTED_HICACHE_TOTAL_GB=$((HICACHE_SIZE_GB * TP + HICACHE_ALIGNMENT_RESERVE_GB)) - ;; - esac + HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) if [ "$HICACHE_SIZE_GB" -lt 1 ]; then echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 exit 1 fi + PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 exit 1 fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB per rank with ${QWEN35_HICACHE_BUDGET_MODE} budgeting across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" + echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" CACHE_ARGS=( --page-size 64 --enable-hierarchical-cache @@ -105,13 +92,6 @@ MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS="$CONC" [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 -# New nightlies split graph sizing by phase; keep older published images valid. -SGLANG_SERVER_HELP=$(python3 -m sglang.launch_server --help 2>&1) -CUDA_GRAPH_SIZE_FLAG=--cuda-graph-max-bs -if [[ "$SGLANG_SERVER_HELP" == *--cuda-graph-max-bs-decode* ]]; then - CUDA_GRAPH_SIZE_FLAG=--cuda-graph-max-bs-decode -fi - export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 export NCCL_NVLS_ENABLE=1 @@ -141,7 +121,7 @@ SGLANG_CMD=( --mamba-ssm-dtype bfloat16 --attention-backend trtllm_mha --moe-runner-backend flashinfer_trtllm - "$CUDA_GRAPH_SIZE_FLAG" "$CUDA_GRAPH_MAX_BS" + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" --max-running-requests "$MAX_RUNNING_REQUESTS" --max-prefill-tokens 16384 --chunked-prefill-size 16384 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 136a328562..1dfbe61074 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8259,23 +8259,3 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false -qwen3.5-fp8-b300-sglang-agentic-latest: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:b300-dsxe - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.8 - search-space: - - tp: 4 - ep: 1 - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 32 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index b6821386f6..09e7b20064 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -209,15 +209,6 @@ Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-nego 6. Append a changelog entry selecting all affected keys (wildcards are allowed when intentional), including old/new versions and material runtime changes. 7. Generate each affected family and verify no stale tag survives in its runtime path. -B300 Qwen3.5 FP8 SGLang AgentX uses caller-selected `QWEN35_HICACHE_BUDGET_MODE`. -`launch_b300-dsxe.sh` supplies `combined` for the exact -`nightly-dev-cu13-20260918-20518d85` image and `legacy` for older images; the -benchmark script requires and validates that value. The combined mode divides -`TOTAL_CPU_DRAM_GB` across TP ranks after reserving 1 GB per rank. Legacy mode -retains the target KV, Mamba and draft-pool multiplier. For TP4 with a 1,199 GB -budget, the combined mode supplies `--hicache-size 298` per rank. Qualify a new -image's actual pool allocation before changing this explicit selection. - ## Add or change MTP Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants), [MTP appendix in the model+hardware playbook](../.claude/commands/add-model-hardware.md#appendix--mtp--eagle3-spec-decoding-variant), and current [`*_mtp.sh` siblings](../benchmarks/single_node/fixed_seq_len/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index cbfaf02341..801a453598 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -186,14 +186,6 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 6. 追加选择全部受影响 key 的 changelog 条目（有意覆盖多个 key 时可以使用通配符），并列出旧/新版本及实质运行时变更。 7. 生成每个受影响的配置族，确认其运行时路径中没有残留旧 tag。 -B300 Qwen3.5 FP8 SGLang AgentX 通过调用方选择 `QWEN35_HICACHE_BUDGET_MODE`。 -`launch_b300-dsxe.sh` 对精确镜像 `nightly-dev-cu13-20260918-20518d85` 传入 -`combined`，对较旧镜像传入 `legacy`；benchmark 脚本要求并验证该值。 -combined 模式先为每个 rank 预留 1 GB，再按 TP rank 分配 `TOTAL_CPU_DRAM_GB`。 -legacy 模式保留目标 KV、Mamba 和 draft 池的内存倍率。TP4 的预算为 1,199 GB 时， -combined 模式为每个 rank 传入 `--hicache-size 298`。修改此显式选择前，必须验证 -新镜像的实际内存池分配。 - ## 添加或修改 MTP 来源：[`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants)、[模型+硬件 playbook 的 MTP 附录](../.claude/commands/add-model-hardware.md#appendix--mtp--eagle3-spec-decoding-variant)和现有 [`*_mtp.sh` 同类项](../benchmarks/single_node/fixed_seq_len/)。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3fb102e9ae..ff429eaa22 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8193,7 +8193,6 @@ - config-keys: - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - - qwen3.5-fp8-b300-sglang-agentic-latest scenario-type: - agentic-coding description: @@ -8208,3 +8207,11 @@ description: - "Add the measured colocated 1P1D concurrency-48 point with 298 GB HiCache per prefill rank. B300 Slurm 4093 completed 2683 profiled requests with zero errors or grace cancellations at 29720.62 logical tok/s/GPU and 181.40 p90 interactivity, extending the published FP8 frontier between the C32 and C64 candidates. This fast result remains subject to the fresh official full sweep and accuracy evaluations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Retain only the new measured disaggregated qualification scope. Keep latest-image aggregate controls as comparison evidence without publishing a sparse replacement for the existing aggregate curves; the dashboard selects the latest run per curve scope, so a lone new aggregate point would remove still-competitive published endpoints. Remove provisional aggregate configuration and its unused launcher, budget, and documentation changes. Preserve the existing aggregate FP8 scopes alongside the expanded PD curve." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index d98fdd310f..a631cad3bb 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -265,15 +265,6 @@ find . -name '.nfs*' -delete 2>/dev/null || true exit "$SRT_JOB_RC" else - if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && - "$PRECISION" == "fp8" && "$FRAMEWORK" == "sglang" && "$SPEC_DECODING" == "mtp" ]]; then - check_env_vars IMAGE - export QWEN35_HICACHE_BUDGET_MODE=legacy - if [[ "$IMAGE" == "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85" ]]; then - export QWEN35_HICACHE_BUDGET_MODE=combined - fi - fi - # AgentX trace datasets need a writable persistent cache. Keep the host and # container paths separate so the cache remains valid with # --no-container-mount-home. From 3f1f9661a258aa7dff81aa275729e9ad09f9b7b9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:17:33 -0500 Subject: [PATCH 13/34] style(b300): normalize final config newline --- configs/nvidia-master.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1dfbe61074..b1cf69125e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8258,4 +8258,3 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false - From 06749eb3b03915a5421703f652587727df002baa Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 16:58:15 -0500 Subject: [PATCH 14/34] fix: require validated B300 Qwen3.5 AgentX power telemetry --- ...p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml | 10 +++ ...p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml | 10 +++ ...p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml | 10 +++ ...p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml | 10 +++ ...p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml | 10 +++ perf-changelog.yaml | 8 +++ runners/launch_b300-dsxe.sh | 24 ++++--- runners/test_b300_srt_status.py | 68 ++++++++++++++----- 8 files changed, 126 insertions(+), 24 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml index 9d2f24ed00..4e3837bad7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml @@ -178,3 +178,13 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml index 6e71a251c5..047f1a4ac2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml @@ -178,3 +178,13 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml index db27c1a56c..227c7b1d2f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml @@ -178,3 +178,13 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml index 4f22f1a950..9f307066df 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml @@ -178,3 +178,13 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml index 4c6ddac732..b8d381d36e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml @@ -178,3 +178,13 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ff429eaa22..e98bc7a45e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8215,3 +8215,11 @@ description: - "Retain only the new measured disaggregated qualification scope. Keep latest-image aggregate controls as comparison evidence without publishing a sparse replacement for the existing aggregate curves; the dashboard selects the latest run per curve scope, so a lone new aggregate point would remove still-competitive published endpoints. Remove provisional aggregate configuration and its unused launcher, budget, and documentation changes. Preserve the existing aggregate FP8 scopes alongside the expanded PD curve." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Require native SRT DCGM power telemetry and measurement windows for B300 Qwen3.5 FP8 AgentX throughput runs. Collect and validate the producer contract before archiving artifacts, retain diagnostics when power validation fails, and skip throughput power validation for eval-only jobs. The initial canonical run lacked this contract and cannot qualify as a valid power result. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index a631cad3bb..d7cad8bd68 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -102,14 +102,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' ' "$_RECIPE_SRC"; then USES_DCGM_POWER=1 fi -if [[ "$USES_DCGM_POWER" == "1" && ( - "${IS_AGENTIC}" == "1" || - "$MODEL_PREFIX" != "dsv4" || - "$PRECISION" != "fp4" || - ( "$FRAMEWORK" != "dynamo-sglang" && "$FRAMEWORK" != "dynamo-vllm" ) -) ]]; then - echo "Error: B300 dcgm-power is limited to fixed-sequence DSV4 FP4 dynamo-sglang/vllm" >&2 - exit 1 +if [[ "$USES_DCGM_POWER" == "1" ]]; then + if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + : # AgentX uses the native SRT measurement-window contract and adapter. + elif [[ "$IS_AGENTIC" != "1" && "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && ( "$FRAMEWORK" == "dynamo-sglang" || "$FRAMEWORK" == "dynamo-vllm" ) ]]; then + : # Existing fixed-sequence telemetry path. + else + echo "Error: B300 dcgm-power supports fixed-sequence DSV4 FP4 and Qwen3.5 FP8 dynamo-sglang AgentX" >&2 + exit 1 + fi fi SRT_REPO_DIR="srt-slurm" @@ -227,6 +228,13 @@ if [[ "$USES_DCGM_POWER" == "1" ]]; then cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt" fi +if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && "${EVAL_ONLY}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$GITHUB_WORKSPACE" \ + "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$SRT_SLURM_COMMIT" \ + "${POWER_CONCURRENCIES[@]}" || SRT_JOB_RC=$? +fi + cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py index c9b5a836b1..08e691e92b 100644 --- a/runners/test_b300_srt_status.py +++ b/runners/test_b300_srt_status.py @@ -1,5 +1,6 @@ """A completed Slurm allocation must not hide a failed benchmark.""" +import json import os import shutil import subprocess @@ -18,18 +19,20 @@ def _stub(directory: Path, name: str, body: str) -> None: @pytest.mark.parametrize( - "accounting,expected_status", + "accounting,expected_status,power_mode", [ - ("COMPLETED|0:0", 0), - ("FAILED|1:0", 1), - ("COMPLETED|0:9", 1), - ("CANCELLED by 123|0:15", 1), - ("DELAYED", 0), - ("", 1), + ("COMPLETED|0:0", 0, "off"), + ("FAILED|1:0", 1, "off"), + ("COMPLETED|0:9", 1, "off"), + ("CANCELLED by 123|0:15", 1, "off"), + ("DELAYED", 0, "off"), + ("", 1, "off"), + ("COMPLETED|0:0", 1, "missing"), + ("COMPLETED|0:0", 0, "eval"), ], ) def test_b300_collects_artifacts_before_returning_slurm_status( - tmp_path: Path, accounting: str, expected_status: int + tmp_path: Path, accounting: str, expected_status: int, power_mode: str ) -> None: """Run the actual Qwen launcher path with only external services stubbed.""" binaries = tmp_path / "bin" @@ -40,11 +43,11 @@ def test_b300_collects_artifacts_before_returning_slurm_status( "make", "srtctl", "flock", - "unsquashfs", "squeue", "sleep", ): _stub(binaries, name, "exit 0") + _stub(binaries, "unsquashfs", "exit 0") _stub( binaries, "sacct", @@ -54,6 +57,10 @@ def test_b300_collects_artifacts_before_returning_slurm_status( [[ -f "$MOCK_SACCT_COUNT" ]] && read -r count < "$MOCK_SACCT_COUNT" count=$((count + 1)) printf '%s\n' "$count" > "$MOCK_SACCT_COUNT" +if [[ "$*" == *JobIDRaw* ]]; then + printf '42|%s\n' "$MOCK_ACCOUNTING" + exit 0 +fi if [[ "$MOCK_ACCOUNTING" == DELAYED ]]; then case "$count" in 1) exit 0 ;; @@ -71,11 +78,18 @@ def test_b300_collects_artifacts_before_returning_slurm_status( "tail", 'for arg in "$@"; do [[ -f "$arg" ]] && cat "$arg"; done; exit 0', ) + _stub(binaries, "sha256sum", 'printf "%064d fixture-image\\n" 0') workspace = tmp_path / "workspace" workspace.mkdir() # Keep the real launcher-owned profiles and renderer in the fixture checkout. shutil.copytree(ROOT / "runners", workspace / "runners") (workspace / "benchmarks").mkdir() + if power_mode != "off": + recipe = workspace / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text( + "telemetry:\n enabled: true\n dcgm_exporter:\n container_image: dcgm-exporter\n" + ) shutil.copyfile( ROOT / "benchmarks/benchmark_lib.sh", workspace / "benchmarks/benchmark_lib.sh" ) @@ -87,11 +101,13 @@ def test_b300_collects_artifacts_before_returning_slurm_status( mkdir -p "$1/recipes" cd "$1" || return 1 printf 'name: fixture\n' > recipes/test.yaml + SRT_SLURM_COMMIT=1111111111111111111111111111111111111111 + printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/power-producer-sha.txt" } apply_srt_recipe() { mkdir -p outputs/42/logs cp -R "$MOCK_FIXTURE/." outputs/42/logs/ - printf '{"diagnostic":"retained"}\n' > "$GITHUB_WORKSPACE/aggregate_conc1.json" + printf '{"diagnostic":"retained","disagg":true,"num_prefill_gpu":4,"num_decode_gpu":4}\n' > "$GITHUB_WORKSPACE/aggregate_conc1.json" printf '✅ Job 42\n' } """) @@ -113,8 +129,9 @@ def test_b300_collects_artifacts_before_returning_slurm_status( "MOCK_SACCT_COUNT": str(tmp_path / "sacct-count"), "MOCK_FIXTURE": str(fixture), "GITHUB_WORKSPACE": str(workspace), - "EVAL_ONLY": "false", - "IS_AGENTIC": "0", + "EVAL_ONLY": "true" if power_mode == "eval" else "false", + "IS_AGENTIC": "1" if power_mode != "off" else "0", + "CONC_LIST": "1", "IS_MULTINODE": "true", "RUN_EVAL": "true", "SLURM_PARTITION": "batch_1", @@ -156,6 +173,10 @@ def test_b300_collects_artifacts_before_returning_slurm_status( if [[ "$*" == "-p /data/home/sa-gha-runner/squash" ]]; then return 0; fi command mkdir "$@" } +test() { + if [[ "$1" == -r && "$2" == /data/* ]]; then return 0; fi + builtin test "$@" +} builtin source "$1/runners/launch_b300-dsxe.sh" """, "bash", @@ -169,9 +190,16 @@ def test_b300_collects_artifacts_before_returning_slurm_status( timeout=15, ) assert result.returncode == expected_status, result.stdout + result.stderr - assert ( - workspace / "aggregate_conc1.json" - ).read_text() == '{"diagnostic":"retained"}\n' + aggregate = json.loads((workspace / "aggregate_conc1.json").read_text()) + assert aggregate["diagnostic"] == "retained" + if power_mode == "missing": + validation = json.loads( + (workspace / "LOGS/agentic/conc_1/power_validation.json").read_text() + ) + assert validation["power_valid"] is False + assert validation["reasons"] == ["formal_benchmark_result_missing"] + else: + assert not (workspace / "LOGS/agentic/conc_1/power_validation.json").exists() assert (workspace / "results_eval.json").read_text() == '{"eval":"retained"}\n' assert (workspace / "LOGS/sweep_42.log").read_text() == "benchmark diagnostics\n" with tarfile.open(workspace / "multinode_server_logs.tar.gz") as archive: @@ -180,5 +208,13 @@ def test_b300_collects_artifacts_before_returning_slurm_status( in archive.getnames() ) assert not (workspace / "srt-slurm/outputs").exists() - expected_queries = 3 if accounting == "DELAYED" else 10 if accounting == "" else 1 + expected_queries = ( + 2 + if power_mode == "missing" + else 3 + if accounting == "DELAYED" + else 10 + if accounting == "" + else 1 + ) assert int((tmp_path / "sacct-count").read_text()) == expected_queries From ebb3b093fe2b8ae23e84dac4f19df79438cc347c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:00:29 -0500 Subject: [PATCH 15/34] fix: bind B300 power windows to matrix concurrency --- runners/launch_b300-dsxe.sh | 6 ++++++ runners/test_b300_srt_status.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index d7cad8bd68..f6ce177d58 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -181,6 +181,12 @@ fi sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" +if [[ "$USES_DCGM_POWER" == "1" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + python "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \ + "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 +fi + # Weights live on node-local MODEL_ROOT, which this login host cannot stat, so # srtctl's preflight model.path check is always skipped. Runtime loading still # validates the path on the compute nodes. diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py index 08e691e92b..fd7e6a533b 100644 --- a/runners/test_b300_srt_status.py +++ b/runners/test_b300_srt_status.py @@ -100,7 +100,7 @@ def test_b300_collects_artifacts_before_returning_slurm_status( setup_srt_slurm() { mkdir -p "$1/recipes" cd "$1" || return 1 - printf 'name: fixture\n' > recipes/test.yaml + printf 'name: fixture\nbenchmark:\n type: custom\n' > recipes/test.yaml SRT_SLURM_COMMIT=1111111111111111111111111111111111111111 printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/power-producer-sha.txt" } @@ -190,6 +190,11 @@ def test_b300_collects_artifacts_before_returning_slurm_status( timeout=15, ) assert result.returncode == expected_status, result.stdout + result.stderr + if power_mode != "off": + import yaml + + emitted = yaml.safe_load((workspace / "srt-slurm/recipes/test.yaml").read_text()) + assert emitted["benchmark"]["concurrencies"] == [1] aggregate = json.loads((workspace / "aggregate_conc1.json").read_text()) assert aggregate["diagnostic"] == "retained" if power_mode == "missing": From 434c5bee95ce20427bbb236632cb5bbcdd3337b8 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:03:13 -0500 Subject: [PATCH 16/34] perf: qualify B300 FP8 write-through PD at concurrency 96 --- ...located-c96-write-through-mtp-hicache.yaml | 190 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 8 + 3 files changed, 216 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml new file mode 100644 index 0000000000..4fb7ba3cc4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml @@ -0,0 +1,190 @@ +# Measured fast candidate; full official sweep and accuracy gates remain required. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '2:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 192 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 192 + cuda-graph-max-bs-decode: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b1cf69125e..184fef8f0b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8258,3 +8258,21 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 96 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e98bc7a45e..f2c14f0cf7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8223,3 +8223,11 @@ description: - Require native SRT DCGM power telemetry and measurement windows for B300 Qwen3.5 FP8 AgentX throughput runs. Collect and validate the producer contract before archiving artifacts, retain diagnostics when power validation fails, and skip throughput power validation for eval-only jobs. The initial canonical run lacked this contract and cannot qualify as a valid power result. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Add concurrency 96 with 298 GB HiCache per prefill rank and first-use write-through. Slurm 4105 measured 46358.88 logical tok/s/GPU at 94.75 p90 interactivity with 5048 profiled requests, zero errors and two grace cancellations, strictly dominating two published FP8 frontier points. The trace submission and duration coverage passed; this diagnostic still requires the fresh full official sweep, real accuracy evals and validated power telemetry. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From ddc3ac18564274f7e9e53a3d6da8d1d364f1a122 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:11:18 -0500 Subject: [PATCH 17/34] style: format B300 power contract fixture --- runners/test_b300_srt_status.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py index fd7e6a533b..65d6eec40c 100644 --- a/runners/test_b300_srt_status.py +++ b/runners/test_b300_srt_status.py @@ -193,7 +193,9 @@ def test_b300_collects_artifacts_before_returning_slurm_status( if power_mode != "off": import yaml - emitted = yaml.safe_load((workspace / "srt-slurm/recipes/test.yaml").read_text()) + emitted = yaml.safe_load( + (workspace / "srt-slurm/recipes/test.yaml").read_text() + ) assert emitted["benchmark"]["concurrencies"] == [1] aggregate = json.loads((workspace / "aggregate_conc1.json").read_text()) assert aggregate["diagnostic"] == "retained" From 0b07767c3c6fdf673e743398c59aa27dc39295a1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:23:41 -0500 Subject: [PATCH 18/34] fix: declare the B300 AgentX formal power window contract --- ...p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml | 3 ++ ...p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml | 3 ++ ...p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml | 3 ++ ...p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml | 3 ++ ...p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml | 3 ++ ...located-c96-write-through-mtp-hicache.yaml | 3 ++ perf-changelog.yaml | 8 ++++ runners/test_b300_srt_status.py | 47 ++++++++++++++++++- 8 files changed, 71 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml index 4e3837bad7..a7af1817a8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml @@ -170,6 +170,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '16' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml index 047f1a4ac2..1be2815a08 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml @@ -170,6 +170,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '24' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml index 227c7b1d2f..6f13fc2a39 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml @@ -170,6 +170,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml index 9f307066df..4e3e4c138d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml @@ -170,6 +170,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml index b8d381d36e..cbcba7f2a2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml @@ -170,6 +170,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '64' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml index 4fb7ba3cc4..31cac4932a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml @@ -170,6 +170,9 @@ benchmark: env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '96' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f2c14f0cf7..54e4ea9819 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8231,3 +8231,11 @@ description: - Add concurrency 96 with 298 GB HiCache per prefill rank and first-use write-through. Slurm 4105 measured 46358.88 logical tok/s/GPU at 94.75 p90 interactivity with 5048 profiled requests, zero errors and two grace cancellations, strictly dominating two published FP8 frontier points. The trace submission and duration coverage passed; this diagnostic still requires the fresh full official sweep, real accuracy evals and validated power telemetry. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Supply the explicit custom benchmark type, measured concurrency and result-root fields required by the AgentX formal power-window adapter. The pinned native SRT producer supplies only the window directory; required telemetry alone did not satisfy the full contract. Verify all six recipe environments through the real window writer before fresh runtime qualification. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py index 65d6eec40c..152d52e158 100644 --- a/runners/test_b300_srt_status.py +++ b/runners/test_b300_srt_status.py @@ -8,6 +8,7 @@ from pathlib import Path import pytest +import yaml ROOT = Path(__file__).resolve().parents[1] @@ -191,8 +192,6 @@ def test_b300_collects_artifacts_before_returning_slurm_status( ) assert result.returncode == expected_status, result.stdout + result.stderr if power_mode != "off": - import yaml - emitted = yaml.safe_load( (workspace / "srt-slurm/recipes/test.yaml").read_text() ) @@ -225,3 +224,47 @@ def test_b300_collects_artifacts_before_returning_slurm_status( else 1 ) assert int((tmp_path / "sacct-count").read_text()) == expected_queries + + +@pytest.mark.parametrize("concurrency", [16, 24, 32, 48, 64, 96]) +def test_b300_recipe_publishes_native_power_window( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, concurrency: int +) -> None: + """Bridge the actual recipe env and native producer DIR to the real adapter.""" + from infx.results.agentic.power_adapter import write_multinode_power_window + + recipes = ( + ROOT / "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx" + ) + recipe_path = next(recipes.glob(f"*-colocated-c{concurrency}-*hicache.yaml")) + recipe = yaml.safe_load(recipe_path.read_text()) + for name, value in recipe["benchmark"]["env"].items(): + monkeypatch.setenv(name, str(value)) + + # Native SRT provides DIR; /logs is the container mount of the host log root. + log_root = tmp_path / "logs" + window_dir = log_root / "power/windows" + result_dir = log_root / "agentic" / f"conc_{concurrency}" + window_dir.mkdir(parents=True) + result_dir.mkdir(parents=True) + assert recipe["benchmark"]["env"]["SRT_MEASUREMENT_WINDOW_RESULT_ROOT"] == "/logs" + monkeypatch.setenv("SRT_MEASUREMENT_WINDOW_DIR", str(window_dir)) + monkeypatch.setenv("SRT_MEASUREMENT_WINDOW_RESULT_ROOT", str(log_root)) + + assert ( + write_multinode_power_window( + result_dir=result_dir, + concurrency=concurrency, + state="running", + require_power=True, + ) + == 0 + ) + window = json.loads( + (window_dir / f"agentic_power_concurrency_{concurrency}.json").read_text() + ) + assert window["status"] == "running" + assert window["concurrency"] == concurrency + assert window["result_path"] == ( + f"agentic/conc_{concurrency}/agentic_power_concurrency_{concurrency}.json" + ) From a9e1e3ec7020c66c2a2feeb165788abfac7069ac Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:51:44 -0500 Subject: [PATCH 19/34] docs: consolidate the B300 FP8 performance changelog --- perf-changelog.yaml | 122 ++------------------------------------------ 1 file changed, 4 insertions(+), 118 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f49e77074c..cd84ff79c7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8303,122 +8303,8 @@ scenario-type: - agentic-coding description: - - 'Add a starter Qwen3.5-397B FP8 B300 disaggregated AgentX recipe using Dynamo - and SGLang through srt-slurm: 1P1D TP4, NEXTN MTP, and concurrencies 1, 4, 8, - 16.' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Place the 1P1D TP4/TP4 B300 AgentX starter on one eight-GPU node with disjoint - GPU groups; add concurrency 32 and match aggregate per-concurrency request limits, - decode graph sizing, scheduler interval, and streaming cadence. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Use the Mamba radix-cache strategy argument declared by the pinned SGLang nightly for prefill and decode roles. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Test B300 Qwen3.5 FP8 disaggregation on the latest amd64 CUDA 13 SGLang nightly, 20260918-20518d85. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Pin Dynamo 805a77f with compatibility handling for the latest SGLang argument and model configuration APIs. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Use the supported intra-node NVLink transport and allocator for colocated B300 prefill/decode; the equal-TP real-verification diagnostic passes four short and 16K-input probes after RDMA source-memory registration failed. Record verified model and runtime identities; performance qualification remains pending. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Replace unmeasured starter points with the measured concurrency-32 1P1D TP4/TP4 HiCache candidate (72 GB per prefill rank). B300 Slurm 4060 completed 1986 profiled requests without errors at 21258.99 logical tok/s/GPU and 224.89 p90 interactivity, numerically extending the published FP8 frontier. This 1200-second fast diagnostic is not an official result; full sweep, applicable accuracy evals, and optimized same-image aggregate comparisons remain required. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Add the NVIDIA vendor wheel index to the recipe frontend and worker environments so aisimulate can resolve a verified vendor wheel directly instead of its fragile placeholder downloader; retain normal pip hash and package validation. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Verify the Slurm allocation terminal state and exit code after log streaming, preserve benchmark and eval artifacts, then fail the B300 launcher when the allocation failed or its final status cannot be verified. Prevent failed disaggregation runs from appearing successful solely because they left the queue. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - "Prepare the expanded B300 FP8 qualification curve from measured fast candidates: colocated 1P1D TP4/TP4 at concurrency 16, 24, 32 and 64, plus TP4 aggregate concurrency 32. Use 72 GB HiCache per prefill rank at C16-C32 and 298 GB at C64; use the latest nightly combined HiCache budget for the aggregate candidate while retaining older-image budgeting. These short measurements require a fresh full official sweep and applicable accuracy evaluations before admission." - - Select the explicit decode CUDA graph flag exposed by new SGLang versions, preserving the older flag for historical images. Scope terminal Slurm status verification to the B300 generic PD path and preserve artifacts before returning failure; run the updated launcher tests through the real runtime-profile renderer. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - "Add the measured colocated 1P1D concurrency-48 point with 298 GB HiCache per prefill rank. B300 Slurm 4093 completed 2683 profiled requests with zero errors or grace cancellations at 29720.62 logical tok/s/GPU and 181.40 p90 interactivity, extending the published FP8 frontier between the C32 and C64 candidates. This fast result remains subject to the fresh official full sweep and accuracy evaluations." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - "Retain only the new measured disaggregated qualification scope. Keep latest-image aggregate controls as comparison evidence without publishing a sparse replacement for the existing aggregate curves; the dashboard selects the latest run per curve scope, so a lone new aggregate point would remove still-competitive published endpoints. Remove provisional aggregate configuration and its unused launcher, budget, and documentation changes. Preserve the existing aggregate FP8 scopes alongside the expanded PD curve." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Require native SRT DCGM power telemetry and measurement windows for B300 Qwen3.5 FP8 AgentX throughput runs. Collect and validate the producer contract before archiving artifacts, retain diagnostics when power validation fails, and skip throughput power validation for eval-only jobs. The initial canonical run lacked this contract and cannot qualify as a valid power result. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Add concurrency 96 with 298 GB HiCache per prefill rank and first-use write-through. Slurm 4105 measured 46358.88 logical tok/s/GPU at 94.75 p90 interactivity with 5048 profiled requests, zero errors and two grace cancellations, strictly dominating two published FP8 frontier points. The trace submission and duration coverage passed; this diagnostic still requires the fresh full official sweep, real accuracy evals and validated power telemetry. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 - -- config-keys: - - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg - scenario-type: - - agentic-coding - description: - - Supply the explicit custom benchmark type, measured concurrency and result-root fields required by the AgentX formal power-window adapter. The pinned native SRT producer supplies only the window directory; required telemetry alone did not satisfy the full contract. Verify all six recipe environments through the real window writer before fresh runtime qualification. + - Add six B300 Qwen3.5-397B-A17B FP8 AgentX recipes using colocated 1P1D TP4/TP4 Dynamo plus SGLang through srt-slurm, with all eight allocated GPUs active. Use 72 GB HiCache per prefill rank at concurrency 16, 24 and 32; use 298 GB at concurrency 48, 64 and 96, with first-use write-through at 96 and selective write-through elsewhere. Keep existing aggregate publication scopes intact and exclude losing 2P1D experiments. + - Pin SGLang nightly-dev-cu13-20260918-20518d85 and Dynamo 805a77f053d807b0d8def5d27f674a6df0ed839e, use the supported Mamba cache flags and colocated Mooncake intra-node NVLink transport, and resolve verified NVIDIA vendor wheels through the vendor index. Keep HiCache on prefill only and use NEXTN with repository golden acceptance for throughput and real verification for accuracy evals. + - Require native DCGM power telemetry, explicit custom-benchmark measurement-window fields and exact matrix concurrency injection. Verify terminal Slurm accounting and the complete producer power contract, preserve diagnostics before returning failures, and skip throughput power validation for eval-only jobs. Skip login-host model preflight for node-local weights; retain runtime loading and health checks. + - Select the six provisional points from measured published-frontier and same-image aggregate comparisons, including the concurrency-96 write-through improvement. Require a fresh full official sweep with six canonical 3600-second profiles and six full applicable accuracy evals before admission; short diagnostics and the earlier narrow run with invalid power do not substitute for this qualification. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From 88b0e1ebcd90ed01d8f3f9e9fdf9b9c79a54649b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 18:09:51 -0500 Subject: [PATCH 20/34] Avoid unverified host-memory pressure for B300 H298 recipes --- .../agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml | 2 ++ .../agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml | 2 ++ ...gg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml | 2 ++ perf-changelog.yaml | 1 + 4 files changed, 7 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml index 4e3e4c138d..60ec8e8e5e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml @@ -159,6 +159,8 @@ roles: max-running-requests: 96 cuda-graph-max-bs-decode: 48 sbatch_directives: + # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. + exclude: dsxe-sa-b300-prd0-gpu-16 mem: '0' cpus-per-task: '144' srun_options: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml index cbcba7f2a2..f564f93616 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml @@ -159,6 +159,8 @@ roles: max-running-requests: 128 cuda-graph-max-bs-decode: 64 sbatch_directives: + # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. + exclude: dsxe-sa-b300-prd0-gpu-16 mem: '0' cpus-per-task: '144' srun_options: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml index 31cac4932a..7bc509d9e1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml @@ -159,6 +159,8 @@ roles: max-running-requests: 192 cuda-graph-max-bs-decode: 64 sbatch_directives: + # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. + exclude: dsxe-sa-b300-prd0-gpu-16 mem: '0' cpus-per-task: '144' srun_options: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cb853a7959..66ea934549 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8369,4 +8369,5 @@ - Pin SGLang nightly-dev-cu13-20260918-20518d85 and Dynamo 805a77f053d807b0d8def5d27f674a6df0ed839e, use the supported Mamba cache flags and colocated Mooncake intra-node NVLink transport, and resolve verified NVIDIA vendor wheels through the vendor index. Keep HiCache on prefill only and use NEXTN with repository golden acceptance for throughput and real verification for accuracy evals. - Require native DCGM power telemetry, explicit custom-benchmark measurement-window fields and exact matrix concurrency injection. Verify terminal Slurm accounting and the complete producer power contract, preserve diagnostics before returning failures, and skip throughput power validation for eval-only jobs. Skip login-host model preflight for node-local weights; retain runtime loading and health checks. - Select the six provisional points from measured published-frontier and same-image aggregate comparisons, including the concurrency-96 write-through improvement. Require a fresh full official sweep with six canonical 3600-second profiles and six full applicable accuracy evals before admission; short diagnostics and the earlier narrow run with invalid power do not substitute for this qualification. + - Temporarily exclude gpu-16 for the three H298 recipes after a host-memory OOM and a separately owned 1.63 TB preexisting tmpfs allocation were observed there. Current clearance could not be verified while another allocation was active; no shared files or other jobs were changed. Other nodes remain eligible, and H72 recipes are unaffected. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From a55052c2fcf0652c9c1ca1035ae1084abe4caf9a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 00:24:28 -0500 Subject: [PATCH 21/34] fix: allow complete B300 AgentX runs on eligible nodes --- .../agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml | 3 ++- .../agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml | 3 ++- .../agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml | 3 ++- .../agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml | 4 ++-- .../agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml | 4 ++-- ...-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml | 4 ++-- configs/nvidia-master.yaml | 2 ++ perf-changelog.yaml | 3 ++- 8 files changed, 16 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml index a7af1817a8..561e1fd864 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml @@ -19,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '2:00:00' + time_limit: '4:00:00' health_check: max_attempts: 1440 interval_seconds: 10 @@ -159,6 +159,7 @@ roles: max-running-requests: 32 cuda-graph-max-bs-decode: 16 sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-11 mem: '0' cpus-per-task: '144' srun_options: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml index 1be2815a08..46a22456dc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml @@ -19,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '2:00:00' + time_limit: '4:00:00' health_check: max_attempts: 1440 interval_seconds: 10 @@ -159,6 +159,7 @@ roles: max-running-requests: 48 cuda-graph-max-bs-decode: 24 sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-11 mem: '0' cpus-per-task: '144' srun_options: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml index 6f13fc2a39..78a16050cd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml @@ -19,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '2:00:00' + time_limit: '4:00:00' health_check: max_attempts: 1440 interval_seconds: 10 @@ -159,6 +159,7 @@ roles: max-running-requests: 64 cuda-graph-max-bs-decode: 32 sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-11 mem: '0' cpus-per-task: '144' srun_options: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml index 60ec8e8e5e..94cb7493a9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml @@ -19,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '2:00:00' + time_limit: '4:00:00' health_check: max_attempts: 1440 interval_seconds: 10 @@ -160,7 +160,7 @@ roles: cuda-graph-max-bs-decode: 48 sbatch_directives: # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. - exclude: dsxe-sa-b300-prd0-gpu-16 + exclude: dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16 mem: '0' cpus-per-task: '144' srun_options: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml index f564f93616..59da0ea431 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml @@ -19,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '2:00:00' + time_limit: '4:00:00' health_check: max_attempts: 1440 interval_seconds: 10 @@ -160,7 +160,7 @@ roles: cuda-graph-max-bs-decode: 64 sbatch_directives: # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. - exclude: dsxe-sa-b300-prd0-gpu-16 + exclude: dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16 mem: '0' cpus-per-task: '144' srun_options: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml index 7bc509d9e1..2f34806d38 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml @@ -19,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '2:00:00' + time_limit: '4:00:00' health_check: max_attempts: 1440 interval_seconds: 10 @@ -160,7 +160,7 @@ roles: cuda-graph-max-bs-decode: 64 sbatch_directives: # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. - exclude: dsxe-sa-b300-prd0-gpu-16 + exclude: dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16 mem: '0' cpus-per-task: '144' srun_options: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 500c6ee1f6..b12fe06aa0 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8250,6 +8250,8 @@ dsv41flash-fp4-h200-sglang-agentic-dspark: search-space: - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16] } +# Recipes allow four hours for canonical warmup, one-hour profiling and artifact processing. +# Temporary node exclusions cover observed startup/resource failures, not measured performance. qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e654ceb114..870337fe7e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8415,5 +8415,6 @@ - Pin SGLang nightly-dev-cu13-20260918-20518d85 and Dynamo 805a77f053d807b0d8def5d27f674a6df0ed839e, use the supported Mamba cache flags and colocated Mooncake intra-node NVLink transport, and resolve verified NVIDIA vendor wheels through the vendor index. Keep HiCache on prefill only and use NEXTN with repository golden acceptance for throughput and real verification for accuracy evals. - Require native DCGM power telemetry, explicit custom-benchmark measurement-window fields and exact matrix concurrency injection. Verify terminal Slurm accounting and the complete producer power contract, preserve diagnostics before returning failures, and skip throughput power validation for eval-only jobs. Skip login-host model preflight for node-local weights; retain runtime loading and health checks. - Select the six provisional points from measured published-frontier and same-image aggregate comparisons, including the concurrency-96 write-through improvement. Require a fresh full official sweep with six canonical 3600-second profiles and six full applicable accuracy evals before admission; short diagnostics and the earlier narrow run with invalid power do not substitute for this qualification. - - Temporarily exclude gpu-16 for the three H298 recipes after a host-memory OOM and a separately owned 1.63 TB preexisting tmpfs allocation were observed there. Current clearance could not be verified while another allocation was active; no shared files or other jobs were changed. Other nodes remain eligible, and H72 recipes are unaffected. + - Temporarily exclude gpu-11 for all six recipes after three official jobs repeatedly failed during prefill TP3 model loading there; the underlying node-specific cause remains unproven. Also exclude gpu-16 for the three H298 recipes after a host-memory OOM and a separately owned 1.63 TB preexisting tmpfs allocation were observed there, with current clearance unverified. These exclusions address observed startup/resource failures; no shared files, node settings or other jobs were changed. + - Allow a four-hour Slurm allocation for canonical warmup, the unchanged 3600-second profile and required artifact processing. Full high-concurrency warmup exceeded the earlier short-run allowance; an owner-only extension request for the existing job was denied, so the longer envelope applies only to new jobs. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From 48a9c308dedaafa228b74bc99ad0f37159a37e47 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 01:46:13 -0500 Subject: [PATCH 22/34] perf: retain B300 FP8 candidates supported by canonical qualification --- ...p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml | 195 ------------------ ...p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml | 195 ------------------ ...located-c96-write-through-mtp-hicache.yaml | 195 ------------------ configs/nvidia-master.yaml | 54 ----- perf-changelog.yaml | 7 +- runners/test_b300_srt_status.py | 44 ---- 6 files changed, 3 insertions(+), 687 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml deleted file mode 100644 index 94cb7493a9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# Measured fast candidate; full official sweep and accuracy gates remain required. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 96 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 96 - cuda-graph-max-bs-decode: 48 -sbatch_directives: - # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. - exclude: dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16 - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml deleted file mode 100644 index 59da0ea431..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# Measured fast candidate; full official sweep and accuracy gates remain required. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 128 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 128 - cuda-graph-max-bs-decode: 64 -sbatch_directives: - # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. - exclude: dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16 - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '64' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml deleted file mode 100644 index 2f34806d38..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# Measured fast candidate; full official sweep and accuracy gates remain required. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 192 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 192 - cuda-graph-max-bs-decode: 64 -sbatch_directives: - # Prior node-local tmpfs pressure caused an H298 host OOM; clearance is unverified. - exclude: dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16 - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '96' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b12fe06aa0..e983597bb8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8323,57 +8323,3 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 48 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 64 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c64-mtp-hicache.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 96 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c96-write-through-mtp-hicache.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 870337fe7e..2ee01922d4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8411,10 +8411,9 @@ scenario-type: - agentic-coding description: - - Add six B300 Qwen3.5-397B-A17B FP8 AgentX recipes using colocated 1P1D TP4/TP4 Dynamo plus SGLang through srt-slurm, with all eight allocated GPUs active. Use 72 GB HiCache per prefill rank at concurrency 16, 24 and 32; use 298 GB at concurrency 48, 64 and 96, with first-use write-through at 96 and selective write-through elsewhere. Keep existing aggregate publication scopes intact and exclude losing 2P1D experiments. + - Add B300 Qwen3.5-397B-A17B FP8 AgentX recipes at concurrency 16, 24 and 32 using colocated 1P1D TP4/TP4 Dynamo plus SGLang through srt-slurm, with all eight allocated GPUs active and 72 GB HiCache per prefill rank. Preserve the published aggregate curves. Exclude losing 2P1D experiments and high-concurrency candidates that did not qualify under canonical measurement. - Pin SGLang nightly-dev-cu13-20260918-20518d85 and Dynamo 805a77f053d807b0d8def5d27f674a6df0ed839e, use the supported Mamba cache flags and colocated Mooncake intra-node NVLink transport, and resolve verified NVIDIA vendor wheels through the vendor index. Keep HiCache on prefill only and use NEXTN with repository golden acceptance for throughput and real verification for accuracy evals. - Require native DCGM power telemetry, explicit custom-benchmark measurement-window fields and exact matrix concurrency injection. Verify terminal Slurm accounting and the complete producer power contract, preserve diagnostics before returning failures, and skip throughput power validation for eval-only jobs. Skip login-host model preflight for node-local weights; retain runtime loading and health checks. - - Select the six provisional points from measured published-frontier and same-image aggregate comparisons, including the concurrency-96 write-through improvement. Require a fresh full official sweep with six canonical 3600-second profiles and six full applicable accuracy evals before admission; short diagnostics and the earlier narrow run with invalid power do not substitute for this qualification. - - Temporarily exclude gpu-11 for all six recipes after three official jobs repeatedly failed during prefill TP3 model loading there; the underlying node-specific cause remains unproven. Also exclude gpu-16 for the three H298 recipes after a host-memory OOM and a separately owned 1.63 TB preexisting tmpfs allocation were observed there, with current clearance unverified. These exclusions address observed startup/resource failures; no shared files, node settings or other jobs were changed. - - Allow a four-hour Slurm allocation for canonical warmup, the unchanged 3600-second profile and required artifact processing. Full high-concurrency warmup exceeded the earlier short-run allowance; an owner-only extension request for the existing job was denied, so the longer envelope applies only to new jobs. + - The initial full sweep qualified concurrency 32 at 20629 total tokens/s/GPU and p90 interactivity 200.58, strictly improving a published aggregate frontier point. Short-run gains at higher concurrency did not establish canonical qualification. Require a fresh complete sweep of the retained three canonical 3600-second profiles and all three applicable full accuracy evals; initial C16/C24 attempts failed before measurement. + - Temporarily exclude gpu-11 after three official jobs repeatedly failed during prefill TP3 model loading there; the underlying cause remains unproven, and a later eval on that node succeeded. Allow four hours for canonical warmup, unchanged one-hour profiling and artifact processing. No shared files, node settings or other users' jobs were changed. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py index 152d52e158..8d961a3da7 100644 --- a/runners/test_b300_srt_status.py +++ b/runners/test_b300_srt_status.py @@ -224,47 +224,3 @@ def test_b300_collects_artifacts_before_returning_slurm_status( else 1 ) assert int((tmp_path / "sacct-count").read_text()) == expected_queries - - -@pytest.mark.parametrize("concurrency", [16, 24, 32, 48, 64, 96]) -def test_b300_recipe_publishes_native_power_window( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, concurrency: int -) -> None: - """Bridge the actual recipe env and native producer DIR to the real adapter.""" - from infx.results.agentic.power_adapter import write_multinode_power_window - - recipes = ( - ROOT / "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx" - ) - recipe_path = next(recipes.glob(f"*-colocated-c{concurrency}-*hicache.yaml")) - recipe = yaml.safe_load(recipe_path.read_text()) - for name, value in recipe["benchmark"]["env"].items(): - monkeypatch.setenv(name, str(value)) - - # Native SRT provides DIR; /logs is the container mount of the host log root. - log_root = tmp_path / "logs" - window_dir = log_root / "power/windows" - result_dir = log_root / "agentic" / f"conc_{concurrency}" - window_dir.mkdir(parents=True) - result_dir.mkdir(parents=True) - assert recipe["benchmark"]["env"]["SRT_MEASUREMENT_WINDOW_RESULT_ROOT"] == "/logs" - monkeypatch.setenv("SRT_MEASUREMENT_WINDOW_DIR", str(window_dir)) - monkeypatch.setenv("SRT_MEASUREMENT_WINDOW_RESULT_ROOT", str(log_root)) - - assert ( - write_multinode_power_window( - result_dir=result_dir, - concurrency=concurrency, - state="running", - require_power=True, - ) - == 0 - ) - window = json.loads( - (window_dir / f"agentic_power_concurrency_{concurrency}.json").read_text() - ) - assert window["status"] == "running" - assert window["concurrency"] == concurrency - assert window["result_path"] == ( - f"agentic/conc_{concurrency}/agentic_power_concurrency_{concurrency}.json" - ) From 0bbd8facb2d9ea1c61b36208c93f86012eaf3edd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 14:01:48 -0500 Subject: [PATCH 23/34] feat(models): acquire verified original BF16 MTP shards atomically --- configs/models/qwen3.5-397b-mtp-bf16.json | 59 ++++++ docs/configuration-procedures.md | 45 ++++ docs/configuration-procedures_zh.md | 40 ++++ infx/models/__init__.py | 1 + infx/models/acquire_mtp.py | 220 ++++++++++++++++++++ utils/test_acquire_mtp.py | 240 ++++++++++++++++++++++ 6 files changed, 605 insertions(+) create mode 100644 configs/models/qwen3.5-397b-mtp-bf16.json create mode 100644 infx/models/__init__.py create mode 100644 infx/models/acquire_mtp.py create mode 100644 utils/test_acquire_mtp.py diff --git a/configs/models/qwen3.5-397b-mtp-bf16.json b/configs/models/qwen3.5-397b-mtp-bf16.json new file mode 100644 index 0000000000..ab2eadb3ab --- /dev/null +++ b/configs/models/qwen3.5-397b-mtp-bf16.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "repo": "Qwen/Qwen3.5-397B-A17B", + "revision": "8472618112abcbd45acbcdc58436aff4233c23f7", + "files": [ + { + "name": "config.json", + "size": 4177, + "sha256": "3ae7fa89c2f7d1354096418ddaf1331e9e0898a8ca11e804fb6dbaa087efb7da" + }, + { + "name": "generation_config.json", + "size": 244, + "sha256": "303aba891d66ab63908a7b3cc9163bcb835fdf8b9f6301c73216f3f1eb3992dd" + }, + { + "name": "model.safetensors-00091-of-00094.safetensors", + "size": 9638517720, + "sha256": "3633797c27c32b194da65eaff2c5c27eb184bd547577c4124498bb8ae51e670f" + }, + { + "name": "model.safetensors-00092-of-00094.safetensors", + "size": 9658656328, + "sha256": "66cd7e6aecdcd4c69bfc7e8c072cbc4ae3771897785279a0ff2da6a0eb4a8211" + }, + { + "name": "model.safetensors-00093-of-00094.safetensors", + "size": 9663818112, + "sha256": "15f424c1a22772238bb0efcf53291f29ec00f5c08d84b644ae54ee31486475e1" + }, + { + "name": "model.safetensors-00094-of-00094.safetensors", + "size": 4741119112, + "sha256": "75b5c72a8dc8f07946dab6bdccec6663f332b24c9be8c59c215e0eabd5204467" + }, + { + "name": "model.safetensors.index.json", + "size": 303440, + "sha256": "407d6a184a29469034ad92bcfaa6b72b582f8ad6afcbfad18a38a1c59ed296f8" + }, + { + "name": "tokenizer.json", + "size": 12807982, + "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42" + }, + { + "name": "tokenizer_config.json", + "size": 16710, + "sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8" + } + ], + "mtp_prefix": "mtp.", + "mtp_tensor_count": 1553, + "mtp_bytes": 13191136256, + "required_bf16_tensors": [ + "lm_head.weight", + "model.language_model.embed_tokens.weight" + ] +} diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 4c6abfccfa..32336c7120 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -408,6 +408,51 @@ those paths on `vllm` only. GPU sweep and eval evidence is required before calling any of these arms validated. +### Acquire the original Qwen3.5 BF16 MTP subset + +[`infx.models.acquire_mtp`](../infx/models/acquire_mtp.py) prepares an original-weight +MTP asset using the pinned [manifest](../configs/models/qwen3.5-397b-mtp-bf16.json). +It downloads four unmodified shards (91–94, about 33.7 GB) and support metadata from +`Qwen/Qwen3.5-397B-A17B` revision `8472618112abcbd45acbcdc58436aff4233c23f7`. +Manifest digests come from the immutable HF revision: upstream LFS SHA256 for large +files and verified original bytes for metadata. No tensors are converted, repacked, +or dequantized. This directory is an MTP subset, **not a complete target model**. + +The caller chooses the shared destination and lock timeout explicitly: + +```bash +python -m infx.models.acquire_mtp \ + --manifest configs/models/qwen3.5-397b-mtp-bf16.json \ + --destination /shared/models/Qwen3.5-397B-MTP-BF16-8472618 \ + --lock-timeout 3600 +``` + +The helper serializes callers with a destination-specific POSIX lock. It checks every +original file's size and SHA256, all selected header/index entries, all 1,553 original +BF16 MTP tensors, and the BF16 embedding/head. It preserves the full upstream index +as `model.safetensors.index.json.original`, computes the filtered index's `total_size` +from tensor offsets, and atomically publishes the validated directory with +`subset-provenance.json`. Warm reuse revalidates the complete asset, including hashes +and manifest identity. An existing incomplete or conflicting destination fails without +repair or replacement. Interrupted temporary staging directories are never accepted as +complete; a subsequent attempt downloads into a new staging directory. Verification +requires reading the full 33.7 GB on warm reuse too. The helper requires POSIX locking +and atomic directory rename on the caller's shared filesystem. + +When integrating a qualified recipe, its existing pool launcher should acquire the +asset before submission, pass `--mount HOST_PATH /draft-model` through +`write_srt_cluster_config`, and preserve `subset-provenance.json` plus the resolved +host/container paths in benchmark artifacts. Do not put personal paths in recipes. +For SGLang `20518d85`, the supported separate-draft arguments are: +`speculative-draft-model-path: /draft-model`, the exact revision above, +`speculative-draft-model-quantization: unquant`, +`speculative-draft-kv-cache-dtype: bf16`, and +`speculative-moe-runner-backend: flashinfer_trtllm`. +Omitting `unquant` can inherit the FP8 target's quantization. Separate draft KV dtype +does not guarantee mixed-dtype HiCache compatibility; retain the recipe's independently +validated cache/retraction configuration. This helper alone does not change any launcher +or qualify a serving recipe; real accuracy and performance tests remain required. + ## Validate Run the smallest checks that cover the edited layers. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index f877035f18..ed879e8549 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -354,6 +354,46 @@ offload），prefill 分块上限设为 4096，即 vLLM H100 配方在 80 GB 显 在获得 GPU sweep 与 eval 证据之前，不得将这些配方视为已验证。 +### 获取 Qwen3.5 原始 BF16 MTP 子集 + +[`infx.models.acquire_mtp`](../infx/models/acquire_mtp.py) 根据固定的 +[清单](../configs/models/qwen3.5-397b-mtp-bf16.json) 准备原始权重 MTP 资源。 +它从 `Qwen/Qwen3.5-397B-A17B` 的提交 +`8472618112abcbd45acbcdc58436aff4233c23f7` 下载未经修改的第 91–94 个分片 +（约 33.7 GB）及配套元数据。清单使用不可变 HF 提交的摘要：大文件采用上游 +LFS SHA256，元数据采用已验证原始字节的 SHA256。工具不会转换、重新打包或 +反量化张量。该目录是 MTP 子集，**不是完整的目标模型**。 + +调用方必须明确指定共享目录和锁等待时间： + +```bash +python -m infx.models.acquire_mtp \ + --manifest configs/models/qwen3.5-397b-mtp-bf16.json \ + --destination /shared/models/Qwen3.5-397B-MTP-BF16-8472618 \ + --lock-timeout 3600 +``` + +工具使用目标目录专属的 POSIX 锁串行处理调用方，检查每个原始文件的大小和 +SHA256、所选分片的头部与索引条目、全部 1,553 个原始 BF16 MTP 张量，以及 +BF16 embedding/head。完整上游索引保存在 `model.safetensors.index.json.original`； +过滤索引的 `total_size` 根据张量偏移量计算。验证完成后，工具将目录及 +`subset-provenance.json` 原子发布。复用时仍验证全部文件摘要与清单身份。 +已有但不完整或冲突的目录会直接报错，不会修复或覆盖。中断留下的临时目录 +不视为完整资源；重试使用新的暂存目录下载。复用也需要读取全部约 33.7 GB。 +共享文件系统必须支持 POSIX 锁和原子目录重命名。 + +后续集成已验证配方时，应由现有集群启动器在提交前获取资源，通过 +`write_srt_cluster_config` 传入 `--mount HOST_PATH /draft-model`，并在测试产物中 +保留 `subset-provenance.json` 及实际宿主机/容器路径。配方不得依赖个人目录。 +SGLang `20518d85` 的独立 draft 参数为 +`speculative-draft-model-path: /draft-model`、上述固定 revision、 +`speculative-draft-model-quantization: unquant`、 +`speculative-draft-kv-cache-dtype: bf16` 和 +`speculative-moe-runner-backend: flashinfer_trtllm`。 +省略 `unquant` 可能继承 FP8 目标模型的量化配置。独立 draft KV 类型并不保证 +混合类型 HiCache 兼容；必须保留配方已独立验证的缓存和回退配置。 +本工具本身不修改任何启动器，也不构成配方验证；仍需真实准确率和性能测试。 + ## 验证 运行覆盖被修改层的最小检查。 diff --git a/infx/models/__init__.py b/infx/models/__init__.py new file mode 100644 index 0000000000..bfe31762fa --- /dev/null +++ b/infx/models/__init__.py @@ -0,0 +1 @@ +"""Verified model asset acquisition.""" diff --git a/infx/models/acquire_mtp.py b/infx/models/acquire_mtp.py new file mode 100644 index 0000000000..7361f200f0 --- /dev/null +++ b/infx/models/acquire_mtp.py @@ -0,0 +1,220 @@ +"""Acquire original checkpoint shards as a verified, atomic MTP-only asset.""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +import math +import shutil +import struct +import tempfile +import time +import urllib.request +from pathlib import Path +from typing import Annotated + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +INDEX = "model.safetensors.index.json" +PROVENANCE = "subset-provenance.json" +Sha256 = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")] +Filename = Annotated[str, Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")] + + +class FileSpec(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + name: Filename + size: Annotated[int, Field(gt=0)] + sha256: Sha256 + + +class Manifest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + schema_version: Annotated[int, Field(ge=1, le=1)] + repo: Annotated[str, Field(pattern=r"^[\w.-]+/[\w.-]+$")] + revision: Annotated[str, Field(pattern=r"^[0-9a-f]{40}$")] + files: list[FileSpec] + mtp_prefix: str + mtp_tensor_count: Annotated[int, Field(gt=0)] + mtp_bytes: Annotated[int, Field(gt=0)] + required_bf16_tensors: list[str] + + @model_validator(mode="after") + def check_files(self) -> Manifest: + names = [f.name for f in self.files] + if len(names) != len(set(names)) or not self.mtp_prefix: + raise ValueError("Duplicate filenames or empty MTP prefix") + if not {INDEX, "config.json"}.issubset(names): + raise ValueError("Original index and config are required") + if PROVENANCE in names or INDEX + ".original" in names: + raise ValueError("Manifest uses a reserved output filename") + return self + + +def _json_bytes(value: object) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def _sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def _download(url: str, destination: Path) -> None: + # URLs are constructed exclusively from the validated HF repo/revision/file. + with ( + urllib.request.urlopen(url, timeout=180) as response, # noqa: S310 + destination.open("xb") as output, + ): + shutil.copyfileobj(response, output, length=8 * 1024 * 1024) + + +def _original_path(directory: Path, name: str) -> Path: + return directory / (name + ".original" if name == INDEX else name) + + +def _verify_files(directory: Path, manifest: Manifest) -> None: + for spec in manifest.files: + path = _original_path(directory, spec.name) + if path.is_symlink() or not path.is_file(): + raise ValueError(f"Missing or non-regular original file: {spec.name}") + if path.stat().st_size != spec.size or _sha256(path) != spec.sha256: + raise ValueError(f"Size/SHA256 mismatch: {spec.name}") + + +def _filtered_index(directory: Path, manifest: Manifest) -> dict: + original = json.loads(_original_path(directory, INDEX).read_bytes()) + weight_map = original["weight_map"] + shards = {f.name for f in manifest.files if f.name.endswith(".safetensors")} + selected = {name: shard for name, shard in weight_map.items() if shard in shards} + tensors = {} + total_size = 0 + for shard in sorted(shards): + path = directory / shard + with path.open("rb") as stream: + raw_length = stream.read(8) + if len(raw_length) != 8: + raise ValueError(f"Truncated safetensors prefix: {shard}") + header_size = struct.unpack(" min(100_000_000, path.stat().st_size - 8): + raise ValueError(f"Invalid safetensors header length: {shard}") + header = json.loads(stream.read(header_size)) + spans = [] + for name, tensor in header.items(): + if name == "__metadata__": + continue + start, end = tensor["data_offsets"] + if start < 0 or end < start or end > path.stat().st_size - 8 - header_size: + raise ValueError(f"Invalid tensor offsets: {name}") + if name in tensors or selected.get(name) != shard: + raise ValueError(f"Header/index tensor mismatch: {name}") + tensors[name] = tensor + spans.append((start, end)) + total_size += end - start + cursor = 0 + for start, end in sorted(spans): + if start != cursor: + raise ValueError(f"Non-contiguous tensor data: {shard}") + cursor = end + if cursor != path.stat().st_size - 8 - header_size: + raise ValueError(f"Unindexed tensor data: {shard}") + if set(tensors) != set(selected): + raise ValueError("Selected index references missing tensors") + expected_mtp = {name for name in weight_map if name.startswith(manifest.mtp_prefix)} + actual_mtp = {name for name in tensors if name.startswith(manifest.mtp_prefix)} + if expected_mtp != actual_mtp or len(actual_mtp) != manifest.mtp_tensor_count: + raise ValueError("Incomplete MTP tensor set") + mtp_bytes = sum( + tensors[n]["data_offsets"][1] - tensors[n]["data_offsets"][0] for n in actual_mtp + ) + if mtp_bytes != manifest.mtp_bytes: + raise ValueError("MTP tensor byte count mismatch") + for name in actual_mtp | set(manifest.required_bf16_tensors): + tensor = tensors.get(name) + if tensor is None or tensor["dtype"] != "BF16": + raise ValueError(f"Required original BF16 tensor missing or wrong dtype: {name}") + if math.prod(tensor["shape"]) * 2 != tensor["data_offsets"][1] - tensor["data_offsets"][0]: + raise ValueError(f"Invalid BF16 tensor shape: {name}") + return { + "metadata": {**original.get("metadata", {}), "total_size": total_size}, + "weight_map": selected, + } + + +def _provenance(manifest: Manifest, filtered: bytes) -> dict: + return { + "artifact_type": "original-bf16-mtp-subset-not-full-target", + "manifest": manifest.model_dump(), + "manifest_sha256": hashlib.sha256(_json_bytes(manifest.model_dump())).hexdigest(), + "filtered_index_sha256": hashlib.sha256(filtered).hexdigest(), + "tensor_conversion": False, + } + + +def _validate(directory: Path, manifest: Manifest) -> tuple[bytes, dict]: + _verify_files(directory, manifest) + filtered = _json_bytes(_filtered_index(directory, manifest)) + return filtered, _provenance(manifest, filtered) + + +def acquire(manifest: Manifest, destination: Path, *, lock_timeout: float) -> Path: + """Verify/reuse or publish a new asset; never repair or overwrite an existing one.""" + if not math.isfinite(lock_timeout) or lock_timeout < 0: + raise ValueError("lock_timeout must be finite and nonnegative") + destination = destination.absolute() + destination.parent.mkdir(parents=True, exist_ok=True) + lock_path = destination.with_name(destination.name + ".download.lock") + with lock_path.open("a") as lock: + deadline = time.monotonic() + lock_timeout + while True: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if time.monotonic() >= deadline: + raise TimeoutError(f"Timed out locking {destination}") from None + time.sleep(min(0.1, max(0, deadline - time.monotonic()))) + if destination.is_symlink(): + raise ValueError("Destination must not be a symlink") + if destination.exists(): + filtered, provenance = _validate(destination, manifest) + if (destination / INDEX).read_bytes() != filtered or json.loads( + (destination / PROVENANCE).read_bytes() + ) != provenance: + raise ValueError("Existing destination identity or generated index differs") + expected = {_original_path(destination, f.name).name for f in manifest.files} | { + INDEX, + PROVENANCE, + } + if {p.name for p in destination.iterdir()} != expected: + raise ValueError("Existing destination has unexpected files") + return destination + # Same parent/filesystem guarantees readers see either absent or complete. + with tempfile.TemporaryDirectory( + prefix=f".{destination.name}.staging-", dir=destination.parent + ) as staging: + stage = Path(staging) + for spec in manifest.files: + url = f"https://huggingface.co/{manifest.repo}/resolve/{manifest.revision}/{spec.name}" + _download(url, _original_path(stage, spec.name)) + filtered, provenance = _validate(stage, manifest) + (stage / INDEX).write_bytes(filtered) + (stage / PROVENANCE).write_bytes(_json_bytes(provenance)) + stage.rename(destination) + return destination + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--lock-timeout", type=float, required=True) + args = parser.parse_args() + manifest = Manifest.model_validate_json(args.manifest.read_bytes()) + print(acquire(manifest, args.destination, lock_timeout=args.lock_timeout)) + + +if __name__ == "__main__": + main() diff --git a/utils/test_acquire_mtp.py b/utils/test_acquire_mtp.py new file mode 100644 index 0000000000..cc98a7c13f --- /dev/null +++ b/utils/test_acquire_mtp.py @@ -0,0 +1,240 @@ +"""Behavioral acquisition tests with tiny original safetensors and mocked HTTP.""" + +import hashlib +import json +import struct +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from infx.models import acquire_mtp as acquisition + + +def fixture_asset(*, dtype="BF16", missing_mtp=False): + # Four 1-element BF16 tensors: the selected payload is exactly eight bytes. + names = ["mtp.weight", "embed.weight", "head.weight", "target.weight"] + header = { + name: { + "dtype": dtype if name == "mtp.weight" else "BF16", + "shape": [1], + "data_offsets": [2 * i, 2 * i + 2], + } + for i, name in enumerate(names) + } + encoded = json.dumps(header).encode() + encoded += b" " * (-len(encoded) % 8) + shard = struct.pack(" Date: Sat, 19 Sep 2026 14:03:41 -0500 Subject: [PATCH 24/34] test(models): reject corrupted provenance and unexpected asset files --- utils/test_acquire_mtp.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/utils/test_acquire_mtp.py b/utils/test_acquire_mtp.py index cc98a7c13f..f27caf47c4 100644 --- a/utils/test_acquire_mtp.py +++ b/utils/test_acquire_mtp.py @@ -117,7 +117,9 @@ def bad_download(url, destination): assert not list(tmp_path.glob(".draft.staging-*")) -@pytest.mark.parametrize("kind", ["incomplete", "corrupt", "identity", "index"]) +@pytest.mark.parametrize( + "kind", ["incomplete", "corrupt", "identity", "index", "provenance", "unexpected"] +) def test_rejects_existing_destination_without_repair(tmp_path, monkeypatch, kind): manifest, files = fixture_asset() install_download(monkeypatch, files) @@ -129,8 +131,14 @@ def test_rejects_existing_destination_without_repair(tmp_path, monkeypatch, kind (dest / "part.safetensors").write_bytes(b"broken") elif kind == "identity": manifest = manifest.model_copy(update={"revision": "b" * 40}) - else: + elif kind == "index": (dest / "model.safetensors.index.json").write_text("{}") + elif kind == "provenance": + provenance = json.loads((dest / "subset-provenance.json").read_bytes()) + provenance["tensor_conversion"] = True + (dest / "subset-provenance.json").write_text(json.dumps(provenance)) + else: + (dest / "unexpected.safetensors").write_bytes(b"not part of the verified asset") before = {p.name: p.read_bytes() for p in dest.iterdir()} with pytest.raises(ValueError): acquisition.acquire(manifest, dest, lock_timeout=1) From 8ba25226a5f9dd821fe8824faec47f456c58e80c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 14:23:04 -0500 Subject: [PATCH 25/34] feat(b300): stage original BF16 MTP assets for Qwen FP8 recipes --- ...p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml | 8 +++++ ...p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml | 8 +++++ ...p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml | 8 +++++ docs/configuration-procedures.md | 14 ++++---- docs/configuration-procedures_zh.md | 10 +++--- perf-changelog.yaml | 2 +- runners/launch_b300-dsxe.sh | 23 +++++++++++- runners/test_b300_srt_status.py | 36 +++++++++++++++++++ 8 files changed, 97 insertions(+), 12 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml index 561e1fd864..334b5de43c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml @@ -154,6 +154,13 @@ roles: speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-model-quantization: unquant + speculative-draft-kv-cache-dtype: bf16 + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor + disaggregation-decode-extra-slots: 0 scheduler-recv-interval: 10 stream-interval: 50 max-running-requests: 32 @@ -167,6 +174,7 @@ srun_options: container-remap-root: '' benchmark: type: custom + concurrencies: [16] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml index 46a22456dc..bf962ea753 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml @@ -154,6 +154,13 @@ roles: speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-model-quantization: unquant + speculative-draft-kv-cache-dtype: bf16 + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor + disaggregation-decode-extra-slots: 0 scheduler-recv-interval: 10 stream-interval: 50 max-running-requests: 48 @@ -167,6 +174,7 @@ srun_options: container-remap-root: '' benchmark: type: custom + concurrencies: [24] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml index 78a16050cd..95f69be190 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml @@ -154,6 +154,13 @@ roles: speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-model-quantization: unquant + speculative-draft-kv-cache-dtype: bf16 + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor + disaggregation-decode-extra-slots: 0 scheduler-recv-interval: 10 stream-interval: 50 max-running-requests: 64 @@ -167,6 +174,7 @@ srun_options: container-remap-root: '' benchmark: type: custom + concurrencies: [32] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 32336c7120..ff66bd46b7 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -439,10 +439,12 @@ complete; a subsequent attempt downloads into a new staging directory. Verificat requires reading the full 33.7 GB on warm reuse too. The helper requires POSIX locking and atomic directory rename on the caller's shared filesystem. -When integrating a qualified recipe, its existing pool launcher should acquire the -asset before submission, pass `--mount HOST_PATH /draft-model` through -`write_srt_cluster_config`, and preserve `subset-provenance.json` plus the resolved -host/container paths in benchmark artifacts. Do not put personal paths in recipes. +The B300 DSXE launcher acquires this asset before submitting Qwen3.5 FP8 +Dynamo/SGLang AgentX jobs. It uses a manifest-hash directory under its existing +`WRITABLE_MODELS_DIR`, mounts it at `/draft-model` through `write_srt_cluster_config`, +and includes the manifest, provenance and resolved host/container paths in +`draft-model-provenance/` inside the server-log archive. Recipes do not depend on +personal directories. For SGLang `20518d85`, the supported separate-draft arguments are: `speculative-draft-model-path: /draft-model`, the exact revision above, `speculative-draft-model-quantization: unquant`, @@ -450,8 +452,8 @@ For SGLang `20518d85`, the supported separate-draft arguments are: `speculative-moe-runner-backend: flashinfer_trtllm`. Omitting `unquant` can inherit the FP8 target's quantization. Separate draft KV dtype does not guarantee mixed-dtype HiCache compatibility; retain the recipe's independently -validated cache/retraction configuration. This helper alone does not change any launcher -or qualify a serving recipe; real accuracy and performance tests remain required. +validated cache/retraction configuration. Acquisition does not qualify a serving recipe; +real accuracy and performance tests remain required. ## Validate diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index ed879e8549..85c2c7303f 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -382,9 +382,11 @@ BF16 embedding/head。完整上游索引保存在 `model.safetensors.index.json. 不视为完整资源；重试使用新的暂存目录下载。复用也需要读取全部约 33.7 GB。 共享文件系统必须支持 POSIX 锁和原子目录重命名。 -后续集成已验证配方时，应由现有集群启动器在提交前获取资源，通过 -`write_srt_cluster_config` 传入 `--mount HOST_PATH /draft-model`，并在测试产物中 -保留 `subset-provenance.json` 及实际宿主机/容器路径。配方不得依赖个人目录。 +B300 DSXE 启动器在提交 Qwen3.5 FP8 Dynamo/SGLang AgentX 任务前获取该资源， +使用现有 `WRITABLE_MODELS_DIR` 下以清单哈希命名的目录，通过 +`write_srt_cluster_config` 挂载到 `/draft-model`，并在服务端日志归档的 +`draft-model-provenance/` 中保存清单、来源记录及实际宿主机/容器路径。 +配方不依赖个人目录。 SGLang `20518d85` 的独立 draft 参数为 `speculative-draft-model-path: /draft-model`、上述固定 revision、 `speculative-draft-model-quantization: unquant`、 @@ -392,7 +394,7 @@ SGLang `20518d85` 的独立 draft 参数为 `speculative-moe-runner-backend: flashinfer_trtllm`。 省略 `unquant` 可能继承 FP8 目标模型的量化配置。独立 draft KV 类型并不保证 混合类型 HiCache 兼容；必须保留配方已独立验证的缓存和回退配置。 -本工具本身不修改任何启动器，也不构成配方验证；仍需真实准确率和性能测试。 +资源获取本身不构成配方验证；仍需真实准确率和性能测试。 ## 验证 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 93c4d5bbdc..60fab0c367 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8456,5 +8456,5 @@ - Add B300 Qwen3.5-397B-A17B FP8 AgentX disaggregation with colocated TP4 prefill and TP4 decode on eight GPUs at concurrency 16, 24 and 32. - Use Dynamo/SGLang nightly 20260918-20518d85 with NEXTN MTP and Mooncake NVLink transfers. - - Enable 72 GB per-rank prefill HiCache. + - Enable 72 GB per-rank prefill HiCache and use original BF16 MTP draft weights and draft KV. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index f6ce177d58..b9b152be9f 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -151,9 +151,27 @@ export ISL="$ISL" export OSL="$OSL" SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" +SRT_MODEL_MOUNTS=() +DRAFT_PROVENANCE_DIR="" +if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + DRAFT_MANIFEST="$GITHUB_WORKSPACE/configs/models/qwen3.5-397b-mtp-bf16.json" + DRAFT_MANIFEST_SHA=$(sha256sum "$DRAFT_MANIFEST") || exit 1 + DRAFT_CACHE_DIR="$WRITABLE_MODELS_DIR/qwen3.5-original-mtp-${DRAFT_MANIFEST_SHA%% *}" + PYTHONPATH="$GITHUB_WORKSPACE" python -m infx.models.acquire_mtp \ + --manifest "$DRAFT_MANIFEST" --destination "$DRAFT_CACHE_DIR" \ + --lock-timeout 3600 || exit 1 + SRT_MODEL_MOUNTS+=(--mount "$DRAFT_CACHE_DIR" /draft-model) + DRAFT_PROVENANCE_DIR="$GITHUB_WORKSPACE/draft-model-provenance" + mkdir -p "$DRAFT_PROVENANCE_DIR" || exit 1 + cp "$DRAFT_MANIFEST" "$DRAFT_PROVENANCE_DIR/manifest.json" || exit 1 + cp "$DRAFT_CACHE_DIR/subset-provenance.json" "$DRAFT_PROVENANCE_DIR/" || exit 1 + printf '%s\n' "$DRAFT_CACHE_DIR" > "$DRAFT_PROVENANCE_DIR/host-path.txt" + printf '%s\n' /draft-model > "$DRAFT_PROVENANCE_DIR/container-path.txt" +fi + echo "Creating srtslurm.yaml configuration..." write_srt_cluster_config b300-dsxe srtslurm.yaml "$USES_DCGM_POWER" \ - --var MODEL_ROOT "$MODEL_ROOT" || exit 1 + --var MODEL_ROOT "$MODEL_ROOT" "${SRT_MODEL_MOUNTS[@]}" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml @@ -241,6 +259,9 @@ if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && "${EVAL_ONLY}" != "tr "${POWER_CONCURRENCIES[@]}" || SRT_JOB_RC=$? fi +if [[ -n "$DRAFT_PROVENANCE_DIR" ]]; then + cp -r "$DRAFT_PROVENANCE_DIR" "$LOGS_DIR/draft-model-provenance" || exit 1 +fi cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py index 8d961a3da7..1428fc01c1 100644 --- a/runners/test_b300_srt_status.py +++ b/runners/test_b300_srt_status.py @@ -4,6 +4,7 @@ import os import shutil import subprocess +import sys import tarfile from pathlib import Path @@ -49,6 +50,25 @@ def test_b300_collects_artifacts_before_returning_slurm_status( ): _stub(binaries, name, "exit 0") _stub(binaries, "unsquashfs", "exit 0") + _stub( + binaries, + "python", + r""" +if [[ "$1" == -m && "$2" == infx.models.acquire_mtp ]]; then + shift 2 + while [[ "$#" -gt 0 ]]; do + case "$1" in + --destination) destination="$2"; shift 2 ;; + *) shift 2 ;; + esac + done + mkdir -p "$destination" + printf '{"artifact_type":"fixture-original-draft"}\n' > "$destination/subset-provenance.json" + exit 0 +fi +exec "$MOCK_REAL_PYTHON" "$@" +""", + ) _stub( binaries, "sacct", @@ -85,6 +105,10 @@ def test_b300_collects_artifacts_before_returning_slurm_status( # Keep the real launcher-owned profiles and renderer in the fixture checkout. shutil.copytree(ROOT / "runners", workspace / "runners") (workspace / "benchmarks").mkdir() + (workspace / "configs/models").mkdir(parents=True) + (workspace / "configs/models/qwen3.5-397b-mtp-bf16.json").write_text( + '{"source":"fixture-original-draft"}\n' + ) if power_mode != "off": recipe = workspace / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" recipe.parent.mkdir(parents=True) @@ -99,6 +123,7 @@ def test_b300_collects_artifacts_before_returning_slurm_status( with (workspace / "runners/slurm_utils.sh").open("a") as helpers: helpers.write(r""" setup_srt_slurm() { + WRITABLE_MODELS_DIR="$MOCK_DRAFT_CACHE" mkdir -p "$1/recipes" cd "$1" || return 1 printf 'name: fixture\nbenchmark:\n type: custom\n' > recipes/test.yaml @@ -126,6 +151,8 @@ def test_b300_collects_artifacts_before_returning_slurm_status( env = { **os.environ, "PATH": f"{binaries}:{os.environ['PATH']}", + "MOCK_REAL_PYTHON": sys.executable, + "MOCK_DRAFT_CACHE": str(tmp_path / "models"), "MOCK_ACCOUNTING": accounting, "MOCK_SACCT_COUNT": str(tmp_path / "sacct-count"), "MOCK_FIXTURE": str(fixture), @@ -196,6 +223,15 @@ def test_b300_collects_artifacts_before_returning_slurm_status( (workspace / "srt-slurm/recipes/test.yaml").read_text() ) assert emitted["benchmark"]["concurrencies"] == [1] + profile = yaml.safe_load((workspace / "srt-slurm/srtslurm.yaml").read_text()) + cache = tmp_path / "models" / ("qwen3.5-original-mtp-" + "0" * 64) + assert profile["default_mounts"][str(cache)] == "/draft-model" + draft_artifact = workspace / "LOGS/draft-model-provenance" + assert json.loads((draft_artifact / "subset-provenance.json").read_text()) == { + "artifact_type": "fixture-original-draft" + } + assert (draft_artifact / "host-path.txt").read_text().strip() == str(cache) + assert (draft_artifact / "container-path.txt").read_text().strip() == "/draft-model" aggregate = json.loads((workspace / "aggregate_conc1.json").read_text()) assert aggregate["diagnostic"] == "retained" if power_mode == "missing": From b73f48805accd27c9af2757b4b298c82d6371c6c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 15:15:45 -0500 Subject: [PATCH 26/34] feat(b300): prepare C48 disaggregation with 64 decode slots --- ...p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml | 203 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 5 +- 3 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml new file mode 100644 index 0000000000..ecd3d20a41 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml @@ -0,0 +1,203 @@ +# C48 candidate screened with canonical warmup; full official sweep and evals required. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 96 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 48 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 48 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index dd3449114a..a8b5680249 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8449,3 +8449,21 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 48 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 60fab0c367..0f8fbe5df9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8454,7 +8454,8 @@ - agentic-coding description: - Add B300 Qwen3.5-397B-A17B FP8 AgentX disaggregation with colocated TP4 prefill - and TP4 decode on eight GPUs at concurrency 16, 24 and 32. + and TP4 decode on eight GPUs at concurrency 16, 24, 32 and 48. - Use Dynamo/SGLang nightly 20260918-20518d85 with NEXTN MTP and Mooncake NVLink transfers. - - Enable 72 GB per-rank prefill HiCache and use original BF16 MTP draft weights and draft KV. + - Use prefill HiCache (72 GB/rank at C16-C32; 298 GB/rank with first-use write-through + at C48), 64 decode slots at C48, and original BF16 MTP weights and draft KV. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From b986eddc246c01add718baa7fc18c06d5652b631 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 15:48:41 -0500 Subject: [PATCH 27/34] perf(b300): add screened TP2 EP2 write-through candidate --- ...2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml | 204 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 11 +- 3 files changed, 228 insertions(+), 5 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml new file mode 100644 index 0000000000..f3024fdf5d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 32 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a8b5680249..8a38f37e68 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8467,3 +8467,21 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 32 + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0f8fbe5df9..1e2435317e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8453,9 +8453,10 @@ scenario-type: - agentic-coding description: - - Add B300 Qwen3.5-397B-A17B FP8 AgentX disaggregation with colocated TP4 prefill - and TP4 decode on eight GPUs at concurrency 16, 24, 32 and 48. - - Use Dynamo/SGLang nightly 20260918-20518d85 with NEXTN MTP and Mooncake NVLink transfers. - - Use prefill HiCache (72 GB/rank at C16-C32; 298 GB/rank with first-use write-through - at C48), 64 decode slots at C48, and original BF16 MTP weights and draft KV. + - Add B300 Qwen3.5-397B-A17B FP8 AgentX disaggregation with colocated TP4 prefill/decode + at C16/C24/C32/C48 and TP2+EP2 prefill/decode at C32. + - Use Dynamo/SGLang nightly 20260918-20518d85, NEXTN with original BF16 draft weights/KV, + and Mooncake NVLink transfers. + - Use prefill HiCache, including 298 GB/rank write-through and 64 decode slots for + TP4 C48 and TP2+EP2 C32. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From 481e1ac61b292ab48901c81c0c1c1ba3ea968ebe Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 15:51:54 -0500 Subject: [PATCH 28/34] perf(b300): select measured C24 decode capacity --- ...p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml index bf962ea753..3e31b2a49c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml @@ -1,4 +1,4 @@ -# Measured fast candidate; full official sweep and accuracy gates remain required. +# Colocated TP4 prefill and decode with prefill HiCache. schema: 2 name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache model: @@ -94,7 +94,7 @@ roles: mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 - max-running-requests: 48 + max-running-requests: 32 enable-hierarchical-cache: true hicache-size: 72 hicache-io-backend: kernel @@ -154,19 +154,19 @@ roles: speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 speculative-draft-model-path: /draft-model speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 - speculative-draft-model-quantization: unquant speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant speculative-moe-runner-backend: flashinfer_trtllm disaggregation-decode-retraction-backup: cpu_tensor - disaggregation-decode-extra-slots: 0 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 48 - cuda-graph-max-bs-decode: 24 sbatch_directives: - exclude: dsxe-sa-b300-prd0-gpu-11 + exclude: dsxe-sa-b300-prd0-gpu-[11,16] mem: '0' cpus-per-task: '144' srun_options: @@ -174,7 +174,6 @@ srun_options: container-remap-root: '' benchmark: type: custom - concurrencies: [24] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace @@ -190,6 +189,8 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 24 telemetry: enabled: true collect_interval_ms: 1000 @@ -200,3 +201,4 @@ telemetry: dcgm_exporter: container_image: dcgm-exporter port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 From 3d6c4fe93852b965344f5a472c411f0052a72da6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 16:42:00 -0500 Subject: [PATCH 29/34] perf: expand B300 FP8 disaggregated AgentX frontier --- ...1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml | 204 ++++++++++++++++++ ...2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml | 204 ++++++++++++++++++ ...2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml | 204 ++++++++++++++++++ ...p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml | 204 ++++++++++++++++++ ...p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml} | 24 ++- ...tp4ep4-tp4-colocated-c32-mtp-hicache.yaml} | 26 +-- configs/nvidia-master.yaml | 78 ++++++- perf-changelog.yaml | 8 +- 8 files changed, 922 insertions(+), 30 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/{disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml => disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml} (95%) rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/{disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml => disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml} (94%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml new file mode 100644 index 0000000000..fbcc7645bc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated prefill and decode with prefill HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '8' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 8 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml new file mode 100644 index 0000000000..f495974f9f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '2:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '24' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 24 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml new file mode 100644 index 0000000000..543c407565 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '2:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 48 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '40' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 40 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml new file mode 100644 index 0000000000..b2f652e186 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated prefill and decode with prefill HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 96 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 48 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '40' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 40 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml similarity index 95% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml index 334b5de43c..9df219d66c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml @@ -1,6 +1,6 @@ -# Measured fast candidate; full official sweep and accuracy gates remain required. +# Colocated prefill and decode with prefill HiCache. schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -154,19 +154,19 @@ roles: speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + disaggregation-decode-extra-slots: 0 speculative-draft-model-path: /draft-model speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 - speculative-draft-model-quantization: unquant speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant speculative-moe-runner-backend: flashinfer_trtllm disaggregation-decode-retraction-backup: cpu_tensor - disaggregation-decode-extra-slots: 0 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 16 sbatch_directives: - exclude: dsxe-sa-b300-prd0-gpu-11 + exclude: dsxe-sa-b300-prd0-gpu-[11,16] mem: '0' cpus-per-task: '144' srun_options: @@ -174,13 +174,12 @@ srun_options: container-remap-root: '' benchmark: type: custom - concurrencies: [16] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '16' + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '8' SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' @@ -190,6 +189,8 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 8 telemetry: enabled: true collect_interval_ms: 1000 @@ -200,3 +201,4 @@ telemetry: dcgm_exporter: container_image: dcgm-exporter port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml similarity index 94% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml index 95f69be190..edc29371ea 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml @@ -1,6 +1,6 @@ -# Fast AgentX candidate from Slurm 4060; full official sweep and accuracy gates remain required. +# Colocated prefill and decode with prefill HiCache. schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache +name: qwen35-b300-disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -69,7 +69,7 @@ roles: quantization: fp8 kv-cache-dtype: fp8_e4m3 tensor-parallel-size: 4 - expert-parallel-size: 1 + expert-parallel-size: 4 data-parallel-size: 1 mamba-ssm-dtype: bfloat16 attention-backend: trtllm_mha @@ -94,7 +94,7 @@ roles: mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 - max-running-requests: 64 + max-running-requests: 32 enable-hierarchical-cache: true hicache-size: 72 hicache-io-backend: kernel @@ -154,19 +154,19 @@ roles: speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 speculative-draft-model-path: /draft-model speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 - speculative-draft-model-quantization: unquant speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant speculative-moe-runner-backend: flashinfer_trtllm disaggregation-decode-retraction-backup: cpu_tensor - disaggregation-decode-extra-slots: 0 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 32 sbatch_directives: - exclude: dsxe-sa-b300-prd0-gpu-11 + exclude: dsxe-sa-b300-prd0-gpu-[11,16] mem: '0' cpus-per-task: '144' srun_options: @@ -174,7 +174,6 @@ srun_options: container-remap-root: '' benchmark: type: custom - concurrencies: [32] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace @@ -190,6 +189,8 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 32 telemetry: enabled: true collect_interval_ms: 1000 @@ -200,3 +201,4 @@ telemetry: dcgm_exporter: container_image: dcgm-exporter port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8a38f37e68..756205798d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8400,19 +8400,37 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: kv-offload-backend: name: hicache conc-list: - - 16 + - 8 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c16-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 8 + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -8437,13 +8455,31 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: name: hicache conc-list: - 32 + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 40 prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c32-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml decode: num-worker: 1 tp: 4 @@ -8485,3 +8521,39 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 2 ep: 2 dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 24 + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 40 + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1e2435317e..bc2cfd4d74 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8453,10 +8453,10 @@ scenario-type: - agentic-coding description: - - Add B300 Qwen3.5-397B-A17B FP8 AgentX disaggregation with colocated TP4 prefill/decode - at C16/C24/C32/C48 and TP2+EP2 prefill/decode at C32. + - Add B300 Qwen3.5-397B-A17B FP8 AgentX disaggregation at C8/C24/C32/C40/C48 with + colocated TP2/TP4 prefill and decode, including expert-parallel variants. - Use Dynamo/SGLang nightly 20260918-20518d85, NEXTN with original BF16 draft weights/KV, and Mooncake NVLink transfers. - - Use prefill HiCache, including 298 GB/rank write-through and 64 decode slots for - TP4 C48 and TP2+EP2 C32. + - Use 72 or 298 GB/rank prefill HiCache, write-through at TP4 C40/C48 and TP2+EP2 C24/C32/C40, + and 32 or 64 decode slots. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From 331938b05c3c02a0ec2d84ee58d0ffe4af14f264 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 16:44:31 -0500 Subject: [PATCH 30/34] fix: allow full B300 TP2 sweep startup and drain time --- .../disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml | 2 +- .../disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml index f495974f9f..b8ee9371fa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml @@ -19,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '2:00:00' + time_limit: '4:00:00' health_check: max_attempts: 1440 interval_seconds: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml index 543c407565..e4af57fb63 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml @@ -19,7 +19,7 @@ dynamo: source: rev: 805a77f053d807b0d8def5d27f674a6df0ed839e slurm: - time_limit: '2:00:00' + time_limit: '4:00:00' health_check: max_attempts: 1440 interval_seconds: 10 From 6971261e8090326a18eefe0cf79a4410a01e4d8e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 17:25:03 -0500 Subject: [PATCH 31/34] perf: select B300 1P2D throughput frontier point --- ...p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml | 203 ------------------ ...ep2-tp2ep2-colocated-c48-mtp-hicache.yaml} | 12 +- configs/nvidia-master.yaml | 28 +-- perf-changelog.yaml | 10 +- 4 files changed, 16 insertions(+), 237 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/{disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml => disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml} (96%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml deleted file mode 100644 index ecd3d20a41..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml +++ /dev/null @@ -1,203 +0,0 @@ -# C48 candidate screened with canonical warmup; full official sweep and evals required. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 96 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - speculative-draft-model-path: /draft-model - speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 - speculative-draft-kv-cache-dtype: bf16 - speculative-draft-model-quantization: unquant - speculative-moe-runner-backend: flashinfer_trtllm - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - exclude: dsxe-sa-b300-prd0-gpu-[11,16] - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 48 -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml similarity index 96% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml index b8ee9371fa..a4a88c6223 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml @@ -1,6 +1,6 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. +# Colocated TP2+EP2 prefill and two decode workers with write-through HiCache. schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache +name: qwen35-b300-disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -102,7 +102,7 @@ roles: hicache-write-policy: write_through decode: nodes: colocate - workers: 1 + workers: 2 gpus: 2 env: PYTHONUNBUFFERED: '1' @@ -179,7 +179,7 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '24' + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' @@ -190,7 +190,7 @@ benchmark: AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' concurrencies: - - 24 + - 48 telemetry: enabled: true collect_interval_ms: 1000 @@ -201,4 +201,4 @@ telemetry: dcgm_exporter: container_image: dcgm-exporter port: 9401 - command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 756205798d..deaadffda3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8485,24 +8485,6 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 48 - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c48-mtp-hicache.yaml - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -8526,14 +8508,14 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: kv-offload-backend: name: hicache conc-list: - - 24 + - 40 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c24-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml decode: num-worker: 1 tp: 2 @@ -8544,16 +8526,16 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: kv-offload-backend: name: hicache conc-list: - - 40 + - 48 prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml decode: - num-worker: 1 + num-worker: 2 tp: 2 ep: 2 dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bc2cfd4d74..005cfd42bd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8453,10 +8453,10 @@ scenario-type: - agentic-coding description: - - Add B300 Qwen3.5-397B-A17B FP8 AgentX disaggregation at C8/C24/C32/C40/C48 with - colocated TP2/TP4 prefill and decode, including expert-parallel variants. - - Use Dynamo/SGLang nightly 20260918-20518d85, NEXTN with original BF16 draft weights/KV, - and Mooncake NVLink transfers. - - Use 72 or 298 GB/rank prefill HiCache, write-through at TP4 C40/C48 and TP2+EP2 C24/C32/C40, + - Add B300 Qwen3.5-397B-A17B FP8 AgentX recipes at C8/C24/C32/C40/C48, using + colocated TP2/TP4 and expert-parallel 1P1D plus TP2+EP2 1P2D. + - Use SGLang nightly 20260918-20518d85, Dynamo/SRT Slurm, Mooncake NVLink, + and NEXTN with original BF16 draft weights and KV. + - Use 72 or 298 GB/rank prefill HiCache, first-use write-through for higher loads, and 32 or 64 decode slots. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 From ed6d08161bb58b28832aec529da0559c6958b52d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 18:10:06 -0500 Subject: [PATCH 32/34] perf: select B300 1P2D C32 frontier point --- ...ep2-tp2ep2-colocated-c32-mtp-hicache.yaml} | 30 +++++++++---------- configs/nvidia-master.yaml | 14 ++++----- 2 files changed, 22 insertions(+), 22 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/{disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml => disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml} (92%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml similarity index 92% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml index b2f652e186..9fc100de04 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml @@ -1,6 +1,6 @@ -# Colocated prefill and decode with prefill HiCache. +# Colocated TP2+EP2 prefill and two decode workers with write-through HiCache. schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache +name: qwen35-b300-disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -44,7 +44,7 @@ roles: prefill: nodes: 1 workers: 1 - gpus: 4 + gpus: 2 env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -68,8 +68,8 @@ roles: trust-remote-code: true quantization: fp8 kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 + tensor-parallel-size: 2 + expert-parallel-size: 2 data-parallel-size: 1 mamba-ssm-dtype: bfloat16 attention-backend: trtllm_mha @@ -94,7 +94,7 @@ roles: mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 - max-running-requests: 96 + max-running-requests: 64 enable-hierarchical-cache: true hicache-size: 298 hicache-io-backend: kernel @@ -102,8 +102,8 @@ roles: hicache-write-policy: write_through decode: nodes: colocate - workers: 1 - gpus: 4 + workers: 2 + gpus: 2 env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -127,8 +127,8 @@ roles: trust-remote-code: true quantization: fp8 kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 + tensor-parallel-size: 2 + expert-parallel-size: 2 data-parallel-size: 1 mamba-ssm-dtype: bfloat16 attention-backend: trtllm_mha @@ -139,7 +139,7 @@ roles: enable-cache-report: true enable-metrics: true context-length: 262144 - mem-fraction-static: 0.8 + mem-fraction-static: 0.88 chunked-prefill-size: 16384 max-prefill-tokens: 16384 page-size: 64 @@ -157,7 +157,7 @@ roles: scheduler-recv-interval: 10 stream-interval: 50 max-running-requests: 64 - cuda-graph-max-bs-decode: 48 + cuda-graph-max-bs-decode: 32 disaggregation-decode-extra-slots: 0 speculative-draft-model-path: /draft-model speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 @@ -179,7 +179,7 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '40' + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' @@ -190,7 +190,7 @@ benchmark: AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' concurrencies: - - 40 + - 32 telemetry: enabled: true collect_interval_ms: 1000 @@ -201,4 +201,4 @@ telemetry: dcgm_exporter: container_image: dcgm-exporter port: 9401 - command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index deaadffda3..b7f176139b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8472,18 +8472,18 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: kv-offload-backend: name: hicache conc-list: - - 40 + - 32 prefill: num-worker: 1 - tp: 4 - ep: 1 + tp: 2 + ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c40-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml decode: - num-worker: 1 - tp: 4 - ep: 1 + num-worker: 2 + tp: 2 + ep: 2 dp-attn: false - spec-decoding: mtp kv-offloading: dram From b5f72ca69e9c7df6b7beebd5c49edb86ad68e30c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 18:12:30 -0500 Subject: [PATCH 33/34] perf: add B300 1P2D C64 frontier point --- ...2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml | 204 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 2 +- 3 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml new file mode 100644 index 0000000000..4863b954ac --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP2+EP2 prefill and two decode workers with write-through HiCache. +schema: 2 +name: qwen35-b300-disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 2 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '64' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 64 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b7f176139b..ce7b6e48a9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8539,3 +8539,21 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 2 ep: 2 dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 64 + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml + decode: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 005cfd42bd..e421b4ac06 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8453,7 +8453,7 @@ scenario-type: - agentic-coding description: - - Add B300 Qwen3.5-397B-A17B FP8 AgentX recipes at C8/C24/C32/C40/C48, using + - Add B300 Qwen3.5-397B-A17B FP8 AgentX recipes at C8/C24/C32/C40/C48/C64, using colocated TP2/TP4 and expert-parallel 1P1D plus TP2+EP2 1P2D. - Use SGLang nightly 20260918-20518d85, Dynamo/SRT Slurm, Mooncake NVLink, and NEXTN with original BF16 draft weights and KV. From f6953be8bde5359ea7fa608a3f522c9c5e83336a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 19 Sep 2026 21:11:54 -0500 Subject: [PATCH 34/34] perf: refine B300 FP8 disaggregation frontier candidates --- ...1d-tp4-tp4-colocated-c12-mtp-hicache.yaml} | 26 +-- ...2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml | 204 ------------------ configs/nvidia-master.yaml | 26 +-- perf-changelog.yaml | 4 +- 4 files changed, 19 insertions(+), 241 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/{disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml => disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml} (94%) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml similarity index 94% rename from benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml rename to benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml index fbcc7645bc..84646b6f35 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml @@ -1,6 +1,6 @@ # Colocated prefill and decode with prefill HiCache. schema: 2 -name: qwen35-b300-disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache model: path: qwen3.5-fp8 container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 @@ -44,7 +44,7 @@ roles: prefill: nodes: 1 workers: 1 - gpus: 2 + gpus: 4 env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -68,7 +68,7 @@ roles: trust-remote-code: true quantization: fp8 kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 + tensor-parallel-size: 4 expert-parallel-size: 1 data-parallel-size: 1 mamba-ssm-dtype: bfloat16 @@ -94,16 +94,16 @@ roles: mamba-track-interval: 2048 scheduler-recv-interval: 10 stream-interval: 50 - max-running-requests: 64 + max-running-requests: 32 enable-hierarchical-cache: true - hicache-size: 298 + hicache-size: 72 hicache-io-backend: kernel hicache-mem-layout: page_first hicache-write-policy: write_through_selective decode: nodes: colocate workers: 1 - gpus: 2 + gpus: 4 env: PYTHONUNBUFFERED: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -127,7 +127,7 @@ roles: trust-remote-code: true quantization: fp8 kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 + tensor-parallel-size: 4 expert-parallel-size: 1 data-parallel-size: 1 mamba-ssm-dtype: bfloat16 @@ -139,7 +139,7 @@ roles: enable-cache-report: true enable-metrics: true context-length: 262144 - mem-fraction-static: 0.88 + mem-fraction-static: 0.8 chunked-prefill-size: 16384 max-prefill-tokens: 16384 page-size: 64 @@ -156,8 +156,8 @@ roles: speculative-num-draft-tokens: 4 scheduler-recv-interval: 10 stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 32 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 disaggregation-decode-extra-slots: 0 speculative-draft-model-path: /draft-model speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 @@ -179,7 +179,7 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '8' + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '12' SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs PORT: '8000' IS_MULTINODE: 'true' @@ -190,7 +190,7 @@ benchmark: AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' concurrencies: - - 8 + - 12 telemetry: enabled: true collect_interval_ms: 1000 @@ -201,4 +201,4 @@ telemetry: dcgm_exporter: container_image: dcgm-exporter port: 9401 - command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml deleted file mode 100644 index 4863b954ac..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml +++ /dev/null @@ -1,204 +0,0 @@ -# Colocated TP2+EP2 prefill and two decode workers with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - frameworks: - dynamo: 1.6.0 - sglang: 0.0.0.dev1+g20518d851 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 2 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.88 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 32 - disaggregation-decode-extra-slots: 0 - speculative-draft-model-path: /draft-model - speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 - speculative-draft-kv-cache-dtype: bf16 - speculative-draft-model-quantization: unquant - speculative-moe-runner-backend: flashinfer_trtllm - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - exclude: dsxe-sa-b300-prd0-gpu-[11,16] - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '64' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 64 -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 83ed225610..464aefe1a6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8418,17 +8418,17 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: kv-offload-backend: name: hicache conc-list: - - 8 + - 12 prefill: num-worker: 1 - tp: 2 + tp: 4 ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2-tp2-colocated-c8-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml decode: num-worker: 1 - tp: 2 + tp: 4 ep: 1 dp-attn: false - spec-decoding: mtp @@ -8539,21 +8539,3 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: tp: 2 ep: 2 dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 64 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c64-mtp-hicache.yaml - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c3fbc5d1e7..6a4d5e65ee 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8461,8 +8461,8 @@ scenario-type: - agentic-coding description: - - Add B300 Qwen3.5-397B-A17B FP8 AgentX recipes at C8/C24/C32/C40/C48/C64, using - colocated TP2/TP4 and expert-parallel 1P1D plus TP2+EP2 1P2D. + - Add B300 Qwen3.5-397B-A17B FP8 AgentX recipes at C8/C12/C24/C32/C40/C48, using + colocated TP4 and TP2+EP2 1P1D/1P2D with tuned prefill expert parallelism. - Use SGLang nightly 20260918-20518d85, Dynamo/SRT Slurm, Mooncake NVLink, and NEXTN with original BF16 draft weights and KV. - Use 72 or 298 GB/rank prefill HiCache, first-use write-through for higher loads,