Add B200 Qwen3.5 397B FP8 disaggregated AgentX recipes

benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml

@@ -0,0 +1,175 @@
+# Experimental; retain only measured improvements over the current Pareto frontier.
+schema: 2
+name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp
+model:
+  path: qwen3.5-fp8
+  container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85
+  precision: fp8
+engine: sglang
+dynamo:
+  install: true
+  source:
+    rev: 805a77f053d807b0d8def5d27f674a6df0ed839e
+resources:
+  gpu_type: b200
+  gpus_per_node: 8
+slurm:
+  time_limit: '4:00:00'
+frontend:
+  type: dynamo
+  nginx_container: nginx-sqsh
+  nginx_session_affinity: true
+  nginx_session_affinity_header: X-Dynamo-Session-ID
+  enable_multiple_frontends: true
+  num_additional_frontends: 1
+  env:
+    PIP_BREAK_SYSTEM_PACKAGES: '1'
+    PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com
+  args:
+    router-mode: kv
+    router-session-affinity-ttl-secs: '3600'
+roles:
+  prefill:
+    nodes: 1
+    workers: 1
+    gpus: 4
+    env:
+      PYTHONUNBUFFERED: '1'
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+      PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com
+      SGLANG_ENABLE_SPEC_V2: '1'
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
+      NCCL_NVLS_ENABLE: '1'
+      SGL_ENABLE_JIT_DEEPGEMM: 'false'
+      SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+      PYTHONNOUSERSITE: '1'
+      MC_INTRANODE_NVLINK: '1'
+      SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK
+    args:
+      served-model-name: Qwen/Qwen3.5-397B-A17B-FP8
+      trust-remote-code: true
+      tensor-parallel-size: 4
+      expert-parallel-size: 1
+      data-parallel-size: 1
+      quantization: fp8
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      linear-attn-decode-backend: flashinfer
+      page-size: 64
+      mem-fraction-static: 0.8
+      context-length: 262144
+      chunked-prefill-size: 16384
+      max-prefill-tokens: 16384
+      max-running-requests: 32
+      reasoning-parser: qwen3
+      tool-call-parser: qwen3_coder
+      enable-metrics: true
+      enable-cache-report: true
+      disaggregation-mode: prefill
+      disaggregation-transfer-backend: mooncake
+      disable-cuda-graph: true
+      enable-symm-mem: true
+      scheduler-recv-interval: 10
+      stream-interval: 50
+      mamba-radix-cache-strategy: extra_buffer
+      mamba-track-interval: 2048
+      enable-hierarchical-cache: true
+      hicache-size: 104
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first
+      hicache-write-policy: write_through_selective
+  decode:
+    nodes: colocate
+    workers: 1
+    gpus: 4
+    env:
+      PYTHONUNBUFFERED: '1'
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+      PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com
+      SGLANG_ENABLE_SPEC_V2: '1'
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
+      NCCL_NVLS_ENABLE: '1'
+      SGL_ENABLE_JIT_DEEPGEMM: 'false'
+      SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+      PYTHONNOUSERSITE: '1'
+      MC_INTRANODE_NVLINK: '1'
+      SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK
+    args:
+      served-model-name: Qwen/Qwen3.5-397B-A17B-FP8
+      trust-remote-code: true
+      tensor-parallel-size: 4
+      expert-parallel-size: 1
+      data-parallel-size: 1
+      quantization: fp8
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      linear-attn-decode-backend: flashinfer
+      page-size: 64
+      mem-fraction-static: 0.8
+      context-length: 262144
+      chunked-prefill-size: 16384
+      max-prefill-tokens: 16384
+      max-running-requests: 32
+      reasoning-parser: qwen3
+      tool-call-parser: qwen3_coder
+      enable-metrics: true
+      enable-cache-report: true
+      disaggregation-mode: decode
+      disaggregation-transfer-backend: mooncake
+      disable-radix-cache: true
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs-decode: 16
+      stream-interval: 50
+      enable-symm-mem: true
+      scheduler-recv-interval: 10
+      mamba-radix-cache-strategy: no_buffer
+      mamba-track-interval: 128
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+sbatch_directives:
+  mem: '0'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+telemetry:
+  enabled: true
+  collect_interval_ms: 1000
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  concurrencies:
+  - 16
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom
+    SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '16'
+    SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs
+    PORT: '8000'
+    IS_MULTINODE: 'true'
+    EVAL_FRAMEWORK: lm-eval
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_MAX_CONTEXT_LENGTH: '262144'
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'

benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml

@@ -0,0 +1,175 @@
+# Experimental; retain only measured improvements over the current Pareto frontier.
+schema: 2
+name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp
+model:
+  path: qwen3.5-fp8
+  container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85
+  precision: fp8
+engine: sglang
+dynamo:
+  install: true
+  source:
+    rev: 805a77f053d807b0d8def5d27f674a6df0ed839e
+resources:
+  gpu_type: b200
+  gpus_per_node: 8
+slurm:
+  time_limit: '4:00:00'
+frontend:
+  type: dynamo
+  nginx_container: nginx-sqsh
+  nginx_session_affinity: true
+  nginx_session_affinity_header: X-Dynamo-Session-ID
+  enable_multiple_frontends: true
+  num_additional_frontends: 1
+  env:
+    PIP_BREAK_SYSTEM_PACKAGES: '1'
+    PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com
+  args:
+    router-mode: kv
+    router-session-affinity-ttl-secs: '3600'
+roles:
+  prefill:
+    nodes: 1
+    workers: 1
+    gpus: 4
+    env:
+      PYTHONUNBUFFERED: '1'
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+      PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com
+      SGLANG_ENABLE_SPEC_V2: '1'
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
+      NCCL_NVLS_ENABLE: '1'
+      SGL_ENABLE_JIT_DEEPGEMM: 'false'
+      SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+      PYTHONNOUSERSITE: '1'
+      MC_INTRANODE_NVLINK: '1'
+      SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK
+    args:
+      served-model-name: Qwen/Qwen3.5-397B-A17B-FP8
+      trust-remote-code: true
+      tensor-parallel-size: 4
+      expert-parallel-size: 1
+      data-parallel-size: 1
+      quantization: fp8
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      linear-attn-decode-backend: flashinfer
+      page-size: 64
+      mem-fraction-static: 0.8
+      context-length: 262144
+      chunked-prefill-size: 16384
+      max-prefill-tokens: 16384
+      max-running-requests: 48
+      reasoning-parser: qwen3
+      tool-call-parser: qwen3_coder
+      enable-metrics: true
+      enable-cache-report: true
+      disaggregation-mode: prefill
+      disaggregation-transfer-backend: mooncake
+      disable-cuda-graph: true
+      enable-symm-mem: true
+      scheduler-recv-interval: 10
+      stream-interval: 50
+      mamba-radix-cache-strategy: extra_buffer
+      mamba-track-interval: 2048
+      enable-hierarchical-cache: true
+      hicache-size: 104
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first
+      hicache-write-policy: write_through_selective
+  decode:
+    nodes: colocate
+    workers: 1
+    gpus: 4
+    env:
+      PYTHONUNBUFFERED: '1'
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+      PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com
+      SGLANG_ENABLE_SPEC_V2: '1'
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
+      NCCL_NVLS_ENABLE: '1'
+      SGL_ENABLE_JIT_DEEPGEMM: 'false'
+      SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+      PYTHONNOUSERSITE: '1'
+      MC_INTRANODE_NVLINK: '1'
+      SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK
+    args:
+      served-model-name: Qwen/Qwen3.5-397B-A17B-FP8
+      trust-remote-code: true
+      tensor-parallel-size: 4
+      expert-parallel-size: 1
+      data-parallel-size: 1
+      quantization: fp8
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      linear-attn-decode-backend: flashinfer
+      page-size: 64
+      mem-fraction-static: 0.8
+      context-length: 262144
+      chunked-prefill-size: 16384
+      max-prefill-tokens: 16384
+      max-running-requests: 48
+      reasoning-parser: qwen3
+      tool-call-parser: qwen3_coder
+      enable-metrics: true
+      enable-cache-report: true
+      disaggregation-mode: decode
+      disaggregation-transfer-backend: mooncake
+      disable-radix-cache: true
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs-decode: 24
+      stream-interval: 50
+      enable-symm-mem: true
+      scheduler-recv-interval: 10
+      mamba-radix-cache-strategy: no_buffer
+      mamba-track-interval: 128
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+sbatch_directives:
+  mem: '0'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+telemetry:
+  enabled: true
+  collect_interval_ms: 1000
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  concurrencies:
+  - 24
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom
+    SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '24'
+    SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs
+    PORT: '8000'
+    IS_MULTINODE: 'true'
+    EVAL_FRAMEWORK: lm-eval
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_MAX_CONTEXT_LENGTH: '262144'
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'