diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml new file mode 100644 index 000000000..f3024fdf5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 32 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml new file mode 100644 index 000000000..e4af57fb6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 48 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '40' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 40 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml new file mode 100644 index 000000000..84646b6f3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated prefill and decode with prefill HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + enable-hierarchical-cache: true + hicache-size: 72 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '12' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 12 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml new file mode 100644 index 000000000..3e31b2a49 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP4 prefill and decode with prefill HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + enable-hierarchical-cache: true + hicache-size: 72 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '24' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 24 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml new file mode 100644 index 000000000..9df219d66 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated prefill and decode with prefill HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + enable-hierarchical-cache: true + hicache-size: 72 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '8' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 8 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml new file mode 100644 index 000000000..edc29371e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated prefill and decode with prefill HiCache. +schema: 2 +name: qwen35-b300-disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 4 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + enable-hierarchical-cache: true + hicache-size: 72 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 32 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5,6,7 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml new file mode 100644 index 000000000..9fc100de0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP2+EP2 prefill and two decode workers with write-through HiCache. +schema: 2 +name: qwen35-b300-disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 2 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 32 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml new file mode 100644 index 000000000..a4a88c622 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml @@ -0,0 +1,204 @@ +# Colocated TP2+EP2 prefill and two decode workers with write-through HiCache. +schema: 2 +name: qwen35-b300-disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache +model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + frameworks: + dynamo: 1.6.0 + sglang: 0.0.0.dev1+g20518d851 +dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e +slurm: + time_limit: '4:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 2 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + disaggregation-decode-extra-slots: 0 + speculative-draft-model-path: /draft-model + speculative-draft-model-revision: 8472618112abcbd45acbcdc58436aff4233c23f7 + speculative-draft-kv-cache-dtype: bf16 + speculative-draft-model-quantization: unquant + speculative-moe-runner-backend: flashinfer_trtllm + disaggregation-decode-retraction-backup: cpu_tensor +sbatch_directives: + exclude: dsxe-sa-b300-prd0-gpu-[11,16] + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom + SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' + SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + concurrencies: + - 48 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: dcgm-exporter --collect-interval=100 --address :{port} --devices g:0,1,2,3,4,5 diff --git a/configs/models/qwen3.5-397b-mtp-bf16.json b/configs/models/qwen3.5-397b-mtp-bf16.json new file mode 100644 index 000000000..ab2eadb3a --- /dev/null +++ b/configs/models/qwen3.5-397b-mtp-bf16.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "repo": "Qwen/Qwen3.5-397B-A17B", + "revision": "8472618112abcbd45acbcdc58436aff4233c23f7", + "files": [ + { + "name": "config.json", + "size": 4177, + "sha256": "3ae7fa89c2f7d1354096418ddaf1331e9e0898a8ca11e804fb6dbaa087efb7da" + }, + { + "name": "generation_config.json", + "size": 244, + "sha256": "303aba891d66ab63908a7b3cc9163bcb835fdf8b9f6301c73216f3f1eb3992dd" + }, + { + "name": "model.safetensors-00091-of-00094.safetensors", + "size": 9638517720, + "sha256": "3633797c27c32b194da65eaff2c5c27eb184bd547577c4124498bb8ae51e670f" + }, + { + "name": "model.safetensors-00092-of-00094.safetensors", + "size": 9658656328, + "sha256": "66cd7e6aecdcd4c69bfc7e8c072cbc4ae3771897785279a0ff2da6a0eb4a8211" + }, + { + "name": "model.safetensors-00093-of-00094.safetensors", + "size": 9663818112, + "sha256": "15f424c1a22772238bb0efcf53291f29ec00f5c08d84b644ae54ee31486475e1" + }, + { + "name": "model.safetensors-00094-of-00094.safetensors", + "size": 4741119112, + "sha256": "75b5c72a8dc8f07946dab6bdccec6663f332b24c9be8c59c215e0eabd5204467" + }, + { + "name": "model.safetensors.index.json", + "size": 303440, + "sha256": "407d6a184a29469034ad92bcfaa6b72b582f8ad6afcbfad18a38a1c59ed296f8" + }, + { + "name": "tokenizer.json", + "size": 12807982, + "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42" + }, + { + "name": "tokenizer_config.json", + "size": 16710, + "sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8" + } + ], + "mtp_prefix": "mtp.", + "mtp_tensor_count": 1553, + "mtp_bytes": 13191136256, + "required_bf16_tensors": [ + "lm_head.weight", + "model.language_model.embed_tokens.weight" + ] +} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c5d5edac6..464aefe1a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8375,3 +8375,167 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: dp-attn: false kv-offload-backend: name: hicache + +# Recipes allow four hours for canonical warmup, one-hour profiling and artifact processing. +# Temporary node exclusions cover observed startup/resource failures, not measured performance. +qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:b300-dsxe + precision: fp8 + framework: dynamo-sglang + router: + name: dynamo-router + version: 805a77f053d807b0d8def5d27f674a6df0ed839e + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.8 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 8 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c8-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 12 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 24 + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 32 + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 32 + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml + decode: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 32 + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 40 + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml + decode: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: + name: hicache + conc-list: + - 48 + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p2d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml + decode: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 4c6abfccf..ff66bd46b 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -408,6 +408,53 @@ those paths on `vllm` only. GPU sweep and eval evidence is required before calling any of these arms validated. +### Acquire the original Qwen3.5 BF16 MTP subset + +[`infx.models.acquire_mtp`](../infx/models/acquire_mtp.py) prepares an original-weight +MTP asset using the pinned [manifest](../configs/models/qwen3.5-397b-mtp-bf16.json). +It downloads four unmodified shards (91–94, about 33.7 GB) and support metadata from +`Qwen/Qwen3.5-397B-A17B` revision `8472618112abcbd45acbcdc58436aff4233c23f7`. +Manifest digests come from the immutable HF revision: upstream LFS SHA256 for large +files and verified original bytes for metadata. No tensors are converted, repacked, +or dequantized. This directory is an MTP subset, **not a complete target model**. + +The caller chooses the shared destination and lock timeout explicitly: + +```bash +python -m infx.models.acquire_mtp \ + --manifest configs/models/qwen3.5-397b-mtp-bf16.json \ + --destination /shared/models/Qwen3.5-397B-MTP-BF16-8472618 \ + --lock-timeout 3600 +``` + +The helper serializes callers with a destination-specific POSIX lock. It checks every +original file's size and SHA256, all selected header/index entries, all 1,553 original +BF16 MTP tensors, and the BF16 embedding/head. It preserves the full upstream index +as `model.safetensors.index.json.original`, computes the filtered index's `total_size` +from tensor offsets, and atomically publishes the validated directory with +`subset-provenance.json`. Warm reuse revalidates the complete asset, including hashes +and manifest identity. An existing incomplete or conflicting destination fails without +repair or replacement. Interrupted temporary staging directories are never accepted as +complete; a subsequent attempt downloads into a new staging directory. Verification +requires reading the full 33.7 GB on warm reuse too. The helper requires POSIX locking +and atomic directory rename on the caller's shared filesystem. + +The B300 DSXE launcher acquires this asset before submitting Qwen3.5 FP8 +Dynamo/SGLang AgentX jobs. It uses a manifest-hash directory under its existing +`WRITABLE_MODELS_DIR`, mounts it at `/draft-model` through `write_srt_cluster_config`, +and includes the manifest, provenance and resolved host/container paths in +`draft-model-provenance/` inside the server-log archive. Recipes do not depend on +personal directories. +For SGLang `20518d85`, the supported separate-draft arguments are: +`speculative-draft-model-path: /draft-model`, the exact revision above, +`speculative-draft-model-quantization: unquant`, +`speculative-draft-kv-cache-dtype: bf16`, and +`speculative-moe-runner-backend: flashinfer_trtllm`. +Omitting `unquant` can inherit the FP8 target's quantization. Separate draft KV dtype +does not guarantee mixed-dtype HiCache compatibility; retain the recipe's independently +validated cache/retraction configuration. Acquisition does not qualify a serving recipe; +real accuracy and performance tests remain required. + ## Validate Run the smallest checks that cover the edited layers. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index f877035f1..85c2c7303 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -354,6 +354,48 @@ offload），prefill 分块上限设为 4096，即 vLLM H100 配方在 80 GB 显 在获得 GPU sweep 与 eval 证据之前，不得将这些配方视为已验证。 +### 获取 Qwen3.5 原始 BF16 MTP 子集 + +[`infx.models.acquire_mtp`](../infx/models/acquire_mtp.py) 根据固定的 +[清单](../configs/models/qwen3.5-397b-mtp-bf16.json) 准备原始权重 MTP 资源。 +它从 `Qwen/Qwen3.5-397B-A17B` 的提交 +`8472618112abcbd45acbcdc58436aff4233c23f7` 下载未经修改的第 91–94 个分片 +（约 33.7 GB）及配套元数据。清单使用不可变 HF 提交的摘要：大文件采用上游 +LFS SHA256，元数据采用已验证原始字节的 SHA256。工具不会转换、重新打包或 +反量化张量。该目录是 MTP 子集，**不是完整的目标模型**。 + +调用方必须明确指定共享目录和锁等待时间： + +```bash +python -m infx.models.acquire_mtp \ + --manifest configs/models/qwen3.5-397b-mtp-bf16.json \ + --destination /shared/models/Qwen3.5-397B-MTP-BF16-8472618 \ + --lock-timeout 3600 +``` + +工具使用目标目录专属的 POSIX 锁串行处理调用方，检查每个原始文件的大小和 +SHA256、所选分片的头部与索引条目、全部 1,553 个原始 BF16 MTP 张量，以及 +BF16 embedding/head。完整上游索引保存在 `model.safetensors.index.json.original`； +过滤索引的 `total_size` 根据张量偏移量计算。验证完成后，工具将目录及 +`subset-provenance.json` 原子发布。复用时仍验证全部文件摘要与清单身份。 +已有但不完整或冲突的目录会直接报错，不会修复或覆盖。中断留下的临时目录 +不视为完整资源；重试使用新的暂存目录下载。复用也需要读取全部约 33.7 GB。 +共享文件系统必须支持 POSIX 锁和原子目录重命名。 + +B300 DSXE 启动器在提交 Qwen3.5 FP8 Dynamo/SGLang AgentX 任务前获取该资源， +使用现有 `WRITABLE_MODELS_DIR` 下以清单哈希命名的目录，通过 +`write_srt_cluster_config` 挂载到 `/draft-model`，并在服务端日志归档的 +`draft-model-provenance/` 中保存清单、来源记录及实际宿主机/容器路径。 +配方不依赖个人目录。 +SGLang `20518d85` 的独立 draft 参数为 +`speculative-draft-model-path: /draft-model`、上述固定 revision、 +`speculative-draft-model-quantization: unquant`、 +`speculative-draft-kv-cache-dtype: bf16` 和 +`speculative-moe-runner-backend: flashinfer_trtllm`。 +省略 `unquant` 可能继承 FP8 目标模型的量化配置。独立 draft KV 类型并不保证 +混合类型 HiCache 兼容；必须保留配方已独立验证的缓存和回退配置。 +资源获取本身不构成配方验证；仍需真实准确率和性能测试。 + ## 验证 运行覆盖被修改层的最小检查。 diff --git a/infx/models/__init__.py b/infx/models/__init__.py new file mode 100644 index 000000000..bfe31762f --- /dev/null +++ b/infx/models/__init__.py @@ -0,0 +1 @@ +"""Verified model asset acquisition.""" diff --git a/infx/models/acquire_mtp.py b/infx/models/acquire_mtp.py new file mode 100644 index 000000000..7361f200f --- /dev/null +++ b/infx/models/acquire_mtp.py @@ -0,0 +1,220 @@ +"""Acquire original checkpoint shards as a verified, atomic MTP-only asset.""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +import math +import shutil +import struct +import tempfile +import time +import urllib.request +from pathlib import Path +from typing import Annotated + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +INDEX = "model.safetensors.index.json" +PROVENANCE = "subset-provenance.json" +Sha256 = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")] +Filename = Annotated[str, Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")] + + +class FileSpec(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + name: Filename + size: Annotated[int, Field(gt=0)] + sha256: Sha256 + + +class Manifest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + schema_version: Annotated[int, Field(ge=1, le=1)] + repo: Annotated[str, Field(pattern=r"^[\w.-]+/[\w.-]+$")] + revision: Annotated[str, Field(pattern=r"^[0-9a-f]{40}$")] + files: list[FileSpec] + mtp_prefix: str + mtp_tensor_count: Annotated[int, Field(gt=0)] + mtp_bytes: Annotated[int, Field(gt=0)] + required_bf16_tensors: list[str] + + @model_validator(mode="after") + def check_files(self) -> Manifest: + names = [f.name for f in self.files] + if len(names) != len(set(names)) or not self.mtp_prefix: + raise ValueError("Duplicate filenames or empty MTP prefix") + if not {INDEX, "config.json"}.issubset(names): + raise ValueError("Original index and config are required") + if PROVENANCE in names or INDEX + ".original" in names: + raise ValueError("Manifest uses a reserved output filename") + return self + + +def _json_bytes(value: object) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def _sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def _download(url: str, destination: Path) -> None: + # URLs are constructed exclusively from the validated HF repo/revision/file. + with ( + urllib.request.urlopen(url, timeout=180) as response, # noqa: S310 + destination.open("xb") as output, + ): + shutil.copyfileobj(response, output, length=8 * 1024 * 1024) + + +def _original_path(directory: Path, name: str) -> Path: + return directory / (name + ".original" if name == INDEX else name) + + +def _verify_files(directory: Path, manifest: Manifest) -> None: + for spec in manifest.files: + path = _original_path(directory, spec.name) + if path.is_symlink() or not path.is_file(): + raise ValueError(f"Missing or non-regular original file: {spec.name}") + if path.stat().st_size != spec.size or _sha256(path) != spec.sha256: + raise ValueError(f"Size/SHA256 mismatch: {spec.name}") + + +def _filtered_index(directory: Path, manifest: Manifest) -> dict: + original = json.loads(_original_path(directory, INDEX).read_bytes()) + weight_map = original["weight_map"] + shards = {f.name for f in manifest.files if f.name.endswith(".safetensors")} + selected = {name: shard for name, shard in weight_map.items() if shard in shards} + tensors = {} + total_size = 0 + for shard in sorted(shards): + path = directory / shard + with path.open("rb") as stream: + raw_length = stream.read(8) + if len(raw_length) != 8: + raise ValueError(f"Truncated safetensors prefix: {shard}") + header_size = struct.unpack(" min(100_000_000, path.stat().st_size - 8): + raise ValueError(f"Invalid safetensors header length: {shard}") + header = json.loads(stream.read(header_size)) + spans = [] + for name, tensor in header.items(): + if name == "__metadata__": + continue + start, end = tensor["data_offsets"] + if start < 0 or end < start or end > path.stat().st_size - 8 - header_size: + raise ValueError(f"Invalid tensor offsets: {name}") + if name in tensors or selected.get(name) != shard: + raise ValueError(f"Header/index tensor mismatch: {name}") + tensors[name] = tensor + spans.append((start, end)) + total_size += end - start + cursor = 0 + for start, end in sorted(spans): + if start != cursor: + raise ValueError(f"Non-contiguous tensor data: {shard}") + cursor = end + if cursor != path.stat().st_size - 8 - header_size: + raise ValueError(f"Unindexed tensor data: {shard}") + if set(tensors) != set(selected): + raise ValueError("Selected index references missing tensors") + expected_mtp = {name for name in weight_map if name.startswith(manifest.mtp_prefix)} + actual_mtp = {name for name in tensors if name.startswith(manifest.mtp_prefix)} + if expected_mtp != actual_mtp or len(actual_mtp) != manifest.mtp_tensor_count: + raise ValueError("Incomplete MTP tensor set") + mtp_bytes = sum( + tensors[n]["data_offsets"][1] - tensors[n]["data_offsets"][0] for n in actual_mtp + ) + if mtp_bytes != manifest.mtp_bytes: + raise ValueError("MTP tensor byte count mismatch") + for name in actual_mtp | set(manifest.required_bf16_tensors): + tensor = tensors.get(name) + if tensor is None or tensor["dtype"] != "BF16": + raise ValueError(f"Required original BF16 tensor missing or wrong dtype: {name}") + if math.prod(tensor["shape"]) * 2 != tensor["data_offsets"][1] - tensor["data_offsets"][0]: + raise ValueError(f"Invalid BF16 tensor shape: {name}") + return { + "metadata": {**original.get("metadata", {}), "total_size": total_size}, + "weight_map": selected, + } + + +def _provenance(manifest: Manifest, filtered: bytes) -> dict: + return { + "artifact_type": "original-bf16-mtp-subset-not-full-target", + "manifest": manifest.model_dump(), + "manifest_sha256": hashlib.sha256(_json_bytes(manifest.model_dump())).hexdigest(), + "filtered_index_sha256": hashlib.sha256(filtered).hexdigest(), + "tensor_conversion": False, + } + + +def _validate(directory: Path, manifest: Manifest) -> tuple[bytes, dict]: + _verify_files(directory, manifest) + filtered = _json_bytes(_filtered_index(directory, manifest)) + return filtered, _provenance(manifest, filtered) + + +def acquire(manifest: Manifest, destination: Path, *, lock_timeout: float) -> Path: + """Verify/reuse or publish a new asset; never repair or overwrite an existing one.""" + if not math.isfinite(lock_timeout) or lock_timeout < 0: + raise ValueError("lock_timeout must be finite and nonnegative") + destination = destination.absolute() + destination.parent.mkdir(parents=True, exist_ok=True) + lock_path = destination.with_name(destination.name + ".download.lock") + with lock_path.open("a") as lock: + deadline = time.monotonic() + lock_timeout + while True: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if time.monotonic() >= deadline: + raise TimeoutError(f"Timed out locking {destination}") from None + time.sleep(min(0.1, max(0, deadline - time.monotonic()))) + if destination.is_symlink(): + raise ValueError("Destination must not be a symlink") + if destination.exists(): + filtered, provenance = _validate(destination, manifest) + if (destination / INDEX).read_bytes() != filtered or json.loads( + (destination / PROVENANCE).read_bytes() + ) != provenance: + raise ValueError("Existing destination identity or generated index differs") + expected = {_original_path(destination, f.name).name for f in manifest.files} | { + INDEX, + PROVENANCE, + } + if {p.name for p in destination.iterdir()} != expected: + raise ValueError("Existing destination has unexpected files") + return destination + # Same parent/filesystem guarantees readers see either absent or complete. + with tempfile.TemporaryDirectory( + prefix=f".{destination.name}.staging-", dir=destination.parent + ) as staging: + stage = Path(staging) + for spec in manifest.files: + url = f"https://huggingface.co/{manifest.repo}/resolve/{manifest.revision}/{spec.name}" + _download(url, _original_path(stage, spec.name)) + filtered, provenance = _validate(stage, manifest) + (stage / INDEX).write_bytes(filtered) + (stage / PROVENANCE).write_bytes(_json_bytes(provenance)) + stage.rename(destination) + return destination + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--lock-timeout", type=float, required=True) + args = parser.parse_args() + manifest = Manifest.model_validate_json(args.manifest.read_bytes()) + print(acquire(manifest, args.destination, lock_timeout=args.lock_timeout)) + + +if __name__ == "__main__": + main() diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5b4309f5..6a4d5e65e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8455,3 +8455,16 @@ description: - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - Add B300 Qwen3.5-397B-A17B FP8 AgentX recipes at C8/C12/C24/C32/C40/C48, using + colocated TP4 and TP2+EP2 1P1D/1P2D with tuned prefill expert parallelism. + - Use SGLang nightly 20260918-20518d85, Dynamo/SRT Slurm, Mooncake NVLink, + and NEXTN with original BF16 draft weights and KV. + - Use 72 or 298 GB/rank prefill HiCache, first-use write-through for higher loads, + and 32 or 64 decode slots. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index d49b28b41..b9b152be9 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -102,14 +102,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' ' "$_RECIPE_SRC"; then USES_DCGM_POWER=1 fi -if [[ "$USES_DCGM_POWER" == "1" && ( - "${IS_AGENTIC}" == "1" || - "$MODEL_PREFIX" != "dsv4" || - "$PRECISION" != "fp4" || - ( "$FRAMEWORK" != "dynamo-sglang" && "$FRAMEWORK" != "dynamo-vllm" ) -) ]]; then - echo "Error: B300 dcgm-power is limited to fixed-sequence DSV4 FP4 dynamo-sglang/vllm" >&2 - exit 1 +if [[ "$USES_DCGM_POWER" == "1" ]]; then + if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + : # AgentX uses the native SRT measurement-window contract and adapter. + elif [[ "$IS_AGENTIC" != "1" && "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && ( "$FRAMEWORK" == "dynamo-sglang" || "$FRAMEWORK" == "dynamo-vllm" ) ]]; then + : # Existing fixed-sequence telemetry path. + else + echo "Error: B300 dcgm-power supports fixed-sequence DSV4 FP4 and Qwen3.5 FP8 dynamo-sglang AgentX" >&2 + exit 1 + fi fi SRT_REPO_DIR="srt-slurm" @@ -150,9 +151,27 @@ export ISL="$ISL" export OSL="$OSL" SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" +SRT_MODEL_MOUNTS=() +DRAFT_PROVENANCE_DIR="" +if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + DRAFT_MANIFEST="$GITHUB_WORKSPACE/configs/models/qwen3.5-397b-mtp-bf16.json" + DRAFT_MANIFEST_SHA=$(sha256sum "$DRAFT_MANIFEST") || exit 1 + DRAFT_CACHE_DIR="$WRITABLE_MODELS_DIR/qwen3.5-original-mtp-${DRAFT_MANIFEST_SHA%% *}" + PYTHONPATH="$GITHUB_WORKSPACE" python -m infx.models.acquire_mtp \ + --manifest "$DRAFT_MANIFEST" --destination "$DRAFT_CACHE_DIR" \ + --lock-timeout 3600 || exit 1 + SRT_MODEL_MOUNTS+=(--mount "$DRAFT_CACHE_DIR" /draft-model) + DRAFT_PROVENANCE_DIR="$GITHUB_WORKSPACE/draft-model-provenance" + mkdir -p "$DRAFT_PROVENANCE_DIR" || exit 1 + cp "$DRAFT_MANIFEST" "$DRAFT_PROVENANCE_DIR/manifest.json" || exit 1 + cp "$DRAFT_CACHE_DIR/subset-provenance.json" "$DRAFT_PROVENANCE_DIR/" || exit 1 + printf '%s\n' "$DRAFT_CACHE_DIR" > "$DRAFT_PROVENANCE_DIR/host-path.txt" + printf '%s\n' /draft-model > "$DRAFT_PROVENANCE_DIR/container-path.txt" +fi + echo "Creating srtslurm.yaml configuration..." write_srt_cluster_config b300-dsxe srtslurm.yaml "$USES_DCGM_POWER" \ - --var MODEL_ROOT "$MODEL_ROOT" || exit 1 + --var MODEL_ROOT "$MODEL_ROOT" "${SRT_MODEL_MOUNTS[@]}" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml @@ -180,6 +199,12 @@ fi sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" +if [[ "$USES_DCGM_POWER" == "1" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + python "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \ + "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 +fi + # Weights live on node-local MODEL_ROOT, which this login host cannot stat, so # srtctl's preflight model.path check is always skipped. Runtime loading still # validates the path on the compute nodes. @@ -205,29 +230,9 @@ echo "Extracted JOB_ID: $JOB_ID" LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" -while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 -done - -( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 - done -) & -POLL_PID=$! - -echo "Tailing LOG_FILE: $LOG_FILE" - -# -F follows by name and polls; inotify does not work on NFS. -tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null - -wait $POLL_PID +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? +verify_slurm_job_status "$JOB_ID" || SRT_JOB_RC=$? set -x @@ -247,6 +252,16 @@ if [[ "$USES_DCGM_POWER" == "1" ]]; then cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt" fi +if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && "${EVAL_ONLY}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$GITHUB_WORKSPACE" \ + "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$SRT_SLURM_COMMIT" \ + "${POWER_CONCURRENCIES[@]}" || SRT_JOB_RC=$? +fi + +if [[ -n "$DRAFT_PROVENANCE_DIR" ]]; then + cp -r "$DRAFT_PROVENANCE_DIR" "$LOGS_DIR/draft-model-provenance" || exit 1 +fi cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . @@ -281,6 +296,8 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true +# Preserve diagnostics and eval outputs before propagating a failed allocation. +exit "$SRT_JOB_RC" else # AgentX trace datasets need a writable persistent cache. Keep the host and diff --git a/runners/test_b300_srt_status.py b/runners/test_b300_srt_status.py new file mode 100644 index 000000000..1428fc01c --- /dev/null +++ b/runners/test_b300_srt_status.py @@ -0,0 +1,262 @@ +"""A completed Slurm allocation must not hide a failed benchmark.""" + +import json +import os +import shutil +import subprocess +import sys +import tarfile +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def _stub(directory: Path, name: str, body: str) -> None: + path = directory / name + path.write_text("#!/bin/bash\n" + body + "\n") + path.chmod(0o755) + + +@pytest.mark.parametrize( + "accounting,expected_status,power_mode", + [ + ("COMPLETED|0:0", 0, "off"), + ("FAILED|1:0", 1, "off"), + ("COMPLETED|0:9", 1, "off"), + ("CANCELLED by 123|0:15", 1, "off"), + ("DELAYED", 0, "off"), + ("", 1, "off"), + ("COMPLETED|0:0", 1, "missing"), + ("COMPLETED|0:0", 0, "eval"), + ], +) +def test_b300_collects_artifacts_before_returning_slurm_status( + tmp_path: Path, accounting: str, expected_status: int, power_mode: str +) -> None: + """Run the actual Qwen launcher path with only external services stubbed.""" + binaries = tmp_path / "bin" + binaries.mkdir() + for name in ( + "curl", + "uv", + "make", + "srtctl", + "flock", + "squeue", + "sleep", + ): + _stub(binaries, name, "exit 0") + _stub(binaries, "unsquashfs", "exit 0") + _stub( + binaries, + "python", + r""" +if [[ "$1" == -m && "$2" == infx.models.acquire_mtp ]]; then + shift 2 + while [[ "$#" -gt 0 ]]; do + case "$1" in + --destination) destination="$2"; shift 2 ;; + *) shift 2 ;; + esac + done + mkdir -p "$destination" + printf '{"artifact_type":"fixture-original-draft"}\n' > "$destination/subset-provenance.json" + exit 0 +fi +exec "$MOCK_REAL_PYTHON" "$@" +""", + ) + _stub( + binaries, + "sacct", + r""" +[[ " $* " == *" -X "* ]] || exit 2 +count=0 +[[ -f "$MOCK_SACCT_COUNT" ]] && read -r count < "$MOCK_SACCT_COUNT" +count=$((count + 1)) +printf '%s\n' "$count" > "$MOCK_SACCT_COUNT" +if [[ "$*" == *JobIDRaw* ]]; then + printf '42|%s\n' "$MOCK_ACCOUNTING" + exit 0 +fi +if [[ "$MOCK_ACCOUNTING" == DELAYED ]]; then + case "$count" in + 1) exit 0 ;; + 2) printf 'RUNNING|0:0\n' ;; + *) printf 'COMPLETED|0:0\n' ;; + esac +else + printf '%s\n' "$MOCK_ACCOUNTING" +fi +""", + ) + # Tail's long-running process is a clock collaborator; logs remain real. + _stub( + binaries, + "tail", + 'for arg in "$@"; do [[ -f "$arg" ]] && cat "$arg"; done; exit 0', + ) + _stub(binaries, "sha256sum", 'printf "%064d fixture-image\\n" 0') + workspace = tmp_path / "workspace" + workspace.mkdir() + # Keep the real launcher-owned profiles and renderer in the fixture checkout. + shutil.copytree(ROOT / "runners", workspace / "runners") + (workspace / "benchmarks").mkdir() + (workspace / "configs/models").mkdir(parents=True) + (workspace / "configs/models/qwen3.5-397b-mtp-bf16.json").write_text( + '{"source":"fixture-original-draft"}\n' + ) + if power_mode != "off": + recipe = workspace / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text( + "telemetry:\n enabled: true\n dcgm_exporter:\n container_image: dcgm-exporter\n" + ) + shutil.copyfile( + ROOT / "benchmarks/benchmark_lib.sh", workspace / "benchmarks/benchmark_lib.sh" + ) + (workspace / "infx").symlink_to(ROOT / "infx", target_is_directory=True) + # Stub remote checkout and submission after loading the real shared helpers. + with (workspace / "runners/slurm_utils.sh").open("a") as helpers: + helpers.write(r""" +setup_srt_slurm() { + WRITABLE_MODELS_DIR="$MOCK_DRAFT_CACHE" + mkdir -p "$1/recipes" + cd "$1" || return 1 + printf 'name: fixture\nbenchmark:\n type: custom\n' > recipes/test.yaml + SRT_SLURM_COMMIT=1111111111111111111111111111111111111111 + printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/power-producer-sha.txt" +} +apply_srt_recipe() { + mkdir -p outputs/42/logs + cp -R "$MOCK_FIXTURE/." outputs/42/logs/ + printf '{"diagnostic":"retained","disagg":true,"num_prefill_gpu":4,"num_decode_gpu":4}\n' > "$GITHUB_WORKSPACE/aggregate_conc1.json" + printf '✅ Job 42\n' +} +""") + activation = workspace / ".venv/bin/activate" + activation.parent.mkdir(parents=True) + activation.write_text(":\n") + fixture = tmp_path / "fixture" + (fixture / "agentic/conc_1/aiperf_artifacts").mkdir(parents=True) + (fixture / "eval_results").mkdir() + (fixture / "sweep_42.log").write_text("benchmark diagnostics\n") + (fixture / "agentic/conc_1/aiperf_artifacts/profile_export_aiperf.json").write_text( + '{"metadata":{"submission_valid":false}}\n' + ) + (fixture / "eval_results/results_eval.json").write_text('{"eval":"retained"}\n') + env = { + **os.environ, + "PATH": f"{binaries}:{os.environ['PATH']}", + "MOCK_REAL_PYTHON": sys.executable, + "MOCK_DRAFT_CACHE": str(tmp_path / "models"), + "MOCK_ACCOUNTING": accounting, + "MOCK_SACCT_COUNT": str(tmp_path / "sacct-count"), + "MOCK_FIXTURE": str(fixture), + "GITHUB_WORKSPACE": str(workspace), + "EVAL_ONLY": "true" if power_mode == "eval" else "false", + "IS_AGENTIC": "1" if power_mode != "off" else "0", + "CONC_LIST": "1", + "IS_MULTINODE": "true", + "RUN_EVAL": "true", + "SLURM_PARTITION": "batch_1", + "SLURM_ACCOUNT": "benchmark", + "FRAMEWORK": "dynamo-sglang", + "MODEL_PREFIX": "qwen3.5", + "PRECISION": "fp8", + "MODEL": "Qwen/Qwen3.5-397B-A17B-FP8", + "SPEC_DECODING": "mtp", + "CONFIG_FILE": "recipes/test.yaml", + "RUNNER_NAME": "b300-dsxe_08", + "IMAGE": "fixture-image", + "ENROOT_IMPORT_TIME_LIMIT": "00:05:00", + "SALLOC_TIME_LIMIT": "00:59:00", + "RESULT_FILENAME": "aggregate", + "ISL": "1", + "OSL": "1", + } + result = subprocess.run( + [ + "bash", + "-c", + r""" +# GNU-only command options in this Linux launcher are irrelevant to the +# completion contract; keep the test runnable on macOS too. +grep() { + if [[ "$1" == -oP ]]; then cat >/dev/null; printf '42\n'; else command grep "$@"; fi +} +sed() { + if [[ "$1" == -i && "$(uname)" == Darwin ]]; then + shift; command sed -i '' "$@" + else + command sed "$@" + fi +} +# The launcher writes the fixed cluster cache directory once; the mock image +# probe does not need it, and the portable test must not touch /data. +mkdir() { + if [[ "$*" == "-p /data/home/sa-gha-runner/squash" ]]; then return 0; fi + command mkdir "$@" +} +test() { + if [[ "$1" == -r && "$2" == /data/* ]]; then return 0; fi + builtin test "$@" +} +builtin source "$1/runners/launch_b300-dsxe.sh" +""", + "bash", + str(workspace), + ], + cwd=workspace, + env=env, + text=True, + capture_output=True, + check=False, + timeout=15, + ) + assert result.returncode == expected_status, result.stdout + result.stderr + if power_mode != "off": + emitted = yaml.safe_load( + (workspace / "srt-slurm/recipes/test.yaml").read_text() + ) + assert emitted["benchmark"]["concurrencies"] == [1] + profile = yaml.safe_load((workspace / "srt-slurm/srtslurm.yaml").read_text()) + cache = tmp_path / "models" / ("qwen3.5-original-mtp-" + "0" * 64) + assert profile["default_mounts"][str(cache)] == "/draft-model" + draft_artifact = workspace / "LOGS/draft-model-provenance" + assert json.loads((draft_artifact / "subset-provenance.json").read_text()) == { + "artifact_type": "fixture-original-draft" + } + assert (draft_artifact / "host-path.txt").read_text().strip() == str(cache) + assert (draft_artifact / "container-path.txt").read_text().strip() == "/draft-model" + aggregate = json.loads((workspace / "aggregate_conc1.json").read_text()) + assert aggregate["diagnostic"] == "retained" + if power_mode == "missing": + validation = json.loads( + (workspace / "LOGS/agentic/conc_1/power_validation.json").read_text() + ) + assert validation["power_valid"] is False + assert validation["reasons"] == ["formal_benchmark_result_missing"] + else: + assert not (workspace / "LOGS/agentic/conc_1/power_validation.json").exists() + assert (workspace / "results_eval.json").read_text() == '{"eval":"retained"}\n' + assert (workspace / "LOGS/sweep_42.log").read_text() == "benchmark diagnostics\n" + with tarfile.open(workspace / "multinode_server_logs.tar.gz") as archive: + assert ( + "./agentic/conc_1/aiperf_artifacts/profile_export_aiperf.json" + in archive.getnames() + ) + assert not (workspace / "srt-slurm/outputs").exists() + expected_queries = ( + 2 + if power_mode == "missing" + else 3 + if accounting == "DELAYED" + else 10 + if accounting == "" + else 1 + ) + assert int((tmp_path / "sacct-count").read_text()) == expected_queries diff --git a/utils/test_acquire_mtp.py b/utils/test_acquire_mtp.py new file mode 100644 index 000000000..f27caf47c --- /dev/null +++ b/utils/test_acquire_mtp.py @@ -0,0 +1,248 @@ +"""Behavioral acquisition tests with tiny original safetensors and mocked HTTP.""" + +import hashlib +import json +import struct +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from infx.models import acquire_mtp as acquisition + + +def fixture_asset(*, dtype="BF16", missing_mtp=False): + # Four 1-element BF16 tensors: the selected payload is exactly eight bytes. + names = ["mtp.weight", "embed.weight", "head.weight", "target.weight"] + header = { + name: { + "dtype": dtype if name == "mtp.weight" else "BF16", + "shape": [1], + "data_offsets": [2 * i, 2 * i + 2], + } + for i, name in enumerate(names) + } + encoded = json.dumps(header).encode() + encoded += b" " * (-len(encoded) % 8) + shard = struct.pack("