diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh new file mode 100755 index 0000000000..cd90152d9e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# TEMPORARY: verify the SGLang source tree carrying the DSV4 Mooncake +# external-linker optimizations before any server starts. +# +# The recipes put ${SGLANG_SRC}/python on PYTHONPATH so the servers import the +# reviewed branch instead of the container's installed SGLang, while still +# using the container's compiled kernels. If the runner failed to clone the +# tree, PYTHONPATH silently resolves to the stock package and the job would +# benchmark the wrong code, so fail loudly here instead. +# +# Delete this script, its runner wiring, and the recipes' setup_script and +# environment keys once the optimizations ship in the pinned image. +set -euo pipefail + +SGLANG_SRC="${SGLANG_MOONCAKE_OPT_SRC:-/configs/sglang-mooncake-opt}" +INIT="${SGLANG_SRC}/python/sglang/__init__.py" + +if [ ! -f "${INIT}" ]; then + echo "ERROR: SGLang optimization source missing at ${INIT}." >&2 + echo "The runner must clone it before submitting; refusing to run against" >&2 + echo "the container's stock SGLang." >&2 + exit 1 +fi + +python3 - "${SGLANG_SRC}" <<'PYEOF' +import sys +from pathlib import Path + +root = Path(sys.argv[1]) / "python" +required = { + "SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL": "sglang/srt/environ.py", + "SGLANG_MOONCAKE_STORE_CONTRIBUTOR": "sglang/srt/environ.py", +} +for token, relative in sorted(required.items()): + path = root / relative + if token not in path.read_text(): + raise SystemExit(f"ERROR: {token} not found in {path}; wrong revision") +print(f"[sglang-mooncake-opt] verified source tree at {root}") +PYEOF + +COMPAT_SHIM="${SGLANG_MOONCAKE_OPT_COMPAT:-/configs/sglang-server-args-compat}/sitecustomize.py" +if [ ! -f "${COMPAT_SHIM}" ]; then + echo "ERROR: Dynamo compatibility shim missing at ${COMPAT_SHIM}." >&2 + exit 1 +fi + +resolved=$(PYTHONPATH="${SGLANG_SRC}/python:${PYTHONPATH:-}" python3 -c 'import sglang; print(sglang.__file__)') +case "${resolved}" in + "${SGLANG_SRC}"/*) echo "[sglang-mooncake-opt] sglang resolves to ${resolved}" ;; + *) echo "ERROR: sglang resolves to ${resolved}, not ${SGLANG_SRC}" >&2; exit 1 ;; +esac diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-server-args-compat/sitecustomize.py b/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-server-args-compat/sitecustomize.py new file mode 100644 index 0000000000..4ad5d3c77b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-server-args-compat/sitecustomize.py @@ -0,0 +1,27 @@ +"""TEMPORARY: restore ServerArgs.get_model_config for the pinned Dynamo wheel. + +The reviewed SGLang branch these recipes import moved model-config resolution +out of ServerArgs, but the pinned Dynamo wheel still calls +``server_args.get_model_config()`` while parsing worker arguments. Re-attach +the accessor so the wheel keeps working against the newer source tree. + +This file is picked up because its directory is on PYTHONPATH, so it is +imported by every interpreter in the job, including ones that never import +SGLang. Failing to import SGLang there is expected and must stay silent. + +Remove this directory, its PYTHONPATH entry, and the rest of the temporary +source override once the optimizations ship in the pinned image. +""" + +try: + from sglang.srt.arg_groups.model_override_base import model_config_of + from sglang.srt.server_args import ServerArgs +except Exception: # noqa: BLE001 - non-SGLang interpreters legitimately fail here + pass +else: + if not hasattr(ServerArgs, "get_model_config"): + + def get_model_config(self): + return model_config_of(self) + + ServerArgs.get_model_config = get_model_config diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml index 15885fb9af..e3223dac1d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml @@ -18,7 +18,7 @@ identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9" dynamo: install: true @@ -32,20 +32,35 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# TEMPORARY: the Mooncake external-linker optimizations this Pareto point +# depends on are not in a released SGLang image yet. The gb300-nv runner clones +# the reviewed branch into the srt-slurm configs directory and the servers +# import it from there. The setup script aborts the job when that tree is +# missing, so a run can never silently fall back to the container's SGLang. +# Remove both keys once the optimizations ship in the pinned image. +setup_script: dsv4-gb300-sglang-mooncake-opt.sh + +environment: + PYTHONPATH: /configs/sglang-server-args-compat:/configs/sglang-mooncake-opt/python + resources: gpu_type: gb300 gpus_per_node: 4 services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 +- name: etcd + type: etcd + placement: + node: dedicated +- name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 frontend: type: dynamo nginx_session_affinity: true @@ -100,6 +115,15 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + SGLANG_EXTERNAL_LINKER_WRITE_THROUGH_THRESHOLD: '1' + SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL: '32768' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -125,7 +149,7 @@ roles: page-size: 256 swa-full-tokens-ratio: 0.02 max-running-requests: 256 - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' @@ -135,11 +159,11 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - + prefill-chunk-size-per-request: 2048 + disaggregation-defer-partial-kv-transfer: true + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 workers: 1 @@ -175,6 +199,13 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MOONCAKE_PROTOCOL: rdma + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -195,12 +226,13 @@ roles: enable-w4a4-mxfp4-megamoe: true disaggregation-transfer-backend: mooncake disaggregation-mode: decode + mooncake-store-contributor: true load-balance-method: total_tokens mem-fraction-static: 0.9 page-size: 256 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml index 00ce318b0d..df7ff86113 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml @@ -18,7 +18,7 @@ identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9" dynamo: install: true @@ -32,20 +32,35 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# TEMPORARY: the Mooncake external-linker optimizations this Pareto point +# depends on are not in a released SGLang image yet. The gb300-nv runner clones +# the reviewed branch into the srt-slurm configs directory and the servers +# import it from there. The setup script aborts the job when that tree is +# missing, so a run can never silently fall back to the container's SGLang. +# Remove both keys once the optimizations ship in the pinned image. +setup_script: dsv4-gb300-sglang-mooncake-opt.sh + +environment: + PYTHONPATH: /configs/sglang-server-args-compat:/configs/sglang-mooncake-opt/python + resources: gpu_type: gb300 gpus_per_node: 4 services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 +- name: etcd + type: etcd + placement: + node: dedicated +- name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 frontend: type: dynamo nginx_session_affinity: true @@ -100,6 +115,15 @@ roles: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + SGLANG_EXTERNAL_LINKER_WRITE_THROUGH_THRESHOLD: '1' + SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL: '32768' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -125,7 +149,7 @@ roles: page-size: 256 swa-full-tokens-ratio: 0.02 max-running-requests: 256 - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' @@ -135,11 +159,11 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - + prefill-chunk-size-per-request: 2048 + disaggregation-defer-partial-kv-transfer: true + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 workers: 1 @@ -177,6 +201,13 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_DSV4_MHC_PREWARM: '1' + MOONCAKE_PROTOCOL: rdma + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -197,12 +228,13 @@ roles: enable-w4a4-mxfp4-megamoe: true disaggregation-transfer-backend: mooncake disaggregation-mode: decode + mooncake-store-contributor: true load-balance-method: total_tokens mem-fraction-static: 0.9 page-size: 256 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml index f0e66a32d8..e3ce10ea1d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml @@ -18,7 +18,7 @@ identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9" dynamo: install: true @@ -32,20 +32,35 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# TEMPORARY: the Mooncake external-linker optimizations this Pareto point +# depends on are not in a released SGLang image yet. The gb300-nv runner clones +# the reviewed branch into the srt-slurm configs directory and the servers +# import it from there. The setup script aborts the job when that tree is +# missing, so a run can never silently fall back to the container's SGLang. +# Remove both keys once the optimizations ship in the pinned image. +setup_script: dsv4-gb300-sglang-mooncake-opt.sh + +environment: + PYTHONPATH: /configs/sglang-server-args-compat:/configs/sglang-mooncake-opt/python + resources: gpu_type: gb300 gpus_per_node: 4 services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 +- name: etcd + type: etcd + placement: + node: dedicated +- name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 frontend: type: dynamo nginx_session_affinity: true @@ -100,6 +115,15 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + SGLANG_EXTERNAL_LINKER_WRITE_THROUGH_THRESHOLD: '1' + SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL: '32768' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -125,7 +149,7 @@ roles: page-size: 256 swa-full-tokens-ratio: 0.02 max-running-requests: 512 - cuda-graph-max-bs: 512 + cuda-graph-max-bs-decode: 512 chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' @@ -135,11 +159,11 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - + prefill-chunk-size-per-request: 2048 + disaggregation-defer-partial-kv-transfer: true + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 workers: 1 @@ -175,6 +199,13 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MOONCAKE_PROTOCOL: rdma + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -195,12 +226,13 @@ roles: enable-w4a4-mxfp4-megamoe: true disaggregation-transfer-backend: mooncake disaggregation-mode: decode + mooncake-store-contributor: true load-balance-method: total_tokens mem-fraction-static: 0.9 page-size: 256 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 512 + cuda-graph-max-bs-decode: 512 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml index e569bd41cd..838a57e040 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml @@ -18,7 +18,7 @@ identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9" dynamo: install: true @@ -32,20 +32,35 @@ health_check: max_attempts: 1440 interval_seconds: 10 +# TEMPORARY: the Mooncake external-linker optimizations this Pareto point +# depends on are not in a released SGLang image yet. The gb300-nv runner clones +# the reviewed branch into the srt-slurm configs directory and the servers +# import it from there. The setup script aborts the job when that tree is +# missing, so a run can never silently fall back to the container's SGLang. +# Remove both keys once the optimizations ship in the pinned image. +setup_script: dsv4-gb300-sglang-mooncake-opt.sh + +environment: + PYTHONPATH: /configs/sglang-server-args-compat:/configs/sglang-mooncake-opt/python + resources: gpu_type: gb300 gpus_per_node: 4 services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 +- name: etcd + type: etcd + placement: + node: dedicated +- name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 frontend: type: dynamo nginx_session_affinity: true @@ -104,6 +119,15 @@ roles: SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + MOONCAKE_PROTOCOL: rdma + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + SGLANG_EXTERNAL_LINKER_WRITE_THROUGH_THRESHOLD: '1' + SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL: '32768' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -129,7 +153,7 @@ roles: page-size: 256 swa-full-tokens-ratio: 0.02 max-running-requests: 1024 - cuda-graph-max-bs: 1024 + cuda-graph-max-bs-decode: 1024 chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' @@ -139,11 +163,11 @@ roles: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - + prefill-chunk-size-per-request: 2048 + disaggregation-defer-partial-kv-transfer: true + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' decode: nodes: 4 workers: 1 @@ -179,6 +203,13 @@ roles: SGLANG_LOG_MS: '1' SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + MOONCAKE_PROTOCOL: rdma + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + MOONCAKE_STANDALONE_STORAGE: '0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' args: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -199,12 +230,13 @@ roles: enable-w4a4-mxfp4-megamoe: true disaggregation-transfer-backend: mooncake disaggregation-mode: decode + mooncake-store-contributor: true load-balance-method: total_tokens mem-fraction-static: 0.9 page-size: 256 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 192 + cuda-graph-max-bs-decode: 192 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f28e74de34..b41cc9ff3d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6452,7 +6452,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml" dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21 + image: lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb300-nv @@ -6469,7 +6469,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [480] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake } prefill: num-worker: 1 tp: 8 @@ -6485,7 +6485,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [960] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake } prefill: num-worker: 2 tp: 8 @@ -6501,7 +6501,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [1440] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake } prefill: num-worker: 3 tp: 8 @@ -6517,7 +6517,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - spec-decoding: draft_model conc-list: [1920] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake } prefill: num-worker: 4 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a112ba56db..fc1fb8b420 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8018,6 +8018,30 @@ - "Re-pin from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 to lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b (2026-09-08 cu13 dev nightly, digest sha256:9a352a35c973a2357372e85f3bcb5388b6b3c46c1329165987260f3b089647dc; Docker Hub last pushed 2026-09-08T01:40:59Z, tag commit sgl-project/sglang@20ca564b). The 2026-09-07 build carries an unguarded kv_index_translator.translate_dcp_read_ids call on the DSA fp8 KV read path that the EAGLE draft backend never binds, so GLM-5.2 MTP runs crash intermittently with AttributeError (observed on the MI355X FP8 sibling in run 34173459478 after 74 minutes of serving). sgl-project/sglang#38318 (merged 2026-09-07T20:03Z) adds the None guard and is six commits behind 20ca564b." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2863 +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Move the GB300 DeepSeek-V4-Pro AgentX disaggregated ladder from hierarchical cache to the Mooncake unified-cache external linker, let decode ranks contribute host DRAM to the store, and keep only a sparse SWA checkpoint per 32768-token interval so the same DRAM holds about twice as many prefixes." + - "Prefill now splits a request into 2048-token chunks and defers intermediate chunk KV transfer. Image bumped to lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9, the first pinned image whose sglang-kernel satisfies the minimum the reviewed branch asserts." + - "Measured against the previous configuration at the same golden acceptance length: total tok/s/GPU +13.2% at 1P1D, +18.4% at 2P1D, +17.5% at 4P1D, with mean TTFT -21.6%, -30.7%, -37.3%. Steady-state prefix cache hit rate is 96.0-96.2% across all three points." + - "The runner also pins this cluster's RDMA devices for the Mooncake store: without an explicit device list the client routes store transfers over NVLink for same-domain peers, whose address lookup fails and aborts every worker during the store warmup put. Temporary: the serving change is still in review as sgl-project/sglang#39694, so the runner clones that branch into the srt-slurm configs directory and the servers import it through PYTHONPATH while using the container's compiled kernels. A setup script aborts the job when that tree is missing or is the wrong revision, so a run cannot silently fall back to the container's SGLang. Remove the clone, the setup script, and the recipes' PYTHONPATH once the change ships in the pinned image." + - "将 GB300 DeepSeek-V4-Pro AgentX 分离式配置从 hierarchical cache 切换到 Mooncake unified-cache external linker,让 decode rank 也贡献主机 DRAM 容量,并按 32768 token 间隔只保留稀疏 SWA checkpoint,使同样的 DRAM 可容纳约两倍前缀。" + - "Prefill 改为按 2048 token 切分单个请求并延迟中间 chunk 的 KV 传输,镜像升级到 lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9,这是首个 sglang-kernel 版本满足该分支最低要求的固定镜像。" + - "在相同 golden acceptance length 下相比原配置实测:1P1D、2P1D、4P1D 的单卡总吞吐分别提升 13.2%、18.4%、17.5%,平均 TTFT 分别下降 21.6%、30.7%、37.3%,三个点的稳态前缀命中率均为 96.0%-96.2%。" + - "runner 同时为 Mooncake store 指定该集群的 RDMA 设备:缺少显式设备列表时,store 的传输会对同 NVLink 域的对端走 NVLink,地址查找失败并在 warmup put 阶段让所有 worker 退出。临时方案:服务端改动仍在 sgl-project/sglang#39694 评审中,因此 runner 将该分支克隆到 srt-slurm 的 configs 目录,服务进程通过 PYTHONPATH 导入,同时继续使用容器内编译好的 kernel。配套 setup 脚本在该源码树缺失或版本不符时直接让任务失败,避免静默回退到容器自带的 SGLang。待改动进入固定镜像后应移除克隆、setup 脚本和 recipe 中的 PYTHONPATH。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3187 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Replace the removed cuda-graph-max-bs recipe key with cuda-graph-max-bs-decode in every prefill and decode role across the GB300 DSV4 AgentX Mooncake C480-C1920 ladder, preserving each role's existing graph batch-size value. Run 35101789051 failed before server readiness because the reviewed SGLang branch split the option into decode/prefill variants and rejected the legacy spelling as ambiguous." + - "在 GB300 DSV4 AgentX Mooncake C480-C1920 阶梯的所有 prefill 与 decode role 中,将已移除的 cuda-graph-max-bs 配方键替换为 cuda-graph-max-bs-decode,并保留各 role 现有的 CUDA graph batch-size 数值。运行 35101789051 在服务就绪前失败,因为评审中的 SGLang 分支已将该选项拆分为 decode/prefill 两个变体,并因歧义拒绝旧写法。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3187 + - config-keys: - dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark scenario-type: @@ -8082,3 +8106,11 @@ - "Capture only full decode CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=0 with cudagraph_mode FULL_DECODE_ONLY, as the MiniMax-M3 gfx942 arm does) and restore --moe-backend aiter: on gfx942 every worker segfaulted during piecewise graph capture with both the Triton W4A16 MoE kernel (run 35305045778) and the auto-selected unfused Triton kernel (run 35306398350), so the capture mode rather than the MoE kernel is the failing piece; prefill runs eagerly" - "仅捕获完整的 decode CUDA graph(VLLM_USE_BREAKABLE_CUDAGRAPH=0 并设置 cudagraph_mode FULL_DECODE_ONLY,与 MiniMax-M3 gfx942 配方一致)并恢复 --moe-backend aiter:在 gfx942 上,无论使用 Triton W4A16 MoE 内核(运行 35305045778)还是自动选择的未融合 Triton 内核(运行 35306398350),所有 worker 都在 piecewise graph 捕获期间段错误,说明问题在于捕获模式而非 MoE 内核;prefill 以 eager 方式运行" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3247 +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Drive the decode ranks' passive Mooncake capacity contribution from the server flag --mooncake-store-contributor instead of the SGLANG_MOONCAKE_STORE_CONTRIBUTOR environment variable, and repin the reviewed SGLang branch to the revision that builds the contributor where the KV pools are built. The environment variable was read by the Mooncake transfer manager, so the contribution silently disappeared whenever KV transfer used another backend, and setting it on a prefill rank did nothing without an error; the flag is rejected on non-decode roles and when combined with the external linker. Recipe behaviour at every ladder point is unchanged." + - "将 decode rank 向 Mooncake store 贡献主机 DRAM 的开关从环境变量 SGLANG_MOONCAKE_STORE_CONTRIBUTOR 改为服务端参数 --mooncake-store-contributor,并把评审中的 SGLang 分支重新固定到在构建 KV pool 处创建该 contributor 的版本。原环境变量由 Mooncake 传输管理器读取,因此只要 KV 传输改用其他后端,容量贡献就会静默消失;把它设在 prefill rank 上则毫无作用且不报错。新参数在非 decode 角色或与 external linker 同时开启时会直接报错。各并发点的配方行为不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3187 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 4031483ed5..6a729767d2 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -215,6 +215,29 @@ rm -rf "$SRT_REPO_DIR" setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1 +# TEMPORARY: the Mooncake external-linker optimizations the DSV4 AgentX +# disaggregated recipes rely on are not in a released SGLang image yet, so +# clone the reviewed branch into configs/ (mounted at /configs) and let the +# servers import it through PYTHONPATH while still using the container's +# compiled kernels. The recipes' setup script aborts the job if this tree is +# missing. Drop this block, the setup script, and the recipes' PYTHONPATH once +# the change ships in the pinned image. +if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-sglang" && "$MODEL_PREFIX" == "dsv4" ]]; then + SGLANG_MOONCAKE_OPT_URL="https://github.com/weireweire/sglang.git" + SGLANG_MOONCAKE_OPT_PIN="7b18eddd006a0166c5d092dfb399ca7d136494fc" + git init configs/sglang-mooncake-opt || exit 1 + git -C configs/sglang-mooncake-opt fetch --depth 1 \ + "$SGLANG_MOONCAKE_OPT_URL" "$SGLANG_MOONCAKE_OPT_PIN" || exit 1 + git -C configs/sglang-mooncake-opt checkout --detach FETCH_HEAD || exit 1 + + # The Mooncake store transfers from host buffers that only the RDMA + # transport registers on the fly. Without an explicit device list the + # client falls back to NVLink for same-domain peers, whose address lookup + # then fails and aborts every worker during the store warmup put. These + # are this cluster's RDMA devices; other clusters name theirs differently. + MOONCAKE_STORE_DEVICES="mlx5_0,mlx5_1,mlx5_2,mlx5_3" +fi + if [[ "$FRAMEWORK" == "dynamo-trt" && "$MODEL_PREFIX" == "dsv4" ]]; then SRT_SLURM_MODEL_PREFIX="deepseek-ai/DeepSeek-V4-Pro" fi @@ -337,6 +360,10 @@ SRTCTL_APPLY_ARGS=( -f "$CONFIG_FILE" --tags "gb300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) +if [[ -n "${MOONCAKE_STORE_DEVICES:-}" ]]; then + SRTCTL_APPLY_ARGS+=(--set "roles.prefill.env.MOONCAKE_DEVICE=$MOONCAKE_STORE_DEVICES") + SRTCTL_APPLY_ARGS+=(--set "roles.decode.env.MOONCAKE_DEVICE=$MOONCAKE_STORE_DEVICES") +fi if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" ) || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && ( "$FRAMEWORK" == "dynamo-trt" || "$USES_DCGM_POWER" == "1" ) ) || ( "$USES_DCGM_POWER" == "1" && "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ) ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi