From e95e3abbcaa3b820560fb8bdb8718385ba3e23f0 Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Wed, 16 Sep 2026 21:24:59 +0800
Subject: [PATCH 01/10] perf(agentx): move GB300 DSV4 AgentX disagg to Mooncake
 external linker

Switch the GB300 DeepSeek-V4-Pro AgentX disaggregated Pareto ladder from
hierarchical cache to the Mooncake unified-cache external linker, let decode
ranks contribute host DRAM to the store, and retain only a sparse SWA
checkpoint per 32768-token interval so the same DRAM holds about twice as
many prefixes. Prefill splits a request into 2048-token chunks, defers
intermediate chunk KV transfer, and raises mem-fraction-static to 0.92.

The serving change is still in review as sgl-project/sglang#39694 and is not
in a released image, so the runner clones that branch into the srt-slurm
configs directory and the servers import it through PYTHONPATH while using
the container's compiled kernels. A setup script aborts the job when the tree
is missing or is the wrong revision, so a run cannot silently fall back to
the container's SGLang. This clone, the setup script, and the recipes'
PYTHONPATH are temporary and come out once the change ships in the image.
---
 .../configs/dsv4-gb300-sglang-mooncake-opt.sh | 45 +++++++++++++
 ...gg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml | 66 ++++++++++++++-----
 ...gg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml | 66 ++++++++++++++-----
 ...g-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml | 66 ++++++++++++++-----
 ...g-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml | 66 ++++++++++++++-----
 configs/nvidia-master.yaml                    | 10 +--
 perf-changelog.yaml                           | 15 +++++
 runners/launch_gb300-nv.sh                    | 16 +++++
 8 files changed, 277 insertions(+), 73 deletions(-)
 create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh

diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh
new file mode 100755
index 0000000000..f62ecaf977
--- /dev/null
+++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# TEMPORARY: verify the SGLang source tree carrying the DSV4 Mooncake
+# external-linker optimizations before any server starts.
+#
+# The recipes put ${SGLANG_SRC}/python on PYTHONPATH so the servers import the
+# reviewed branch instead of the container's installed SGLang, while still
+# using the container's compiled kernels. If the runner failed to clone the
+# tree, PYTHONPATH silently resolves to the stock package and the job would
+# benchmark the wrong code, so fail loudly here instead.
+#
+# Delete this script, its runner wiring, and the recipes' setup_script and
+# environment keys once the optimizations ship in the pinned image.
+set -euo pipefail
+
+SGLANG_SRC="${SGLANG_MOONCAKE_OPT_SRC:-/configs/sglang-mooncake-opt}"
+INIT="${SGLANG_SRC}/python/sglang/__init__.py"
+
+if [ ! -f "${INIT}" ]; then
+    echo "ERROR: SGLang optimization source missing at ${INIT}." >&2
+    echo "The runner must clone it before submitting; refusing to run against" >&2
+    echo "the container's stock SGLang." >&2
+    exit 1
+fi
+
+python3 - "${SGLANG_SRC}" <<'PYEOF'
+import sys
+from pathlib import Path
+
+root = Path(sys.argv[1]) / "python"
+required = {
+    "SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL": "sglang/srt/environ.py",
+    "SGLANG_MOONCAKE_STORE_CONTRIBUTOR": "sglang/srt/environ.py",
+}
+for token, relative in sorted(required.items()):
+    path = root / relative
+    if token not in path.read_text():
+        raise SystemExit(f"ERROR: {token} not found in {path}; wrong revision")
+print(f"[sglang-mooncake-opt] verified source tree at {root}")
+PYEOF
+
+resolved=$(PYTHONPATH="${SGLANG_SRC}/python:${PYTHONPATH:-}" python3 -c 'import sglang; print(sglang.__file__)')
+case "${resolved}" in
+    "${SGLANG_SRC}"/*) echo "[sglang-mooncake-opt] sglang resolves to ${resolved}" ;;
+    *) echo "ERROR: sglang resolves to ${resolved}, not ${SGLANG_SRC}" >&2; exit 1 ;;
+esac
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
index 15885fb9af..de8b5c5e79 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
@@ -18,7 +18,7 @@ identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro-0813"
   container:
-    image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21"
+    image: "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b"
 
 dynamo:
   install: true
@@ -32,20 +32,35 @@ health_check:
   max_attempts: 1440
   interval_seconds: 10
 
+# TEMPORARY: the Mooncake external-linker optimizations this Pareto point
+# depends on are not in a released SGLang image yet. The gb300-nv runner clones
+# the reviewed branch into the srt-slurm configs directory and the servers
+# import it from there. The setup script aborts the job when that tree is
+# missing, so a run can never silently fall back to the container's SGLang.
+# Remove both keys once the optimizations ship in the pinned image.
+setup_script: dsv4-gb300-sglang-mooncake-opt.sh
+
+environment:
+  PYTHONPATH: /configs/sglang-mooncake-opt/python
+
 resources:
   gpu_type: gb300
   gpus_per_node: 4
 services:
-  - name: etcd
-    type: etcd
-    placement:
-      node: dedicated
-  - name: nats
-    type: nats
-    placement:
-      node: dedicated
-    options:
-      max_payload_mb: 32
+- name: etcd
+  type: etcd
+  placement:
+    node: dedicated
+- name: nats
+  type: nats
+  placement:
+    node: dedicated
+  options:
+    max_payload_mb: 32
+- name: mooncake-master
+  type: mooncake-master
+  args:
+  - --eviction_high_watermark_ratio=0.90
 frontend:
   type: dynamo
   nginx_session_affinity: true
@@ -100,6 +115,15 @@ roles:
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
       SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
 
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC: '1'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
+      SGLANG_EXTERNAL_LINKER_WRITE_THROUGH_THRESHOLD: '1'
+      SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL: '32768'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -121,7 +145,7 @@ roles:
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: prefill
       load-balance-method: total_tokens
-      mem-fraction-static: 0.85
+      mem-fraction-static: 0.92
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 256
@@ -135,11 +159,11 @@ roles:
       speculative-num-steps: 1
       speculative-eagle-topk: 1
       speculative-num-draft-tokens: 7
-      enable-hierarchical-cache: true
-      hicache-write-policy: write_back
-      hicache-ratio: 1
-      hicache-io-backend: direct
-
+      prefill-chunk-size-per-request: 2048
+      disaggregation-defer-partial-kv-transfer: true
+      enable-unified-cache-external-linker: true
+      unified-cache-external-linker-backend: mooncake
+      hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
   decode:
     nodes: 4
     workers: 1
@@ -175,6 +199,14 @@ roles:
       SGLANG_LOG_MS: '1'
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
 
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC: '1'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
+      SGLANG_MOONCAKE_STORE_CONTRIBUTOR: '1'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
index 00ce318b0d..db4f6e8d14 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
@@ -18,7 +18,7 @@ identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro-0813"
   container:
-    image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21"
+    image: "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b"
 
 dynamo:
   install: true
@@ -32,20 +32,35 @@ health_check:
   max_attempts: 1440
   interval_seconds: 10
 
+# TEMPORARY: the Mooncake external-linker optimizations this Pareto point
+# depends on are not in a released SGLang image yet. The gb300-nv runner clones
+# the reviewed branch into the srt-slurm configs directory and the servers
+# import it from there. The setup script aborts the job when that tree is
+# missing, so a run can never silently fall back to the container's SGLang.
+# Remove both keys once the optimizations ship in the pinned image.
+setup_script: dsv4-gb300-sglang-mooncake-opt.sh
+
+environment:
+  PYTHONPATH: /configs/sglang-mooncake-opt/python
+
 resources:
   gpu_type: gb300
   gpus_per_node: 4
 services:
-  - name: etcd
-    type: etcd
-    placement:
-      node: dedicated
-  - name: nats
-    type: nats
-    placement:
-      node: dedicated
-    options:
-      max_payload_mb: 32
+- name: etcd
+  type: etcd
+  placement:
+    node: dedicated
+- name: nats
+  type: nats
+  placement:
+    node: dedicated
+  options:
+    max_payload_mb: 32
+- name: mooncake-master
+  type: mooncake-master
+  args:
+  - --eviction_high_watermark_ratio=0.90
 frontend:
   type: dynamo
   nginx_session_affinity: true
@@ -100,6 +115,15 @@ roles:
       SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
       SGLANG_DSV4_MHC_PREWARM: '1'
 
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC: '1'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
+      SGLANG_EXTERNAL_LINKER_WRITE_THROUGH_THRESHOLD: '1'
+      SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL: '32768'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -121,7 +145,7 @@ roles:
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: prefill
       load-balance-method: total_tokens
-      mem-fraction-static: 0.85
+      mem-fraction-static: 0.92
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 256
@@ -135,11 +159,11 @@ roles:
       speculative-num-steps: 1
       speculative-eagle-topk: 1
       speculative-num-draft-tokens: 7
-      enable-hierarchical-cache: true
-      hicache-write-policy: write_back
-      hicache-ratio: 1
-      hicache-io-backend: direct
-
+      prefill-chunk-size-per-request: 2048
+      disaggregation-defer-partial-kv-transfer: true
+      enable-unified-cache-external-linker: true
+      unified-cache-external-linker-backend: mooncake
+      hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
   decode:
     nodes: 4
     workers: 1
@@ -177,6 +201,14 @@ roles:
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
       SGLANG_DSV4_MHC_PREWARM: '1'
 
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC: '1'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
+      SGLANG_MOONCAKE_STORE_CONTRIBUTOR: '1'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
index f0e66a32d8..f9f06cd096 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
@@ -18,7 +18,7 @@ identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro-0813"
   container:
-    image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21"
+    image: "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b"
 
 dynamo:
   install: true
@@ -32,20 +32,35 @@ health_check:
   max_attempts: 1440
   interval_seconds: 10
 
+# TEMPORARY: the Mooncake external-linker optimizations this Pareto point
+# depends on are not in a released SGLang image yet. The gb300-nv runner clones
+# the reviewed branch into the srt-slurm configs directory and the servers
+# import it from there. The setup script aborts the job when that tree is
+# missing, so a run can never silently fall back to the container's SGLang.
+# Remove both keys once the optimizations ship in the pinned image.
+setup_script: dsv4-gb300-sglang-mooncake-opt.sh
+
+environment:
+  PYTHONPATH: /configs/sglang-mooncake-opt/python
+
 resources:
   gpu_type: gb300
   gpus_per_node: 4
 services:
-  - name: etcd
-    type: etcd
-    placement:
-      node: dedicated
-  - name: nats
-    type: nats
-    placement:
-      node: dedicated
-    options:
-      max_payload_mb: 32
+- name: etcd
+  type: etcd
+  placement:
+    node: dedicated
+- name: nats
+  type: nats
+  placement:
+    node: dedicated
+  options:
+    max_payload_mb: 32
+- name: mooncake-master
+  type: mooncake-master
+  args:
+  - --eviction_high_watermark_ratio=0.90
 frontend:
   type: dynamo
   nginx_session_affinity: true
@@ -100,6 +115,15 @@ roles:
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
       SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
 
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC: '1'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
+      SGLANG_EXTERNAL_LINKER_WRITE_THROUGH_THRESHOLD: '1'
+      SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL: '32768'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -121,7 +145,7 @@ roles:
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: prefill
       load-balance-method: total_tokens
-      mem-fraction-static: 0.85
+      mem-fraction-static: 0.92
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 512
@@ -135,11 +159,11 @@ roles:
       speculative-num-steps: 1
       speculative-eagle-topk: 1
       speculative-num-draft-tokens: 7
-      enable-hierarchical-cache: true
-      hicache-write-policy: write_back
-      hicache-ratio: 1
-      hicache-io-backend: direct
-
+      prefill-chunk-size-per-request: 2048
+      disaggregation-defer-partial-kv-transfer: true
+      enable-unified-cache-external-linker: true
+      unified-cache-external-linker-backend: mooncake
+      hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
   decode:
     nodes: 4
     workers: 1
@@ -175,6 +199,14 @@ roles:
       SGLANG_LOG_MS: '1'
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
 
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC: '1'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
+      SGLANG_MOONCAKE_STORE_CONTRIBUTOR: '1'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
index e569bd41cd..3b1da99d9c 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
@@ -18,7 +18,7 @@ identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro-0813"
   container:
-    image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21"
+    image: "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b"
 
 dynamo:
   install: true
@@ -32,20 +32,35 @@ health_check:
   max_attempts: 1440
   interval_seconds: 10
 
+# TEMPORARY: the Mooncake external-linker optimizations this Pareto point
+# depends on are not in a released SGLang image yet. The gb300-nv runner clones
+# the reviewed branch into the srt-slurm configs directory and the servers
+# import it from there. The setup script aborts the job when that tree is
+# missing, so a run can never silently fall back to the container's SGLang.
+# Remove both keys once the optimizations ship in the pinned image.
+setup_script: dsv4-gb300-sglang-mooncake-opt.sh
+
+environment:
+  PYTHONPATH: /configs/sglang-mooncake-opt/python
+
 resources:
   gpu_type: gb300
   gpus_per_node: 4
 services:
-  - name: etcd
-    type: etcd
-    placement:
-      node: dedicated
-  - name: nats
-    type: nats
-    placement:
-      node: dedicated
-    options:
-      max_payload_mb: 32
+- name: etcd
+  type: etcd
+  placement:
+    node: dedicated
+- name: nats
+  type: nats
+  placement:
+    node: dedicated
+  options:
+    max_payload_mb: 32
+- name: mooncake-master
+  type: mooncake-master
+  args:
+  - --eviction_high_watermark_ratio=0.90
 frontend:
   type: dynamo
   nginx_session_affinity: true
@@ -104,6 +119,15 @@ roles:
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
       SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
 
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC: '1'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
+      SGLANG_EXTERNAL_LINKER_WRITE_THROUGH_THRESHOLD: '1'
+      SGLANG_EXTERNAL_LINKER_SWA_RETENTION_INTERVAL: '32768'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -125,7 +149,7 @@ roles:
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: prefill
       load-balance-method: total_tokens
-      mem-fraction-static: 0.85
+      mem-fraction-static: 0.92
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 1024
@@ -139,11 +163,11 @@ roles:
       speculative-num-steps: 1
       speculative-eagle-topk: 1
       speculative-num-draft-tokens: 7
-      enable-hierarchical-cache: true
-      hicache-write-policy: write_back
-      hicache-ratio: 1
-      hicache-io-backend: direct
-
+      prefill-chunk-size-per-request: 2048
+      disaggregation-defer-partial-kv-transfer: true
+      enable-unified-cache-external-linker: true
+      unified-cache-external-linker-backend: mooncake
+      hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
   decode:
     nodes: 4
     workers: 1
@@ -179,6 +203,14 @@ roles:
       SGLANG_LOG_MS: '1'
       SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
 
+      MOONCAKE_PROTOCOL: rdma
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      MOONCAKE_STANDALONE_STORAGE: '0'
+      MC_STORE_CLIENT_METRIC: '1'
+      MC_STORE_CLIENT_METRIC_INTERVAL: '5'
+      MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
+      WITH_NVIDIA_PEERMEM: '0'
+      SGLANG_MOONCAKE_STORE_CONTRIBUTOR: '1'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index eabc690f45..763eda2f6d 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -6647,7 +6647,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg:
           - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77"
           - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml"
 dsv4-fp4-gb300-dynamo-sglang-agentic-disagg:
-  image: lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21
+  image: lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b
   model: deepseek-ai/DeepSeek-V4-Pro-0813
   model-prefix: dsv4
   runner: cluster:gb300-nv
@@ -6664,7 +6664,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg:
       - spec-decoding: draft_model
         conc-list: [480]
         kv-offloading: dram
-        kv-offload-backend: { name: hicache }
+        kv-offload-backend: { name: mooncake }
         prefill:
           num-worker: 1
           tp: 8
@@ -6682,7 +6682,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg:
       - spec-decoding: draft_model
         conc-list: [960]
         kv-offloading: dram
-        kv-offload-backend: { name: hicache }
+        kv-offload-backend: { name: mooncake }
         prefill:
           num-worker: 2
           tp: 8
@@ -6700,7 +6700,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg:
       - spec-decoding: draft_model
         conc-list: [1440]
         kv-offloading: dram
-        kv-offload-backend: { name: hicache }
+        kv-offload-backend: { name: mooncake }
         prefill:
           num-worker: 3
           tp: 8
@@ -6718,7 +6718,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg:
       - spec-decoding: draft_model
         conc-list: [1920]
         kv-offloading: dram
-        kv-offload-backend: { name: hicache }
+        kv-offload-backend: { name: mooncake }
         prefill:
           num-worker: 4
           tp: 8
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 8d11fa35b6..5b5b5813cc 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -7923,3 +7923,18 @@
     - "Bump image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915."
     - "Switch HiCache defaults to --hicache-io-backend kernel and --hicache-mem-layout page_first (from direct / page_first_direct). Ratio 1.5 and write_through are unchanged."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3118
+
+- config-keys:
+      - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Move the GB300 DeepSeek-V4-Pro AgentX disaggregated ladder from hierarchical cache to the Mooncake unified-cache external linker, let decode ranks contribute host DRAM to the store, and keep only a sparse SWA checkpoint per 32768-token interval so the same DRAM holds about twice as many prefixes."
+    - "Prefill now splits a request into 2048-token chunks and defers intermediate chunk KV transfer, and raises mem-fraction-static from 0.85 to 0.92. Image bumped to lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b."
+    - "Measured against the previous configuration at the same golden acceptance length: total tok/s/GPU +13.2% at 1P1D, +18.4% at 2P1D, +17.5% at 4P1D, with mean TTFT -21.6%, -30.7%, -37.3%. Steady-state prefix cache hit rate is 96.0-96.2% across all three points."
+    - "Temporary: the serving change is still in review as sgl-project/sglang#39694, so the runner clones that branch into the srt-slurm configs directory and the servers import it through PYTHONPATH while using the container's compiled kernels. A setup script aborts the job when that tree is missing or is the wrong revision, so a run cannot silently fall back to the container's SGLang. Remove the clone, the setup script, and the recipes' PYTHONPATH once the change ships in the pinned image."
+    - "将 GB300 DeepSeek-V4-Pro AgentX 分离式配置从 hierarchical cache 切换到 Mooncake unified-cache external linker，让 decode rank 也贡献主机 DRAM 容量，并按 32768 token 间隔只保留稀疏 SWA checkpoint，使同样的 DRAM 可容纳约两倍前缀。"
+    - "Prefill 改为按 2048 token 切分单个请求并延迟中间 chunk 的 KV 传输，mem-fraction-static 从 0.85 提升到 0.92，镜像升级到 lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b。"
+    - "在相同 golden acceptance length 下相比原配置实测：1P1D、2P1D、4P1D 的单卡总吞吐分别提升 13.2%、18.4%、17.5%，平均 TTFT 分别下降 21.6%、30.7%、37.3%，三个点的稳态前缀命中率均为 96.0%-96.2%。"
+    - "临时方案：服务端改动仍在 sgl-project/sglang#39694 评审中，因此 runner 将该分支克隆到 srt-slurm 的 configs 目录，服务进程通过 PYTHONPATH 导入，同时继续使用容器内编译好的 kernel。配套 setup 脚本在该源码树缺失或版本不符时直接让任务失败，避免静默回退到容器自带的 SGLang。待改动进入固定镜像后应移除克隆、setup 脚本和 recipe 中的 PYTHONPATH。"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3187
diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh
index 0dc0da179c..0fa4b37de7 100644
--- a/runners/launch_gb300-nv.sh
+++ b/runners/launch_gb300-nv.sh
@@ -215,6 +215,22 @@ rm -rf "$SRT_REPO_DIR"
 
 setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1
 
+# TEMPORARY: the Mooncake external-linker optimizations the DSV4 AgentX
+# disaggregated recipes rely on are not in a released SGLang image yet, so
+# clone the reviewed branch into configs/ (mounted at /configs) and let the
+# servers import it through PYTHONPATH while still using the container's
+# compiled kernels. The recipes' setup script aborts the job if this tree is
+# missing. Drop this block, the setup script, and the recipes' PYTHONPATH once
+# the change ships in the pinned image.
+if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-sglang" && "$MODEL_PREFIX" == "dsv4" ]]; then
+    SGLANG_MOONCAKE_OPT_URL="https://github.com/weireweire/sglang.git"
+    SGLANG_MOONCAKE_OPT_PIN="d2cf19e69fe7f9d69a7504012c3617fe6669acbd"
+    git init configs/sglang-mooncake-opt || exit 1
+    git -C configs/sglang-mooncake-opt fetch --depth 1 \
+        "$SGLANG_MOONCAKE_OPT_URL" "$SGLANG_MOONCAKE_OPT_PIN" || exit 1
+    git -C configs/sglang-mooncake-opt checkout --detach FETCH_HEAD || exit 1
+fi
+
 if [[ "$FRAMEWORK" == "dynamo-trt" && "$MODEL_PREFIX" == "dsv4" ]]; then
     SRT_SLURM_MODEL_PREFIX="deepseek-ai/DeepSeek-V4-Pro"
 fi

From 56a0d35fdd4d2525d62806e94310a0ca87478b2e Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Thu, 17 Sep 2026 01:17:46 +0800
Subject: [PATCH 02/10] fix(agentx): use the split CUDA-graph batch-size flag
 for GB300 DSV4 AgentX

The reviewed SGLang branch these recipes import dropped the deprecated
--cuda-graph-max-bs alias, so argparse now rejects it as an ambiguous prefix
of --cuda-graph-max-bs-decode and --cuda-graph-max-bs-prefill and every
server exits before startup. Rename the key to the flag the alias mapped to,
keeping each point's value unchanged.
---
 .../agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml     | 4 ++--
 .../agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml     | 4 ++--
 .../agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml    | 4 ++--
 .../agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml    | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
index de8b5c5e79..2e3e8d2491 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
@@ -149,7 +149,7 @@ roles:
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 256
-      cuda-graph-max-bs: 256
+      cuda-graph-max-bs-decode: 256
       chunked-prefill-size: 65536
       disable-flashinfer-autotune: true
       model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
@@ -232,7 +232,7 @@ roles:
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 3072
-      cuda-graph-max-bs: 256
+      cuda-graph-max-bs-decode: 256
       disable-flashinfer-autotune: true
       model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
       kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
index db4f6e8d14..6ea5796b1e 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
@@ -149,7 +149,7 @@ roles:
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 256
-      cuda-graph-max-bs: 256
+      cuda-graph-max-bs-decode: 256
       chunked-prefill-size: 65536
       disable-flashinfer-autotune: true
       model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
@@ -234,7 +234,7 @@ roles:
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 3072
-      cuda-graph-max-bs: 256
+      cuda-graph-max-bs-decode: 256
       disable-flashinfer-autotune: true
       model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
       kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
index f9f06cd096..9fa2dc2966 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
@@ -149,7 +149,7 @@ roles:
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 512
-      cuda-graph-max-bs: 512
+      cuda-graph-max-bs-decode: 512
       chunked-prefill-size: 65536
       disable-flashinfer-autotune: true
       model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
@@ -232,7 +232,7 @@ roles:
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 3072
-      cuda-graph-max-bs: 512
+      cuda-graph-max-bs-decode: 512
       disable-flashinfer-autotune: true
       model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
       kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
index 3b1da99d9c..2710a8d297 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
@@ -153,7 +153,7 @@ roles:
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 1024
-      cuda-graph-max-bs: 1024
+      cuda-graph-max-bs-decode: 1024
       chunked-prefill-size: 65536
       disable-flashinfer-autotune: true
       model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
@@ -236,7 +236,7 @@ roles:
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 3072
-      cuda-graph-max-bs: 192
+      cuda-graph-max-bs-decode: 192
       disable-flashinfer-autotune: true
       model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
       kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}'

From 3c1a3d6d37d1c007174771f4af49bbb2982de54b Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Thu, 17 Sep 2026 09:20:54 +0800
Subject: [PATCH 03/10] fix(agentx): restore the ServerArgs accessor the pinned
 Dynamo wheel needs

The reviewed SGLang branch moved model-config resolution out of ServerArgs,
but the pinned Dynamo wheel still calls server_args.get_model_config() while
parsing worker arguments, so every worker aborted with AttributeError. Ship
the same compatibility shim the local validation used, put it ahead of the
source tree on PYTHONPATH, and have the setup script fail when it is absent.
---
 .../configs/dsv4-gb300-sglang-mooncake-opt.sh |  6 +++++
 .../sitecustomize.py                          | 27 +++++++++++++++++++
 ...gg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml |  2 +-
 ...gg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml |  2 +-
 ...g-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml |  2 +-
 ...g-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml |  2 +-
 6 files changed, 37 insertions(+), 4 deletions(-)
 create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/sglang-server-args-compat/sitecustomize.py

diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh
index f62ecaf977..cd90152d9e 100755
--- a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh
+++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-gb300-sglang-mooncake-opt.sh
@@ -38,6 +38,12 @@ for token, relative in sorted(required.items()):
 print(f"[sglang-mooncake-opt] verified source tree at {root}")
 PYEOF
 
+COMPAT_SHIM="${SGLANG_MOONCAKE_OPT_COMPAT:-/configs/sglang-server-args-compat}/sitecustomize.py"
+if [ ! -f "${COMPAT_SHIM}" ]; then
+    echo "ERROR: Dynamo compatibility shim missing at ${COMPAT_SHIM}." >&2
+    exit 1
+fi
+
 resolved=$(PYTHONPATH="${SGLANG_SRC}/python:${PYTHONPATH:-}" python3 -c 'import sglang; print(sglang.__file__)')
 case "${resolved}" in
     "${SGLANG_SRC}"/*) echo "[sglang-mooncake-opt] sglang resolves to ${resolved}" ;;
diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-server-args-compat/sitecustomize.py b/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-server-args-compat/sitecustomize.py
new file mode 100644
index 0000000000..4ad5d3c77b
--- /dev/null
+++ b/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-server-args-compat/sitecustomize.py
@@ -0,0 +1,27 @@
+"""TEMPORARY: restore ServerArgs.get_model_config for the pinned Dynamo wheel.
+
+The reviewed SGLang branch these recipes import moved model-config resolution
+out of ServerArgs, but the pinned Dynamo wheel still calls
+``server_args.get_model_config()`` while parsing worker arguments. Re-attach
+the accessor so the wheel keeps working against the newer source tree.
+
+This file is picked up because its directory is on PYTHONPATH, so it is
+imported by every interpreter in the job, including ones that never import
+SGLang. Failing to import SGLang there is expected and must stay silent.
+
+Remove this directory, its PYTHONPATH entry, and the rest of the temporary
+source override once the optimizations ship in the pinned image.
+"""
+
+try:
+    from sglang.srt.arg_groups.model_override_base import model_config_of
+    from sglang.srt.server_args import ServerArgs
+except Exception:  # noqa: BLE001 - non-SGLang interpreters legitimately fail here
+    pass
+else:
+    if not hasattr(ServerArgs, "get_model_config"):
+
+        def get_model_config(self):
+            return model_config_of(self)
+
+        ServerArgs.get_model_config = get_model_config
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
index 2e3e8d2491..e95499a111 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
@@ -41,7 +41,7 @@ health_check:
 setup_script: dsv4-gb300-sglang-mooncake-opt.sh
 
 environment:
-  PYTHONPATH: /configs/sglang-mooncake-opt/python
+  PYTHONPATH: /configs/sglang-server-args-compat:/configs/sglang-mooncake-opt/python
 
 resources:
   gpu_type: gb300
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
index 6ea5796b1e..84a315bcc6 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
@@ -41,7 +41,7 @@ health_check:
 setup_script: dsv4-gb300-sglang-mooncake-opt.sh
 
 environment:
-  PYTHONPATH: /configs/sglang-mooncake-opt/python
+  PYTHONPATH: /configs/sglang-server-args-compat:/configs/sglang-mooncake-opt/python
 
 resources:
   gpu_type: gb300
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
index 9fa2dc2966..55a8991a9d 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
@@ -41,7 +41,7 @@ health_check:
 setup_script: dsv4-gb300-sglang-mooncake-opt.sh
 
 environment:
-  PYTHONPATH: /configs/sglang-mooncake-opt/python
+  PYTHONPATH: /configs/sglang-server-args-compat:/configs/sglang-mooncake-opt/python
 
 resources:
   gpu_type: gb300
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
index 2710a8d297..e571bc5a9d 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
@@ -41,7 +41,7 @@ health_check:
 setup_script: dsv4-gb300-sglang-mooncake-opt.sh
 
 environment:
-  PYTHONPATH: /configs/sglang-mooncake-opt/python
+  PYTHONPATH: /configs/sglang-server-args-compat:/configs/sglang-mooncake-opt/python
 
 resources:
   gpu_type: gb300

From a89365ff42356a332f16772f2e58a7443ca431c9 Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Thu, 17 Sep 2026 10:29:38 +0800
Subject: [PATCH 04/10] fix(agentx): pin an image whose sglang-kernel meets the
 branch minimum

The reviewed SGLang branch asserts sglang-kernel >= 0.4.7 at engine startup,
while the previously pinned image ships 0.4.6.post1, so every worker raised
before launching subprocesses. Move the GB300 DSV4 AgentX disaggregated
ladder to the first image that carries a new enough kernel.
---
 .../agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml     | 2 +-
 .../agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml     | 2 +-
 .../agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml    | 2 +-
 .../agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml    | 2 +-
 configs/nvidia-master.yaml                                    | 2 +-
 perf-changelog.yaml                                           | 4 ++--
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
index e95499a111..d3f101e4ac 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
@@ -18,7 +18,7 @@ identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro-0813"
   container:
-    image: "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b"
+    image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9"
 
 dynamo:
   install: true
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
index 84a315bcc6..75815c974d 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
@@ -18,7 +18,7 @@ identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro-0813"
   container:
-    image: "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b"
+    image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9"
 
 dynamo:
   install: true
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
index 55a8991a9d..6d9ac19dab 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
@@ -18,7 +18,7 @@ identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro-0813"
   container:
-    image: "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b"
+    image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9"
 
 dynamo:
   install: true
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
index e571bc5a9d..88d6ab6ee9 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
@@ -18,7 +18,7 @@ identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro-0813"
   container:
-    image: "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b"
+    image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9"
 
 dynamo:
   install: true
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index ef0eec7ee3..b18e9a8da0 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -6647,7 +6647,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg:
           - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77"
           - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml"
 dsv4-fp4-gb300-dynamo-sglang-agentic-disagg:
-  image: lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b
+  image: lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9
   model: deepseek-ai/DeepSeek-V4-Pro-0813
   model-prefix: dsv4
   runner: cluster:gb300-nv
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index fe37ea26af..de6dbd33f0 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -7995,11 +7995,11 @@
     - agentic-coding
   description:
     - "Move the GB300 DeepSeek-V4-Pro AgentX disaggregated ladder from hierarchical cache to the Mooncake unified-cache external linker, let decode ranks contribute host DRAM to the store, and keep only a sparse SWA checkpoint per 32768-token interval so the same DRAM holds about twice as many prefixes."
-    - "Prefill now splits a request into 2048-token chunks and defers intermediate chunk KV transfer, and raises mem-fraction-static from 0.85 to 0.92. Image bumped to lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b."
+    - "Prefill now splits a request into 2048-token chunks and defers intermediate chunk KV transfer, and raises mem-fraction-static from 0.85 to 0.92. Image bumped to lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9, the first pinned image whose sglang-kernel satisfies the minimum the reviewed branch asserts."
     - "Measured against the previous configuration at the same golden acceptance length: total tok/s/GPU +13.2% at 1P1D, +18.4% at 2P1D, +17.5% at 4P1D, with mean TTFT -21.6%, -30.7%, -37.3%. Steady-state prefix cache hit rate is 96.0-96.2% across all three points."
     - "Temporary: the serving change is still in review as sgl-project/sglang#39694, so the runner clones that branch into the srt-slurm configs directory and the servers import it through PYTHONPATH while using the container's compiled kernels. A setup script aborts the job when that tree is missing or is the wrong revision, so a run cannot silently fall back to the container's SGLang. Remove the clone, the setup script, and the recipes' PYTHONPATH once the change ships in the pinned image."
     - "将 GB300 DeepSeek-V4-Pro AgentX 分离式配置从 hierarchical cache 切换到 Mooncake unified-cache external linker，让 decode rank 也贡献主机 DRAM 容量，并按 32768 token 间隔只保留稀疏 SWA checkpoint，使同样的 DRAM 可容纳约两倍前缀。"
-    - "Prefill 改为按 2048 token 切分单个请求并延迟中间 chunk 的 KV 传输，mem-fraction-static 从 0.85 提升到 0.92，镜像升级到 lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b。"
+    - "Prefill 改为按 2048 token 切分单个请求并延迟中间 chunk 的 KV 传输，mem-fraction-static 从 0.85 提升到 0.92，镜像升级到 lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9，这是首个 sglang-kernel 版本满足该分支最低要求的固定镜像。"
     - "在相同 golden acceptance length 下相比原配置实测：1P1D、2P1D、4P1D 的单卡总吞吐分别提升 13.2%、18.4%、17.5%，平均 TTFT 分别下降 21.6%、30.7%、37.3%，三个点的稳态前缀命中率均为 96.0%-96.2%。"
     - "临时方案：服务端改动仍在 sgl-project/sglang#39694 评审中，因此 runner 将该分支克隆到 srt-slurm 的 configs 目录，服务进程通过 PYTHONPATH 导入，同时继续使用容器内编译好的 kernel。配套 setup 脚本在该源码树缺失或版本不符时直接让任务失败，避免静默回退到容器自带的 SGLang。待改动进入固定镜像后应移除克隆、setup 脚本和 recipe 中的 PYTHONPATH。"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3187

From 6b47d5d08fc4a3ac2d3871b68f138b9d58adbff4 Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Thu, 17 Sep 2026 12:02:20 +0800
Subject: [PATCH 05/10] fix(agentx): keep prefill mem-fraction-static at the
 validated 0.85

Raising it to 0.92 runs the prefill workers out of device memory at the top
concurrency point: the DSV4 indexer's transient allocation fails with only a
few GiB free, warmup never completes, and the run aborts before profiling.
Revert to the value this ladder has been validated at; a higher fraction can
return once a run demonstrates it holds at every point.
---
 .../agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml     | 2 +-
 .../agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml     | 2 +-
 .../agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml    | 2 +-
 .../agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml    | 2 +-
 perf-changelog.yaml                                           | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
index d3f101e4ac..efd6ae4425 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
@@ -145,7 +145,7 @@ roles:
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: prefill
       load-balance-method: total_tokens
-      mem-fraction-static: 0.92
+      mem-fraction-static: 0.85
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 256
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
index 75815c974d..7baf4534f0 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
@@ -145,7 +145,7 @@ roles:
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: prefill
       load-balance-method: total_tokens
-      mem-fraction-static: 0.92
+      mem-fraction-static: 0.85
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 256
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
index 6d9ac19dab..b4699b152a 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
@@ -145,7 +145,7 @@ roles:
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: prefill
       load-balance-method: total_tokens
-      mem-fraction-static: 0.92
+      mem-fraction-static: 0.85
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 512
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
index 88d6ab6ee9..7fae6b0a2d 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
@@ -149,7 +149,7 @@ roles:
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: prefill
       load-balance-method: total_tokens
-      mem-fraction-static: 0.92
+      mem-fraction-static: 0.85
       page-size: 256
       swa-full-tokens-ratio: 0.02
       max-running-requests: 1024
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index de6dbd33f0..004dff637d 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -7995,11 +7995,11 @@
     - agentic-coding
   description:
     - "Move the GB300 DeepSeek-V4-Pro AgentX disaggregated ladder from hierarchical cache to the Mooncake unified-cache external linker, let decode ranks contribute host DRAM to the store, and keep only a sparse SWA checkpoint per 32768-token interval so the same DRAM holds about twice as many prefixes."
-    - "Prefill now splits a request into 2048-token chunks and defers intermediate chunk KV transfer, and raises mem-fraction-static from 0.85 to 0.92. Image bumped to lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9, the first pinned image whose sglang-kernel satisfies the minimum the reviewed branch asserts."
+    - "Prefill now splits a request into 2048-token chunks and defers intermediate chunk KV transfer. Image bumped to lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9, the first pinned image whose sglang-kernel satisfies the minimum the reviewed branch asserts."
     - "Measured against the previous configuration at the same golden acceptance length: total tok/s/GPU +13.2% at 1P1D, +18.4% at 2P1D, +17.5% at 4P1D, with mean TTFT -21.6%, -30.7%, -37.3%. Steady-state prefix cache hit rate is 96.0-96.2% across all three points."
     - "Temporary: the serving change is still in review as sgl-project/sglang#39694, so the runner clones that branch into the srt-slurm configs directory and the servers import it through PYTHONPATH while using the container's compiled kernels. A setup script aborts the job when that tree is missing or is the wrong revision, so a run cannot silently fall back to the container's SGLang. Remove the clone, the setup script, and the recipes' PYTHONPATH once the change ships in the pinned image."
     - "将 GB300 DeepSeek-V4-Pro AgentX 分离式配置从 hierarchical cache 切换到 Mooncake unified-cache external linker，让 decode rank 也贡献主机 DRAM 容量，并按 32768 token 间隔只保留稀疏 SWA checkpoint，使同样的 DRAM 可容纳约两倍前缀。"
-    - "Prefill 改为按 2048 token 切分单个请求并延迟中间 chunk 的 KV 传输，mem-fraction-static 从 0.85 提升到 0.92，镜像升级到 lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9，这是首个 sglang-kernel 版本满足该分支最低要求的固定镜像。"
+    - "Prefill 改为按 2048 token 切分单个请求并延迟中间 chunk 的 KV 传输，镜像升级到 lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9，这是首个 sglang-kernel 版本满足该分支最低要求的固定镜像。"
     - "在相同 golden acceptance length 下相比原配置实测：1P1D、2P1D、4P1D 的单卡总吞吐分别提升 13.2%、18.4%、17.5%，平均 TTFT 分别下降 21.6%、30.7%、37.3%，三个点的稳态前缀命中率均为 96.0%-96.2%。"
     - "临时方案：服务端改动仍在 sgl-project/sglang#39694 评审中，因此 runner 将该分支克隆到 srt-slurm 的 configs 目录，服务进程通过 PYTHONPATH 导入，同时继续使用容器内编译好的 kernel。配套 setup 脚本在该源码树缺失或版本不符时直接让任务失败，避免静默回退到容器自带的 SGLang。待改动进入固定镜像后应移除克隆、setup 脚本和 recipe 中的 PYTHONPATH。"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3187

From db313582b7f83e61da90dc73a8514848b72ccff3 Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Thu, 17 Sep 2026 15:23:56 +0800
Subject: [PATCH 06/10] fix(changelog): restore the list-item header the main
 merge dropped

---
 perf-changelog.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 3205fc29ed..793d9163e7 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -8004,7 +8004,8 @@
     - "Re-pin from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 to lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b (2026-09-08 cu13 dev nightly, digest sha256:9a352a35c973a2357372e85f3bcb5388b6b3c46c1329165987260f3b089647dc; Docker Hub last pushed 2026-09-08T01:40:59Z, tag commit sgl-project/sglang@20ca564b). The 2026-09-07 build carries an unguarded kv_index_translator.translate_dcp_read_ids call on the DSA fp8 KV read path that the EAGLE draft backend never binds, so GLM-5.2 MTP runs crash intermittently with AttributeError (observed on the MI355X FP8 sibling in run 34173459478 after 74 minutes of serving). sgl-project/sglang#38318 (merged 2026-09-07T20:03Z) adds the None guard and is six commits behind 20ca564b."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2863
 
-      - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg
   scenario-type:
     - agentic-coding
   description:

From a39207e8dd83c84b80376749fc1b4d7351203cf5 Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Fri, 18 Sep 2026 10:07:10 +0800
Subject: [PATCH 07/10] fix(agentx): keep GB300 DSV4 AgentX KV transfers on
 RDMA while the store is on

The Mooncake store issues its warmup put from a transient host buffer that no
transport has registered. Where the prefill and decode nodes share an NVLink
domain, forcing MNNVL routes that put through the NVLink transport, which
cannot resolve the address, so every worker aborts before serving with
"Warmup put failed after 10 attempts". Stop forcing MNNVL for these recipes;
KV transfer falls back to RDMA, which registers the buffer on the fly.
---
 .../disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml     | 10 ++++++++--
 .../disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml     | 10 ++++++++--
 .../disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml    | 10 ++++++++--
 .../disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml    | 10 ++++++++--
 4 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
index efd6ae4425..509014c322 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
@@ -104,7 +104,10 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      MC_FORCE_MNNVL: '1'
+      # The Mooncake store's warmup put comes from an unregistered host
+    # buffer, which the NVLink transport cannot resolve when peers share
+    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
+    MC_FORCE_MNNVL: '0'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
@@ -188,7 +191,10 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      MC_FORCE_MNNVL: '1'
+      # The Mooncake store's warmup put comes from an unregistered host
+    # buffer, which the NVLink transport cannot resolve when peers share
+    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
+    MC_FORCE_MNNVL: '0'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
index 7baf4534f0..406d37449b 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
@@ -103,7 +103,10 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      MC_FORCE_MNNVL: '1'
+      # The Mooncake store's warmup put comes from an unregistered host
+    # buffer, which the NVLink transport cannot resolve when peers share
+    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
+    MC_FORCE_MNNVL: '0'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
@@ -189,7 +192,10 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      MC_FORCE_MNNVL: '1'
+      # The Mooncake store's warmup put comes from an unregistered host
+    # buffer, which the NVLink transport cannot resolve when peers share
+    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
+    MC_FORCE_MNNVL: '0'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
index b4699b152a..22af9bee5e 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
@@ -104,7 +104,10 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      MC_FORCE_MNNVL: '1'
+      # The Mooncake store's warmup put comes from an unregistered host
+    # buffer, which the NVLink transport cannot resolve when peers share
+    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
+    MC_FORCE_MNNVL: '0'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
@@ -188,7 +191,10 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      MC_FORCE_MNNVL: '1'
+      # The Mooncake store's warmup put comes from an unregistered host
+    # buffer, which the NVLink transport cannot resolve when peers share
+    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
+    MC_FORCE_MNNVL: '0'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
index 7fae6b0a2d..77dfce3704 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
@@ -108,7 +108,10 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      MC_FORCE_MNNVL: '1'
+      # The Mooncake store's warmup put comes from an unregistered host
+    # buffer, which the NVLink transport cannot resolve when peers share
+    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
+    MC_FORCE_MNNVL: '0'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
@@ -192,7 +195,10 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      MC_FORCE_MNNVL: '1'
+      # The Mooncake store's warmup put comes from an unregistered host
+    # buffer, which the NVLink transport cannot resolve when peers share
+    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
+    MC_FORCE_MNNVL: '0'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'

From 4e52d2fe6f7d2a60d3a118ad7ff8860f2926676c Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Fri, 18 Sep 2026 10:55:21 +0800
Subject: [PATCH 08/10] fix(agentx): pin the cluster's RDMA devices for the
 Mooncake store

Dropping MOONCAKE_DEVICE for portability left the store client without a
device list, so it routed transfers over NVLink for peers in the same domain.
That path cannot resolve the unregistered host buffer the warmup put comes
from, and every worker aborted with "Warmup put failed after 10 attempts".
Restore MC_FORCE_MNNVL for KV transfer and have the runner, which is already
cluster-specific, inject this cluster's devices instead of hardcoding names
that differ elsewhere. Verified on the same cluster: with the device list all
eight ranks report "Mooncake store warmup successfully".
---
 .../disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml    | 10 ++--------
 .../disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml    | 10 ++--------
 .../disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml   | 10 ++--------
 .../disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml   | 10 ++--------
 perf-changelog.yaml                                   |  4 ++--
 runners/launch_gb300-nv.sh                            | 11 +++++++++++
 6 files changed, 21 insertions(+), 34 deletions(-)

diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
index 509014c322..efd6ae4425 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
@@ -104,10 +104,7 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      # The Mooncake store's warmup put comes from an unregistered host
-    # buffer, which the NVLink transport cannot resolve when peers share
-    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
-    MC_FORCE_MNNVL: '0'
+      MC_FORCE_MNNVL: '1'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
@@ -191,10 +188,7 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      # The Mooncake store's warmup put comes from an unregistered host
-    # buffer, which the NVLink transport cannot resolve when peers share
-    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
-    MC_FORCE_MNNVL: '0'
+      MC_FORCE_MNNVL: '1'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
index 406d37449b..7baf4534f0 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
@@ -103,10 +103,7 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      # The Mooncake store's warmup put comes from an unregistered host
-    # buffer, which the NVLink transport cannot resolve when peers share
-    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
-    MC_FORCE_MNNVL: '0'
+      MC_FORCE_MNNVL: '1'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
@@ -192,10 +189,7 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      # The Mooncake store's warmup put comes from an unregistered host
-    # buffer, which the NVLink transport cannot resolve when peers share
-    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
-    MC_FORCE_MNNVL: '0'
+      MC_FORCE_MNNVL: '1'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
index 22af9bee5e..b4699b152a 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
@@ -104,10 +104,7 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      # The Mooncake store's warmup put comes from an unregistered host
-    # buffer, which the NVLink transport cannot resolve when peers share
-    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
-    MC_FORCE_MNNVL: '0'
+      MC_FORCE_MNNVL: '1'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
@@ -191,10 +188,7 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      # The Mooncake store's warmup put comes from an unregistered host
-    # buffer, which the NVLink transport cannot resolve when peers share
-    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
-    MC_FORCE_MNNVL: '0'
+      MC_FORCE_MNNVL: '1'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
index 77dfce3704..7fae6b0a2d 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
@@ -108,10 +108,7 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      # The Mooncake store's warmup put comes from an unregistered host
-    # buffer, which the NVLink transport cannot resolve when peers share
-    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
-    MC_FORCE_MNNVL: '0'
+      MC_FORCE_MNNVL: '1'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
@@ -195,10 +192,7 @@ roles:
       NCCL_MNNVL_ENABLE: '1'
       NCCL_CUMEM_ENABLE: '1'
       SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
-      # The Mooncake store's warmup put comes from an unregistered host
-    # buffer, which the NVLink transport cannot resolve when peers share
-    # an NVLink domain. Keep KV transfers on RDMA while the store is on.
-    MC_FORCE_MNNVL: '0'
+      MC_FORCE_MNNVL: '1'
       NCCL_TIMEOUT: '100000'
       NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
       SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 793d9163e7..0a27e57dd6 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -8012,11 +8012,11 @@
     - "Move the GB300 DeepSeek-V4-Pro AgentX disaggregated ladder from hierarchical cache to the Mooncake unified-cache external linker, let decode ranks contribute host DRAM to the store, and keep only a sparse SWA checkpoint per 32768-token interval so the same DRAM holds about twice as many prefixes."
     - "Prefill now splits a request into 2048-token chunks and defers intermediate chunk KV transfer. Image bumped to lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9, the first pinned image whose sglang-kernel satisfies the minimum the reviewed branch asserts."
     - "Measured against the previous configuration at the same golden acceptance length: total tok/s/GPU +13.2% at 1P1D, +18.4% at 2P1D, +17.5% at 4P1D, with mean TTFT -21.6%, -30.7%, -37.3%. Steady-state prefix cache hit rate is 96.0-96.2% across all three points."
-    - "Temporary: the serving change is still in review as sgl-project/sglang#39694, so the runner clones that branch into the srt-slurm configs directory and the servers import it through PYTHONPATH while using the container's compiled kernels. A setup script aborts the job when that tree is missing or is the wrong revision, so a run cannot silently fall back to the container's SGLang. Remove the clone, the setup script, and the recipes' PYTHONPATH once the change ships in the pinned image."
+    - "The runner also pins this cluster's RDMA devices for the Mooncake store: without an explicit device list the client routes store transfers over NVLink for same-domain peers, whose address lookup fails and aborts every worker during the store warmup put. Temporary: the serving change is still in review as sgl-project/sglang#39694, so the runner clones that branch into the srt-slurm configs directory and the servers import it through PYTHONPATH while using the container's compiled kernels. A setup script aborts the job when that tree is missing or is the wrong revision, so a run cannot silently fall back to the container's SGLang. Remove the clone, the setup script, and the recipes' PYTHONPATH once the change ships in the pinned image."
     - "将 GB300 DeepSeek-V4-Pro AgentX 分离式配置从 hierarchical cache 切换到 Mooncake unified-cache external linker，让 decode rank 也贡献主机 DRAM 容量，并按 32768 token 间隔只保留稀疏 SWA checkpoint，使同样的 DRAM 可容纳约两倍前缀。"
     - "Prefill 改为按 2048 token 切分单个请求并延迟中间 chunk 的 KV 传输，镜像升级到 lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9，这是首个 sglang-kernel 版本满足该分支最低要求的固定镜像。"
     - "在相同 golden acceptance length 下相比原配置实测：1P1D、2P1D、4P1D 的单卡总吞吐分别提升 13.2%、18.4%、17.5%，平均 TTFT 分别下降 21.6%、30.7%、37.3%，三个点的稳态前缀命中率均为 96.0%-96.2%。"
-    - "临时方案：服务端改动仍在 sgl-project/sglang#39694 评审中，因此 runner 将该分支克隆到 srt-slurm 的 configs 目录，服务进程通过 PYTHONPATH 导入，同时继续使用容器内编译好的 kernel。配套 setup 脚本在该源码树缺失或版本不符时直接让任务失败，避免静默回退到容器自带的 SGLang。待改动进入固定镜像后应移除克隆、setup 脚本和 recipe 中的 PYTHONPATH。"
+    - "runner 同时为 Mooncake store 指定该集群的 RDMA 设备：缺少显式设备列表时，store 的传输会对同 NVLink 域的对端走 NVLink，地址查找失败并在 warmup put 阶段让所有 worker 退出。临时方案：服务端改动仍在 sgl-project/sglang#39694 评审中，因此 runner 将该分支克隆到 srt-slurm 的 configs 目录，服务进程通过 PYTHONPATH 导入，同时继续使用容器内编译好的 kernel。配套 setup 脚本在该源码树缺失或版本不符时直接让任务失败，避免静默回退到容器自带的 SGLang。待改动进入固定镜像后应移除克隆、setup 脚本和 recipe 中的 PYTHONPATH。"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3187
 
 - config-keys:
diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh
index 176a4c8124..e9ddc5d040 100644
--- a/runners/launch_gb300-nv.sh
+++ b/runners/launch_gb300-nv.sh
@@ -229,6 +229,13 @@ if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-sglang" && "$MODEL_PREFIX"
     git -C configs/sglang-mooncake-opt fetch --depth 1 \
         "$SGLANG_MOONCAKE_OPT_URL" "$SGLANG_MOONCAKE_OPT_PIN" || exit 1
     git -C configs/sglang-mooncake-opt checkout --detach FETCH_HEAD || exit 1
+
+    # The Mooncake store transfers from host buffers that only the RDMA
+    # transport registers on the fly. Without an explicit device list the
+    # client falls back to NVLink for same-domain peers, whose address lookup
+    # then fails and aborts every worker during the store warmup put. These
+    # are this cluster's RDMA devices; other clusters name theirs differently.
+    MOONCAKE_STORE_DEVICES="mlx5_0,mlx5_1,mlx5_2,mlx5_3"
 fi
 
 if [[ "$FRAMEWORK" == "dynamo-trt" && "$MODEL_PREFIX" == "dsv4" ]]; then
@@ -353,6 +360,10 @@ SRTCTL_APPLY_ARGS=(
     -f "$CONFIG_FILE"
     --tags "gb300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)"
 )
+if [[ -n "${MOONCAKE_STORE_DEVICES:-}" ]]; then
+    SRTCTL_APPLY_ARGS+=(--set "roles.prefill.env.MOONCAKE_DEVICE=$MOONCAKE_STORE_DEVICES")
+    SRTCTL_APPLY_ARGS+=(--set "roles.decode.env.MOONCAKE_DEVICE=$MOONCAKE_STORE_DEVICES")
+fi
 if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" ) || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && ( "$FRAMEWORK" == "dynamo-trt" || "$USES_DCGM_POWER" == "1" ) ) || ( "$USES_DCGM_POWER" == "1" && "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ) ]]; then
     SRTCTL_APPLY_ARGS+=(--no-preflight)
 fi

From 0c5805e5a720f102657ccb9479a0693a9843277b Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Fri, 18 Sep 2026 16:00:39 +0800
Subject: [PATCH 09/10] refactor(agentx): drive the decode store contributor
 from a server flag

The pinned SGLang branch now creates the passive decode-side Mooncake
contributor in build_kv_cache behind --mooncake-store-contributor instead of
hooking the Mooncake transfer manager behind an environment variable, so the
capacity contribution no longer depends on Mooncake also being the KV transfer
backend. Repin the branch and swap the recipes' decode env knob for the flag.
---
 .../agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml       | 2 +-
 .../agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml       | 2 +-
 .../agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml      | 2 +-
 .../agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml      | 2 +-
 runners/launch_gb300-nv.sh                                      | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
index efd6ae4425..e3223dac1d 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml
@@ -206,7 +206,6 @@ roles:
       MC_STORE_CLIENT_METRIC_INTERVAL: '5'
       MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
       WITH_NVIDIA_PEERMEM: '0'
-      SGLANG_MOONCAKE_STORE_CONTRIBUTOR: '1'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -227,6 +226,7 @@ roles:
       enable-w4a4-mxfp4-megamoe: true
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: decode
+      mooncake-store-contributor: true
       load-balance-method: total_tokens
       mem-fraction-static: 0.9
       page-size: 256
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
index 7baf4534f0..df7ff86113 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml
@@ -208,7 +208,6 @@ roles:
       MC_STORE_CLIENT_METRIC_INTERVAL: '5'
       MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
       WITH_NVIDIA_PEERMEM: '0'
-      SGLANG_MOONCAKE_STORE_CONTRIBUTOR: '1'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -229,6 +228,7 @@ roles:
       enable-w4a4-mxfp4-megamoe: true
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: decode
+      mooncake-store-contributor: true
       load-balance-method: total_tokens
       mem-fraction-static: 0.9
       page-size: 256
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
index b4699b152a..e3ce10ea1d 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml
@@ -206,7 +206,6 @@ roles:
       MC_STORE_CLIENT_METRIC_INTERVAL: '5'
       MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
       WITH_NVIDIA_PEERMEM: '0'
-      SGLANG_MOONCAKE_STORE_CONTRIBUTOR: '1'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -227,6 +226,7 @@ roles:
       enable-w4a4-mxfp4-megamoe: true
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: decode
+      mooncake-store-contributor: true
       load-balance-method: total_tokens
       mem-fraction-static: 0.9
       page-size: 256
diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
index 7fae6b0a2d..838a57e040 100644
--- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
+++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml
@@ -210,7 +210,6 @@ roles:
       MC_STORE_CLIENT_METRIC_INTERVAL: '5'
       MC_ENABLE_DEST_DEVICE_AFFINITY: '1'
       WITH_NVIDIA_PEERMEM: '0'
-      SGLANG_MOONCAKE_STORE_CONTRIBUTOR: '1'
     args:
       host: 0.0.0.0
       served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
@@ -231,6 +230,7 @@ roles:
       enable-w4a4-mxfp4-megamoe: true
       disaggregation-transfer-backend: mooncake
       disaggregation-mode: decode
+      mooncake-store-contributor: true
       load-balance-method: total_tokens
       mem-fraction-static: 0.9
       page-size: 256
diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh
index e9ddc5d040..f543c3faf5 100644
--- a/runners/launch_gb300-nv.sh
+++ b/runners/launch_gb300-nv.sh
@@ -224,7 +224,7 @@ setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1
 # the change ships in the pinned image.
 if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-sglang" && "$MODEL_PREFIX" == "dsv4" ]]; then
     SGLANG_MOONCAKE_OPT_URL="https://github.com/weireweire/sglang.git"
-    SGLANG_MOONCAKE_OPT_PIN="d2cf19e69fe7f9d69a7504012c3617fe6669acbd"
+    SGLANG_MOONCAKE_OPT_PIN="0488f31582ca8b1a5c3a0ba2b1ed2a29e4fb890c"
     git init configs/sglang-mooncake-opt || exit 1
     git -C configs/sglang-mooncake-opt fetch --depth 1 \
         "$SGLANG_MOONCAKE_OPT_URL" "$SGLANG_MOONCAKE_OPT_PIN" || exit 1

From 9fcb8a6871f543d2be2415d43e46a49f571f12a4 Mon Sep 17 00:00:00 2001
From: weireweire <20922698+weireweire@users.noreply.github.com>
Date: Fri, 18 Sep 2026 17:43:03 +0800
Subject: [PATCH 10/10] chore(agentx): repin the reviewed SGLang branch

The branch was rewritten to drop trailing metadata from its commit messages;
the tree is byte-identical to the previous pin.
---
 runners/launch_gb300-nv.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh
index f543c3faf5..6a729767d2 100644
--- a/runners/launch_gb300-nv.sh
+++ b/runners/launch_gb300-nv.sh
@@ -224,7 +224,7 @@ setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1
 # the change ships in the pinned image.
 if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-sglang" && "$MODEL_PREFIX" == "dsv4" ]]; then
     SGLANG_MOONCAKE_OPT_URL="https://github.com/weireweire/sglang.git"
-    SGLANG_MOONCAKE_OPT_PIN="0488f31582ca8b1a5c3a0ba2b1ed2a29e4fb890c"
+    SGLANG_MOONCAKE_OPT_PIN="7b18eddd006a0166c5d092dfb399ca7d136494fc"
     git init configs/sglang-mooncake-opt || exit 1
     git -C configs/sglang-mooncake-opt fetch --depth 1 \
         "$SGLANG_MOONCAKE_OPT_URL" "$SGLANG_MOONCAKE_OPT_PIN" || exit 1