diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 11e40a70e..c22d93365 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -61,6 +61,10 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 # inter-turn idle gaps can reuse a socket exactly as the server closes it. # Outlast the client pool so the race cannot occur. export SGLANG_TIMEOUT_KEEP_ALIVE=900 +# /health does a 1-token generate. First real-MTP (eval-only) request can +# JIT mha_varlen for ~40s; the default 20s timeout marks the server +# UnHealthy and GSM8K then hits a dead port (mi355x-amds_09 2026-09-09). +export SGLANG_HEALTH_CHECK_TIMEOUT=120 # The DSA indexer's top-k v2 kernel (default since v0.5.14) is JIT-compiled # from CUDA-only source (cooperative_groups.h) and cannot build for gfx950; # v1 dispatches to the precompiled HIP op in sgl-kernel (upstream MI355X CI @@ -91,9 +95,9 @@ if agentic_kv_offload_enabled; then # env-var override for maximum throughput on nodes with >4 TB DRAM. HICACHE_RATIO="${HICACHE_RATIO:-1.5}" fi - # write_through_selective skips DRAM writes for non-reusable KV blocks, - # reducing host-bus traffic without affecting the cache hit rate. - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" + # Keep write_through as the validated baseline. Override + # HICACHE_WRITE_POLICY explicitly for selective-write experiments. + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" case "$KV_OFFLOAD_BACKEND" in diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 18007db0d..b7dcaa358 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1709,7 +1709,7 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: # c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b05aba27c..04089e135 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6948,6 +6948,16 @@ - "Add --compilation-config cudagraph_mode=FULL_DECODE_ONLY to the serve command. The upstream nightly does not torch-compile MiniMaxM3SparseForConditionalGeneration, so with VLLM_USE_BREAKABLE_CUDAGRAPH=0 the default FULL_AND_PIECEWISE mode aborts at engine init with piecewise CUDA graphs unavailable (first sweep, run 34174124043, eval cell); full decode-only graphs are what the MI355X MiniMax-M3 sibling runs on its nightly (#2825)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2873 +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." + - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." + - "Restore HiCache write_through as the default write policy to optimize GLM-5.2-MXFP4 output interactivity and per-GPU throughput in the MI355X AgentX configuration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 + - config-keys: - qwen3.5-fp8-b200-sglang scenario-type: @@ -6957,7 +6967,6 @@ - "Tune the 8k/1k serving recipe with --mamba-full-memory-ratio 0.37 and --linear-attn-prefill-backend flashinfer; raise max-prefill-tokens and chunked-prefill-size from 16384 to 32768 and mem-fraction-static from 0.8 to 0.86." - "Expand the TP8 and TP4/EP1 sweep coverage through concurrency 640." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2866 - - config-keys: - dsv4-fp4-b200-sglang-agentic-hicache-mtp