refactor: standardize srt-slurm cluster configuration across launchers

docs/configuration-procedures.md

@@ -33,6 +33,27 @@ git submodule update --init
 
 To upgrade, fetch and check out the desired commit inside the relevant submodule, then commit the updated submodule pointer in InferenceX. Benchmark workflows already initialize submodules. Slurm launchers make a local Git clone for each job so recipe staging and runtime writes do not modify the submodule, and record the actual commit for result provenance. NVIDIA setup clones locally; TileRT setup fetches its pinned fork commit over the network.
 
+### Cluster profiles
+
+Launchers that use srt-slurm keep their cluster configuration in
+[`runners/srt-slurm/<launcher>.yaml`](../runners/srt-slurm/). The native settings
+(GPU count, scheduling directives, aliases, and mounts) are separate from workload recipes.
+Only launchers with an existing srt-slurm path have a profile. Both B200 Nscale
+srt-slurm paths share one profile, with path-specific container aliases supplied by
+the launcher.
+
+Call `write_srt_cluster_config <profile> srtslurm.yaml <uses_power>` from
+[`runners/slurm_utils.sh`](../runners/slurm_utils.sh) after staging images and paths.
+It writes the job-local config before `make setup`. `${NAME}` placeholders receive
+explicit `--var NAME VALUE` inputs, never implicit process-environment substitution.
+Optional `--model ALIAS PATH`, `--container ALIAS PATH`, and `--mount HOST CONTAINER`
+arguments add or override mapping entries. Power jobs add the staged DCGM image through
+the same writer. Missing variables fail before writing; values are substituted into
+parsed YAML scalars so quotes and punctuation remain data, not YAML or shell syntax.
+
+Keep model selection, cache preparation, and workload-dependent time limits in the
+launcher. Do not add profiles for non-srt-slurm launchers or change their routing here.
+
 ## Procedure index
 
 1. [Prepare a worktree](#prepare-a-worktree)

infx/srt_slurm/cluster_config.py

@@ -0,0 +1,64 @@
+"""Render a runner's srt-slurm cluster profile with explicit job-local inputs."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from string import Template
+from typing import Any
+
+import yaml
+
+
+def render_cluster_config(
+    profile: dict[str, Any],
+    variables: dict[str, str],
+    overrides: dict[str, list[list[str]]],
+) -> dict[str, Any]:
+    """Substitute YAML scalars, then add caller-owned model/container/mount mappings."""
+
+    def expand(value: Any) -> Any:
+        if isinstance(value, str):
+            return Template(value).substitute(variables)
+        if isinstance(value, dict):
+            return {expand(key): expand(item) for key, item in value.items()}
+        if isinstance(value, list):
+            return [expand(item) for item in value]
+        return value
+
+    config = expand(profile)
+    for section, pairs in overrides.items():
+        if pairs:
+            config.setdefault(section, {}).update(pairs)
+    return config
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("profile", type=Path)
+    parser.add_argument("output", type=Path)
+    parser.add_argument("--var", nargs=2, action="append", default=[], metavar=("NAME", "VALUE"))
+    parser.add_argument("--model", nargs=2, action="append", default=[], metavar=("ALIAS", "PATH"))
+    parser.add_argument(
+        "--container", nargs=2, action="append", default=[], metavar=("ALIAS", "PATH")
+    )
+    parser.add_argument(
+        "--mount", nargs=2, action="append", default=[], metavar=("HOST", "CONTAINER")
+    )
+    args = parser.parse_args()
+    try:
+        profile = yaml.safe_load(args.profile.read_text())
+        if not isinstance(profile, dict):
+            raise ValueError("Cluster profile must be a YAML mapping")
+        config = render_cluster_config(
+            profile,
+            dict(args.var),
+            {"model_paths": args.model, "containers": args.container, "default_mounts": args.mount},
+        )
+        args.output.write_text(yaml.safe_dump(config, sort_keys=False))
+    except (OSError, ValueError, KeyError, yaml.YAMLError) as exc:
+        parser.error(str(exc))
+
+
+if __name__ == "__main__":
+    main()

runners/launch_b200-nscale-slurm.sh

@@ -306,14 +306,15 @@ run_native_srt_lane() {
     import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" || exit 1
 
     PREFILL_SQUASH_FILE=""
-    TILERT_CONTAINER_BLOCK=""
+    SRT_CLUSTER_ARGS=()
     if [[ $FRAMEWORK == "tilert" ]]; then
         : "${PREFILL_IMAGE:?PREFILL_IMAGE is required for TileRT prefill}"
         PREFILL_SQUASH_FILE="$SQUASH_DIR/$(echo "$PREFILL_IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
         import_squash "$PREFILL_SQUASH_FILE" "$PREFILL_IMAGE" || exit 1
-        TILERT_CONTAINER_BLOCK="
-  tilert-decode: ${SQUASH_FILE}
-  tilert-prefill: ${PREFILL_SQUASH_FILE}"
+        SRT_CLUSTER_ARGS+=(
+            --container tilert-decode "$SQUASH_FILE"
+            --container tilert-prefill "$PREFILL_SQUASH_FILE"
+        )
     fi
 
     if [[ "$USES_DCGM_POWER" == "1" ]]; then
@@ -330,54 +331,28 @@ run_native_srt_lane() {
 
     # Persistent caches for aiperf's dataset mmap files and the HF trace dataset;
     # the container paths are referenced by the agentic recipes' benchmark.env.
-    DEFAULT_MOUNTS_BLOCK=""
     if [[ "$IS_AGENTIC" == "1" ]]; then
         mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH"
         chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true
-        DEFAULT_MOUNTS_BLOCK="default_mounts:
-  ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache
-  ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache"
+        SRT_CLUSTER_ARGS+=(
+            --mount "$AIPERF_MMAP_CACHE_HOST_PATH" /aiperf_mmap_cache
+            --mount "$HF_HUB_CACHE_HOST_PATH" /hf_hub_cache
+        )
     fi
     if [[ $FRAMEWORK == "tilert" ]]; then
         TILERT_WEIGHTS_HOST_PATH="/data/home/sa-shared/gharunners/tilert-cache"
         mkdir -p "$TILERT_WEIGHTS_HOST_PATH"
-        DEFAULT_MOUNTS_BLOCK="${DEFAULT_MOUNTS_BLOCK}
-  ${GITHUB_WORKSPACE}: /infmax-workspace
-  ${TILERT_WEIGHTS_HOST_PATH}: ${TILERT_WEIGHTS_HOST_PATH}"
+        SRT_CLUSTER_ARGS+=(
+            --mount "$GITHUB_WORKSPACE" /infmax-workspace
+            --mount "$TILERT_WEIGHTS_HOST_PATH" "$TILERT_WEIGHTS_HOST_PATH"
+        )
     fi
 
     SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}"
     echo "Creating srtslurm.yaml configuration..."
-    cat > srtslurm.yaml <<EOF
-# SRT SLURM Configuration for B200 nscale
-
-# Default SLURM settings
-default_account: "${SLURM_ACCOUNT}"
-default_partition: "${SLURM_PARTITION}"
-default_time_limit: "4:00:00"
-# Resource defaults
-gpus_per_node: 8
-network_interface: ""
-# Path to srtctl repo root (where the configs live)
-srtctl_root: "${SRTCTL_ROOT}"
-# Model path aliases
-model_paths:
-  "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}"
-# Container aliases
-containers:
-  dynamo-vllm: "${SQUASH_FILE}"
-  dynamo-sglang: "${SQUASH_FILE}"
-  "${IMAGE}": "${SQUASH_FILE}"
-  nginx-sqsh: "${NGINX_SQUASH_FILE}"
-${TILERT_CONTAINER_BLOCK}
-use_exclusive_sbatch_directive: true
-${DEFAULT_MOUNTS_BLOCK}
-EOF
-
-    if [[ "$USES_DCGM_POWER" == "1" ]]; then
-        sed -i "/^  nginx-sqsh:/a\\  dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml
-        grep -q "^  dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; }
-    fi
+    write_srt_cluster_config b200-nscale-slurm srtslurm.yaml "$USES_DCGM_POWER" \
+        --model "$SRT_SLURM_MODEL_PREFIX" "$MODEL_PATH" \
+        "${SRT_CLUSTER_ARGS[@]}" || exit 1
 
     echo "Generated srtslurm.yaml:"
     cat srtslurm.yaml
@@ -652,49 +627,24 @@ run_multinode_srt() {
     # Persistent Lustre caches for aiperf's dataset mmap files and the HF trace
     # dataset; the container paths are referenced by the agentic recipes'
     # benchmark.env.
-    DEFAULT_MOUNTS_BLOCK=""
+    SRT_CLUSTER_ARGS=()
     if [[ "$IS_AGENTIC" == "1" ]]; then
         HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache"
         mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH"
         chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true
-        DEFAULT_MOUNTS_BLOCK="default_mounts:
-  ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache
-  ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache"
+        SRT_CLUSTER_ARGS+=(
+            --mount "$AIPERF_MMAP_CACHE_HOST_PATH" /aiperf_mmap_cache
+            --mount "$HF_HUB_CACHE_HOST_PATH" /hf_hub_cache
+        )
     fi
 
     SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}"
     echo "Creating srtslurm.yaml configuration..."
-    cat > srtslurm.yaml <<EOF
-# SRT SLURM Configuration for B200
-
-# Default SLURM settings
-default_account: "${SLURM_ACCOUNT}"
-default_partition: "${SLURM_PARTITION}"
-default_time_limit: "4:00:00"
-# Resource defaults
-gpus_per_node: 8
-network_interface: ""
-# Path to srtctl repo root (where the configs live)
-srtctl_root: "${SRTCTL_ROOT}"
-# Model path aliases
-model_paths:
-  "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}"
-# Container aliases
-containers:
-  dynamo-trtllm: "${SQUASH_FILE}"
-  dynamo-sglang: "${SQUASH_FILE}"
-  dynamo-vllm: "${SQUASH_FILE}"
-  sglang-v0.5.11-cu130: "${SQUASH_FILE}"
-  "${IMAGE}": "${SQUASH_FILE}"
-  nginx-sqsh: "${NGINX_SQUASH_FILE}"
-use_exclusive_sbatch_directive: true
-${DEFAULT_MOUNTS_BLOCK}
-EOF
-
-    if [[ "$USES_DCGM_POWER" == "1" ]]; then
-        sed -i "/^  nginx-sqsh:/a\\  dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml
-        grep -q "^  dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; }
-    fi
+    write_srt_cluster_config b200-nscale-slurm srtslurm.yaml "$USES_DCGM_POWER" \
+        --model "$SRT_SLURM_MODEL_PREFIX" "$MODEL_PATH" \
+        --container dynamo-trtllm "$SQUASH_FILE" \
+        --container sglang-v0.5.11-cu130 "$SQUASH_FILE" \
+        "${SRT_CLUSTER_ARGS[@]}" || exit 1
 
     echo "Generated srtslurm.yaml:"
     cat srtslurm.yaml

runners/launch_b300-dsxe.sh

@@ -45,32 +45,6 @@ STAGED_MODELS=(
     Qwen3.8-2.4T-A95B-FP8
 )
 
-# srt-slurm recipes refer to models by alias (model.path in the recipe yaml). Every
-# alias below is written into srtslurm.yaml, so no per-model branching is needed.
-# Several aliases map to the same directory because recipes are not consistent.
-declare -A MODEL_ALIASES=(
-    [dsr1]="DeepSeek-R1-0528-NVFP4-v2"
-    [dsr1-fp8]="DeepSeek-R1-0528"
-    [deepseek-v4-pro]="DeepSeek-V4-Pro"
-    [deepseek-ai/DeepSeek-V4-Pro]="DeepSeek-V4-Pro"
-    [glm-5.2-fp4]="GLM-5.2-NVFP4"
-    [glm-5.2-fp8]="GLM-5.2-FP8"
-    [nvidia/GLM-5.2-NVFP4]="GLM-5.2-NVFP4"
-    [zai-org/GLM-5.2-FP8]="GLM-5.2-FP8"
-    [kimi-k2.6-nvfp4]="Kimi-K2.6-NVFP4"
-    [kimi-k3]="Kimi-K3"
-    [kimik3]="Kimi-K3"
-    [moonshotai/Kimi-K3]="Kimi-K3"
-    [minimax-m3-nvfp4]="MiniMax-M3-NVFP4"
-    [nvidia/MiniMax-M3-NVFP4]="MiniMax-M3-NVFP4"
-    [minimax-m3-mxfp8]="MiniMax-M3-MXFP8"
-    [MiniMaxAI/MiniMax-M3-MXFP8]="MiniMax-M3-MXFP8"
-    [qwen3.5-fp4]="Qwen3.5-397B-A17B-NVFP4-V2"
-    [qwen3.5-fp8]="Qwen3.5-397B-A17B-FP8"
-    [nvidia/Qwen3.5-397B-A17B-NVFP4-V2]="Qwen3.5-397B-A17B-NVFP4-V2"
-)
-
-
 mkdir -p "$SQUASH_DIR"
 set -x
 
@@ -177,32 +151,8 @@ export OSL="$OSL"
 
 SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}"
 echo "Creating srtslurm.yaml configuration..."
-{
-    cat <<EOF
-# SRT SLURM Configuration for B300 DSXE (generated by launch_b300-dsxe.sh)
-default_account: "${SLURM_ACCOUNT}"
-default_partition: "${SLURM_PARTITION}"
-gpus_per_node: 8
-network_interface: ""
-srtctl_root: "${SRTCTL_ROOT}"
-model_paths:
-EOF
-    for alias in "${!MODEL_ALIASES[@]}"; do
-        printf '  "%s": "%s/%s"\n' "$alias" "$MODEL_ROOT" "${MODEL_ALIASES[$alias]}"
-    done | sort
-    cat <<EOF
-containers:
-  dynamo-trtllm: "${SQUASH_FILE}"
-  dynamo-sglang: "${SQUASH_FILE}"
-  dynamo-vllm: "${SQUASH_FILE}"
-  "${IMAGE}": "${SQUASH_FILE}"
-  nginx-sqsh: "${NGINX_SQUASH_FILE}"
-EOF
-    if [[ "$USES_DCGM_POWER" == "1" ]]; then
-        printf '  dcgm-exporter: "%s"\n' "$DCGM_EXPORTER_SQSH"
-    fi
-    echo "use_exclusive_sbatch_directive: true"
-} > srtslurm.yaml
+write_srt_cluster_config b300-dsxe srtslurm.yaml "$USES_DCGM_POWER" \
+    --var MODEL_ROOT "$MODEL_ROOT" || exit 1
 
 echo "Generated srtslurm.yaml:"
 cat srtslurm.yaml

runners/launch_gb200-nv.sh

@@ -177,7 +177,7 @@ fi
 # Recipes name HF model IDs; resolve them to pre-staged paths so the shared
 # cluster does not re-download. SRT_SLURM_MODEL_PREFIX must match the recipe's
 # model.path alias.
-MODEL_PATHS_EXTRA=""
+SRT_CLUSTER_ARGS=()
 if [[ $FRAMEWORK == "dynamo-sglang" ]]; then
     export CONFIG_DIR="/mnt/lustre01/artifacts/sglang-configs/1k1k"
     if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then
@@ -255,7 +255,7 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then
         # Lustre sibling is the FP8 checkpoint, so the CamelCase path is deliberate.
         export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro"
         export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro"
-        MODEL_PATHS_EXTRA='  "deepseek-v4-pro-mxfp4": "/mnt/lustre01/models/DeepSeek-V4-Pro"'
+        SRT_CLUSTER_ARGS+=(--model deepseek-v4-pro-mxfp4 "/mnt/lustre01/models/DeepSeek-V4-Pro")
     elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then
         export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5-NVFP4"
         export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4"
@@ -442,66 +442,27 @@ fi
 # Persistent Lustre caches for aiperf's dataset mmap files (~65 GB per corpus,
 # re-tokenized from scratch without it) and the HF trace dataset; the container
 # paths are referenced by the agentic recipes' benchmark.env.
-DEFAULT_MOUNTS_BLOCK=""
 if [[ "$IS_AGENTIC" == "1" ]]; then
     AIPERF_MMAP_CACHE_HOST_PATH="/mnt/lustre01/users-public/sa-shared/ai-perf-cache"
     HF_HUB_CACHE_HOST_PATH="/mnt/lustre01/users-public/sa-shared/hf-hub-cache"
     mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH"
     chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true
-    DEFAULT_MOUNTS_BLOCK="default_mounts:
-  ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache
-  ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache"
+    SRT_CLUSTER_ARGS+=(
+        --mount "$AIPERF_MMAP_CACHE_HOST_PATH" /aiperf_mmap_cache
+        --mount "$HF_HUB_CACHE_HOST_PATH" /hf_hub_cache
+    )
     if uses_watchtower_shared_fs && [[ "$MODEL_PREFIX" == "glm5.2" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then
         DYNAMO_WHEELS_CACHE_HOST_PATH="${SHARED_BASE}/dynamo-wheels"
         mkdir -p "$DYNAMO_WHEELS_CACHE_HOST_PATH"
         chmod 777 "$DYNAMO_WHEELS_CACHE_HOST_PATH" 2>/dev/null || true
-        DEFAULT_MOUNTS_BLOCK+="
-  ${DYNAMO_WHEELS_CACHE_HOST_PATH}: /configs/dynamo-wheels"
+        SRT_CLUSTER_ARGS+=(--mount "$DYNAMO_WHEELS_CACHE_HOST_PATH" /configs/dynamo-wheels)
     fi
 fi
 
 echo "Creating srtslurm.yaml configuration..."
-cat > srtslurm.yaml <<EOF
-# SRT SLURM Configuration for GB200
-
-# Default SLURM settings
-default_account: "${SLURM_ACCOUNT}"
-default_partition: "${SLURM_PARTITION}"
-default_time_limit: "6:00:00"
-
-# Resource defaults
-gpus_per_node: 4
-network_interface: ""
-
-# Path to srtctl repo root (where the configs live)
-srtctl_root: "${SRTCTL_ROOT}"
-
-# Model path aliases
-model_paths:
-  "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}"
-${MODEL_PATHS_EXTRA}
-containers:
-  dynamo-trtllm: ${SQUASH_FILE}
-  dynamo-sglang: ${SQUASH_FILE}
-  "${IMAGE}": ${SQUASH_FILE}
-  nginx-sqsh: ${NGINX_SQUASH_FILE}
-# srtctl defaults this to true, which adds #SBATCH --segment=<total_nodes>.
-# On watchtower the whole batch partition (blue-cn01-18) is a single NVL72
-# rack, so segment contiguity buys nothing for MNNVL — but it DOES make
-# jobs unschedulable when the partition is fragmented: Slurm backfills a
-# non-contiguous node set, fails segment placement at start, and the job
-# dies with "CANCELLED Reason=Resources" at RunTime=0 (hit by the first
-# gb200 agentic run, job 18582). Mirror launch_gb300-nv.sh and disable.
-use_segment_sbatch_directive: false
-${DEFAULT_MOUNTS_BLOCK}
-EOF
-
-# Appended via sed so non-power lanes' generated yaml stays byte-identical.
-if [[ "$USES_DCGM_POWER" == "1" ]]; then
-    sed -i "/^  nginx-sqsh:/a\\  dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml
-    # sed's append is a silent no-op if the anchor drifts.
-    grep -q "^  dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; }
-fi
+write_srt_cluster_config gb200-nv srtslurm.yaml "$USES_DCGM_POWER" \
+    --model "$SRT_SLURM_MODEL_PREFIX" "$MODEL_PATH" \
+    "${SRT_CLUSTER_ARGS[@]}" || exit 1
 
 echo "Generated srtslurm.yaml:"
 cat srtslurm.yaml