From ef59705d2bcd77575fdde68f84534ba02923c034 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 17 Sep 2026 18:00:45 -0500 Subject: [PATCH] refactor: standardize srt-slurm cluster profiles across launchers --- docs/configuration-procedures.md | 21 ++++ infx/srt_slurm/cluster_config.py | 64 +++++++++++ runners/launch_b200-nscale-slurm.sh | 102 +++++------------- runners/launch_b300-dsxe.sh | 54 +--------- runners/launch_gb200-nv.sh | 59 ++--------- runners/launch_gb300-nv.sh | 51 ++------- runners/launch_h100-dgxc-slurm.sh | 29 +---- runners/launch_h200-dgxc-slurm.sh | 52 ++------- runners/slurm_utils.sh | 23 ++++ runners/srt-slurm/b200-nscale-slurm.yaml | 12 +++ runners/srt-slurm/b300-dsxe.yaml | 32 ++++++ runners/srt-slurm/gb200-nv.yaml | 13 +++ runners/srt-slurm/gb300-nv.yaml | 18 ++++ runners/srt-slurm/h100-dgxc-slurm.yaml | 15 +++ runners/srt-slurm/h200-dgxc-slurm.yaml | 19 ++++ utils/test_srt_cluster_config.py | 129 +++++++++++++++++++++++ 16 files changed, 402 insertions(+), 291 deletions(-) create mode 100644 infx/srt_slurm/cluster_config.py create mode 100644 runners/srt-slurm/b200-nscale-slurm.yaml create mode 100644 runners/srt-slurm/b300-dsxe.yaml create mode 100644 runners/srt-slurm/gb200-nv.yaml create mode 100644 runners/srt-slurm/gb300-nv.yaml create mode 100644 runners/srt-slurm/h100-dgxc-slurm.yaml create mode 100644 runners/srt-slurm/h200-dgxc-slurm.yaml create mode 100644 utils/test_srt_cluster_config.py diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index dc3498d46e..09e7b20064 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -33,6 +33,27 @@ git submodule update --init To upgrade, fetch and check out the desired commit inside the relevant submodule, then commit the updated submodule pointer in InferenceX. Benchmark workflows already initialize submodules. Slurm launchers make a local Git clone for each job so recipe staging and runtime writes do not modify the submodule, and record the actual commit for result provenance. NVIDIA setup clones locally; TileRT setup fetches its pinned fork commit over the network. +### Cluster profiles + +Launchers that use srt-slurm keep their cluster configuration in +[`runners/srt-slurm/.yaml`](../runners/srt-slurm/). The native settings +(GPU count, scheduling directives, aliases, and mounts) are separate from workload recipes. +Only launchers with an existing srt-slurm path have a profile. Both B200 Nscale +srt-slurm paths share one profile, with path-specific container aliases supplied by +the launcher. + +Call `write_srt_cluster_config srtslurm.yaml ` from +[`runners/slurm_utils.sh`](../runners/slurm_utils.sh) after staging images and paths. +It writes the job-local config before `make setup`. `${NAME}` placeholders receive +explicit `--var NAME VALUE` inputs, never implicit process-environment substitution. +Optional `--model ALIAS PATH`, `--container ALIAS PATH`, and `--mount HOST CONTAINER` +arguments add or override mapping entries. Power jobs add the staged DCGM image through +the same writer. Missing variables fail before writing; values are substituted into +parsed YAML scalars so quotes and punctuation remain data, not YAML or shell syntax. + +Keep model selection, cache preparation, and workload-dependent time limits in the +launcher. Do not add profiles for non-srt-slurm launchers or change their routing here. + ## Procedure index 1. [Prepare a worktree](#prepare-a-worktree) diff --git a/infx/srt_slurm/cluster_config.py b/infx/srt_slurm/cluster_config.py new file mode 100644 index 0000000000..4ce1154400 --- /dev/null +++ b/infx/srt_slurm/cluster_config.py @@ -0,0 +1,64 @@ +"""Render a runner's srt-slurm cluster profile with explicit job-local inputs.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from string import Template +from typing import Any + +import yaml + + +def render_cluster_config( + profile: dict[str, Any], + variables: dict[str, str], + overrides: dict[str, list[list[str]]], +) -> dict[str, Any]: + """Substitute YAML scalars, then add caller-owned model/container/mount mappings.""" + + def expand(value: Any) -> Any: + if isinstance(value, str): + return Template(value).substitute(variables) + if isinstance(value, dict): + return {expand(key): expand(item) for key, item in value.items()} + if isinstance(value, list): + return [expand(item) for item in value] + return value + + config = expand(profile) + for section, pairs in overrides.items(): + if pairs: + config.setdefault(section, {}).update(pairs) + return config + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("profile", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--var", nargs=2, action="append", default=[], metavar=("NAME", "VALUE")) + parser.add_argument("--model", nargs=2, action="append", default=[], metavar=("ALIAS", "PATH")) + parser.add_argument( + "--container", nargs=2, action="append", default=[], metavar=("ALIAS", "PATH") + ) + parser.add_argument( + "--mount", nargs=2, action="append", default=[], metavar=("HOST", "CONTAINER") + ) + args = parser.parse_args() + try: + profile = yaml.safe_load(args.profile.read_text()) + if not isinstance(profile, dict): + raise ValueError("Cluster profile must be a YAML mapping") + config = render_cluster_config( + profile, + dict(args.var), + {"model_paths": args.model, "containers": args.container, "default_mounts": args.mount}, + ) + args.output.write_text(yaml.safe_dump(config, sort_keys=False)) + except (OSError, ValueError, KeyError, yaml.YAMLError) as exc: + parser.error(str(exc)) + + +if __name__ == "__main__": + main() diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index da2b680b2b..0e1b694d89 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -306,14 +306,15 @@ run_native_srt_lane() { import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" || exit 1 PREFILL_SQUASH_FILE="" - TILERT_CONTAINER_BLOCK="" + SRT_CLUSTER_ARGS=() if [[ $FRAMEWORK == "tilert" ]]; then : "${PREFILL_IMAGE:?PREFILL_IMAGE is required for TileRT prefill}" PREFILL_SQUASH_FILE="$SQUASH_DIR/$(echo "$PREFILL_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" import_squash "$PREFILL_SQUASH_FILE" "$PREFILL_IMAGE" || exit 1 - TILERT_CONTAINER_BLOCK=" - tilert-decode: ${SQUASH_FILE} - tilert-prefill: ${PREFILL_SQUASH_FILE}" + SRT_CLUSTER_ARGS+=( + --container tilert-decode "$SQUASH_FILE" + --container tilert-prefill "$PREFILL_SQUASH_FILE" + ) fi if [[ "$USES_DCGM_POWER" == "1" ]]; then @@ -330,54 +331,28 @@ run_native_srt_lane() { # Persistent caches for aiperf's dataset mmap files and the HF trace dataset; # the container paths are referenced by the agentic recipes' benchmark.env. - DEFAULT_MOUNTS_BLOCK="" if [[ "$IS_AGENTIC" == "1" ]]; then mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true - DEFAULT_MOUNTS_BLOCK="default_mounts: - ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache - ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache" + SRT_CLUSTER_ARGS+=( + --mount "$AIPERF_MMAP_CACHE_HOST_PATH" /aiperf_mmap_cache + --mount "$HF_HUB_CACHE_HOST_PATH" /hf_hub_cache + ) fi if [[ $FRAMEWORK == "tilert" ]]; then TILERT_WEIGHTS_HOST_PATH="/data/home/sa-shared/gharunners/tilert-cache" mkdir -p "$TILERT_WEIGHTS_HOST_PATH" - DEFAULT_MOUNTS_BLOCK="${DEFAULT_MOUNTS_BLOCK} - ${GITHUB_WORKSPACE}: /infmax-workspace - ${TILERT_WEIGHTS_HOST_PATH}: ${TILERT_WEIGHTS_HOST_PATH}" + SRT_CLUSTER_ARGS+=( + --mount "$GITHUB_WORKSPACE" /infmax-workspace + --mount "$TILERT_WEIGHTS_HOST_PATH" "$TILERT_WEIGHTS_HOST_PATH" + ) fi SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." - cat > srtslurm.yaml <&2; exit 1; } - fi + write_srt_cluster_config b200-nscale-slurm srtslurm.yaml "$USES_DCGM_POWER" \ + --model "$SRT_SLURM_MODEL_PREFIX" "$MODEL_PATH" \ + "${SRT_CLUSTER_ARGS[@]}" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml @@ -652,49 +627,24 @@ run_multinode_srt() { # Persistent Lustre caches for aiperf's dataset mmap files and the HF trace # dataset; the container paths are referenced by the agentic recipes' # benchmark.env. - DEFAULT_MOUNTS_BLOCK="" + SRT_CLUSTER_ARGS=() if [[ "$IS_AGENTIC" == "1" ]]; then HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true - DEFAULT_MOUNTS_BLOCK="default_mounts: - ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache - ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache" + SRT_CLUSTER_ARGS+=( + --mount "$AIPERF_MMAP_CACHE_HOST_PATH" /aiperf_mmap_cache + --mount "$HF_HUB_CACHE_HOST_PATH" /hf_hub_cache + ) fi SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." - cat > srtslurm.yaml <&2; exit 1; } - fi + write_srt_cluster_config b200-nscale-slurm srtslurm.yaml "$USES_DCGM_POWER" \ + --model "$SRT_SLURM_MODEL_PREFIX" "$MODEL_PATH" \ + --container dynamo-trtllm "$SQUASH_FILE" \ + --container sglang-v0.5.11-cu130 "$SQUASH_FILE" \ + "${SRT_CLUSTER_ARGS[@]}" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 665577c171..d49b28b417 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -45,32 +45,6 @@ STAGED_MODELS=( Qwen3.8-2.4T-A95B-FP8 ) -# srt-slurm recipes refer to models by alias (model.path in the recipe yaml). Every -# alias below is written into srtslurm.yaml, so no per-model branching is needed. -# Several aliases map to the same directory because recipes are not consistent. -declare -A MODEL_ALIASES=( - [dsr1]="DeepSeek-R1-0528-NVFP4-v2" - [dsr1-fp8]="DeepSeek-R1-0528" - [deepseek-v4-pro]="DeepSeek-V4-Pro" - [deepseek-ai/DeepSeek-V4-Pro]="DeepSeek-V4-Pro" - [glm-5.2-fp4]="GLM-5.2-NVFP4" - [glm-5.2-fp8]="GLM-5.2-FP8" - [nvidia/GLM-5.2-NVFP4]="GLM-5.2-NVFP4" - [zai-org/GLM-5.2-FP8]="GLM-5.2-FP8" - [kimi-k2.6-nvfp4]="Kimi-K2.6-NVFP4" - [kimi-k3]="Kimi-K3" - [kimik3]="Kimi-K3" - [moonshotai/Kimi-K3]="Kimi-K3" - [minimax-m3-nvfp4]="MiniMax-M3-NVFP4" - [nvidia/MiniMax-M3-NVFP4]="MiniMax-M3-NVFP4" - [minimax-m3-mxfp8]="MiniMax-M3-MXFP8" - [MiniMaxAI/MiniMax-M3-MXFP8]="MiniMax-M3-MXFP8" - [qwen3.5-fp4]="Qwen3.5-397B-A17B-NVFP4-V2" - [qwen3.5-fp8]="Qwen3.5-397B-A17B-FP8" - [nvidia/Qwen3.5-397B-A17B-NVFP4-V2]="Qwen3.5-397B-A17B-NVFP4-V2" -) - - mkdir -p "$SQUASH_DIR" set -x @@ -177,32 +151,8 @@ export OSL="$OSL" SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." -{ - cat < srtslurm.yaml +write_srt_cluster_config b300-dsxe srtslurm.yaml "$USES_DCGM_POWER" \ + --var MODEL_ROOT "$MODEL_ROOT" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index fcb64de7fd..034091435c 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -177,7 +177,7 @@ fi # Recipes name HF model IDs; resolve them to pre-staged paths so the shared # cluster does not re-download. SRT_SLURM_MODEL_PREFIX must match the recipe's # model.path alias. -MODEL_PATHS_EXTRA="" +SRT_CLUSTER_ARGS=() if [[ $FRAMEWORK == "dynamo-sglang" ]]; then export CONFIG_DIR="/mnt/lustre01/artifacts/sglang-configs/1k1k" if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then @@ -255,7 +255,7 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then # Lustre sibling is the FP8 checkpoint, so the CamelCase path is deliberate. export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" - MODEL_PATHS_EXTRA=' "deepseek-v4-pro-mxfp4": "/mnt/lustre01/models/DeepSeek-V4-Pro"' + SRT_CLUSTER_ARGS+=(--model deepseek-v4-pro-mxfp4 "/mnt/lustre01/models/DeepSeek-V4-Pro") elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" @@ -442,66 +442,27 @@ fi # Persistent Lustre caches for aiperf's dataset mmap files (~65 GB per corpus, # re-tokenized from scratch without it) and the HF trace dataset; the container # paths are referenced by the agentic recipes' benchmark.env. -DEFAULT_MOUNTS_BLOCK="" if [[ "$IS_AGENTIC" == "1" ]]; then AIPERF_MMAP_CACHE_HOST_PATH="/mnt/lustre01/users-public/sa-shared/ai-perf-cache" HF_HUB_CACHE_HOST_PATH="/mnt/lustre01/users-public/sa-shared/hf-hub-cache" mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true - DEFAULT_MOUNTS_BLOCK="default_mounts: - ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache - ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache" + SRT_CLUSTER_ARGS+=( + --mount "$AIPERF_MMAP_CACHE_HOST_PATH" /aiperf_mmap_cache + --mount "$HF_HUB_CACHE_HOST_PATH" /hf_hub_cache + ) if uses_watchtower_shared_fs && [[ "$MODEL_PREFIX" == "glm5.2" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then DYNAMO_WHEELS_CACHE_HOST_PATH="${SHARED_BASE}/dynamo-wheels" mkdir -p "$DYNAMO_WHEELS_CACHE_HOST_PATH" chmod 777 "$DYNAMO_WHEELS_CACHE_HOST_PATH" 2>/dev/null || true - DEFAULT_MOUNTS_BLOCK+=" - ${DYNAMO_WHEELS_CACHE_HOST_PATH}: /configs/dynamo-wheels" + SRT_CLUSTER_ARGS+=(--mount "$DYNAMO_WHEELS_CACHE_HOST_PATH" /configs/dynamo-wheels) fi fi echo "Creating srtslurm.yaml configuration..." -cat > srtslurm.yaml <. -# On watchtower the whole batch partition (blue-cn01-18) is a single NVL72 -# rack, so segment contiguity buys nothing for MNNVL — but it DOES make -# jobs unschedulable when the partition is fragmented: Slurm backfills a -# non-contiguous node set, fails segment placement at start, and the job -# dies with "CANCELLED Reason=Resources" at RunTime=0 (hit by the first -# gb200 agentic run, job 18582). Mirror launch_gb300-nv.sh and disable. -use_segment_sbatch_directive: false -${DEFAULT_MOUNTS_BLOCK} -EOF - -# Appended via sed so non-power lanes' generated yaml stays byte-identical. -if [[ "$USES_DCGM_POWER" == "1" ]]; then - sed -i "/^ nginx-sqsh:/a\\ dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml - # sed's append is a silent no-op if the anchor drifts. - grep -q "^ dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; } -fi +write_srt_cluster_config gb200-nv srtslurm.yaml "$USES_DCGM_POWER" \ + --model "$SRT_SLURM_MODEL_PREFIX" "$MODEL_PATH" \ + "${SRT_CLUSTER_ARGS[@]}" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 4031483ed5..4df406d3a2 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -254,51 +254,12 @@ SRT_DEFAULT_TIME_LIMIT="4:00:00" if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "dsv4" && ( "$FRAMEWORK" == "dynamo-sglang" || "$FRAMEWORK" == "dynamo-trt" ) ]]; then SRT_DEFAULT_TIME_LIMIT="8:00:00" fi -cat > srtslurm.yaml <&2; exit 1; } -fi +write_srt_cluster_config gb300-nv srtslurm.yaml "$USES_DCGM_POWER" \ + --model "$SRT_SLURM_MODEL_PREFIX" "$MODEL_PATH" \ + --var SRT_DEFAULT_TIME_LIMIT "$SRT_DEFAULT_TIME_LIMIT" \ + --var AIPERF_MMAP_CACHE_HOST_PATH "$AIPERF_MMAP_CACHE_HOST_PATH" \ + --var HF_HUB_CACHE_HOST_PATH "$HF_HUB_CACHE_HOST_PATH" \ + --var DYNAMO_WHEELS_CACHE_HOST_PATH "$DYNAMO_WHEELS_CACHE_HOST_PATH" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index ebaeec13a9..858eae872f 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -82,32 +82,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." - cat > srtslurm.yaml < srtslurm.yaml <&2; exit 1; } - fi + write_srt_cluster_config h200-dgxc-slurm srtslurm.yaml "$USES_DCGM_POWER" \ + --model "$SRT_SLURM_MODEL_PREFIX" "$MODEL_PATH" \ + --var CONTAINER_KEY "$CONTAINER_KEY" \ + --model "$MODEL_PREFIX" "$MODEL_PATH" \ + --var SRT_DEFAULT_TIME_LIMIT "$SRT_DEFAULT_TIME_LIMIT" \ + --var AIPERF_MMAP_CACHE_HOST_PATH "$AIPERF_MMAP_CACHE_HOST_PATH" \ + --var HF_HUB_CACHE_MOUNT "$SRT_HF_HUB_CACHE_MOUNT" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 79f9175915..edb6d2a66a 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -8,6 +8,29 @@ SRTCTL_EVAL_ARGS=( --set 'post_eval.command=["bash", "{infmax_workspace}/benchmarks/multi_node/srt_eval.sh", "{endpoint}", "{infmax_workspace}"]' ) +# Write a job-local cluster config; profiles contain only native srt-slurm settings. +write_srt_cluster_config() { + if [[ $# -lt 3 || -z "$1" || -z "$2" || ( "$3" != 0 && "$3" != 1 ) ]]; then + echo "Usage: write_srt_cluster_config profile output uses_power (0 or 1) [overrides...]" >&2 + return 1 + fi + check_env_vars SLURM_ACCOUNT SLURM_PARTITION SRTCTL_ROOT SQUASH_FILE NGINX_SQUASH_FILE IMAGE + local profile="$1" output="$2" uses_power="$3" + shift 3 + local power_args=() + if [[ "$uses_power" == 1 ]]; then + check_env_vars DCGM_EXPORTER_SQSH + power_args=(--container dcgm-exporter "$DCGM_EXPORTER_SQSH") + fi + PYTHONPATH="$INFERENCEX_SLURM_UTILS_DIR/..${PYTHONPATH:+:$PYTHONPATH}" \ + python3 -m infx.srt_slurm.cluster_config \ + "$INFERENCEX_SLURM_UTILS_DIR/srt-slurm/${profile}.yaml" "$output" \ + --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ + --var SRTCTL_ROOT "$SRTCTL_ROOT" --var SQUASH_FILE "$SQUASH_FILE" \ + --var NGINX_SQUASH_FILE "$NGINX_SQUASH_FILE" --var IMAGE "$IMAGE" \ + "$@" "${power_args[@]}" +} + # Leaves the caller in the checkout, matching the launchers' installation flow. # Every recipe is owned by InferenceX; srt-slurm 2 no longer ships recipes/. setup_srt_slurm() { diff --git a/runners/srt-slurm/b200-nscale-slurm.yaml b/runners/srt-slurm/b200-nscale-slurm.yaml new file mode 100644 index 0000000000..85874828e6 --- /dev/null +++ b/runners/srt-slurm/b200-nscale-slurm.yaml @@ -0,0 +1,12 @@ +default_account: ${SLURM_ACCOUNT} +default_partition: ${SLURM_PARTITION} +default_time_limit: "4:00:00" +gpus_per_node: 8 +network_interface: "" +srtctl_root: ${SRTCTL_ROOT} +containers: + dynamo-vllm: ${SQUASH_FILE} + dynamo-sglang: ${SQUASH_FILE} + "${IMAGE}": ${SQUASH_FILE} + nginx-sqsh: ${NGINX_SQUASH_FILE} +use_exclusive_sbatch_directive: true diff --git a/runners/srt-slurm/b300-dsxe.yaml b/runners/srt-slurm/b300-dsxe.yaml new file mode 100644 index 0000000000..94684e1941 --- /dev/null +++ b/runners/srt-slurm/b300-dsxe.yaml @@ -0,0 +1,32 @@ +default_account: ${SLURM_ACCOUNT} +default_partition: ${SLURM_PARTITION} +gpus_per_node: 8 +network_interface: "" +srtctl_root: ${SRTCTL_ROOT} +model_paths: + dsr1: ${MODEL_ROOT}/DeepSeek-R1-0528-NVFP4-v2 + dsr1-fp8: ${MODEL_ROOT}/DeepSeek-R1-0528 + deepseek-v4-pro: ${MODEL_ROOT}/DeepSeek-V4-Pro + deepseek-ai/DeepSeek-V4-Pro: ${MODEL_ROOT}/DeepSeek-V4-Pro + glm-5.2-fp4: ${MODEL_ROOT}/GLM-5.2-NVFP4 + glm-5.2-fp8: ${MODEL_ROOT}/GLM-5.2-FP8 + nvidia/GLM-5.2-NVFP4: ${MODEL_ROOT}/GLM-5.2-NVFP4 + zai-org/GLM-5.2-FP8: ${MODEL_ROOT}/GLM-5.2-FP8 + kimi-k2.6-nvfp4: ${MODEL_ROOT}/Kimi-K2.6-NVFP4 + kimi-k3: ${MODEL_ROOT}/Kimi-K3 + kimik3: ${MODEL_ROOT}/Kimi-K3 + moonshotai/Kimi-K3: ${MODEL_ROOT}/Kimi-K3 + minimax-m3-nvfp4: ${MODEL_ROOT}/MiniMax-M3-NVFP4 + nvidia/MiniMax-M3-NVFP4: ${MODEL_ROOT}/MiniMax-M3-NVFP4 + minimax-m3-mxfp8: ${MODEL_ROOT}/MiniMax-M3-MXFP8 + MiniMaxAI/MiniMax-M3-MXFP8: ${MODEL_ROOT}/MiniMax-M3-MXFP8 + qwen3.5-fp4: ${MODEL_ROOT}/Qwen3.5-397B-A17B-NVFP4-V2 + qwen3.5-fp8: ${MODEL_ROOT}/Qwen3.5-397B-A17B-FP8 + nvidia/Qwen3.5-397B-A17B-NVFP4-V2: ${MODEL_ROOT}/Qwen3.5-397B-A17B-NVFP4-V2 +containers: + dynamo-trtllm: ${SQUASH_FILE} + dynamo-sglang: ${SQUASH_FILE} + dynamo-vllm: ${SQUASH_FILE} + "${IMAGE}": ${SQUASH_FILE} + nginx-sqsh: ${NGINX_SQUASH_FILE} +use_exclusive_sbatch_directive: true diff --git a/runners/srt-slurm/gb200-nv.yaml b/runners/srt-slurm/gb200-nv.yaml new file mode 100644 index 0000000000..311242a2ff --- /dev/null +++ b/runners/srt-slurm/gb200-nv.yaml @@ -0,0 +1,13 @@ +default_account: ${SLURM_ACCOUNT} +default_partition: ${SLURM_PARTITION} +default_time_limit: "6:00:00" +gpus_per_node: 4 +network_interface: "" +srtctl_root: ${SRTCTL_ROOT} +containers: + dynamo-trtllm: ${SQUASH_FILE} + dynamo-sglang: ${SQUASH_FILE} + "${IMAGE}": ${SQUASH_FILE} + nginx-sqsh: ${NGINX_SQUASH_FILE} +# Watchtower's single NVL72 rack does not require contiguous segment placement. +use_segment_sbatch_directive: false diff --git a/runners/srt-slurm/gb300-nv.yaml b/runners/srt-slurm/gb300-nv.yaml new file mode 100644 index 0000000000..d09e4542a4 --- /dev/null +++ b/runners/srt-slurm/gb300-nv.yaml @@ -0,0 +1,18 @@ +default_account: ${SLURM_ACCOUNT} +default_partition: ${SLURM_PARTITION} +default_time_limit: ${SRT_DEFAULT_TIME_LIMIT} +gpus_per_node: 4 +network_interface: "" +srtctl_root: ${SRTCTL_ROOT} +default_mounts: + "${AIPERF_MMAP_CACHE_HOST_PATH}": /aiperf_mmap_cache + "${HF_HUB_CACHE_HOST_PATH}": /hf_hub_cache + "${DYNAMO_WHEELS_CACHE_HOST_PATH}": /configs/dynamo-wheels +containers: + dynamo-trtllm: ${SQUASH_FILE} + dynamo-sglang: ${SQUASH_FILE} + v0.5.11: ${SQUASH_FILE} + v0.5.13.post1: ${SQUASH_FILE} + "${IMAGE}": ${SQUASH_FILE} + nginx-sqsh: ${NGINX_SQUASH_FILE} +use_segment_sbatch_directive: false diff --git a/runners/srt-slurm/h100-dgxc-slurm.yaml b/runners/srt-slurm/h100-dgxc-slurm.yaml new file mode 100644 index 0000000000..1188df5c55 --- /dev/null +++ b/runners/srt-slurm/h100-dgxc-slurm.yaml @@ -0,0 +1,15 @@ +default_account: ${SLURM_ACCOUNT} +default_partition: ${SLURM_PARTITION} +default_time_limit: "6:00:00" +gpus_per_node: 8 +network_interface: "" +srtctl_root: ${SRTCTL_ROOT} +containers: + dynamo-trtllm: ${SQUASH_FILE} + dynamo-sglang: ${SQUASH_FILE} + nginx-sqsh: ${NGINX_SQUASH_FILE} + latest: ${SQUASH_FILE} + "${CONTAINER_KEY}": ${SQUASH_FILE} +use_gpus_per_node_directive: true +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: false diff --git a/runners/srt-slurm/h200-dgxc-slurm.yaml b/runners/srt-slurm/h200-dgxc-slurm.yaml new file mode 100644 index 0000000000..d47fb5521a --- /dev/null +++ b/runners/srt-slurm/h200-dgxc-slurm.yaml @@ -0,0 +1,19 @@ +default_account: ${SLURM_ACCOUNT} +default_partition: ${SLURM_PARTITION} +default_time_limit: ${SRT_DEFAULT_TIME_LIMIT} +gpus_per_node: 8 +network_interface: "" +srtctl_root: ${SRTCTL_ROOT} +default_mounts: + "${AIPERF_MMAP_CACHE_HOST_PATH}": /aiperf_mmap_cache + "${HF_HUB_CACHE_MOUNT}": /hf_hub_cache +containers: + dynamo-trtllm: ${SQUASH_FILE} + dynamo-sglang: ${SQUASH_FILE} + dynamo-vllm: ${SQUASH_FILE} + nginx-sqsh: ${NGINX_SQUASH_FILE} + latest: ${SQUASH_FILE} + "${CONTAINER_KEY}": ${SQUASH_FILE} +use_gpus_per_node_directive: true +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: false diff --git a/utils/test_srt_cluster_config.py b/utils/test_srt_cluster_config.py new file mode 100644 index 0000000000..a6229e28b8 --- /dev/null +++ b/utils/test_srt_cluster_config.py @@ -0,0 +1,129 @@ +"""Exercise cluster-profile rendering through the launchers' shared entrypoint.""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from infx.srt_slurm.cluster_config import render_cluster_config + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("power", ["0", "1", "missing-exporter"]) +def test_launcher_writes_job_local_cluster_config(tmp_path: Path, power: str) -> None: + runner_dir = tmp_path / "runners" + profiles = runner_dir / "srt-slurm" + profiles.mkdir(parents=True) + (profiles / "fixture.yaml").write_text( + """default_account: ${SLURM_ACCOUNT} +default_partition: ${SLURM_PARTITION} +srtctl_root: ${SRTCTL_ROOT} +gpus_per_node: 4 +use_segment_sbatch_directive: false +containers: + '${IMAGE}': ${SQUASH_FILE} + nginx-sqsh: ${NGINX_SQUASH_FILE} +default_mounts: + /cache: /old-cache +model_paths: + retained: /models/retained +default_sbatch_directives: + exclude: ['${EXCLUDED_NODE}'] +""" + ) + output = tmp_path / "job config.yaml" + env = { + **os.environ, + "PATH": f"{Path(sys.executable).parent}:{os.environ['PATH']}", + "PYTHONPATH": str(ROOT), + "SLURM_ACCOUNT": "benchmark", + "SLURM_PARTITION": "batch", + "SRTCTL_ROOT": '/shared/job "quoted": 1', + "SQUASH_FILE": '/images/engine "quoted".sqsh', + "NGINX_SQUASH_FILE": "/images/nginx.sqsh", + "IMAGE": "registry/engine:tag", + } + env.pop("DCGM_EXPORTER_SQSH", None) + if power == "1": + env["DCGM_EXPORTER_SQSH"] = "/images/exporter.sqsh" + result = subprocess.run( + [ + "bash", "-c", + 'source "$1"; INFERENCEX_SLURM_UTILS_DIR="$2"; ' + 'write_srt_cluster_config fixture "$3" "$4" ' + '--var EXCLUDED_NODE gpu-b --model selected "/models/price$" ' + '--mount /cache /new-cache --container prefill /images/prefill.sqsh', + "bash", str(ROOT / "runners/slurm_utils.sh"), str(runner_dir), str(output), + "1" if power == "missing-exporter" else power, + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + if power == "missing-exporter": + assert result.returncode != 0 + assert "DCGM_EXPORTER_SQSH" in result.stdout + assert not output.exists() + return + + assert result.returncode == 0, result.stderr + rendered = yaml.safe_load(output.read_text()) + assert rendered == { + "default_account": "benchmark", + "default_partition": "batch", + "srtctl_root": '/shared/job "quoted": 1', + "gpus_per_node": 4, + "use_segment_sbatch_directive": False, + "containers": { + "registry/engine:tag": '/images/engine "quoted".sqsh', + "nginx-sqsh": "/images/nginx.sqsh", + "prefill": "/images/prefill.sqsh", + **({"dcgm-exporter": "/images/exporter.sqsh"} if power == "1" else {}), + }, + "default_mounts": {"/cache": "/new-cache"}, + "model_paths": {"retained": "/models/retained", "selected": "/models/price$"}, + "default_sbatch_directives": {"exclude": ["gpu-b"]}, + } + + +@pytest.mark.parametrize( + ("profile", "error"), + [ + ("srtctl_root: ${MISSING_ROOT}\n", "MISSING_ROOT"), + ("- not-a-mapping\n", "Cluster profile must be a YAML mapping"), + ], +) +def test_invalid_profile_does_not_overwrite_config(tmp_path: Path, profile: str, error: str) -> None: + source = tmp_path / "profile.yaml" + source.write_text(profile) + output = tmp_path / "srtslurm.yaml" + output.write_text("default_partition: existing\n") + result = subprocess.run( + [sys.executable, "-m", "infx.srt_slurm.cluster_config", str(source), str(output)], + cwd=tmp_path, + env={**os.environ, "PYTHONPATH": str(ROOT)}, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 2 + assert error in result.stderr + assert output.read_text() == "default_partition: existing\n" + + +def test_runtime_mounts_create_a_mapping_without_profile_mounts() -> None: + config = render_cluster_config( + {"use_exclusive_sbatch_directive": True}, + {}, + {"default_mounts": [["/job", "/infmax-workspace"], ["/weights", "/weights"]]}, + ) + assert config == { + "use_exclusive_sbatch_directive": True, + "default_mounts": {"/job": "/infmax-workspace", "/weights": "/weights"}, + }