diff --git a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c7..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml new file mode 100644 index 0000000000..dc2242cf92 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml @@ -0,0 +1,32 @@ +# srt-slurm cluster profile for the MI355X AMD Slurm cluster. The login and +# compute nodes share /it-share, so source, output, image, and result paths do +# not require node-local transport. + +cluster: mi355x-amds +default_partition: compute +default_time_limit: "01:00:00" +output_dir: /it-share/gharunners2/srt-slurm/outputs + +gpus_per_node: 8 +accelerator_vendor: amd +network_interface: eno0 + +gpu_sbatch_directive: gres +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: false +runtime_config_transport: shared-filesystem + +default_sbatch_directives: + cpus-per-task: "32" + mem: "128G" + +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + /it-share/hf-hub-cache: /hf_hub_cache + # The launcher appends the exact shared workspace and result roots. + +containers: + sglang-v0.5.16-mi35x: /it-share/gharunners2/srt-slurm/containers/sglang-rocm-v0.5.16-mi35x-20260728.sqsh + +nginx_raise_ulimit: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml new file mode 100644 index 0000000000..dd1f6d766c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml @@ -0,0 +1,149 @@ +# Production-scale MI355X port of the existing Qwen3.5 FP8 1P1D recipe. +# Each role owns one full 8-GPU node; the native SGLang Router provides the +# request plane and AMD MoRI moves KV directly between the P/D workers. + +name: "mi355x-sglang-qwen3.5-fp8-disagg-1p1d-tp4p-tp8d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3.5-397B-A17B-FP8" + container: "sglang-v0.5.16-mi35x" + precision: "fp8" + +identity: + model: + repo: "Qwen/Qwen3.5-397B-A17B-FP8" + container: + image: "lmsysorg/sglang:v0.5.16-rocm720-mi35x" + frameworks: + sglang: "0.5.16" + sglang-router: "0.3.2" + +slurm: + time_limit: "08:00:00" + +# This topology owns every GPU on each MI355X node. Give SGLang the complete +# 128-core cpuset as well: its ROCm image enables per-GPU CPU affinity, which +# maps the eight TP ranks across the full dual-socket CPU topology. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +backend: + type: sglang + prefill_environment: &common_environment + HF_HOME: "/hf_hub_cache" + # Hugging Face stores hub snapshots under $HF_HOME/hub. Keep the explicit + # cache variables on that same path so srt-slurm's prefetch and every + # backend process resolve the identical, current snapshot. + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: "ISOLATION" + SGLANG_USE_AITER: "1" + AITER_LOG_LEVEL: "ERROR" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + decode_environment: *common_environment + sglang_config: + prefill: &common_config + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + tensor-parallel-size: 4 + disaggregation-transfer-backend: mori + disaggregation-ib-device: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + kv-cache-dtype: "fp8_e4m3" + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 262144 + context-length: 16384 + disable-radix-cache: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8] + enable-metrics: true + decode: + <<: *common_config + mem-fraction-static: 0.85 + prefill-round-robin-balance: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in 8 16 32 64 128; do + num_prompts=$((concurrency * 10)) + if ((num_prompts < 16)); then + num_prompts=16 + fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3.5-397B-A17B-FP8 \ + --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3.5-fp8-disagg-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index acf6efef1a..90451b8f3c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -285,9 +285,10 @@ qwen3.5-fp8-mi355x-sglang-disagg: - isl: 8192 osl: 1024 search-space: - # 1P+1D TP4P+TP8D/EP1 baseline (no speculative decoding). - # TP4 prefill saves 4 GPUs vs TP8P while delivering identical decode - # interactivity and 24-31% better throughput/GPU (12 vs 16 GPUs). + # The srt-slurm recipe owns the complete c8-c128 sweep in one Slurm + # allocation so model initialization is paid once instead of per point. + # 1P+1D TP4P+TP8D/EP1 remains the current baseline: TP4 prefill saves + # four GPUs without changing the TP8 decode service. # dp-attn intentionally false: with --enable-dp-attention + # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size, # but is_deepep_class_backend() excludes MoRI, so @@ -295,14 +296,14 @@ qwen3.5-fp8-mi355x-sglang-disagg: # (num_experts - num_shared_slots) % moe_ep_size assertion in # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] + conc-list: [1] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7dfdec714b..72d724b979 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6526,7 +6526,6 @@ - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification." - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2758 - - config-keys: - qwen3.8next-fp8-h100-sglang-agentic-mtp scenario-type: @@ -6537,3 +6536,16 @@ - "Use a float32 Mamba SSM state, as Hopper's flashinfer verify kernel requires, unlike the bfloat16 the Blackwell arms must use." - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2756 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg + scenario-type: + - fixed-seq-len + description: + - "Replace the model-specific amd_utils launcher with native srt-slurm orchestration using SGLang Router and AMD MoRI." + - "Preserve the current one-prefill TP4 plus one-decode TP8 topology and the 8k1k concurrency 8/16/32/64/128 search space." + - "Run all five points in one eight-hour Slurm allocation through the unchanged benchmark_serving.py custom benchmark." + - "Use the existing lmsysorg/sglang:v0.5.16-rocm720-mi35x image, FP8 KV cache, and AITer attention settings." + - "Pin the launcher to SemiAnalysisAI/srt-slurm PR #7 revision 10a58d2ebb4d756c423424049c736b909b72e14f, including DP-aware SGLang P/D rank routing, multi-node rank-offset preservation, and ATOM recipe flag normalization." + - "Reuse validated MI355X Hugging Face cache trees through the standard $HF_HOME/hub layout under a per-model lock." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2628 diff --git a/runners/launch_mi355x-amds-srt.sh b/runners/launch_mi355x-amds-srt.sh new file mode 100755 index 0000000000..eed62c3440 --- /dev/null +++ b/runners/launch_mi355x-amds-srt.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +set -euo pipefail + +# MI355X validation path for the AMD-capable srt-slurm branch. Matrix rows opt +# in explicitly with CONFIG_FILE; all existing MI355X launch behavior remains +# unchanged for every other row. +SRT_SLURM_REPOSITORY="https://github.com/SemiAnalysisAI/srt-slurm.git" +SRT_SLURM_COMMIT="10a58d2ebb4d756c423424049c736b909b72e14f" +SLURM_PARTITION="compute" +SGLANG_IMAGE="lmsysorg/sglang:v0.5.16-rocm720-mi35x" +SHARED_BASE="/it-share/gharunners2/srt-slurm" +SHARED_IMAGE="${SHARED_BASE}/containers/sglang-rocm-v0.5.16-mi35x-20260728.sqsh" +SHARED_HF_CACHE="/it-share/hf-hub-cache" +SHARED_RESULTS="${SHARED_BASE}/results" + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set by Actions}" +: "${RESULT_FILENAME:?RESULT_FILENAME must be set by the benchmark workflow}" +: "${CONFIG_FILE:?CONFIG_FILE must name an srt-slurm recipe}" +: "${MODEL:?MODEL must identify the Hugging Face model}" + +MODEL_CACHE_KEY="models--${MODEL//\//--}" + +CONFIG_PATH="${CONFIG_FILE%%:*}" +LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" +CLUSTER_PROFILE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi355x-amds.yaml" +[[ -f "$LOCAL_RECIPE" ]] || { echo "Missing recipe: $LOCAL_RECIPE" >&2; exit 1; } +[[ -f "$CLUSTER_PROFILE" ]] || { echo "Missing cluster profile: $CLUSTER_PROFILE" >&2; exit 1; } + +RUN_KEY="${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUNNER_NAME:-runner}" +WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" +SRT_REPO_DIR="${WORK_DIR}/srt-slurm" +mkdir -p "$WORK_DIR" "$SHARED_RESULTS" + +# Materialize one immutable, shared squashfs and the requested model. Older +# MI355X jobs populated Hugging Face's cache directly under SHARED_HF_CACHE; +# expose a complete legacy cache through the current $HF_HOME/hub layout before +# snapshot_download so large production checkpoints are reused without copies. +# This job exits normally and never cancels or preempts another job. +STAGE_SCRIPT="${WORK_DIR}/stage-mi355x-runtime.sbatch" +cat > "$STAGE_SCRIPT" <"${SHARED_IMAGE}.lock" +flock -w 2400 9 +if ! unsquashfs -s "$SHARED_IMAGE" >/dev/null 2>&1; then + tmp="${SHARED_IMAGE}.tmp.\${SLURM_JOB_ID}" + rm -f "\$tmp" + local_image="/var/lib/squash/lmsysorg_sglang_v0.5.16-rocm720-mi35x.sqsh" + if unsquashfs -s "\$local_image" >/dev/null 2>&1; then + cp --sparse=always "\$local_image" "\$tmp" + else + enroot import -o "\$tmp" "docker://${SGLANG_IMAGE}" + fi + unsquashfs -s "\$tmp" >/dev/null + mv "\$tmp" "$SHARED_IMAGE" +fi +flock -u 9 +mkdir -p "$SHARED_HF_CACHE/hub" +exec 8>"$SHARED_HF_CACHE/.${MODEL_CACHE_KEY}.stage.lock" +flock -w 2400 8 +legacy_model_dir="$SHARED_HF_CACHE/${MODEL_CACHE_KEY}" +canonical_model_dir="$SHARED_HF_CACHE/hub/${MODEL_CACHE_KEY}" +if [[ ! -e "\$canonical_model_dir" && -f "\$legacy_model_dir/refs/main" && -d "\$legacy_model_dir/snapshots" ]]; then + ln -s "../${MODEL_CACHE_KEY}" "\$canonical_model_dir" +fi +flock -u 8 +srun --nodes=1 --ntasks=1 \ + --container-image="$SHARED_IMAGE" \ + --container-mounts="$SHARED_HF_CACHE:/hf_hub_cache" \ + --container-writable --container-remap-root --no-container-entrypoint \ + --export=ALL,HF_HOME=/hf_hub_cache,HF_HUB_CACHE=/hf_hub_cache/hub,HUGGINGFACE_HUB_CACHE=/hf_hub_cache/hub,MODEL_REPO=${MODEL} \ + python3 -c 'import os; from huggingface_hub import snapshot_download; snapshot_download(os.environ["MODEL_REPO"])' +EOF +STAGE_JOB_ID=$(sbatch --wait --parsable "$STAGE_SCRIPT") +STAGE_JOB_ID="${STAGE_JOB_ID%%;*}" +echo "MI355X runtime prerequisites verified with Slurm job ${STAGE_JOB_ID}" + +git clone "$SRT_SLURM_REPOSITORY" "$SRT_REPO_DIR" +git -C "$SRT_REPO_DIR" checkout "$SRT_SLURM_COMMIT" +ACTUAL_SRT_COMMIT=$(git -C "$SRT_REPO_DIR" rev-parse HEAD) +[[ "$ACTUAL_SRT_COMMIT" == "$SRT_SLURM_COMMIT" ]] || { + echo "srt-slurm checkout mismatch: $ACTUAL_SRT_COMMIT" >&2 + exit 1 +} + +mkdir -p "${SRT_REPO_DIR}/$(dirname "$CONFIG_PATH")" +cp "$LOCAL_RECIPE" "${SRT_REPO_DIR}/${CONFIG_PATH}" +cp "$CLUSTER_PROFILE" "${WORK_DIR}/srtslurm.yaml" +python3 - "${WORK_DIR}/srtslurm.yaml" "$GITHUB_WORKSPACE" "$SHARED_RESULTS" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +workspace, results = sys.argv[2:] +needle = " /it-share/hf-hub-cache: /hf_hub_cache\n" +text = path.read_text() +if text.count(needle) != 1: + raise SystemExit("expected exactly one Hugging Face cache mount") +path.write_text( + text.replace( + needle, + needle + f" {workspace}: /infmax-workspace\n {results}: /results\n", + ) +) +PY + +export PATH="$HOME/.local/bin:$PATH" +cd "$SRT_REPO_DIR" +uv venv --python 3.12 +uv pip install -e . +make setup-compute ARCH=x86_64 +source .venv/bin/activate +export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" +export SRTCTL_RUNTIME_SOURCE_DIR="$SRT_REPO_DIR" + +echo "Submitting ${CONFIG_PATH} with srt-slurm ${SRT_SLURM_COMMIT}" +set +e +SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" \ + --tags "mi355x,inferencex,github-actions,${RUN_KEY}" 2>&1) +SRTCTL_RC=$? +set -e +echo "$SRTCTL_OUTPUT" +if [[ $SRTCTL_RC -ne 0 ]]; then + echo "srtctl apply failed with exit code ${SRTCTL_RC}" >&2 + exit "$SRTCTL_RC" +fi +JOB_ID=$(grep -oE 'Job [0-9]+' <<< "$SRTCTL_OUTPUT" | awk '{print $2}' | tail -1) +[[ -n "$JOB_ID" ]] || { echo "Unable to parse srt-slurm job ID" >&2; exit 1; } +echo "SRT_SLURM_JOB_ID=$JOB_ID" + +while squeue --noheader --jobs "$JOB_ID" | grep -q .; do + squeue --noheader --jobs "$JOB_ID" --format='srt-slurm %i %T %M %R' + sleep 15 +done + +read -r JOB_STATE JOB_EXIT JOB_NODELIST < <( + sacct -X --noheader --parsable2 --jobs "$JOB_ID" \ + --format=State,ExitCode,NodeList | head -1 | tr '|' ' ' +) +echo "srt-slurm job ${JOB_ID}: state=${JOB_STATE} exit=${JOB_EXIT} nodes=${JOB_NODELIST}" + +RESULT_DIR="${SHARED_RESULTS}/${JOB_ID}" +mkdir -p "$GITHUB_WORKSPACE/LOGS" +if [[ -f "$RESULT_DIR/runtime-logs.tar.gz" ]]; then + cp "$RESULT_DIR/runtime-logs.tar.gz" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" +fi +cp -R "$RESULT_DIR/." "$GITHUB_WORKSPACE/LOGS/" + +if [[ "${DISAGG:-false}" == "true" ]]; then + PREFILL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + DECODE_GPUS=$((DECODE_NUM_WORKERS * DECODE_TP)) + TOTAL_GPUS=$((PREFILL_GPUS + DECODE_GPUS)) +else + TOTAL_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP * ${PREFILL_PP_SIZE:-1} * ${PREFILL_PCP_SIZE:-1})) +fi + +shopt -s nullglob +RESULTS=("$RESULT_DIR"/fixed-seq/*.json) +shopt -u nullglob +[[ ${#RESULTS[@]} -gt 0 ]] || { echo "No fixed-sequence results found in $RESULT_DIR" >&2; exit 1; } +for result in "${RESULTS[@]}"; do + concurrency=$(basename "$result" | sed -n 's/.*-c\([0-9][0-9]*\)\.json/\1/p') + [[ -n "$concurrency" ]] || { echo "Cannot parse concurrency from $result" >&2; exit 1; } + if [[ "${DISAGG:-false}" == "true" ]]; then + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}_ctx_${PREFILL_GPUS}_gen_${DECODE_GPUS}.json" + else + output="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_srt-${JOB_ID}_conc${concurrency}_gpus_${TOTAL_GPUS}.json" + fi + cp "$result" "$output" + echo "Collected $output" +done + +if [[ "$JOB_STATE" != COMPLETED || "$JOB_EXIT" != 0:0 ]]; then + echo "srt-slurm validation failed: ${JOB_STATE} (${JOB_EXIT})" >&2 + exit 1 +fi + +printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" +echo "MI355X srt-slurm validation completed successfully" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 49c3af0cf5..a111f19637 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +if [[ -n "${CONFIG_FILE:-}" ]]; then + exec bash "$(dirname "${BASH_SOURCE[0]}")/launch_mi355x-amds-srt.sh" +fi + scancel_sync() { local jobid=$1 local timeout=${2:-600}