From 8d479d1f2bc0e385fd72b14b18de442cd17e3d8c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 8 Sep 2026 14:35:21 -0500 Subject: [PATCH 1/3] fix(runners): import B300 images on the login host [skip-sweep] --- docs/configuration-procedures.md | 15 ++++ docs/configuration-procedures_zh.md | 14 ++++ runners/launch_b300-dsxe.sh | 62 ++++++++------ utils/test_b300_image_import.py | 126 ++++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 25 deletions(-) create mode 100644 utils/test_b300_image_import.py diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index e42027abb5..60b8d9054e 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -112,6 +112,21 @@ The runner-name prefix is load-bearing: workflow routing uses `launch_${RUNNER_N 6. Verify every runner is **Idle** in [repository runner settings](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners) before adding it to sweep traffic. 7. Verify launcher mounts for `_work`, HF cache, staged weights, and squash images from a compute node. Root containers must not leave root-owned files in the shared workspace. +### B300 DSXE image imports + +`launch_b300-dsxe.sh` downloads and converts images on the runner/login host, +before requesting benchmark nodes. Cold imports share a lock to limit login-host +resource use; validated images are published atomically in the shared squash +directory. Enroot defaults to two processors and two download connections. + +Unpacking uses a private directory under `ENROOT_TEMP_PATH` (default `/tmp`). +This must be a local filesystem with enough space for the unpacked image layers, +not Lustre. Set it to a larger local scratch volume when needed. The downloaded +layer cache defaults to `.enroot-cache` in the shared squash directory; +`ENROOT_CACHE_PATH` can override it. `ENROOT_IMPORT_TIME_LIMIT` bounds each import +in minutes (default 120). Compute nodes still need healthy shared-storage access +to read the completed image and benchmark workspace. + ## Register an srt-slurm recipe Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 0ebd17fc68..b41ae9ed1a 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -112,6 +112,20 @@ runner 名称前缀是关键契约:workflow 通过 `launch_${RUNNER_NAME%%_*}. 6. 将 runner 加入 sweep 流量前,在[仓库 runner 设置页](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners)确认每个 runner 都是 **Idle**。 7. 从计算节点验证 launcher 对 `_work`、HF cache、预置权重和 squash 镜像的挂载。root 容器不得在共享 workspace 留下 root 所有的文件。 +### B300 DSXE 镜像导入 + +`launch_b300-dsxe.sh` 在申请基准测试节点之前,先在运行器所在的登录主机上 +下载并转换镜像。未缓存镜像的导入共用一把锁,以限制登录主机的资源占用; +镜像通过验证后,才原子发布到共享 squash 目录。Enroot 默认使用两个处理器 +和两个下载连接。 + +解包使用 `ENROOT_TEMP_PATH`(默认 `/tmp`)下的独立临时目录。该路径必须位于 +本地文件系统,且有足够空间容纳解包后的镜像层,不能使用 Lustre;空间不足时, +应将其指向更大的本地临时存储卷。下载层缓存默认保存在共享 squash 目录下的 +`.enroot-cache`,可通过 `ENROOT_CACHE_PATH` 覆盖。`ENROOT_IMPORT_TIME_LIMIT` +以分钟为单位限制单次导入时长(默认 120)。计算节点仍然需要正常的共享存储 +连接,才能读取完成的镜像和基准测试工作区。 + ## 注册 srt-slurm 配方 映射来源:[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方:[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c229146bfa..93dccc6a4d 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -74,39 +74,51 @@ set -x # when execution reaches it, so moving this inside either branch silently removes # it from the other and the job dies on "command not found" at import time. # -# Import a container image into the shared squash dir. Concurrent callers target the -# same path, so serialize on a per-file lock and skip when a valid squash file exists. -# --time bounds the step; an unbounded srun hangs the job if its step is lost. -# -# The import itself must run on a compute node: enroot builds the squashfs over an -# overlay mount, which the shared filesystem cannot back, and the login host is too -# small to unpack a multi-GB image. Reading the finished file is just I/O, so probe -# it here first -- a warm cache then costs no Slurm allocation at all. The in-srun -# check under the lock stays authoritative, so a stale probe only costs one step. +# Import on the runner/login host; shared storage holds the layer cache and images. +# Enroot's extracted layers and overlay mount must use node-local scratch. +# Serialize cold imports across images to bound login-host CPU and disk usage. import_squash_image() { local image_ref="$1" local sqsh="$2" - local lock="${2}.lock" if unsquashfs -l "$sqsh" > /dev/null 2>&1; then echo "Squash file already present, skipping import: $sqsh" return 0 fi - srun -N 1 -A "$SLURM_ACCOUNT" -p "$SLURM_PARTITION" \ - --time="${ENROOT_IMPORT_TIME_LIMIT:-120}" bash -c " - set -euo pipefail - exec 9>\"$lock\" - flock -w 3600 9 - if unsquashfs -l \"$sqsh\" > /dev/null 2>&1; then + ( + set -uo pipefail + exec 9>"$SQUASH_DIR/.import.lock" || exit 1 + flock -w 7200 9 || { echo "Timed out waiting for the image import lock" >&2; exit 1; } + # Keep the per-image lock compatible with runners on the old launcher. + exec 8>"${sqsh}.lock" || exit 1 + flock -w 7200 8 || { echo "Timed out waiting for $sqsh" >&2; exit 1; } + # Another runner may have populated this image while we waited. + if unsquashfs -l "$sqsh" > /dev/null 2>&1; then + echo "Squash file already present, skipping import: $sqsh" exit 0 fi - rm -f \"$sqsh\" - enroot import -o \"$sqsh\" \"docker://$image_ref\" - unsquashfs -l \"$sqsh\" > /dev/null - " || { echo "Error: enroot import failed for $image_ref -> $sqsh" >&2; exit 1; } - test -r "$sqsh" || { echo "Error: squash file not readable: $sqsh" >&2; exit 1; } + local work_dir partial + work_dir=$(mktemp -d "${ENROOT_TEMP_PATH:-/tmp}/inferencex-enroot.XXXXXX") || exit 1 + partial="${sqsh}.tmp.${work_dir##*/}" + trap 'rm -f -- "$partial"; rm -rf -- "$work_dir"' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + export ENROOT_TEMP_PATH="$work_dir" + export ENROOT_RUNTIME_PATH="$work_dir/runtime" + export ENROOT_DATA_PATH="$work_dir/data" + export ENROOT_CACHE_PATH="${ENROOT_CACHE_PATH:-$SQUASH_DIR/.enroot-cache}" + export ENROOT_MAX_PROCESSORS="${ENROOT_MAX_PROCESSORS:-2}" + export ENROOT_MAX_CONNECTIONS="${ENROOT_MAX_CONNECTIONS:-2}" + + echo "Importing $image_ref on $(hostname) using local scratch $work_dir" + timeout --kill-after=30 "${ENROOT_IMPORT_TIME_LIMIT:-120}m" \ + enroot import -o "$partial" "docker://$image_ref" || exit 1 + unsquashfs -l "$partial" > /dev/null || exit 1 + # Publish only a validated image, without exposing partial imports. + mv -f -- "$partial" "$sqsh" || exit 1 + ) || { echo "Error: enroot import failed for $image_ref -> $sqsh" >&2; return 1; } } if [[ "$IS_MULTINODE" == "true" ]]; then @@ -202,15 +214,15 @@ SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" NGINX_SQUASH_FILE="$SQUASH_DIR/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" # Import containers via enroot -import_squash_image "$IMAGE" "$SQUASH_FILE" -import_squash_image "$NGINX_IMAGE" "$NGINX_SQUASH_FILE" +import_squash_image "$IMAGE" "$SQUASH_FILE" || exit 1 +import_squash_image "$NGINX_IMAGE" "$NGINX_SQUASH_FILE" || exit 1 if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="$SQUASH_DIR/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - import_squash_image "$DCGM_EXPORTER_ENROOT_REF" "$DCGM_EXPORTER_SQSH" + import_squash_image "$DCGM_EXPORTER_ENROOT_REF" "$DCGM_EXPORTER_SQSH" || exit 1 sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" fi @@ -474,7 +486,7 @@ else CONTAINER_MOUNT_DIR=/workspace fi - import_squash_image "$IMAGE" "$SQUASH_FILE" + import_squash_image "$IMAGE" "$SQUASH_FILE" || exit 1 export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" diff --git a/utils/test_b300_image_import.py b/utils/test_b300_image_import.py new file mode 100644 index 0000000000..88889a0953 --- /dev/null +++ b/utils/test_b300_image_import.py @@ -0,0 +1,126 @@ +"""Exercise the launcher's local import, cache publication, and failure paths.""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def write_executable(path: Path, source: str, *, python: bool = False) -> None: + path.write_text(f"#!{sys.executable if python else '/bin/bash'}\n" + source) + path.chmod(0o755) + + +@pytest.fixture +def importer(tmp_path: Path) -> tuple[Path, dict[str, str], Path]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + squash_dir = tmp_path / "squash" + squash_dir.mkdir() + scratch = tmp_path / "scratch" + scratch.mkdir() + # macOS has no util-linux flock or GNU timeout. Keep real advisory locking + # through fcntl; the cluster smoke uses the actual Linux commands. + write_executable( + bin_dir / "flock", + "import fcntl, sys\nfcntl.flock(int(sys.argv[-1]), fcntl.LOCK_EX)\n", + python=True, + ) + write_executable(bin_dir / "timeout", 'shift 2\nexec "$@"\n') + write_executable(bin_dir / "srun", 'echo "unexpected Slurm allocation" >&2\nexit 99\n') + write_executable( + bin_dir / "unsquashfs", + "import pathlib, sys\np = pathlib.Path(sys.argv[2])\n" + "sys.exit(0 if p.exists() and p.read_bytes() == b'valid' else 1)\n", + python=True, + ) + write_executable( + bin_dir / "enroot", + """import os, pathlib, sys, time +active = pathlib.Path(os.environ['IMPORT_LOG'] + '.active') +active.mkdir() # Concurrent cold imports must not overlap on the login host. +try: + with open(os.environ['IMPORT_LOG'], 'a') as log: + log.write(os.environ['ENROOT_TEMP_PATH'] + '\\n') + p = pathlib.Path(sys.argv[3]) + p.write_bytes(b'partial') + time.sleep(0.2) + mode = os.environ.get('IMPORT_MODE', 'success') + if mode == 'failure': + sys.exit(42) + if mode != 'invalid': + p.write_bytes(b'valid') +finally: + active.rmdir() +""", + python=True, + ) + source = (ROOT / "runners/launch_b300-dsxe.sh").read_text() + start = source.index("import_squash_image() {") + end = source.index('\nif [[ "$IS_MULTINODE"', start) + harness = tmp_path / "import.sh" + harness.write_text( + source[start:end] + '\nimport_squash_image example/test:tag "$1" || exit 1\n' + ) + env = { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "SQUASH_DIR": str(squash_dir), + "ENROOT_TEMP_PATH": str(scratch), + "IMPORT_LOG": str(tmp_path / "imports.log"), + } + return harness, env, squash_dir / "image.sqsh" + + +def run_import(importer: tuple[Path, dict[str, str], Path]) -> subprocess.CompletedProcess: + harness, env, squash = importer + return subprocess.run( + ["bash", str(harness), str(squash)], env=env, + capture_output=True, text=True, timeout=10, + ) + + +def test_warm_image_needs_no_import(importer) -> None: + _, env, squash = importer + squash.write_bytes(b"valid") + assert run_import(importer).returncode == 0 + assert not Path(env["IMPORT_LOG"]).exists() + + +@pytest.mark.parametrize("distinct_images", [False, True]) +def test_concurrent_callers_serialize_and_clean_local_scratch(importer, distinct_images: bool) -> None: + harness, env, squash = importer + targets = [squash, squash.with_name("second.sqsh") if distinct_images else squash] + processes = [ + subprocess.Popen( + ["bash", str(harness), str(target)], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + for target in targets + ] + for process in processes: + _, stderr = process.communicate(timeout=10) + assert process.returncode == 0, stderr + assert all(target.read_bytes() == b"valid" for target in targets) + imports = Path(env["IMPORT_LOG"]).read_text().splitlines() + assert len(imports) == (2 if distinct_images else 1) + for path in imports: + assert Path(path).parent == Path(env["ENROOT_TEMP_PATH"]) + assert not Path(path).exists() + assert not list(squash.parent.glob("*.tmp.*")) + + +@pytest.mark.parametrize("mode", ["failure", "invalid"]) +def test_failed_import_never_publishes_partial_image(importer, mode: str) -> None: + _, env, squash = importer + env["IMPORT_MODE"] = mode + squash.write_bytes(b"previous-invalid-image") + result = run_import(importer) + assert result.returncode != 0 + assert squash.read_bytes() == b"previous-invalid-image" + assert not list(squash.parent.glob("*.tmp.*")) + assert not list(Path(env["ENROOT_TEMP_PATH"]).iterdir()) From 96773232f48df0d9e311ca54e4dc4e7771f9ed7f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 8 Sep 2026 14:37:24 -0500 Subject: [PATCH 2/3] test: remove B300 image import tests [skip-sweep] --- utils/test_b300_image_import.py | 126 -------------------------------- 1 file changed, 126 deletions(-) delete mode 100644 utils/test_b300_image_import.py diff --git a/utils/test_b300_image_import.py b/utils/test_b300_image_import.py deleted file mode 100644 index 88889a0953..0000000000 --- a/utils/test_b300_image_import.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Exercise the launcher's local import, cache publication, and failure paths.""" - -import os -import subprocess -import sys -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[1] - - -def write_executable(path: Path, source: str, *, python: bool = False) -> None: - path.write_text(f"#!{sys.executable if python else '/bin/bash'}\n" + source) - path.chmod(0o755) - - -@pytest.fixture -def importer(tmp_path: Path) -> tuple[Path, dict[str, str], Path]: - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - squash_dir = tmp_path / "squash" - squash_dir.mkdir() - scratch = tmp_path / "scratch" - scratch.mkdir() - # macOS has no util-linux flock or GNU timeout. Keep real advisory locking - # through fcntl; the cluster smoke uses the actual Linux commands. - write_executable( - bin_dir / "flock", - "import fcntl, sys\nfcntl.flock(int(sys.argv[-1]), fcntl.LOCK_EX)\n", - python=True, - ) - write_executable(bin_dir / "timeout", 'shift 2\nexec "$@"\n') - write_executable(bin_dir / "srun", 'echo "unexpected Slurm allocation" >&2\nexit 99\n') - write_executable( - bin_dir / "unsquashfs", - "import pathlib, sys\np = pathlib.Path(sys.argv[2])\n" - "sys.exit(0 if p.exists() and p.read_bytes() == b'valid' else 1)\n", - python=True, - ) - write_executable( - bin_dir / "enroot", - """import os, pathlib, sys, time -active = pathlib.Path(os.environ['IMPORT_LOG'] + '.active') -active.mkdir() # Concurrent cold imports must not overlap on the login host. -try: - with open(os.environ['IMPORT_LOG'], 'a') as log: - log.write(os.environ['ENROOT_TEMP_PATH'] + '\\n') - p = pathlib.Path(sys.argv[3]) - p.write_bytes(b'partial') - time.sleep(0.2) - mode = os.environ.get('IMPORT_MODE', 'success') - if mode == 'failure': - sys.exit(42) - if mode != 'invalid': - p.write_bytes(b'valid') -finally: - active.rmdir() -""", - python=True, - ) - source = (ROOT / "runners/launch_b300-dsxe.sh").read_text() - start = source.index("import_squash_image() {") - end = source.index('\nif [[ "$IS_MULTINODE"', start) - harness = tmp_path / "import.sh" - harness.write_text( - source[start:end] + '\nimport_squash_image example/test:tag "$1" || exit 1\n' - ) - env = { - **os.environ, - "PATH": f"{bin_dir}:{os.environ['PATH']}", - "SQUASH_DIR": str(squash_dir), - "ENROOT_TEMP_PATH": str(scratch), - "IMPORT_LOG": str(tmp_path / "imports.log"), - } - return harness, env, squash_dir / "image.sqsh" - - -def run_import(importer: tuple[Path, dict[str, str], Path]) -> subprocess.CompletedProcess: - harness, env, squash = importer - return subprocess.run( - ["bash", str(harness), str(squash)], env=env, - capture_output=True, text=True, timeout=10, - ) - - -def test_warm_image_needs_no_import(importer) -> None: - _, env, squash = importer - squash.write_bytes(b"valid") - assert run_import(importer).returncode == 0 - assert not Path(env["IMPORT_LOG"]).exists() - - -@pytest.mark.parametrize("distinct_images", [False, True]) -def test_concurrent_callers_serialize_and_clean_local_scratch(importer, distinct_images: bool) -> None: - harness, env, squash = importer - targets = [squash, squash.with_name("second.sqsh") if distinct_images else squash] - processes = [ - subprocess.Popen( - ["bash", str(harness), str(target)], env=env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - ) - for target in targets - ] - for process in processes: - _, stderr = process.communicate(timeout=10) - assert process.returncode == 0, stderr - assert all(target.read_bytes() == b"valid" for target in targets) - imports = Path(env["IMPORT_LOG"]).read_text().splitlines() - assert len(imports) == (2 if distinct_images else 1) - for path in imports: - assert Path(path).parent == Path(env["ENROOT_TEMP_PATH"]) - assert not Path(path).exists() - assert not list(squash.parent.glob("*.tmp.*")) - - -@pytest.mark.parametrize("mode", ["failure", "invalid"]) -def test_failed_import_never_publishes_partial_image(importer, mode: str) -> None: - _, env, squash = importer - env["IMPORT_MODE"] = mode - squash.write_bytes(b"previous-invalid-image") - result = run_import(importer) - assert result.returncode != 0 - assert squash.read_bytes() == b"previous-invalid-image" - assert not list(squash.parent.glob("*.tmp.*")) - assert not list(Path(env["ENROOT_TEMP_PATH"]).iterdir()) From fffdaa162944cfac85bbbcaa820ac9a090acec50 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 8 Sep 2026 14:38:30 -0500 Subject: [PATCH 3/3] fix(runners): simplify B300 import to a local call [skip-sweep] --- docs/configuration-procedures.md | 15 -------- docs/configuration-procedures_zh.md | 14 -------- runners/launch_b300-dsxe.sh | 55 ++++++++++------------------- 3 files changed, 18 insertions(+), 66 deletions(-) diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 60b8d9054e..e42027abb5 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -112,21 +112,6 @@ The runner-name prefix is load-bearing: workflow routing uses `launch_${RUNNER_N 6. Verify every runner is **Idle** in [repository runner settings](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners) before adding it to sweep traffic. 7. Verify launcher mounts for `_work`, HF cache, staged weights, and squash images from a compute node. Root containers must not leave root-owned files in the shared workspace. -### B300 DSXE image imports - -`launch_b300-dsxe.sh` downloads and converts images on the runner/login host, -before requesting benchmark nodes. Cold imports share a lock to limit login-host -resource use; validated images are published atomically in the shared squash -directory. Enroot defaults to two processors and two download connections. - -Unpacking uses a private directory under `ENROOT_TEMP_PATH` (default `/tmp`). -This must be a local filesystem with enough space for the unpacked image layers, -not Lustre. Set it to a larger local scratch volume when needed. The downloaded -layer cache defaults to `.enroot-cache` in the shared squash directory; -`ENROOT_CACHE_PATH` can override it. `ENROOT_IMPORT_TIME_LIMIT` bounds each import -in minutes (default 120). Compute nodes still need healthy shared-storage access -to read the completed image and benchmark workspace. - ## Register an srt-slurm recipe Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index b41ae9ed1a..0ebd17fc68 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -112,20 +112,6 @@ runner 名称前缀是关键契约:workflow 通过 `launch_${RUNNER_NAME%%_*}. 6. 将 runner 加入 sweep 流量前,在[仓库 runner 设置页](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners)确认每个 runner 都是 **Idle**。 7. 从计算节点验证 launcher 对 `_work`、HF cache、预置权重和 squash 镜像的挂载。root 容器不得在共享 workspace 留下 root 所有的文件。 -### B300 DSXE 镜像导入 - -`launch_b300-dsxe.sh` 在申请基准测试节点之前,先在运行器所在的登录主机上 -下载并转换镜像。未缓存镜像的导入共用一把锁,以限制登录主机的资源占用; -镜像通过验证后,才原子发布到共享 squash 目录。Enroot 默认使用两个处理器 -和两个下载连接。 - -解包使用 `ENROOT_TEMP_PATH`(默认 `/tmp`)下的独立临时目录。该路径必须位于 -本地文件系统,且有足够空间容纳解包后的镜像层,不能使用 Lustre;空间不足时, -应将其指向更大的本地临时存储卷。下载层缓存默认保存在共享 squash 目录下的 -`.enroot-cache`,可通过 `ENROOT_CACHE_PATH` 覆盖。`ENROOT_IMPORT_TIME_LIMIT` -以分钟为单位限制单次导入时长(默认 120)。计算节点仍然需要正常的共享存储 -连接,才能读取完成的镜像和基准测试工作区。 - ## 注册 srt-slurm 配方 映射来源:[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方:[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 93dccc6a4d..6ff9c2dfa3 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -74,51 +74,32 @@ set -x # when execution reaches it, so moving this inside either branch silently removes # it from the other and the job dies on "command not found" at import time. # -# Import on the runner/login host; shared storage holds the layer cache and images. -# Enroot's extracted layers and overlay mount must use node-local scratch. -# Serialize cold imports across images to bound login-host CPU and disk usage. +# Import a container image into the shared squash dir. Concurrent callers target the +# same path, so serialize on a per-file lock and skip when a valid squash file exists. +# Run the import locally on the runner/login host using its Enroot configuration. import_squash_image() { local image_ref="$1" local sqsh="$2" + local lock="${2}.lock" if unsquashfs -l "$sqsh" > /dev/null 2>&1; then echo "Squash file already present, skipping import: $sqsh" return 0 fi - ( - set -uo pipefail - exec 9>"$SQUASH_DIR/.import.lock" || exit 1 - flock -w 7200 9 || { echo "Timed out waiting for the image import lock" >&2; exit 1; } - # Keep the per-image lock compatible with runners on the old launcher. - exec 8>"${sqsh}.lock" || exit 1 - flock -w 7200 8 || { echo "Timed out waiting for $sqsh" >&2; exit 1; } - # Another runner may have populated this image while we waited. - if unsquashfs -l "$sqsh" > /dev/null 2>&1; then - echo "Squash file already present, skipping import: $sqsh" + bash -c " + set -euo pipefail + exec 9>\"$lock\" + flock -w 3600 9 + if unsquashfs -l \"$sqsh\" > /dev/null 2>&1; then exit 0 fi + rm -f \"$sqsh\" + enroot import -o \"$sqsh\" \"docker://$image_ref\" + unsquashfs -l \"$sqsh\" > /dev/null + " || { echo "Error: enroot import failed for $image_ref -> $sqsh" >&2; exit 1; } - local work_dir partial - work_dir=$(mktemp -d "${ENROOT_TEMP_PATH:-/tmp}/inferencex-enroot.XXXXXX") || exit 1 - partial="${sqsh}.tmp.${work_dir##*/}" - trap 'rm -f -- "$partial"; rm -rf -- "$work_dir"' EXIT - trap 'exit 143' TERM - trap 'exit 130' INT - export ENROOT_TEMP_PATH="$work_dir" - export ENROOT_RUNTIME_PATH="$work_dir/runtime" - export ENROOT_DATA_PATH="$work_dir/data" - export ENROOT_CACHE_PATH="${ENROOT_CACHE_PATH:-$SQUASH_DIR/.enroot-cache}" - export ENROOT_MAX_PROCESSORS="${ENROOT_MAX_PROCESSORS:-2}" - export ENROOT_MAX_CONNECTIONS="${ENROOT_MAX_CONNECTIONS:-2}" - - echo "Importing $image_ref on $(hostname) using local scratch $work_dir" - timeout --kill-after=30 "${ENROOT_IMPORT_TIME_LIMIT:-120}m" \ - enroot import -o "$partial" "docker://$image_ref" || exit 1 - unsquashfs -l "$partial" > /dev/null || exit 1 - # Publish only a validated image, without exposing partial imports. - mv -f -- "$partial" "$sqsh" || exit 1 - ) || { echo "Error: enroot import failed for $image_ref -> $sqsh" >&2; return 1; } + test -r "$sqsh" || { echo "Error: squash file not readable: $sqsh" >&2; exit 1; } } if [[ "$IS_MULTINODE" == "true" ]]; then @@ -214,15 +195,15 @@ SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" NGINX_SQUASH_FILE="$SQUASH_DIR/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" # Import containers via enroot -import_squash_image "$IMAGE" "$SQUASH_FILE" || exit 1 -import_squash_image "$NGINX_IMAGE" "$NGINX_SQUASH_FILE" || exit 1 +import_squash_image "$IMAGE" "$SQUASH_FILE" +import_squash_image "$NGINX_IMAGE" "$NGINX_SQUASH_FILE" if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="$SQUASH_DIR/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - import_squash_image "$DCGM_EXPORTER_ENROOT_REF" "$DCGM_EXPORTER_SQSH" || exit 1 + import_squash_image "$DCGM_EXPORTER_ENROOT_REF" "$DCGM_EXPORTER_SQSH" sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" fi @@ -486,7 +467,7 @@ else CONTAINER_MOUNT_DIR=/workspace fi - import_squash_image "$IMAGE" "$SQUASH_FILE" || exit 1 + import_squash_image "$IMAGE" "$SQUASH_FILE" export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"