From 7019bb26f92453ef4a3e49513ff33e9014cb8fbd Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 9 Sep 2026 01:39:09 -0700 Subject: [PATCH 01/29] feat: add prepared H3 hardware sites and serving matrices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 接入已准备的 H100、H200 与 B200 站点，扩展每档请求数并单独记录全部分配 GPU。AMD 运行时和跨硬件实测尚待完成。 --- .github/workflows/e2e-tests.yml | 22 ++++++++ .github/workflows/h3-video.yml | 52 ++++++++++++++---- experimental/video-generation/README.md | 38 +++++++++---- experimental/video-generation/README_zh.md | 23 ++++++-- experimental/video-generation/ci.py | 53 ++++++++++++++----- .../video-generation/evaluator/mvp_power.py | 2 +- .../evaluator/mvp_serving_smoke.py | 23 ++++---- .../video-generation/runtime-entry.example.sh | 11 ++-- .../video-generation/tests/test_ci.py | 38 +++++++++++++ .../tests/test_mvp_serving_smoke.py | 14 ++--- perf-changelog.yaml | 6 +++ 11 files changed, 225 insertions(+), 57 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 8f0e2a0b32..afb656cee8 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -12,6 +12,16 @@ on: required: false type: boolean default: false + h3-cluster: + description: "Prepared H3 site: h200-dgxc, h100-dgxc, or b200-nscale" + required: false + type: string + default: h200-dgxc + h3-site-config: + description: "Pinned runner-local H3 site JSON; empty uses existing H200 configuration" + required: false + type: string + default: "" h3-reuse-run-ids: description: "Reprocess one or two accepted H3 CI runs (comma-separated); no new H3 generation" required: false @@ -122,6 +132,16 @@ on: required: false type: boolean default: false + h3-cluster: + description: "Prepared H3 site: h200-dgxc, h100-dgxc, or b200-nscale" + required: false + type: string + default: h200-dgxc + h3-site-config: + description: "Pinned runner-local H3 site JSON; empty uses existing H200 configuration" + required: false + type: string + default: "" h3-reuse-run-ids: description: "Reprocess one or two accepted H3 CI runs (comma-separated); no new H3 generation" required: false @@ -234,6 +254,8 @@ jobs: actions: read uses: ./.github/workflows/h3-video.yml with: + cluster: ${{ inputs.h3-cluster }} + site-config: ${{ inputs.h3-site-config }} source-run-ids: ${{ inputs.h3-reuse-run-ids }} inventory-run-id: ${{ inputs.h3-inventory-run-id }} diff --git a/.github/workflows/h3-video.yml b/.github/workflows/h3-video.yml index f0d30bfef4..f0dfafb4f2 100644 --- a/.github/workflows/h3-video.yml +++ b/.github/workflows/h3-video.yml @@ -4,6 +4,17 @@ run-name: H3 video smoke - ${{ github.ref_name }} on: workflow_dispatch: inputs: + cluster: + description: "Prepared H3 hardware site" + required: false + type: choice + options: [h200-dgxc, h100-dgxc, b200-nscale] + default: h200-dgxc + site-config: + description: "Pinned runner-local site JSON; empty uses the existing H200 configuration" + required: false + type: string + default: "" inventory-run-id: description: "Reuse a successful hardware inventory for CPU-only export" required: false @@ -16,6 +27,14 @@ on: default: "" workflow_call: inputs: + cluster: + required: false + type: string + default: h200-dgxc + site-config: + required: false + type: string + default: "" inventory-run-id: required: false type: string @@ -42,6 +61,7 @@ jobs: outputs: priority: ${{ steps.queue.outputs.priority }} queue-token: ${{ steps.queue.outputs.queue-token }} + gpu-model: ${{ steps.queue.outputs.gpu-model }} steps: - name: Authorize manual repository execution uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -68,13 +88,24 @@ jobs: - name: Prepare native queue identity id: queue env: - H3_SITE_CONFIG: ${{ vars.H3_SITE_CONFIG }} + H3_SITE_CONFIG: ${{ inputs.site-config || vars.H3_SITE_CONFIG }} + H3_CLUSTER: ${{ inputs.cluster }} PRIORITY_ENABLED: ${{ vars.PRIORITY_SCHEDULER_ENABLED }} NODE_SLOTS_ENABLED: ${{ vars.NODE_SLOT_SCHEDULER_ENABLED }} H3_SOURCE_RUN_IDS: ${{ inputs.source-run-ids }} H3_INVENTORY_RUN_ID: ${{ inputs.inventory-run-id }} run: | set -euo pipefail + case "$H3_CLUSTER" in + h200-dgxc) echo 'gpu-model=H200' >> "$GITHUB_OUTPUT" ;; + h100-dgxc) echo 'gpu-model=H100' >> "$GITHUB_OUTPUT" ;; + b200-nscale) echo 'gpu-model=B200' >> "$GITHUB_OUTPUT" ;; + *) echo 'Unsupported H3 hardware site.' >&2; exit 1 ;; + esac + if [[ "$H3_CLUSTER" != h200-dgxc && ( -n "$H3_SOURCE_RUN_IDS" || -n "$H3_INVENTORY_RUN_ID" ) ]]; then + echo 'Historical inventory reuse remains specific to the H200 inventory contract.' >&2 + exit 1 + fi if [[ -n "$H3_INVENTORY_RUN_ID" && ( -z "$H3_SOURCE_RUN_IDS" || ! "$H3_INVENTORY_RUN_ID" =~ ^[1-9][0-9]{0,19}$ ) ]]; then echo 'Hardware reuse needs one inventory run ID and explicit source executions.' >&2 exit 1 @@ -88,7 +119,7 @@ jobs: echo 'H3 priority scheduling requires node-slot admission for nodes:1.' >&2 exit 1 fi - scored=$(printf '%s' '[{"runner":"cluster:h200-dgxc","framework":"sglang","node-count":1}]' | + scored=$(jq -nc --arg runner "cluster:$H3_CLUSTER" '[{runner:$runner,framework:"sglang","node-count":1}]' | uv run --no-project --with pyyaml --python 3.12 utils/ci_priority.py) echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" python3 - <<'PY' @@ -105,18 +136,19 @@ jobs: outputs: mode: ${{ steps.execute.outputs.mode }} if: ${{ inputs.inventory-run-id == '' }} - name: p${{ needs.prepare.outputs.priority }} | ${{ inputs.source-run-ids != '' && 'H3 H200 hardware inventory' || 'H3 video H200 smoke' }} + name: p${{ needs.prepare.outputs.priority }} | ${{ inputs.source-run-ids != '' && 'H3 H200 hardware inventory' || format('H3 video {0} smoke', needs.prepare.outputs.gpu-model) }} runs-on: >- ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && - format('["self-hosted","cluster:h200-dgxc","nodes:1",{0},{1}]', + format('["self-hosted","cluster:{2}","nodes:1",{0},{1}]', toJSON(format('ci-job-{0}-{1}', needs.prepare.outputs.priority, needs.prepare.outputs.queue-token)), - toJSON(format('ci-attempt-{0}', github.run_attempt))) || - '["cluster:h200-dgxc"]' + toJSON(format('ci-attempt-{0}', github.run_attempt)), inputs.cluster) || + format('["cluster:{0}"]', inputs.cluster) ) }} - timeout-minutes: 105 + timeout-minutes: 255 env: - H3_SITE_CONFIG: ${{ vars.H3_SITE_CONFIG }} + H3_SITE_CONFIG: ${{ inputs.site-config || vars.H3_SITE_CONFIG }} + H3_CLUSTER: ${{ inputs.cluster }} H3_SOURCE_SHA: ${{ github.sha }} H3_REPOSITORY: ${{ github.repository }} H3_RUN_ID: ${{ github.run_id }} @@ -143,9 +175,11 @@ jobs: python3 - <<'PY' import json, os, sys sys.path.insert(0, 'experimental/video-generation') - from ci import validate_config + from ci import DEFAULT_SITE, validate_config with open(os.environ['H3_SITE_CONFIG']) as stream: config = validate_config(json.load(stream)) + if config.get('site', DEFAULT_SITE)['cluster'] != os.environ['H3_CLUSTER']: + raise ValueError('Prepared site does not match the selected CI hardware cluster') with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: stream.write('mode=' + config['mode'] + '\n') PY diff --git a/experimental/video-generation/README.md b/experimental/video-generation/README.md index 043f6c881e..715e9a542f 100644 --- a/experimental/video-generation/README.md +++ b/experimental/video-generation/README.md @@ -3,7 +3,7 @@ **English** | [中文](README_zh.md) This experimental lane runs the existing H3 supervisor inside InferenceX CI on -SemiAnalysis H200 resources. Its first target is a bounded same-build smoke: +prepared SemiAnalysis NVIDIA resources. Its first target is a bounded same-build smoke: original generated MP4s, full video/audio validation, measured requests, and verified cleanup. It does not publish a native InferenceX database/UI result. Successful executions also publish a [versioned frontend result contract](RESULTS.md) @@ -29,7 +29,9 @@ record, entry-only script and SHA256, container Python, frozen supervisor spec and SHA256, resource limits, task identity, and optional prior allocation receipts. The spec must record actual compute/model-use approval. Selecting the manual H3 route requests only that configured, reviewed workload; dispatch accepts no shell -command, model path, arbitrary config contents, or alternate provider. +command, model path, arbitrary config contents, or alternate provider. The +optional `h3-site-config` dispatch input selects an existing reviewed JSON file; +`h3-cluster` must match its declared site before any allocation. The runner requires Python 3.11+, Git, the Slurm tools, and access to the declared shared paths. PyAV/NumPy and the pinned H3 runtime/model must already be prepared @@ -54,12 +56,16 @@ locks and telemetry continue to use the assigned UUIDs. The adapter recovers task-owned allocation receipts before allocating. Imported receipts must match task identity, Unix ownership, and the scheduler's exact -allocation identity; ambiguous intent blocks another submission. The fixed site -is `main` / `sa-shared`. A new exclusive allocation reserves eight GPUs; +allocation identity; ambiguous intent blocks another submission. The default H200 site +is `main` / `sa-shared`. An explicit `site` records the cluster, partition, account, +and expected GPU model. Currently admitted clusters are `h200-dgxc`, `h100-dgxc`, +and `b200-nscale`; admission is implementation support, not a completed hardware run. +`resources.allocated_gpus` records the full allocation separately from participating +`resources.gpus`; set it to eight on whole-node H100. A paired allocation reserves eight GPUs; the example step selects four GPUs, 32 CPUs, and 1 TiB of host memory. The pinned four-rank loader exceeded 256 GiB during CPU weight staging; 1 TiB is a tested working allowance, not a measured minimum. Charge reserved capacity. -`resources.minutes` is the total allocation cap, at most 90 minutes. The step +`resources.minutes` is the total allocation cap, at most 240 minutes. The step reserves five minutes for outer cleanup, and the supervisor plus ten minutes must fit the allocation. For example: 90-minute allocation, 85-minute step, 75-minute supervisor. Reused allocations need enough remaining time. Preserve @@ -153,11 +159,11 @@ Serving runs require an uncalibrated policy. CPU fixtures test the harness; they do not establish H3 concurrency support or hardware performance. Set the reviewed site configuration to `"mode": "serving-smoke"` for the bounded -C1/C2/C4 matrix. Its plan must contain exactly four measured requests, plus -explicit warmups. It boots the baseline runtime once per cell in one allocation, -runs twelve measured requests total, and stops after a failed cell. Allocation -GPU count equals the requested count; ordinary paired smoke keeps its existing -allocation behavior. `serving-smoke.json`, `gpu/cN/` and `report/index.html` retain +C1/C2/C4 matrix. Its plan contains 4–200 measured requests per cell, plus +explicit warmups. It boots the baseline runtime once per cell in one allocation +and stops after a failed cell. Allocation GPU count defaults to the participating +count; `resources.allocated_gpus` declares a larger required allocation explicitly. +Ordinary paired smoke keeps its existing allocation behavior. `serving-smoke.json`, `gpu/cN/` and `report/index.html` retain the matrix, original request/media/telemetry evidence and playable report. An interrupted attempt is counted separately from an unstarted request. This mode skips the paired frontend export and cannot claim regression acceptance. @@ -202,3 +208,15 @@ bash -n runtime-entry.example.sh [Test H3 Video](../../.github/workflows/test-h3-video.yml) runs these CPU checks and workflow linting on relevant changes. They make no model, scheduler, or GPU calls. Real CI execution and artifact inspection are separate acceptance evidence. + +## Cross-hardware serving matrices + +The existing `serving-smoke` mode accepts 4–200 measured requests per concurrency +from `plan.cases × plan.repetitions`; concurrency remains 1, 2, and 4. Twenty per +cell produces sixty measured requests plus three separate warmups when +`warmup_runs: 1`. Failures and unstarted requests remain in the declared denominator. +Freeze identical model files, prompts/seeds, video settings, and quality requirements +across sites. Record different runtime builds and deployment topology explicitly. +Small-sample percentiles are preliminary; this closed-loop sweep does not establish +sustainable open-loop arrival capacity. AMD runtime/device admission is not yet +implemented. Missing sites and measurements must not be represented by fixture data. diff --git a/experimental/video-generation/README_zh.md b/experimental/video-generation/README_zh.md index a44fe887b1..a7b2e9db8b 100644 --- a/experimental/video-generation/README_zh.md +++ b/experimental/video-generation/README_zh.md @@ -46,12 +46,12 @@ launcher 当作进入脚本。 设备枚举不一致时启动失败；归属锁和遥测仍使用分配的 UUID。 adapter 在申请资源前恢复本任务的分配收据。导入的收据必须匹配任务标识、Unix -所有者和调度器中的精确分配身份；提交结果不明确时禁止重复申请。固定站点是 -`main` / `sa-shared`。新建独占分配预留八张 GPU；示例 step 使用四张 +所有者和调度器中的精确分配身份；提交结果不明确时禁止重复申请。默认 H200 站点是 +`main` / `sa-shared`，其他站点通过 `site` 明确记录，见下文。新建独占分配预留八张 GPU；示例 step 使用四张 GPU、32 个 CPU 和 1 TiB 主机内存。固定版本的四 rank 加载器在 CPU 暂存权重时 超过了 256 GiB；1 TiB 是实际运行验证过的额度，并非测得的最低需求。预算按预留容量计算。 -`resources.minutes` 是整个分配的时间上限，最多 90 分钟。step 为外层清理 +`resources.minutes` 是整个分配的时间上限，最多 240 分钟。step 为外层清理 预留五分钟，supervisor 的上限加十分钟必须不超过分配上限。例如：分配 90 分钟、step 85 分钟、supervisor 75 分钟。复用分配必须有足够剩余时间。 保留准备好的 rootfs，仅清理属于本次任务的进程和 step，仅释放本次执行拥有的 @@ -164,3 +164,20 @@ bash -n runtime-entry.example.sh artifact 检查是独立的验收证据。 编译缓存保留在持久化存储中，不上传为测量证据。 + +## 跨硬件服务测量 + +现有 `serving-smoke` 模式支持每档 4–200 条测量请求，数量由 +`plan.cases × plan.repetitions` 决定；并发档位保持 1、2、4。每档 20 条 +产生 60 条测量请求，`warmup_runs: 1` 时另有 3 条独立预热。失败和未启动 +请求保留在预先确定的分母中。跨硬件固定相同模型文件、提示词/种子、视频规格 +和质量要求，明确记录运行时构建与部署拓扑差异。小样本分位数属于初步结果， +闭环并发扫描不能证明持续开放到达负载下的服务容量。 + +默认配置保持 H200 的 `main` / `sa-shared`。可选 `site` 明确记录 `cluster`、 +`partition`、`account` 和 `gpu_model`；当前允许 `h200-dgxc`、`h100-dgxc`、 +`b200-nscale`。允许配置不等于实测通过。通过 `h3-cluster` 选择站点， +`h3-site-config` 指向 runner 上已准备并审核的 JSON 文件；二者必须匹配。 +`resources.allocated_gpus` 单独记录全部分配卡数，`resources.gpus` 记录实际参与卡数； +H100 整节点分配需记录 8 张卡。总分配上限提高至 240 分钟，保留原有清理余量。 +AMD 的运行时与设备接入尚未实现，不可用的硬件或指标不能用 fixture 数据代替。 diff --git a/experimental/video-generation/ci.py b/experimental/video-generation/ci.py index dcbdea3009..1ab802ba24 100644 --- a/experimental/video-generation/ci.py +++ b/experimental/video-generation/ci.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""InferenceX H200 Slurm adapter for a prepared, trusted H3 runtime. +"""InferenceX Slurm adapter for a prepared, trusted H3 runtime. No SSH, image import, dependency installation, or model download. The submit host and compute node share workspace.host, mounted at workspace.container by @@ -28,6 +28,8 @@ PARTITION = "main" ACCOUNT = "sa-shared" +DEFAULT_SITE = {"cluster": "h200-dgxc", "partition": PARTITION, "account": ACCOUNT, "gpu_model": "H200"} +NVIDIA_CLUSTERS = {"h100-dgxc": "H100", "h200-dgxc": "H200", "b200-nscale": "B200"} NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,79}") SHA = re.compile(r"[0-9a-f]{64}") IDENTITY = ("JobId", "JobName", "Comment", "WorkDir", "Account", "Partition", "UserId") @@ -84,9 +86,15 @@ def host_path(config: dict, container_path: str) -> Path: def validate_config(config: dict) -> dict: - need(set(config) == {"schema_version", "task_id", "workspace", "runtime", "spec", "resources", "allocation_receipts", "mode"}, "Unknown or missing site configuration fields") + required = {"schema_version", "task_id", "workspace", "runtime", "spec", "resources", "allocation_receipts", "mode"} + need(required <= set(config) <= required | {"site"}, "Unknown or missing site configuration fields") + site = config.get("site", DEFAULT_SITE) + need(isinstance(site, dict) and set(site) == set(DEFAULT_SITE), "Invalid site fields") + need(site["cluster"] in NVIDIA_CLUSTERS and site["gpu_model"] == NVIDIA_CLUSTERS[site["cluster"]], "Unsupported cluster or GPU model") + need(all(isinstance(site[key], str) and NAME.fullmatch(site[key]) for key in ("account", "partition")), "Explicit scheduler account and partition required") need(config["schema_version"] == 1 and NAME.fullmatch(config["task_id"]), "Invalid schema_version/task_id") need(config["mode"] in {"smoke", "regression", "serving-smoke"}, "mode must be smoke, regression or serving-smoke") + need(site["cluster"] == "h200-dgxc" or config["mode"] == "serving-smoke", "Cross-hardware sites currently require serving-smoke; paired export remains H200-only") need(set(config["workspace"]) == {"host", "container"}, "Invalid workspace mapping") for value in config["workspace"].values(): path = absolute(value) @@ -99,15 +107,23 @@ def validate_config(config: dict) -> dict: need(set(config["spec"]) == {"path", "sha256"} and SHA.fullmatch(config["spec"]["sha256"]), "Pinned prepared spec required") absolute(config["spec"]["path"]) resources = config["resources"] - need(set(resources) == {"gpus", "cpus", "memory_gb", "minutes"}, "Invalid resource request") - for key, low, high in (("gpus", 1, 8), ("cpus", 1, 128), ("memory_gb", 1, 1400), ("minutes", 10, 90)): - need(type(resources[key]) is int and low <= resources[key] <= high, "Resource outside bounded H200 budget: " + key) + required_resources = {"gpus", "cpus", "memory_gb", "minutes"} + need(required_resources <= set(resources) <= required_resources | {"allocated_gpus"}, "Invalid resource request") + for key, low, high in (("gpus", 1, 8), ("cpus", 1, 128), ("memory_gb", 1, 1400), ("minutes", 10, 240)): + need(type(resources[key]) is int and low <= resources[key] <= high, "Resource outside bounded GPU budget: " + key) + reserved = allocation_gpus(config) + need(type(reserved) is int and resources["gpus"] <= reserved <= 8, "Allocated GPU budget must cover participating GPUs") + need(config["mode"] == "serving-smoke" or reserved == 8, "Paired measurements require a full eight-GPU allocation") need(isinstance(config["allocation_receipts"], list), "allocation_receipts must be a list") for path in config["allocation_receipts"]: absolute(path) return config +def allocation_gpus(config: dict) -> int: + return config["resources"].get("allocated_gpus", config["resources"]["gpus"] if config["mode"] == "serving-smoke" else 8) + + def environment() -> dict[str, str]: # Slurm defaults inherited from the runner must not change this request. # The payload receives an explicit environment allowlist at the srun edge. @@ -136,7 +152,8 @@ def verify_identity(receipt: dict, record: dict, task_id: str) -> None: expected = receipt["identity"] need(set(expected) == set(IDENTITY), "Incomplete allocation ownership receipt") need(all(record.get(key) == expected[key] for key in IDENTITY), "Slurm allocation identity differs from receipt") - need(record["Account"] == ACCOUNT and record["Partition"] == PARTITION, "Allocation is not in the SemiAnalysis H200 pool") + site = receipt.get("site", DEFAULT_SITE) + need(record["Account"] == site["account"] and record["Partition"] == site["partition"], "Allocation differs from its receipted scheduler pool") need(re.fullmatch(r"[^()]+\(" + str(os.getuid()) + r"\)", record["UserId"]), "Allocation Unix owner differs") @@ -175,6 +192,9 @@ def recover(config: dict, result_root: Path, *, node: str | None = None) -> dict continue record = job_record(job) verify_identity(receipt, record, config["task_id"]) + if receipt.get("site", DEFAULT_SITE) != config.get("site", DEFAULT_SITE): + reasons.append({"job_id": job, "reason": "allocation belongs to a different hardware site"}) + continue state = record["JobState"] if state in TERMINAL: reasons.append({"job_id": job, "reason": state}) @@ -203,11 +223,15 @@ def allocate(config: dict, run_dir: Path, *, node: str | None = None) -> dict: need(NAME.fullmatch(job_name), "Invalid runner/job name") comment = "h3:" + nonce request = config["resources"] + site = config.get("site", DEFAULT_SITE) intent = {"task_id": config["task_id"], "job_name": job_name, "comment": comment, "work_dir": str(run_dir), "user_id": os.getuid(), "created_at": now()} write(run_dir / "allocation-intent.json", intent) - placement = ["--gres=gpu:" + str(request["gpus"])] if config["mode"] == "serving-smoke" else ["--exclusive", "--gres=gpu:8"] - argv = ["salloc", "--no-shell", "--no-bell", "--partition=" + PARTITION, "--account=" + ACCOUNT, + reserved = allocation_gpus(config) + placement = ["--gres=gpu:" + str(reserved)] + if reserved == 8: + placement.insert(0, "--exclusive") + argv = ["salloc", "--no-shell", "--no-bell", "--partition=" + site["partition"], "--account=" + site["account"], "--nodes=1", "--ntasks=1", *placement, "--cpus-per-task=" + str(request["cpus"]), "--mem=" + str(request["memory_gb"]) + "G", "--time=" + str(request["minutes"]), "--immediate=30", @@ -223,9 +247,9 @@ def allocate(config: dict, run_dir: Path, *, node: str | None = None) -> dict: job = granted[0] # Save the expected identity before querying, so a lost query can be recovered. user = command(["id", "-un"]).strip() - receipt = {"task_id": config["task_id"], "created_at": now(), "identity": { + receipt = {"task_id": config["task_id"], "created_at": now(), "site": site, "identity": { "JobId": job, "JobName": job_name, "Comment": comment, "WorkDir": str(run_dir), - "Account": ACCOUNT, "Partition": PARTITION, "UserId": f"{user}({os.getuid()})"}} + "Account": site["account"], "Partition": site["partition"], "UserId": f"{user}({os.getuid()})"}} write(run_dir / "allocation.json", receipt) need(result.returncode == 0, "Slurm returned a failure after granting an allocation; reconcile receipt") return receipt @@ -400,9 +424,9 @@ def launch(config: dict, output: Path) -> int: results.mkdir(parents=True, exist_ok=True) control.mkdir(parents=True, exist_ok=True) run_dir = results / f"github-{run_id}-{attempt}" - reserved_gpus = config["resources"]["gpus"] if config["mode"] == "serving-smoke" else 8 + reserved_gpus = allocation_gpus(config) state = {"schema_version": 1, "task_id": config["task_id"], "run_id": run_id, "run_attempt": attempt, - "source_sha": sha, "started_at": now(), "phase": "preparing", "mode": config["mode"], + "source_sha": sha, "started_at": now(), "phase": "preparing", "mode": config["mode"], "site": config.get("site", DEFAULT_SITE), "ci_accepted": False, "release_qualified": False, "persistent_output": str(run_dir), "excluded_cache_paths": list(CACHE_PATHS), "ci": {"repository": os.environ.get("GITHUB_REPOSITORY"), @@ -486,7 +510,8 @@ def launch(config: dict, output: Path) -> int: write(run_dir / "manifest.json", {"schema_version": 1, "task_id": config["task_id"], "git_commit": sha, "ci": state["ci"], "run_id": run_id, "run_attempt": attempt, "slurm_allocation": receipt, "runtime": config["runtime"], "prepared_spec": config["spec"], - "workload_plan": spec.get("plan"), "mode": config["mode"], "resources": state["resources"], "exit_code": code, + "workload_plan": spec.get("plan"), "mode": config["mode"], "site": config.get("site", DEFAULT_SITE), + "resources": state["resources"], "exit_code": code, "evidence": {path: digest(run_dir / path) for path in links if (run_dir / path).is_file()}, "artifact_checksums": "SHA256SUMS", "excluded_persistent_caches": list(CACHE_PATHS)}) collect(run_dir, output) @@ -505,7 +530,7 @@ def enter(run_dir: Path) -> None: need(digest(config["runtime"]["entry"]) == config["runtime"]["entry_sha256"], "Entry changed on compute node") argv = ["/bin/bash", config["runtime"]["entry"], config["runtime"]["python"], str(mapped(config, Path(__file__).parent) / "ci.py"), "--inside", str(mapped(config, run_dir))] - os.execv(argv[0], argv) + os.execve(argv[0], argv, {**os.environ, "H3_EXPECTED_GPU_MODEL": config.get("site", DEFAULT_SITE)["gpu_model"]}) def workload_complete(verified: dict) -> bool: diff --git a/experimental/video-generation/evaluator/mvp_power.py b/experimental/video-generation/evaluator/mvp_power.py index 9926367dd6..3cc02f6b0c 100644 --- a/experimental/video-generation/evaluator/mvp_power.py +++ b/experimental/video-generation/evaluator/mvp_power.py @@ -332,7 +332,7 @@ def analyze_power(role: dict, run: dict, samples: list[dict], events: list[dict] "submit_to_observed_provider_terminal; excludes_client_download_and_decode"), "peak": "maximum_observed_sensor_sample_in_window; not_instantaneous_electrical_peak", "energy_per_valid_clip": "sum_generation_energy_including_failed_or_invalid_completed_attempts_divided_by_technically_valid_clips", - "sensor": "nvidia-smi power.draw; H200 NVML trailing_one_second_average; phase_edges_have_sensor_averaging_uncertainty", + "sensor": "nvidia-smi power.draw; hardware_sensor_averaging_not_calibrated; phase_edges_have_sensor_averaging_uncertainty", "clock_agreement_limit_seconds": _CLOCK_TOLERANCE_SECONDS, "legacy_journal_agreement_limit_seconds": _LEGACY_JOURNAL_TOLERANCE_SECONDS}, "clock_alignment": {"utc_minus_monotonic_seconds": offset, "observed_offset_spread_seconds": spread}, diff --git a/experimental/video-generation/evaluator/mvp_serving_smoke.py b/experimental/video-generation/evaluator/mvp_serving_smoke.py index 7a08483b9f..d877e2aebe 100644 --- a/experimental/video-generation/evaluator/mvp_serving_smoke.py +++ b/experimental/video-generation/evaluator/mvp_serving_smoke.py @@ -20,8 +20,9 @@ def validate_spec(spec: dict) -> dict: frozen = gpu.validate_gpu_job(spec) - if not frozen.get("serving") or len(frozen["plan"]["cases"]) * frozen["plan"]["repetitions"] != 4: - raise ValueError("serving smoke requires exactly four measured requests per configuration") + count = len(frozen["plan"]["cases"]) * frozen["plan"]["repetitions"] + if not frozen.get("serving") or not 4 <= count <= 200: + raise ValueError("serving matrix requires 4–200 measured requests per configuration") return frozen @@ -48,9 +49,9 @@ def _report(root: Path, matrix: dict) -> None: (report / "index.html").write_text( '' f'H3 serving smoke

H3 serving smoke

' - '

One hardware configuration; four measured requests at each concurrency. Warmups are separate. ' + f'

One hardware configuration; {matrix["requests_per_configuration"]} measured requests at each concurrency. Warmups are separate. ' 'Latency is submit → downloaded media for technically valid clips. Throughput is valid clips / delivery wall seconds. ' - 'Four samples do not establish P90/P95 or sustainable capacity. Failed and unstarted requests remain counted.

' + 'Percentiles from small samples are preliminary and do not establish sustainable capacity. Failed and unstarted requests remain counted.

' '

Download summary and raw-evidence links

' '
' '' @@ -64,13 +65,15 @@ def run_matrix(spec: dict, root: Path) -> dict: from .mvp_power import analyze_power spec = validate_spec(spec) + count = len(spec["plan"]["cases"]) * spec["plan"]["repetitions"] deadline = time.monotonic() + spec["limits"]["job_seconds"] matrix = {"schema_version": "1.0.0", "bundle_type": "h3_serving_smoke_matrix", "status": "running", - "started_at": gpu._now(), "plan": spec["plan"], "runtime": spec["baseline"], "gpu_uuids": spec["gpu_uuids"], - "scheduled": 12, "warmup_per_configuration": spec["plan"]["warmup_runs"], + "started_at": gpu._now(), "plan": spec["plan"], "runtime": spec["baseline"], "server": spec["server"], "gpu_uuids": spec["gpu_uuids"], + "scheduled": count * len(CONCURRENCIES), "requests_per_configuration": count, + "warmup_per_configuration": spec["plan"]["warmup_runs"], "ci_accepted": False, "release_qualified": False, "cells": [{"concurrency": concurrency, "status": "not_started", "verified": False, - "completion": {"scheduled": 4, "attempted": 0, "completed": 0, "valid": 0, "failed": 4, "not_started": 4, "unfinished": 0}} + "completion": {"scheduled": count, "attempted": 0, "completed": 0, "valid": 0, "failed": count, "not_started": count, "unfinished": 0}} for concurrency in CONCURRENCIES]} path = root / "serving-smoke.json" gpu._write(path, matrix) @@ -93,19 +96,19 @@ def run_matrix(spec: dict, root: Path) -> dict: if run_path.is_file(): raw = gpu._read(run_path) cell["run"] = {"path": run_path.relative_to(root).as_posix(), "sha256": gpu._hash(run_path)} - summary = _summary(raw["records"], 4, raw["measurement"]["wall_seconds"]) + summary = _summary(raw["records"], count, raw["measurement"]["wall_seconds"]) cell["completion"] = {key: summary[key] for key in ("scheduled", "completed", "valid", "failed")} finished = {r["slot_id"] for r in raw["records"] if r["phase"] == "measurement" and r["attempted"]} journal = directory / "baseline/events.jsonl" events = [json.loads(line) for line in journal.read_text().splitlines()] if journal.exists() else [] started = finished | {event["slot_id"] for event in events if event["event"] == "attempt_started" and event["slot_id"].startswith("measurement-")} - cell["completion"].update(attempted=len(started), not_started=4-len(started), unfinished=len(started-finished)) + cell["completion"].update(attempted=len(started), not_started=count-len(started), unfinished=len(started-finished)) gpu._write(path, matrix) verified = verify_measurement_job(directory, deadline=deadline, require_success=True, serving_smoke=True) run, role = verified["runs"]["baseline"], receipt["roles"]["baseline"] cell.update(status="complete", verified=True, metrics={ "client_ready_p50_seconds": run["serving"]["client_ready_latency_seconds"]["p50"], - "valid_clips_per_second": _summary(run["records"], 4, run["measurement"]["wall_seconds"])["valid_clips_per_second"], + "valid_clips_per_second": _summary(run["records"], count, run["measurement"]["wall_seconds"])["valid_clips_per_second"], "serving": run["serving"], "measurement": run["measurement"], }) samples = [json.loads(line) for line in (directory / role["telemetry_path"]).read_text().splitlines()] diff --git a/experimental/video-generation/runtime-entry.example.sh b/experimental/video-generation/runtime-entry.example.sh index e8421a13a7..2c52d35f0a 100644 --- a/experimental/video-generation/runtime-entry.example.sh +++ b/experimental/video-generation/runtime-entry.example.sh @@ -34,7 +34,7 @@ for part in value.split(','): elif 0 <= bounds[0] <= bounds[1] < 8: ids.extend(range(bounds[0], bounds[1] + 1)) else: - raise SystemExit('GPU range outside the single H200 node') + raise SystemExit('GPU range outside the single eight-GPU node') if len(ids) != len(set(ids)) or not ids or any(i >= 8 for i in ids): raise SystemExit('Invalid global GPU assignment') devices = {} @@ -52,10 +52,13 @@ PY ) h3_gpu_rows=$(nvidia-smi --id="$h3_gpu_uuids" --query-gpu=uuid,name --format=csv,noheader) H3_ASSIGNED_GPU_UUIDS=$(python3 - "$h3_gpu_rows" "$h3_gpu_uuids" <<'PY' -import csv, re, sys +import csv, os, re, sys rows = list(csv.reader(sys.argv[1].splitlines())) -if not rows or any(len(row) != 2 or 'H200' not in row[1] or not re.fullmatch(r'GPU-[0-9a-fA-F-]{36}', row[0].strip()) for row in rows): - raise SystemExit('Assigned hardware is not a physical H200 GPU set') +expected = os.environ.get('H3_EXPECTED_GPU_MODEL', 'H200') +if expected not in {'H100', 'H200', 'B200'}: + raise SystemExit('Unsupported expected NVIDIA GPU model') +if not rows or any(len(row) != 2 or not re.search(r'\b' + expected + r'\b', row[1]) or not re.fullmatch(r'GPU-[0-9a-fA-F-]{36}', row[0].strip()) for row in rows): + raise SystemExit('Assigned hardware does not match expected physical ' + expected + ' GPUs') observed = [row[0].strip() for row in rows] if sorted(observed) != sorted(sys.argv[2].split(',')): raise SystemExit('NVIDIA query differs from assigned physical UUIDs') diff --git a/experimental/video-generation/tests/test_ci.py b/experimental/video-generation/tests/test_ci.py index c8420d13b3..aff9769069 100644 --- a/experimental/video-generation/tests/test_ci.py +++ b/experimental/video-generation/tests/test_ci.py @@ -38,6 +38,44 @@ def save_receipt(root, receipt): return path +def test_h100_site_keeps_full_allocation_separate_from_participating_gpus(tmp_path, monkeypatch): + monkeypatch.setenv("RUNNER_NAME", "h3-test-runner") + cfg = config(tmp_path) + cfg.update(mode="serving-smoke", site={"cluster": "h100-dgxc", "partition": "hpc-gpu-1", "account": "customer", "gpu_model": "H100"}) + cfg["resources"].update(gpus=4, allocated_gpus=8) + ci.validate_config(cfg) + commands = [] + def run(argv, **kwargs): + commands.append(argv) + return SimpleNamespace(stdout="salloc: Granted job allocation 123", stderr="", returncode=0) + monkeypatch.setattr(ci.subprocess, "run", run) + monkeypatch.setattr(ci, "command", lambda argv: "tester") + receipt = ci.allocate(cfg, tmp_path) + assert "--partition=hpc-gpu-1" in commands[0] and "--account=customer" in commands[0] + assert "--exclusive" in commands[0] and "--gres=gpu:8" in commands[0] + assert receipt["site"] == cfg["site"] + _, record = allocation(tmp_path) + record.update(receipt["identity"]) + ci.verify_identity(receipt, record, cfg["task_id"]) + step = ci.step_argv(cfg, receipt, record, tmp_path, tmp_path) + assert "--gpus-per-task=4" in step + record["Account"] = "other" + with pytest.raises(ValueError, match="identity differs"): + ci.verify_identity(receipt, record, cfg["task_id"]) + + +@pytest.mark.parametrize("change", [ + {"site": {"cluster": "h100-dgxc", "partition": "hpc-gpu-1", "account": "customer", "gpu_model": "H200"}}, + {"site": {"cluster": "unknown", "partition": "main", "account": "customer", "gpu_model": "H200"}}, + {"resources": {"gpus": 4, "allocated_gpus": 2, "cpus": 32, "memory_gb": 512, "minutes": 90}}, +]) +def test_invalid_hardware_or_allocation_budget_is_rejected(tmp_path, change): + cfg = config(tmp_path) + cfg.update(change) + with pytest.raises(ValueError): + ci.validate_config(cfg) + + @pytest.mark.parametrize("mode", ["smoke", "serving-smoke"]) def test_allocation_submits_from_receipted_work_directory(tmp_path, monkeypatch, mode): run_dir = tmp_path / "results" diff --git a/experimental/video-generation/tests/test_mvp_serving_smoke.py b/experimental/video-generation/tests/test_mvp_serving_smoke.py index 2cefef538f..ca6c49b78d 100644 --- a/experimental/video-generation/tests/test_mvp_serving_smoke.py +++ b/experimental/video-generation/tests/test_mvp_serving_smoke.py @@ -38,9 +38,10 @@ def test_single_runtime_smoke_cannot_be_accepted_as_paired_evidence(spec, tmp_pa @pytest.mark.parametrize("fail_second", [False, True]) -def test_matrix_preserves_twelve_requests_without_doubling_roles(spec, tmp_path, monkeypatch, fail_second): +@pytest.mark.parametrize("requests", [4, 20]) +def test_matrix_preserves_scheduled_requests_without_doubling_roles(spec, tmp_path, monkeypatch, fail_second, requests): spec["plan"]["cases"] = spec["plan"]["cases"][:1] - spec["plan"]["repetitions"] = 4 + spec["plan"]["repetitions"] = requests spec["serving"] = {"concurrency": 1} submitted = [] def execute(current, directory, *, serving_smoke): @@ -51,15 +52,16 @@ def execute(current, directory, *, serving_smoke): return saved_single(current, directory) monkeypatch.setattr(gpu, "run_gpu_job", execute) result = smoke.run_matrix(spec, tmp_path) - assert result["completion"]["scheduled"] == 12 + assert result["completion"]["scheduled"] == requests * 3 + assert result["requests_per_configuration"] == requests assert [s["serving"]["concurrency"] for s in submitted] == ([1, 2] if fail_second else [1, 2, 4]) assert all(s["plan"] == spec["plan"] for s in submitted) - assert result["completion"]["valid"] == (4 if fail_second else 12) - assert result["completion"]["not_started"] == (8 if fail_second else 0) + assert result["completion"]["valid"] == requests * (1 if fail_second else 3) + assert result["completion"]["not_started"] == (requests * 2 if fail_second else 0) assert result["status"] == ("failed" if fail_second else "complete") assert not result["ci_accepted"] report = (tmp_path / "report/index.html").read_text() - assert report.count("
ConcurrencyStatusScheduledAttemptedValidFailedNot startedDelivery median (s)Valid clips/s