diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 1fb6985aea..8f0e2a0b32 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -7,6 +7,21 @@ permissions: on: workflow_dispatch: inputs: + h3-video: + description: "Run only the configured H3 smoke; manual dispatch, no LLM sweep" + required: false + type: boolean + default: false + h3-reuse-run-ids: + description: "Reprocess one or two accepted H3 CI runs (comma-separated); no new H3 generation" + required: false + type: string + default: "" + h3-inventory-run-id: + description: "Reuse a successful H3 hardware inventory; export on CPU without allocating GPUs" + required: false + type: string + default: "" generate-cli-command: description: "Command passed to generate matrix script" required: false @@ -102,6 +117,21 @@ on: default: "[]" workflow_call: inputs: + h3-video: + description: "Run only the configured H3 smoke; manual dispatch, no LLM sweep" + required: false + type: boolean + default: false + h3-reuse-run-ids: + description: "Reprocess one or two accepted H3 CI runs (comma-separated); no new H3 generation" + required: false + type: string + default: "" + h3-inventory-run-id: + description: "Reuse a successful H3 hardware inventory; export on CPU without allocating GPUs" + required: false + type: string + default: "" generate-cli-command: description: "Command passed to generate matrix script" required: false @@ -197,7 +227,18 @@ on: default: "[]" jobs: + h3-video: + if: ${{ inputs.h3-video }} + permissions: + contents: read + actions: read + uses: ./.github/workflows/h3-video.yml + with: + source-run-ids: ${{ inputs.h3-reuse-run-ids }} + inventory-run-id: ${{ inputs.h3-inventory-run-id }} + get-jobs: + if: ${{ !inputs.h3-video }} runs-on: ubuntu-latest outputs: single-node-config: ${{ steps.get-jobs.outputs.single-node-config }} @@ -753,7 +794,7 @@ jobs: calc-success-rate: needs: [collect-results, collect-evals] - if: ${{ always() }} + if: ${{ always() && !inputs.h3-video }} runs-on: ubuntu-latest env: diff --git a/.github/workflows/h3-video.yml b/.github/workflows/h3-video.yml new file mode 100644 index 0000000000..f0d30bfef4 --- /dev/null +++ b/.github/workflows/h3-video.yml @@ -0,0 +1,212 @@ +name: H3 Video Smoke +run-name: H3 video smoke - ${{ github.ref_name }} + +on: + workflow_dispatch: + inputs: + inventory-run-id: + description: "Reuse a successful hardware inventory for CPU-only export" + required: false + type: string + default: "" + source-run-ids: + description: "One or two accepted H3 run IDs to reprocess; empty runs new generation" + required: false + type: string + default: "" + workflow_call: + inputs: + inventory-run-id: + required: false + type: string + default: "" + source-run-ids: + required: false + type: string + default: "" + +permissions: + contents: read + actions: read + +concurrency: + group: h3-video-${{ github.repository }} + cancel-in-progress: false + +jobs: + prepare: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + H3_SOURCE_SHA: ${{ github.sha }} + outputs: + priority: ${{ steps.queue.outputs.priority }} + queue-token: ${{ steps.queue.outputs.queue-token }} + steps: + - name: Authorize manual repository execution + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + if (context.eventName !== 'workflow_dispatch' || + context.repo.owner !== 'SemiAnalysisAI' || context.repo.repo !== 'InferenceX' || + !context.ref.startsWith('refs/heads/')) { + throw new Error('H3 smoke requires manual dispatch of an InferenceX repository branch.'); + } + for (const username of new Set([context.actor, process.env.GITHUB_TRIGGERING_ACTOR])) { + const {data} = await github.rest.repos.getCollaboratorPermissionLevel({ + ...context.repo, username, + }); + if (!['write', 'maintain', 'admin'].includes(data.permission)) { + throw new Error(username + ' must have write, maintain, or admin permission.'); + } + } + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - name: Prepare native queue identity + id: queue + env: + H3_SITE_CONFIG: ${{ vars.H3_SITE_CONFIG }} + PRIORITY_ENABLED: ${{ vars.PRIORITY_SCHEDULER_ENABLED }} + NODE_SLOTS_ENABLED: ${{ vars.NODE_SLOT_SCHEDULER_ENABLED }} + H3_SOURCE_RUN_IDS: ${{ inputs.source-run-ids }} + H3_INVENTORY_RUN_ID: ${{ inputs.inventory-run-id }} + run: | + set -euo pipefail + if [[ -n "$H3_INVENTORY_RUN_ID" && ( -z "$H3_SOURCE_RUN_IDS" || ! "$H3_INVENTORY_RUN_ID" =~ ^[1-9][0-9]{0,19}$ ) ]]; then + echo 'Hardware reuse needs one inventory run ID and explicit source executions.' >&2 + exit 1 + fi + [[ "$H3_SITE_CONFIG" = /* ]] || { echo 'Set H3_SITE_CONFIG to the reviewed runner-local JSON path.' >&2; exit 1; } + if [[ -n "$H3_SOURCE_RUN_IDS" && ! "$H3_SOURCE_RUN_IDS" =~ ^[1-9][0-9]{0,19}(,[1-9][0-9]{0,19})?$ ]]; then + echo 'Expected one or two comma-separated source run IDs.' >&2 + exit 1 + fi + if [[ "$PRIORITY_ENABLED" == true && "$NODE_SLOTS_ENABLED" != true ]]; then + echo 'H3 priority scheduling requires node-slot admission for nodes:1.' >&2 + exit 1 + fi + scored=$(printf '%s' '[{"runner":"cluster:h200-dgxc","framework":"sglang","node-count":1}]' | + uv run --no-project --with pyyaml --python 3.12 utils/ci_priority.py) + echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" + python3 - <<'PY' + import hashlib + import os + identity = f"{os.environ['GITHUB_RUN_ID']}:{os.environ['GITHUB_RUN_ATTEMPT']}:h3-video" + token = hashlib.sha256(identity.encode()).hexdigest()[:32] + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f"queue-token={token}\n") + PY + + smoke: + needs: prepare + outputs: + mode: ${{ steps.execute.outputs.mode }} + if: ${{ inputs.inventory-run-id == '' }} + name: p${{ needs.prepare.outputs.priority }} | ${{ inputs.source-run-ids != '' && 'H3 H200 hardware inventory' || 'H3 video H200 smoke' }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format('["self-hosted","cluster:h200-dgxc","nodes:1",{0},{1}]', + toJSON(format('ci-job-{0}-{1}', needs.prepare.outputs.priority, needs.prepare.outputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt))) || + '["cluster:h200-dgxc"]' + ) }} + timeout-minutes: 105 + env: + H3_SITE_CONFIG: ${{ vars.H3_SITE_CONFIG }} + H3_SOURCE_SHA: ${{ github.sha }} + H3_REPOSITORY: ${{ github.repository }} + H3_RUN_ID: ${{ github.run_id }} + H3_RUN_ATTEMPT: ${{ github.run_attempt }} + H3_WORKFLOW_REF: ${{ github.workflow_ref }} + H3_WORKFLOW_SHA: ${{ github.workflow_sha }} + H3_ACTOR: ${{ github.actor }} + H3_TRIGGERING_ACTOR: ${{ github.triggering_actor }} + defaults: + run: + working-directory: h3-video-source-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - name: Checkout exact dispatched source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + path: h3-video-source-${{ github.run_id }}-${{ github.run_attempt }} + persist-credentials: false + - name: Execute configured H3 smoke + id: execute + if: ${{ inputs.source-run-ids == '' }} + run: | + set -euo pipefail + python3 - <<'PY' + import json, os, sys + sys.path.insert(0, 'experimental/video-generation') + from ci import validate_config + with open(os.environ['H3_SITE_CONFIG']) as stream: + config = validate_config(json.load(stream)) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write('mode=' + config['mode'] + '\n') + PY + python3 experimental/video-generation/ci.py \ + --config "$H3_SITE_CONFIG" \ + --output "$RUNNER_TEMP/h3-video-$H3_RUN_ID-$H3_RUN_ATTEMPT" + - name: Inspect current H200 hardware and power configuration + if: ${{ inputs.source-run-ids != '' }} + env: + GH_TOKEN: ${{ github.token }} + H3_SOURCE_RUN_IDS: ${{ inputs.source-run-ids }} + run: | + set -euo pipefail + export PYTHONPATH="$PWD" + python3 experimental/video-generation/inventory_ci.py \ + --config "$H3_SITE_CONFIG" \ + --source-run-ids "$H3_SOURCE_RUN_IDS" \ + --output "$RUNNER_TEMP/h3-video-$H3_RUN_ID-$H3_RUN_ATTEMPT" + - name: Preserve H3 evidence, including incomplete attempts + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.source-run-ids != '' && 'h3-hardware' || 'h3-video' }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/h3-video-${{ github.run_id }}-${{ github.run_attempt }}/ + if-no-files-found: error + compression-level: 0 + retention-days: 14 + + export: + needs: [prepare, smoke] + if: ${{ always() && needs.prepare.result == 'success' && needs.smoke.outputs.mode != 'serving-smoke' && (needs.smoke.result == 'success' || (inputs.inventory-run-id != '' && needs.smoke.result == 'skipped')) }} + name: Verify and export retained H3 results + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + PYTHONPATH: ${{ github.workspace }} + GH_TOKEN: ${{ github.token }} + H3_SOURCE_RUN_IDS: ${{ inputs.source-run-ids || github.run_id }} + H3_REUSE: ${{ inputs.source-run-ids != '' }} + H3_INVENTORY_RUN_ID: ${{ inputs.inventory-run-id || github.run_id }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - name: Verify original execution and publish versioned measurements + run: | + set -euo pipefail + hardware_args=() + if [[ "$H3_REUSE" == true ]]; then hardware_args=(--hardware-run-id "$H3_INVENTORY_RUN_ID"); fi + uv run --no-project --python 3.12 --with 'av==16.1.0' --with 'numpy==2.3.5' \ + python experimental/video-generation/export_ci.py \ + --source-run-ids "$H3_SOURCE_RUN_IDS" "${hardware_args[@]}" \ + --output "$RUNNER_TEMP/h3-results" + - name: Preserve exported results and failure evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: h3-results-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/h3-results/ + if-no-files-found: error + compression-level: 0 + retention-days: 14 diff --git a/.github/workflows/test-h3-video.yml b/.github/workflows/test-h3-video.yml new file mode 100644 index 0000000000..669abb573e --- /dev/null +++ b/.github/workflows/test-h3-video.yml @@ -0,0 +1,47 @@ +name: Test H3 Video + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - 'experimental/video-generation/**' + - 'utils/aggregate_power.py' + - '.github/workflows/h3-video.yml' + - '.github/workflows/test-h3-video.yml' + - '.github/workflows/e2e-tests.yml' + push: + paths: + - 'experimental/video-generation/**' + - 'utils/aggregate_power.py' + - '.github/workflows/h3-video.yml' + - '.github/workflows/test-h3-video.yml' + - '.github/workflows/e2e-tests.yml' + +permissions: + contents: read + +jobs: + test: + if: ${{ github.event.pull_request.draft != true }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - name: Test CPU harness and CI control + working-directory: experimental/video-generation + env: + PYTHONPATH: ${{ github.workspace }} + run: >- + uv run --no-project --python 3.12 + --with 'av==16.1.0' --with 'numpy==2.3.5' + --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' python -m pytest -q + - name: Check entry shell syntax + run: bash -n experimental/video-generation/runtime-entry.example.sh + - name: Check workflow wiring + run: >- + go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 -shellcheck= + .github/workflows/h3-video.yml .github/workflows/test-h3-video.yml + .github/workflows/e2e-tests.yml diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index e42027abb5..788c9a712f 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -257,6 +257,20 @@ Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-nego 6. If the file conflicts with `main`, restore the current `main` version and re-append only this branch's entries. Do not hand-merge reordered history. 7. Parse the file and confirm the generated changelog selection includes the intended keys before requesting a sweep. +### Separately dispatched experimental workflows + +An experimental benchmark outside the LLM master configs can record its change without selecting LLM jobs: + +```yaml +- config-keys: [] + workflow-dispatch: h3-video.yml + description: + - "Add a manually dispatched H3 video benchmark" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX +``` + +`workflow-dispatch` must be a local `.yml` or `.yaml` basename under `.github/workflows/`, and that file must declare a `workflow_dispatch` trigger. It requires empty `config-keys` and rejects active LLM eval, scenario, or `append-only` modifiers. Ordinary entries still require real master-config keys. The changelog processor retains the manual entry in metadata but generates no LLM throughput or eval rows for it; ordinary entries in the same diff keep their normal selection. This field does not dispatch the workflow or establish benchmark success. Run the named workflow explicitly and inspect its artifacts. Replace the `XXX` PR-link placeholder when the PR exists. + ## Stop conditions Stop before dispatching GPU work or claiming the configuration complete when any condition below holds. Obtain the missing fact or fix the source mismatch. Do not guess. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 0ebd17fc68..2e0144cd73 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -257,6 +257,20 @@ python -m pytest utils/matrix_logic/ -v 6. 如果文件与 `main` 冲突，恢复当前 `main` 版本，只重新追加本分支条目。不要手动合并已经重排的历史。 7. 请求 sweep 前解析文件，并确认生成的 changelog 选择包含预期 key。 +### 单独手动派发的实验工作流 + +不属于 LLM master config 的实验基准可以记录变更，而不选择任何 LLM 任务： + +```yaml +- config-keys: [] + workflow-dispatch: h3-video.yml + description: + - "Add a manually dispatched H3 video benchmark" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX +``` + +`workflow-dispatch` 必须是 `.github/workflows/` 下本地 `.yml` 或 `.yaml` 文件的名称，文件必须声明 `workflow_dispatch` 触发器。该模式要求 `config-keys` 为空，不允许启用 LLM eval、scenario 或 `append-only` 选项。普通条目仍须使用真实的 master-config key。changelog 处理器会将手动条目保留在元数据中，但不会为它生成 LLM 吞吐或 eval 任务；同一 diff 中的普通条目仍按原规则选择任务。该字段不会自动派发工作流，也不能证明基准运行成功。请显式运行所列工作流并检查产物。PR 创建后，将链接中的 `XXX` 占位符替换为实际编号。 + ## 停止条件 出现以下任何条件时，在派发 GPU 工作或宣称配置完成前停止。取得缺失事实或修复来源不一致；不要猜测。 diff --git a/experimental/video-generation/README.md b/experimental/video-generation/README.md new file mode 100644 index 0000000000..043f6c881e --- /dev/null +++ b/experimental/video-generation/README.md @@ -0,0 +1,204 @@ +# H3 video CI smoke + +**English** | [中文](README_zh.md) + +This experimental lane runs the existing H3 supervisor inside InferenceX CI on +SemiAnalysis H200 resources. Its first target is a bounded same-build smoke: +original generated MP4s, full video/audio validation, measured requests, and +verified cleanup. It does not publish a native InferenceX database/UI result. +Successful executions also publish a [versioned frontend result contract](RESULTS.md) +with validated GPU power/energy, original measurements and a portable power report. + +The runner supports two frozen 16:9 cells at 1344×768 and 24 FPS: a 4-second +request resolves to 107 frames, while an 8-second request resolves to 192 frames. +These counts follow the pinned H3 runtime's temporal rounding. Freeze a new plan +for a changed prompt or duration; retain earlier runs as their original cells. + +A successful smoke job means the configured measurements and evidence completed. +Its uncalibrated regression decision remains inconclusive and +`ci_accepted: false`. No successful H3 run is established by adding these files +or passing CPU tests. + +## Prepare the existing runtime + +Set repository variable `H3_SITE_CONFIG` to the absolute path of a reviewed JSON +file on the `cluster:h200-dgxc` submission runner. Start from +[site.example.json](site.example.json); its placeholders are not executable. +The configuration binds the persistent workspace, existing rootfs/readiness +record, entry-only script and SHA256, container Python, frozen supervisor spec +and SHA256, resource limits, task identity, and optional prior allocation receipts. +The spec must record actual compute/model-use approval. Selecting the manual H3 +route requests only that configured, reviewed workload; dispatch accepts no shell +command, model path, arbitrary config contents, or alternate provider. + +The runner requires Python 3.11+, Git, the Slurm tools, and access to the declared +shared paths. PyAV/NumPy and the pinned H3 runtime/model must already be prepared +inside the existing runtime. A rootfs-created marker alone is not proof of model +compatibility. The adapter checks saved preparation and input identity before +allocation; it does not install packages, import an image, or create a rootfs. + +Adapt [runtime-entry.example.sh](runtime-entry.example.sh) from the saved working +entry command, then pin its digest. It must enter the existing Enroot rootfs, +map `workspace.host` to `/work`, preserve the Slurm step's GPU/CPU binding and +metadata, forward its command, and propagate its exit code. It translates +`SLURM_STEP_GPUS` global IDs to physical UUIDs, exports +`H3_ASSIGNED_GPU_UUIDS`, and retains the original mask in +`H3_ORIGINAL_CUDA_VISIBLE_DEVICES`. In-container admission checks the actual +driver UUIDs against that assignment. Do not invoke an old allocating launcher +as the entry script. + +The pinned SGLang runtime expects numeric device IDs. Each H3 child receives the +selected devices' observed NVML indices, then verifies their ordered CUDA driver +UUIDs before importing SGLang. An enumeration mismatch fails startup; ownership +locks and telemetry continue to use the assigned UUIDs. + +The adapter recovers task-owned allocation receipts before allocating. Imported +receipts must match task identity, Unix ownership, and the scheduler's exact +allocation identity; ambiguous intent blocks another submission. The fixed site +is `main` / `sa-shared`. A new exclusive allocation reserves eight GPUs; +the example step selects four GPUs, 32 CPUs, and 1 TiB of host memory. The pinned +four-rank loader exceeded 256 GiB during CPU weight staging; 1 TiB is a tested +working allowance, not a measured minimum. Charge reserved capacity. +`resources.minutes` is the total allocation cap, at most 90 minutes. The step +reserves five minutes for outer cleanup, and the supervisor plus ten minutes +must fit the allocation. For example: 90-minute allocation, 85-minute step, +75-minute supervisor. Reused allocations need enough remaining time. Preserve +the prepared rootfs; clean only owned processes/steps and release only allocations +this execution owns. + +## Dispatch through InferenceX + +The first run belongs in CI. For a feature branch, use the already registered +End-to-End Tests workflow, which calls the branch's reusable H3 workflow: + +```bash +gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX \ + --ref feat/h3-video-ci -f h3-video=true -f test-name=h3-first-smoke +``` + +Review the branch before dispatch. The original actor and rerun actor must have +write, maintain, or admin permission. The H3 route checks out `github.sha`: +`--ref` selects its workflow and source; the ordinary LLM `ref` input is not +used. It skips LLM matrix generation, all dependent LLM sweeps/collectors, and +their success-rate calculation. External PR events cannot launch this route. +After the standalone workflow is registered on the default branch, +`h3-video.yml` can also be manually dispatched. + +When priority scheduling is enabled, node-slot scheduling must also be enabled. +The queued job requests exactly `nodes:1` plus its native +`ci-job--` and `ci-attempt-` labels on +`cluster:h200-dgxc`. If priority scheduling is disabled, it follows the +repository's native cluster-label route. GitHub admission and Slurm resource +verification remain distinct. This lane uses native workflow permission and +scheduler admission; it does not add an OIDC service or claim independent +hardware attestation. + +To export accepted H3 evidence using a retained hardware inventory, supply one +or two H3 source run IDs and the inventory run ID: + +```bash +gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX \ + --ref feat/h3-video-ci -f h3-video=true \ + -f h3-reuse-run-ids=34291306687,34293342829 \ + -f h3-inventory-run-id=34297499754 -f test-name=h3-power-export +``` + +This path uses hosted CPU export and skips the native H200 job. The original +actor and rerun actor pass the same authorization checks. Hosted export +independently verifies the accepted H3 executions and the completed inventory +job, including artifact seals, Git/CI/Slurm identities, and the same physical GPU +UUIDs. It makes no new GPU queries or model requests. Keep the source artifacts +within GitHub's retention period. + +Omitting `h3-inventory-run-id` records a new inventory through the native Slurm +route. CI verifies the source commits, original artifacts and persistent Slurm +receipts, then inventories the same node and GPU UUIDs using the existing runtime. +The allocation has a fixed ten-minute cap (at most 1.3333 reserved GPU-hours); +existing task allocations are checked for reuse first. The inventory loads no H3 +model and releases its owned allocation after cleanup. Later power limits cannot +establish historical generation settings. + +The retained [inventory run 34297499754](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754) +([raw inventory artifact](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754/artifacts/10083702100)) +completed Slurm **82290.0** on `worker-10`, observing the original four UUIDs at +`2026-09-09T01:02:44Z`. Recorded NVIDIA H200 PCI device/subsystem IDs +`233510DE` / `18BE10DE` identify H200 SXM, with a manufacturer maximum configurable +TDP of **700 W per GPU**. At this later observation, configured, enforced, default +and maximum limits were all 700 W on all four devices. Historical generation +limits remain unknown. The original inventory profile's unknown classification +is preserved; the exporter classifies its original XML with the new producer +commit, accepting PCI IDs with or without `0x`, without querying hardware again. +See [the retained A/A measurements and their limits](RESULTS.md#observed-aa-evidence). + +## Optional serving load + +Add `"serving": {"concurrency": 2, "delivery_deadline_seconds": 300}` to the +reviewed supervisor spec and update its pinned SHA256 in the site configuration. +This example deadline is operator-selected, not a calibrated acceptance gate. +Concurrency accepts 1–32; omitting `serving` preserves serial regression behavior. +The direct client exposes the same options as `--serving-concurrency` and +`--delivery-deadline-seconds`; without `--execute`, it still only previews. + +Each job measures one concurrency against one supervised endpoint. Workers +submit another request after downloading the previous output; media validation +runs separately. Warmup stays serial and separate. To compare loads, repeat the +same frozen prompt/seed/generation plan and runtime on separately recorded jobs, +including an explicit serving-concurrency-1 control. The default mode does not launch a load sweep. Reuse the existing CI allocation/runtime route and retain every +request outcome; uncertain remote completion stops new submissions. + +This measures closed-loop delivery throughput, not a fixed arrival rate or +sustainable serving capacity. Server queue/execution timestamps, actual batch +sizes, multi-replica layouts and full deployment cost remain unavailable. +Serving runs require an uncalibrated policy. CPU fixtures test the harness; +they do not establish H3 concurrency support or hardware performance. + +Set the reviewed site configuration to `"mode": "serving-smoke"` for the bounded +C1/C2/C4 matrix. Its plan must contain exactly four measured requests, plus +explicit warmups. It boots the baseline runtime once per cell in one allocation, +runs twelve measured requests total, and stops after a failed cell. Allocation +GPU count equals the requested count; ordinary paired smoke keeps its existing +allocation behavior. `serving-smoke.json`, `gpu/cN/` and `report/index.html` retain +the matrix, original request/media/telemetry evidence and playable report. An +interrupted attempt is counted separately from an unstarted request. This mode +skips the paired frontend export and cannot claim regression acceptance. + +## Results and local checks + +Every attempt uploads `h3-video--` for 14 days, with compression +disabled for media. The upload runs even after failure and includes the complete +adapter evidence: receipts, original MP4s, telemetry, checksums, and the portable +report when available. Compiler cache subtrees stay on persistent storage and +are excluded from the upload. Missing reports or media remain missing; fixtures never +replace them. Persistent source evidence remains at the configured workspace. +Retain/download the complete artifact before GitHub retention expires. + +The hosted export job publishes `h3-results--` containing +`index.json`, the JSON schema, bilingual metric definitions, and one source +subdirectory per original execution. Each source contains `result.json`, original +media/logs/report, per-GPU power series, phase integration/coverage and +`power-report.html`. A new inventory job publishes `h3-hardware--`; +CPU-only replay downloads the retained inventory instead. The verified raw +inventory is copied into each result. Original CI identities and +checksum seals are preserved separately from exporter identities and new seals. +Missing or invalid telemetry withholds power; export failures retain error logs +and return an unsuccessful status. Original workload failures still upload their +raw evidence even when export cannot start. + +In smoke mode, exit 0 means both roles completed every planned warmup and +measurement with verified timing, fresh valid media, and clean teardown. Exit 1 +means a completed workload contains an invalid outcome; exit 2 means execution +or evidence verification failed. Latency/fidelity thresholds remain separate: +the report can show a failed or inconclusive comparison after a successful smoke. +Regression mode additionally requires the existing calibrated acceptance gate. + +```bash +cd experimental/video-generation +PYTHONPATH=../.. uv run --no-project --python 3.12 \ + --with 'av==16.1.0' --with 'numpy==2.3.5' \ + --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' python -m pytest -q +bash -n runtime-entry.example.sh +``` + +[Test H3 Video](../../.github/workflows/test-h3-video.yml) runs these CPU checks +and workflow linting on relevant changes. They make no model, scheduler, or GPU +calls. Real CI execution and artifact inspection are separate acceptance evidence. diff --git a/experimental/video-generation/README_zh.md b/experimental/video-generation/README_zh.md new file mode 100644 index 0000000000..a44fe887b1 --- /dev/null +++ b/experimental/video-generation/README_zh.md @@ -0,0 +1,166 @@ +# H3 视频 CI 冒烟测试 + +[English](README.md) | **中文** + +此实验任务在 InferenceX CI 内使用 SemiAnalysis H200 资源运行现有 H3 +supervisor。首个目标是有时间上限的同版本冒烟测试：保留实际生成的 MP4， +完整验证视频和音频，记录请求耗时，并验证清理结果。此任务不写入原生 +InferenceX 数据库，也不新增网站页面。 + +runner 支持两种冻结的 16:9 设置，均为 1344×768、24 FPS：请求 4 秒对应 +107 帧，请求 8 秒对应 192 帧。这些帧数遵循固定版本 H3 的时间取整规则。 +修改提示词或时长时应冻结新 plan，历史运行仍保留原始设置。 + +冒烟任务成功表示配置中要求的测量和证据已完成。未经校准的回归判定仍为 +inconclusive，`ci_accepted: false`。新增这些文件或 CPU 测试通过，都不能 +证明已经成功运行 H3。 + +## 准备现有运行环境 + +将仓库变量 `H3_SITE_CONFIG` 设置为 `cluster:h200-dgxc` 提交节点上 +已审核 JSON 文件的绝对路径。以 [site.example.json](site.example.json) 为起点； +其中占位符不可直接执行。配置绑定持久工作目录、现有 rootfs 和准备记录、 +仅负责进入容器的脚本及其 SHA256、容器内 Python、冻结的 supervisor spec +及其 SHA256、资源上限、任务标识，以及可选的历史分配收据。 + +spec 必须记录真实的算力和模型使用批准。选择手动 H3 路径仅请求执行该配置中 +已审核的工作负载；调度参数不接受 shell 命令、模型路径、任意配置内容或其他 +GPU 提供商。 + +提交节点需要 Python 3.11+、Git、Slurm 工具，以及配置中声明的共享路径访问权。 +PyAV/NumPy、固定版本的 H3 运行时和模型必须已在现有环境中准备好。仅表示 +rootfs 已创建的标记不能证明模型兼容。adapter 在申请资源前检查准备记录和 +输入标识；不会安装依赖、导入镜像或新建 rootfs。 + +根据保存的可用进入命令调整 +[runtime-entry.example.sh](runtime-entry.example.sh)，然后固定其摘要。脚本必须 +进入现有 Enroot rootfs，将 `workspace.host` 挂载到 `/work`，保留 Slurm +step 的 GPU/CPU 绑定和元数据，转发传入命令并传回退出码。它将 +`SLURM_STEP_GPUS` 中的全局编号转换为物理 UUID，导出 +`H3_ASSIGNED_GPU_UUIDS`，并在 `H3_ORIGINAL_CUDA_VISIBLE_DEVICES` 中 +保留原设备掩码。容器内准入会核对驱动实际看到的 UUID。不要把会申请资源的旧 +launcher 当作进入脚本。 + +固定版本的 SGLang 运行时要求数字设备编号。每个 H3 子进程使用所选设备的 +实际 NVML 编号，并在导入 SGLang 前核对 CUDA 驱动返回的有序 UUID。 +设备枚举不一致时启动失败；归属锁和遥测仍使用分配的 UUID。 + +adapter 在申请资源前恢复本任务的分配收据。导入的收据必须匹配任务标识、Unix +所有者和调度器中的精确分配身份；提交结果不明确时禁止重复申请。固定站点是 +`main` / `sa-shared`。新建独占分配预留八张 GPU；示例 step 使用四张 +GPU、32 个 CPU 和 1 TiB 主机内存。固定版本的四 rank 加载器在 CPU 暂存权重时 +超过了 256 GiB；1 TiB 是实际运行验证过的额度，并非测得的最低需求。预算按预留容量计算。 + +`resources.minutes` 是整个分配的时间上限，最多 90 分钟。step 为外层清理 +预留五分钟，supervisor 的上限加十分钟必须不超过分配上限。例如：分配 +90 分钟、step 85 分钟、supervisor 75 分钟。复用分配必须有足够剩余时间。 +保留准备好的 rootfs，仅清理属于本次任务的进程和 step，仅释放本次执行拥有的 +分配。 + +## 通过 InferenceX 调度 + +第一次运行必须在 CI 中进行。对于功能分支，使用已注册的 End-to-End Tests +工作流调用该分支的可复用 H3 工作流： + +```bash +gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX \ + --ref feat/h3-video-ci -f h3-video=true -f test-name=h3-first-smoke +``` + +调度前审核分支。首次调度者和重新运行者都必须具有 write、maintain 或 admin +权限。H3 路径检出 `github.sha`：`--ref` 同时选择工作流定义和源码； +普通 LLM 路径的 `ref` 输入不用于 H3。此模式跳过 LLM 矩阵生成、所有依赖它的 +LLM 扫描和收集任务，以及对应的成功率计算。外部 PR 事件不能启动此路径。 +独立工作流在默认分支注册后，也可手动调度 `h3-video.yml`。 + +开启优先级调度时，必须同时开启节点配额调度。排队任务在 +`cluster:h200-dgxc` 上请求唯一的 `nodes:1`，以及原生 +`ci-job--` 和 `ci-attempt-` 标签。关闭优先级 +调度时，沿用仓库的集群标签路径。GitHub 准入和 Slurm 资源验证是两个不同环节。 +此任务使用原生工作流权限和调度准入，不新增 OIDC 服务，也不声称提供独立硬件 +证明。 + +## 可选的服务负载测试 + +在已审核的 supervisor spec 中添加 +`"serving": {"concurrency": 2, "delivery_deadline_seconds": 300}`， +并更新站点配置固定的 spec SHA256。这里的截止时间是操作者设置的示例， +不是经过校准的验收标准。并发数支持 1–32；省略 `serving` 时保持原有串行回归行为。 +直接客户端提供 `--serving-concurrency` 和 `--delivery-deadline-seconds`； +不加 `--execute` 时仍然只预览。 + +每个任务针对一个受监督的服务端点测量一个并发数。工作线程下载完前一个视频 +后提交下一个请求，媒体校验单独进行；预热仍然串行且单独记录。比较不同负载时， +应在独立任务中使用同一份固定的提示词、种子、生成参数和运行时，并包含显式 +设置并发数为 1 的服务模式对照。不自动启动负载扫描。沿用现有 CI 的分配和 +运行环境复用路径，保留每个请求的结果；远端完成状态不明确时停止新增提交。 + +此模式测量闭环交付吞吐，不代表固定到达速率或可持续服务容量。服务端排队与执行 +时间戳、实际 batch 大小、多副本布局和完整部署成本仍为不可用。服务模式要求使用 +未校准的策略。CPU 测试数据只验证测试工具，不能证明 H3 支持并发或具体硬件性能。 + +## 结果与本地检查 + +成功执行还会发布[前端结果契约](RESULTS_zh.md)：版本化的 `result.json`、 +逐卡功率序列、分阶段能量与覆盖率，以及可离线打开的 `power-report.html`。 +`h3-results--` 包含索引、JSON Schema、双语指标说明，以及每次 +原始执行的媒体、测量、日志和报告。原始执行身份与校验文件和本次导出身份分开保留。 +遥测无效时功率和能量为空；导出失败会保留错误日志并返回失败状态。 + +复用已成功的 A/A 结果和已保留的硬件盘点时，在同一个可信入口提供原始 H3 +运行编号（一至两个）以及盘点运行编号： + +```bash +gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX \ + --ref feat/h3-video-ci -f h3-video=true \ + -f h3-reuse-run-ids=34291306687,34293342829 \ + -f h3-inventory-run-id=34297499754 -f test-name=h3-power-export +``` + +此路径使用托管 CPU 导出，跳过原生 H200 作业。首次调度者和重新运行者仍须通过 +相同权限检查。托管导出独立验证已验收的 H3 执行和已完成的硬件盘点作业，核对 +产物校验清单、Git/CI/Slurm 身份及相同物理 GPU UUID。不执行新的 GPU 查询或 +模型请求。源产物必须仍在 GitHub 保留期内。 + +省略 `h3-inventory-run-id` 时，会通过原生 Slurm 路径执行新的盘点。CI 核对源提交、 +原始产物和持久化 Slurm 收据，在相同节点和 GPU UUID 上复用现有容器做只读检查。 +分配上限为十分钟，即最多 1.3333 个预留 GPU 小时；先检查本任务已有分配是否可 +复用。盘点不加载 H3 模型，清理后释放自有分配。后来的功率上限不能补写为历史 +生成时的设置。新盘点上传 `h3-hardware--`；仅使用 CPU 的重放 +则下载已保留的盘点产物。已验证的原始硬件证据会复制到每份结果中。 + +已保留的[盘点运行 34297499754](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754) +（[原始盘点产物](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754/artifacts/10083702100)） +在 `worker-10` 上完成 Slurm **82290.0**，于 `2026-09-09T01:02:44Z` 观测原来的 +四个 UUID。记录的 NVIDIA H200 PCI 设备/子系统编号 `233510DE` / `18BE10DE` +对应 H200 SXM，厂家最大可配置 TDP 为**每张 700 W**。在这次后续观测中，四张卡的 +配置、实际执行、默认及最大功率上限均为 700 W；历史生成时的设置仍未知。 +原始盘点记录中的 unknown 分类保持不变；导出器使用新 producer 提交解析原始 +XML，同时支持带或不带 `0x` 的 PCI 编号，不重新查询硬件。 +[已保留的 A/A 测量及限制](RESULTS_zh.md#已观测的-aa-证据)提供具体数据。 + +原始工作负载失败时仍上传失败证据，但不会启动成功结果导出。 + +每次尝试都会上传 `h3-video--`，保留 14 天，并关闭媒体压缩。 +失败后也执行上传，内容包括 adapter 的完整输出：收据、原始 MP4、遥测、 +校验和，以及可用时的便携报告。缺失的报告或媒体保持缺失，不以测试素材替代。 +原始持久证据保存在配置的工作目录。请在 GitHub 保留期结束前保存完整 artifact。 + +冒烟模式中，返回 0 表示两个角色的全部预热和测量均完成，耗时证据、重新解码后的 +媒体和资源清理均通过验证；返回 1 表示已完成的负载包含无效结果；返回 2 表示执行 +或证据验证失败。延迟/保真度阈值单独报告：冒烟执行成功后，比较结果仍可能失败 +或不确定。回归模式还要求通过现有的校准与验收门槛。 + +```bash +cd experimental/video-generation +PYTHONPATH=../.. uv run --no-project --python 3.12 \ + --with 'av==16.1.0' --with 'numpy==2.3.5' \ + --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' python -m pytest -q +bash -n runtime-entry.example.sh +``` + +[Test H3 Video](../../.github/workflows/test-h3-video.yml) 对相关变更运行上述 CPU +检查和工作流 lint。这些检查不会调用模型、调度器或 GPU。真实 CI 执行和 +artifact 检查是独立的验收证据。 + +编译缓存保留在持久化存储中，不上传为测量证据。 diff --git a/experimental/video-generation/RESULTS.md b/experimental/video-generation/RESULTS.md new file mode 100644 index 0000000000..1379ce01c8 --- /dev/null +++ b/experimental/video-generation/RESULTS.md @@ -0,0 +1,202 @@ +# H3 backend result contract + +**English** | [中文](./RESULTS_zh.md) + +`result.json` is the frontend entry point for a downloaded H3 CI artifact. Its +`schema_version` is **1.0.0**; [result.schema.json](./result.schema.json) describes +its external shape. Reject unknown versions. This contract does not ingest a +result into the InferenceX database or qualify a release. + +The bundle retains original media, raw request records and telemetry, runtime +and client logs, model/runtime identities, Slurm receipts, and the existing +portable `report/index.html`. The exporter adds `result.json`, timestamped +`power/baseline.json` and `power/candidate.json`, and `power-report.html`. +Original files are not rewritten. The publisher refreshes `SHA256SUMS` after +export and preserves the source checksum file separately when reprocessing. +All media/report references are relative to the artifact root and include +SHA256. Unzip the whole artifact before opening either report. + +## Reading a result + +1. Check `status` and `invalid_reasons`. An invalid artifact writes a failed + result and raises an error so CI can preserve logs and fail the export. +2. Read `workload_status` separately from `regression_status`. Valid A/A clips + with an uncalibrated policy can finish successfully while regression is + `inconclusive`. `release_qualified` is always false for this MVP. +3. Read each `roles..metrics.status` and each power phase's `valid` flag. + A complete export can contain unavailable power. Invalid power/energy is + null, never zero. A phase is not made valid by another phase passing. +4. Use `execution.ci` for the **original GPU execution**, including its commit, + run and attempt. `producer` identifies the exporter commit/current CI and + source hashes. Reprocessing an artifact does not create new GPU measurements. +5. `hardware.selected_gpu_count` is the measured device set; + `reserved_gpu_count` comes from Slurm `AllocTRES`. Four selected GPUs in an + eight-GPU allocation means four-board power and eight-GPU compute billing. + +`workload.plan` freezes prompts, seeds, clip geometry/duration/audio format, +steps, repetitions and warmup count. `workload.server` retains runtime settings. +`execution` joins CI, Slurm, GPU UUIDs, model manifest and observed runtime source. +The exporter checks the original complete checksum inventory when present, +rejects unsafe paths/symlinks, reuses `verify_measurement_job`, and verifies the +original report's local references. This is verification of trusted-runner +records and media hashes; the exporter does not rerun media decoding or provide +independent hardware attestation. The earlier full-stream analyses remain bound +to their original media bytes. Trusted GitHub metadata, when supplied, must join +the successful H3 job, run, attempt, URL and execution commit. During same-run +export, only that exact exporter run/attempt/repository/commit may still be +`in_progress`; `workflow_status_at_export` records this pending container workflow +while the completed H3 job and downloaded artifact identities are checked. + +## Metric definitions + +| Metric | Unit and boundary | Validity and limits | +| --- | --- | --- | +| Request latency | Seconds from submission through downloaded and technically validated media | Valid measured clips only; startup and warmup excluded. Retains terminal, download and validation timings. | +| Valid clips/sec | Valid measured clips / recorded measurement wall seconds | Includes failed attempts. Serial mode includes validation; serving mode ends at final delivery/transport failure. Not sustainable capacity. | +| Serving delivery latency | Seconds from submission through downloaded media; P50/P90/P95 | Technically valid measured clips only. P90/P95 require at least 10/20 samples; these floors are not statistical qualification. Failures remain in outcome counts. | +| Deadline goodput | Technically valid, delivered-within-deadline clips / serving wall seconds | Deadline is operator-selected. Attainment fraction divides by all scheduled measured slots, including failures and not-started slots. | +| Completion | Scheduled, attempted, completed, valid, failed, not-started clips | Completed can still be technically invalid. Warmup records remain separate. | +| GPU memory | Observed device-used MiB per selected UUID | Existing role-wide and client-including-warmup peaks retain their original boundaries; not exact allocator peaks. | +| Technical integrity | Full-stream video/audio checks with per-check units | Decode, geometry, duration, timestamps/cadence, motion and sound defects; not semantic or perceptual quality. | +| Paired fidelity | Video PSNR dB, audio spectral cosine and absolute RMS ratio error | Matching original decoded outputs; exact video match has null finite PSNR and `exact_match=true`. | +| GPU power | Timestamped W per GPU and summed selected-GPU W | Board sensor readings, including device memory; exclude host power and unselected GPUs. | +| Average / observed peak power | Integrated J / phase seconds; maximum in-window sensor W | Averages are time weighted. Peaks are sampled observations, not instantaneous electrical peaks. | +| GPU energy / valid clip | Trapezoidal integrated J / technically valid measured clips | Serial request windows exclude download/decode; serving uses one first-submit-to-last-observed-terminal envelope, including intervening idle/download/validation time. Null for invalid coverage or zero valid clips. | + +Each power file has versioned `sample_series`, `windows`, `phases`, `semantics` +and `clock_alignment`. Windows separate startup, each warmup, and each measured +submission-to-observed-provider-terminal interval. They retain the timing source +and uncertainty, exact monotonic bounds, per-GPU sample counts/gaps, covered +seconds/fraction, boundary bracketing, and invalid reasons. Phase aggregates +combine only when every requested contributing window is valid. Full time +series remain downloadable even when derived measurements are withheld. + +In serving mode, overlapping measured request intervals become one envelope; +board energy is integrated once and cannot be attributed per request. Startup +and warmup remain separate. This generation envelope differs from the delivery +throughput window, which includes the final download or transport failure. + +Optional `roles..metrics.serving` retains raw delivery latency samples, +outcome counts, observed submission rate, peak client requests in flight, decoded +video seconds per second and deadline goodput. `records` retains job IDs and +client lifecycle timings. Local validation is outside the delivery window but +can overlap it; polling time is included and is not a server queue measurement. +`execution.deployment` records one endpoint, selected GPUs and server settings; +reserved GPUs remain separate in `hardware`. Server queue/execution timestamps, +actual batching, fixed offered rate and full deployment cost are null. These +additive fields preserve schema version 1.0.0 and are absent/null on older runs. +Comparisons require matched workloads and timing boundaries; serving measurements +remain descriptive and cannot reuse serial regression calibration. + +Integration uses the shared InferenceX trapezoidal power integrator with linear +boundary interpolation and no extrapolation. UUID/ownership, finite readings, +ordered timestamps, phase overlap, clock agreement and maximum-gap checks gate +power. The allowed gap is `3 × requested sampling interval` (3 seconds for these runs). +Legacy UTC event reconstruction must agree with recorded monotonic durations; +its startup window can be withheld when a boundary is not covered. H200 NVML +power readings have a trailing averaging window, so phase edges also have sensor +averaging uncertainty. No energy counter is inferred from sampled watts. + +## TDP and architectural claims + +A generic `NVIDIA H200` name does not prove SXM form factor or 700 W TDP. +`hardware.tdp` stays unavailable until an explicitly verified, sourced hardware +profile joins the same physical UUIDs. When supplied, the exporter records +measured mean/observed-peak fractions of aggregate specification TDP. These are +descriptive ratios, not a configured power limit or a calibrated decision gate. + +A later read-only inventory remains under `later_hardware_observation`, with its +own CI/Slurm identity and time. Its configured/default/enforced/maximum power +limits **do not backfill historical generation settings**. Historical limits +remain unavailable when the original run did not record them. New executions +retain their own per-role `configured_power_limits.by_role` before/after snapshots +of configured, enforced, default and maximum W. Each snapshot has its observation +time and validity; a missing or mismatched UUID/value/time withholds that snapshot. +`same_observed_values` compares the endpoints only and never proves continuous +power-limit stability between them. + +High observed H3 GPU-board power supports a statement about this workload and +hardware configuration only. Claiming architectural differences from LLMs still +requires matched hardware/topology, precision, power limits, sampling/windows, +warmup and load, including LLM prefill/decode separation and repeated comparable +measurements. These A/A points do not establish significance, general video +quality, performance improvement, or release readiness. + +## Export API + +From the repository root, put the repository and `experimental/video-generation` +on `PYTHONPATH`, then call: + +```python +from pathlib import Path +from evaluator.mvp_result import write_result + +write_result( + Path("/absolute/path/to/copied-source-artifact"), + producer={"git_commit": "<40-character exporter commit>", "ci": {"run_id": ""}}, + source_ci=trusted_github_run_metadata, + hardware_profile=optional_later_inventory, +) +``` + +Use a fresh copy. `source_ci` uses GitHub CLI fields `databaseId`, `runAttempt`, +`headSha`, `url`, `status`, `conclusion`, and `jobs`; obtain them independently of +the artifact. The optional profile must bind the same GPU UUIDs, and a verified +TDP needs `status`, `watts_per_gpu`, `hardware_variant`, `source_url`, and evidence. +The publisher owns final checksums, upload, and download acceptance. Do not +modify old CI or measurement identities to label an export as a new benchmark. + +## Observed A/A evidence + +These results reuse the retained eight-second clockwork-fox execution from +[CI run 34293342829](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34293342829), +commit `65699f7c6`, Slurm **82261.0**, with +[original media, telemetry and report](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34293342829/artifacts/10082823150). +Four selected H200 GPUs ran the same runtime revision sequentially: one warmup +and one measured clip per role. + +| Eight-second measured clip | Baseline | Candidate | +| --- | ---: | ---: | +| Submit-to-validated-media latency (s) | 149.768568 | 149.355090 | +| Aggregate mean GPU-board power (W) | 2737.371816 | 2731.735261 | +| Mean / verified aggregate specification TDP | 97.7633% | 97.5620% | +| GPU energy per valid clip (J) | 406794.062298 | 404620.528709 | +| Sampling coverage fraction | 1.0 | 1.0 | +| Maximum observed sampling gap (s) | 1.3303 | 1.3567 | + +Per-GPU mean power was approximately **680–688 W**. Both measured generation +windows were bracketed, with maximum gaps below the 3-second validity limit. +Power and energy cover submission to observed provider completion; the latency +row also includes media transfer and validation. These are different boundaries. +The later [hardware inventory run 34297499754](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754) +([raw inventory artifact](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754/artifacts/10083702100)) +completed Slurm **82290.0** on `worker-10` and observed the original four UUIDs at +`2026-09-09T01:02:44Z`. NVIDIA H200 PCI device/subsystem IDs `233510DE` / `18BE10DE` +identify SXM boards, whose manufacturer maximum configurable TDP is +[700 W each](https://www.nvidia.com/en-us/data-center/h200/), or **2800 W** for the +four selected GPUs. The measured averages are therefore **97.7633% / 97.5620%** +of this specification value. Both exceed a descriptive 90% “near TDP” criterion, +supporting Oren's hypothesis for these measured H3 generation windows. This +criterion is not a calibrated benchmark gate or evidence of an architectural +difference from LLMs. + +The later inventory recorded configured, enforced, default and maximum limits of +700 W on all four GPUs. The original generation settings remain unknown. The +inventory's original profile classified the variant as unknown because its PCI +IDs omitted `0x`; the exporter now reclassifies the retained XML with its own +producer commit, preserving the source profile and making no new GPU query. +Use the [CPU-only replay command](https://github.com/SemiAnalysisAI/InferenceX/blob/a526907b1154901744bc6c058d213face64a3fef/experimental/video-generation/README.md#dispatch-through-inferencex) to reuse +this verified inventory with the original H3 artifacts. + +The earlier four-second A/A execution remains available in +[CI run 34291306687](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34291306687), +Slurm **82260.0**, and its +[original artifact](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34291306687/artifacts/10081961245). +Its baseline measured power and energy are withheld because the ending boundary +is not bracketed by telemetry. This does not invalidate the retained workload +execution or latency/media results. + +Each duration is a separate frozen workload. Do not pool these durations or +interpret their latency difference as a regression. One measured clip per role +provides point estimates only; workload execution passed while regression +remains uncalibrated and inconclusive. diff --git a/experimental/video-generation/RESULTS_zh.md b/experimental/video-generation/RESULTS_zh.md new file mode 100644 index 0000000000..88e233a356 --- /dev/null +++ b/experimental/video-generation/RESULTS_zh.md @@ -0,0 +1,183 @@ +# H3 后端结果契约 + +[English](./RESULTS.md) | **中文** + +`result.json` 是下载后的 H3 CI 产物面向前端的入口。其 `schema_version` +为 **1.0.0**，[result.schema.json](./result.schema.json) 定义对外结构。 +消费者必须拒绝未知版本。此契约不负责写入 InferenceX 数据库，也不代表发布验收。 + +产物保留原始媒体、请求记录、遥测、运行时及客户端日志、模型与运行时身份、 +Slurm 回执，以及原有便携报告 `report/index.html`。导出器新增 `result.json`、 +带时间戳的 `power/baseline.json`、`power/candidate.json` 和 `power-report.html`。 +原文件不会被重写。发布方在导出后更新 `SHA256SUMS`；重处理时另存原始校验文件。 +媒体及报告路径相对产物根目录，并附 SHA256。打开报告前需解压整个产物。 + +## 读取结果 + +1. 先检查 `status` 和 `invalid_reasons`。无效产物会写入失败结果并抛出错误， + 让 CI 保留日志，同时使导出任务失败。 +2. 分开读取 `workload_status` 与 `regression_status`。A/A 视频有效且负载执行 + 成功时，未校准策略仍可令回归结论为 `inconclusive`。此 MVP 的 + `release_qualified` 始终为 false。 +3. 检查 `roles..metrics.status` 和各功率阶段的 `valid`。 + 导出完成不代表所有功率数据都有效。无效功率及能量为 null，不能当作零。 + 一个阶段通过不能使另一个阶段有效。 +4. `execution.ci` 标识**原始 GPU 执行**，包括提交、运行及尝试编号。 + `producer` 标识导出器提交、当前 CI 和源码哈希。重处理不会产生新的 GPU 测量。 +5. `hardware.selected_gpu_count` 是实际测量设备数；`reserved_gpu_count` 来自 + Slurm `AllocTRES`。保留八张卡、使用四张卡时，功率只覆盖四张卡，计算额度 + 则按八张卡的保留时间计费。 + +`workload.plan` 固定提示词、随机种子、视频尺寸/时长/音频格式、步数、重复次数 +和预热次数，`workload.server` 保留运行时设置。`execution` 关联 CI、Slurm、 +GPU UUID、模型清单及实测运行时源码。导出器检查现有完整校验清单，拒绝不安全 +路径及符号链接，复用 `verify_measurement_job`，并检查原报告的本地引用。 +这是对可信运行器记录和媒体哈希的验证；导出器不会重新解码媒体，也不提供独立 +硬件认证。此前完整解码分析仍绑定原始媒体字节。若提供独立获取的 GitHub 元数据， +必须核对成功的 H3 作业、运行编号、尝试编号、URL 及执行提交。同次运行导出时， +只有与导出器运行、尝试、仓库及提交完全一致的工作流才允许仍为 `in_progress`； +`workflow_status_at_export` 保留此状态，同时仍核对已经完成的 H3 作业和下载产物。 + +## 指标定义 + +| 指标 | 单位与边界 | 有效性及限制 | +| --- | --- | --- | +| 请求延迟 | 从提交到下载完成并通过技术校验的秒数 | 只统计有效测量视频，不含启动和预热；保留终态、下载及校验分段时间。 | +| 有效视频数/秒 | 有效测量视频数 / 记录的测量区间墙钟秒数 | 包含失败尝试。串行模式包含校验；服务模式截止于最后一次交付或传输失败。不代表可持续容量。 | +| 服务交付延迟 | 提交到视频下载完成的秒数；P50/P90/P95 | 只统计技术校验有效的测量视频。P90/P95 至少需要 10/20 个样本；此门槛不代表统计资格。失败仍保留在结果计数中。 | +| 截止时间内的有效吞吐 | 技术校验有效且按时交付的视频数 / 服务测量区间秒数 | 截止时间由操作者设置。达标比例的分母为所有计划测量请求，包括失败和未启动请求。 | +| 完成计数 | 计划、尝试、完成、有效、失败、未启动的视频数 | 完成的视频仍可能技术校验失败；预热记录独立保留。 | +| GPU 显存 | 每个选中 UUID 的设备已用 MiB 观测峰值 | 保留原有整个角色及含预热客户端区间，不是精确分配器峰值。 | +| 技术完整性 | 完整视频/音频解码检查，各检查自带单位 | 覆盖解码、几何、时长、时间戳/帧间隔、运动及声音缺陷；不代表语义或感知质量。 | +| 配对输出保真度 | 视频 PSNR dB、音频频谱余弦及 RMS 绝对比例误差 | 比较原始解码输出；视频完全一致时，有限 PSNR 为 null，`exact_match=true`。 | +| GPU 功率 | 每卡带时间戳的 W，及选中 GPU 的功率总和 | 板级传感器读数含设备显存，不含主机及未选中的 GPU。 | +| 平均/观测峰值功率 | 积分能量 J / 阶段秒数；区间内传感器 W 最大值 | 平均值按时间加权；峰值是采样观测值，不是瞬时电气峰值。 | +| 每个有效视频的 GPU 能量 | 梯形积分 J / 技术校验有效的测量视频数 | 串行请求区间不含下载/解码；服务模式对首次提交到最后观测终态的整个区间积分，包含期间的空闲、下载及校验时间。覆盖无效或有效视频数为零时为 null。 | + +每个功率文件具有版本化的 `sample_series`、`windows`、`phases`、`semantics` +和 `clock_alignment`。分别记录启动、每次预热，以及每次测量从提交到观测到 +提供方终态的区间。区间保留时钟来源及不确定性、精确单调时钟边界、每卡采样数 +和间隔、覆盖秒数/比例、边界包围情况及无效原因。只有所有必需区间有效时，才 +汇总该阶段。即使派生指标被保留为空，完整时间序列仍可下载。 + +服务模式把重叠的测量请求区间合并为一个完整区间，板级能量只积分一次，不归因 +到单个请求。启动和预热仍然独立。该生成区间与吞吐的交付区间不同，后者还包含 +最后一次下载或传输失败。 + +可选的 `roles..metrics.serving` 保留原始交付延迟样本、结果计数、观测提交 +速率、客户端同时在途请求峰值、每秒生成视频秒数，以及截止时间内的有效吞吐。 +`records` 保留任务 ID 和客户端生命周期时间。交付区间不等待本地校验结束，但 +两者可以重叠；延迟包含轮询开销，不代表服务端排队时间。 +`execution.deployment` 记录单个端点、选中的 GPU 和服务设置；预留 GPU 仍在 +`hardware` 中单独记录。服务端排队/执行时间戳、实际 batching、固定到达速率和 +完整部署成本为 null。这些兼容新增字段保持 schema 版本 1.0.0，旧运行中不存在或为 null。 +比较时必须保持工作负载和计时边界一致；服务测量仅作描述，不能沿用串行回归校准。 + +积分复用 InferenceX 的梯形功率积分器，边界做线性插值，不做外推。UUID/进程 +归属、有限读数、时间戳顺序、阶段重叠、时钟一致性及最大间隔共同决定有效性。 +允许间隔为 `3 × 请求采样间隔`（本次运行为 3 秒）。旧版 UTC 事件重建必须与记录的单调 +时钟耗时一致；启动边界未被覆盖时，该阶段可能无效。H200 NVML 功率读数有向后 +平均的窗口，阶段边缘还受传感器平均窗口影响。不能从瓦数采样推断能量计数器。 + +## TDP 与架构结论 + +仅凭 `NVIDIA H200` 名称不能确定 SXM 形态或 700 W TDP。只有明确验证、附来源 +且绑定相同物理 UUID 的硬件记录，才能使 `hardware.tdp` 可用。此时导出器提供 +平均功率及观测峰值相对于所选 GPU 总规格 TDP 的比例。这些是描述性比例， +不是配置功率上限，也不是已校准的判定门槛。 + +后续只读硬件盘点保存在 `later_hardware_observation`，保留自身 CI/Slurm 身份和 +时间。其配置、默认、实际执行及最大功率上限**不能回填历史生成设置**。 +原始运行没有记录的历史上限仍为不可用。新运行会在 +`configured_power_limits.by_role` 中保留各角色执行前后的配置、实际执行、默认 +及最大功率上限（W）。每个快照具有观测时间和有效性；UUID、读数或时间不匹配 +时保留该快照为空。`same_observed_values` 只比较两个端点，不能证明中间功率上限 +始终不变。 + +观察到 H3 板级功率较高，只能说明该负载及硬件配置。声称其与 LLM 存在架构差异， +仍需匹配硬件/拓扑、精度、功率上限、采样与测量窗口、预热及负载，区分 LLM +prefill/decode，并进行重复可比测量。这些 A/A 点不证明统计显著性、通用视频 +质量、性能提升或发布就绪。 + +## 导出 API + +从仓库根目录执行，将仓库根目录及 `experimental/video-generation` 加入 +`PYTHONPATH`，然后调用： + +```python +from pathlib import Path +from evaluator.mvp_result import write_result + +write_result( + Path("/absolute/path/to/copied-source-artifact"), + producer={"git_commit": "<40-character exporter commit>", "ci": {"run_id": ""}}, + source_ci=trusted_github_run_metadata, + hardware_profile=optional_later_inventory, +) +``` + +使用新的副本。`source_ci` 使用 GitHub CLI 字段 `databaseId`、`runAttempt`、 +`headSha`、`url`、`status`、`conclusion`、`jobs`，必须独立于产物获取。 +可选硬件记录必须关联相同 GPU UUID；已验证的 TDP 需包含 `status`、 +`watts_per_gpu`、`hardware_variant`、`source_url` 和证据。 +发布方负责最终校验和、上传及下载验收。不得修改旧 CI 或测量身份，将导出伪装 +成新基准运行。 + +## 已观测的 A/A 证据 + +以下结果复用已保留的八秒机械狐狸运行： +[CI run 34293342829](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34293342829)， +提交 `65699f7c6`，Slurm **82261.0**。 +[原始媒体、遥测及报告](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34293342829/artifacts/10082823150) +可下载。四张选中的 H200 GPU 按顺序运行相同运行时版本，每个角色各执行一次预热 +和一次视频测量。 + +| 八秒视频测量 | Baseline | Candidate | +| --- | ---: | ---: | +| 从提交到媒体校验完成的延迟（s） | 149.768568 | 149.355090 | +| GPU 板级平均总功率（W） | 2737.371816 | 2731.735261 | +| 平均功率 / 已验证的总规格 TDP | 97.7633% | 97.5620% | +| 每个有效视频的 GPU 能量（J） | 406794.062298 | 404620.528709 | +| 采样覆盖比例 | 1.0 | 1.0 | +| 最大观测采样间隔（s） | 1.3303 | 1.3567 | + +每张 GPU 的平均功率约为 **680–688 W**。两个测量生成区间的起止边界均被遥测 +包围，最大采样间隔均低于 3 秒有效性上限。功率和能量覆盖提交到观测到提供方 +完成的区间；延迟还包含媒体传输及校验，两者测量边界不同。 + +后续[硬件盘点运行 34297499754](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754) +（[原始盘点产物](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754/artifacts/10083702100)） +在 `worker-10` 上完成 Slurm **82290.0**，于 `2026-09-09T01:02:44Z` 观测原来的 +四个 UUID。NVIDIA H200 PCI 设备/子系统编号 `233510DE` / `18BE10DE` 对应 SXM +板卡，厂家最大可配置 TDP 为[每张 700 W](https://www.nvidia.com/en-us/data-center/h200/)， +四张所选 GPU 合计 **2800 W**。实测平均功率分别为此规格值的 +**97.7633% / 97.5620%**。按描述性的 90%“接近 TDP”标准，两者均满足，因此这些 +H3 生成测量区间支持 Oren 的假设。该标准不是经过校准的基准门槛，也不能证明 +与 LLM 存在架构差异。 + +后续盘点中，四张卡的配置、实际执行、默认及最大功率上限均为 700 W；原始生成 +时的设置仍未知。原始盘点 profile 因 PCI 编号未带 `0x` 而将形态记为 unknown； +导出器现用自身 producer 提交重新解析保留的 XML，保留源 profile，不发起新的 +GPU 查询。可使用[仅 CPU 的重放命令](https://github.com/SemiAnalysisAI/InferenceX/blob/a526907b1154901744bc6c058d213face64a3fef/experimental/video-generation/README_zh.md#结果与本地检查)，将此次已验证 +盘点与原始 H3 产物一起复用。 + +此前四秒 A/A 运行保存在 +[CI run 34291306687](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34291306687)， +Slurm **82260.0**，并有 +[原始产物](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34291306687/artifacts/10081961245)。 +其 baseline 测量区间的结束边界未被遥测包围，因此功率及能量保留为空。 +这不会使已保留的负载执行、延迟和媒体结果失效。 + +每个时长都是单独固定的负载。不能合并这些时长的数据，也不能将两者延迟差异 +解释为回归。每个角色只有一个测量视频，只能提供点估计；负载执行通过，回归 +仍未校准，结论为 inconclusive。 + + +### 并发烟测矩阵 + +经审阅的 site 配置使用 `mode: serving-smoke` 时，在同一个分配中依次运行 +C1/C2/C4，每组恰好四个测量请求，预热单独记录。每组重启同一个 baseline +运行时；任何一组失败即停止。只申请配置指定的 GPU 数量。 +`serving-smoke.json` 汇总状态、吞吐和延迟，`gpu/cN/` 保留原始请求、视频和 +遥测，`report/index.html` 可播放原始媒体。中断请求与未启动请求分别计数。 +该模式不会导出配对回归结果，四个样本也不能认定 P90/P95 或持续服务容量。 diff --git a/experimental/video-generation/ci.py b/experimental/video-generation/ci.py new file mode 100644 index 0000000000..dcbdea3009 --- /dev/null +++ b/experimental/video-generation/ci.py @@ -0,0 +1,611 @@ +#!/usr/bin/env python3 +"""InferenceX H200 Slurm adapter for a prepared, trusted H3 runtime. + +No SSH, image import, dependency installation, or model download. The submit +host and compute node share workspace.host, mounted at workspace.container by +an immutable entry-only wrapper. A receipt, not a username, identifies reuse. +""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from datetime import datetime, timezone +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import signal +import subprocess +import sys +import time +import uuid +from typing import Any, Iterator + +from evaluator.mvp_gpu_job import cuda_devices + +PARTITION = "main" +ACCOUNT = "sa-shared" +NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,79}") +SHA = re.compile(r"[0-9a-f]{64}") +IDENTITY = ("JobId", "JobName", "Comment", "WorkDir", "Account", "Partition", "UserId") +CACHE_PATHS = ("gpu/supervisor/baseline/cache", "gpu/supervisor/candidate/cache", "gpu/supervisor/compare-cache") +CACHE_PATHS += tuple(f"gpu/c{concurrency}/supervisor/baseline/cache" for concurrency in (1, 2, 4)) +TERMINAL = {"COMPLETED", "CANCELLED", "FAILED", "TIMEOUT", "NODE_FAIL", "OUT_OF_MEMORY", "PREEMPTED", "BOOT_FAIL", "DEADLINE"} + + +def need(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def read(path: Path | str) -> Any: + return json.loads(Path(path).read_text()) + + +def write(path: Path | str, value: Any) -> None: + path = Path(path) + temp = path.with_name(path.name + ".tmp") + temp.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n") + temp.replace(path) + + +def digest(path: Path | str) -> str: + value = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def absolute(value: str) -> Path: + need(isinstance(value, str) and re.fullmatch(r"/[A-Za-z0-9_./-]+", value) + and ".." not in Path(value).parts and value != "/" and str(Path(value)) == value, + "Expected a normalized absolute cluster path without whitespace") + return Path(value) + + +def mapped(config: dict, host: Path) -> Path: + return Path(config["workspace"]["container"]) / Path(host).relative_to(config["workspace"]["host"]) + + +def host_path(config: dict, container_path: str) -> Path: + path = absolute(container_path) + mount = Path(config["workspace"]["container"]) + if path.is_relative_to(mount): + return Path(config["workspace"]["host"]) / path.relative_to(mount) + return Path(config["runtime"]["rootfs"]) / path.relative_to("/") + + +def validate_config(config: dict) -> dict: + need(set(config) == {"schema_version", "task_id", "workspace", "runtime", "spec", "resources", "allocation_receipts", "mode"}, "Unknown or missing site configuration fields") + need(config["schema_version"] == 1 and NAME.fullmatch(config["task_id"]), "Invalid schema_version/task_id") + need(config["mode"] in {"smoke", "regression", "serving-smoke"}, "mode must be smoke, regression or serving-smoke") + need(set(config["workspace"]) == {"host", "container"}, "Invalid workspace mapping") + for value in config["workspace"].values(): + path = absolute(value) + need(not path.is_relative_to("/workspace"), "Use the declared persistent mount, not /workspace") + runtime = config["runtime"] + need(set(runtime) == {"entry", "entry_sha256", "rootfs", "ready_marker", "python"}, "Invalid runtime contract") + for key in ("entry", "rootfs", "ready_marker", "python"): + absolute(runtime[key]) + need(SHA.fullmatch(runtime["entry_sha256"]), "Runtime entry SHA256 required") + need(set(config["spec"]) == {"path", "sha256"} and SHA.fullmatch(config["spec"]["sha256"]), "Pinned prepared spec required") + absolute(config["spec"]["path"]) + resources = config["resources"] + need(set(resources) == {"gpus", "cpus", "memory_gb", "minutes"}, "Invalid resource request") + for key, low, high in (("gpus", 1, 8), ("cpus", 1, 128), ("memory_gb", 1, 1400), ("minutes", 10, 90)): + need(type(resources[key]) is int and low <= resources[key] <= high, "Resource outside bounded H200 budget: " + key) + need(isinstance(config["allocation_receipts"], list), "allocation_receipts must be a list") + for path in config["allocation_receipts"]: + absolute(path) + return config + + +def environment() -> dict[str, str]: + # Slurm defaults inherited from the runner must not change this request. + # The payload receives an explicit environment allowlist at the srun edge. + env = {key: value for key, value in os.environ.items() + if not key.startswith(("SLURM_", "SBATCH_", "SALLOC_", "SRUN_", "SQUEUE_", "SCANCEL_"))} + env.update(TZ="UTC", LC_ALL="C", PYTHONDONTWRITEBYTECODE="1") + return env + + +def command(argv: list[str], timeout: float = 30) -> str: + return subprocess.run(argv, text=True, capture_output=True, check=True, + timeout=timeout, env=environment()).stdout + + +def fields(raw: str) -> dict[str, str]: + return dict(re.findall(r"(?:^|\s)([A-Za-z][A-Za-z0-9_/:]*)=(\S+)", raw)) + + +def job_record(job_id: str) -> dict[str, str]: + need(re.fullmatch(r"[0-9]+", str(job_id)), "Invalid Slurm job ID") + return fields(command(["scontrol", "show", "job", "-o", str(job_id)])) + + +def verify_identity(receipt: dict, record: dict, task_id: str) -> None: + need(receipt.get("task_id") == task_id, "Allocation belongs to another task") + expected = receipt["identity"] + need(set(expected) == set(IDENTITY), "Incomplete allocation ownership receipt") + need(all(record.get(key) == expected[key] for key in IDENTITY), "Slurm allocation identity differs from receipt") + need(record["Account"] == ACCOUNT and record["Partition"] == PARTITION, "Allocation is not in the SemiAnalysis H200 pool") + need(re.fullmatch(r"[^()]+\(" + str(os.getuid()) + r"\)", record["UserId"]), "Allocation Unix owner differs") + + +def capacity(record: dict, resources: dict, timestamp: datetime | None = None) -> str | None: + need(record.get("NumNodes") == "1" and NAME.fullmatch(record.get("NodeList", "")), "Reuse requires one explicit node") + tres = dict(item.split("=", 1) for item in record["AllocTRES"].split(",")) + memory = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)([KMGT]?)", tres.get("mem", "")) + need(memory is not None, "Unrecognized allocated memory") + memory_gb = float(memory[1]) * {"K": 1 / 1048576, "M": 1 / 1024, "": 1 / 1024, "G": 1, "T": 1024}[memory[2]] + end = datetime.fromisoformat(record["EndTime"]).replace(tzinfo=timezone.utc) + remaining = (end - (timestamp or datetime.now(timezone.utc))).total_seconds() + if remaining < resources["minutes"] * 60 - 300 + 30: + return "insufficient remaining allocation time" + if int(tres.get("gres/gpu", "0")) < resources["gpus"] or int(record["NumCPUs"]) < resources["cpus"] or memory_gb < resources["memory_gb"]: + return "insufficient allocated GPU/CPU/memory capacity" + return None + + +def recover(config: dict, result_root: Path, *, node: str | None = None) -> dict: + paths = set(result_root.glob("*/allocation.json")) | {Path(p) for p in config["allocation_receipts"]} + # A crash between intent and acknowledgment must be reconciled, not retried. + for intent in result_root.glob("*/allocation-intent.json"): + need(intent.with_name("allocation.json").exists(), f"Unresolved allocation intent: {intent}; reconcile Slurm before another submission") + active = set(command(["squeue", "--all", "--noheader", "--user=" + str(os.getuid()), "--format=%i"]).split()) + reasons = [] + waiting = [] + for path in sorted(paths): + receipt = read(path) + need(receipt.get("task_id") == config["task_id"], "Saved allocation receipt belongs to another task") + identity = receipt["identity"] + need(set(identity) == set(IDENTITY), "Incomplete allocation ownership receipt") + job = identity["JobId"] + need(re.fullmatch(r"[0-9]+", job), "Invalid saved job ID") + if job not in active: + reasons.append({"receipt": str(path), "job_id": job, "reason": "inactive in successful scheduler snapshot"}) + continue + record = job_record(job) + verify_identity(receipt, record, config["task_id"]) + state = record["JobState"] + if state in TERMINAL: + reasons.append({"job_id": job, "reason": state}) + continue + if state != "RUNNING": + waiting.append({"job_id": job, "state": state}) + continue + if node is not None and record["NodeList"] != node: + reasons.append({"job_id": job, "reason": "allocation is on a different physical node"}) + continue + reason = capacity(record, config["resources"]) + if reason: + reasons.append({"job_id": job, "reason": reason}) + continue + steps = command(["squeue", "--steps", "--noheader", "--jobs=" + job, "--format=%i|%N"]) + return {"action": "reuse", "receipt": receipt, "record": record, "active_steps": steps, "reasons": reasons} + if waiting: + return {"action": "wait", "jobs": waiting, "reasons": reasons} + return {"action": "allocate", "reasons": reasons or [{"reason": "no saved allocations for this task"}]} + + +def allocate(config: dict, run_dir: Path, *, node: str | None = None) -> dict: + need(node is None or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]{0,252}", node), "Invalid target node") + nonce = uuid.uuid4().hex + job_name = os.environ.get("RUNNER_NAME", "h3-" + config["task_id"]) + need(NAME.fullmatch(job_name), "Invalid runner/job name") + comment = "h3:" + nonce + request = config["resources"] + intent = {"task_id": config["task_id"], "job_name": job_name, "comment": comment, + "work_dir": str(run_dir), "user_id": os.getuid(), "created_at": now()} + write(run_dir / "allocation-intent.json", intent) + placement = ["--gres=gpu:" + str(request["gpus"])] if config["mode"] == "serving-smoke" else ["--exclusive", "--gres=gpu:8"] + argv = ["salloc", "--no-shell", "--no-bell", "--partition=" + PARTITION, "--account=" + ACCOUNT, + "--nodes=1", "--ntasks=1", *placement, + "--cpus-per-task=" + str(request["cpus"]), "--mem=" + str(request["memory_gb"]) + "G", + "--time=" + str(request["minutes"]), "--immediate=30", + "--job-name=" + job_name, "--comment=" + comment, "--chdir=" + str(run_dir)] + if node is not None: + argv.append("--nodelist=" + node) + write(run_dir / "allocation-command.json", argv) + # With --no-shell, Slurm records the caller's cwd rather than --chdir. + result = subprocess.run(argv, text=True, capture_output=True, timeout=45, env=environment(), cwd=run_dir) + (run_dir / "salloc.log").write_text(result.stdout + result.stderr) + granted = re.findall(r"Granted job allocation ([0-9]+)", result.stdout + result.stderr) + need(len(set(granted)) == 1, "No unambiguous Slurm acknowledgment; allocation intent retained for reconciliation") + job = granted[0] + # Save the expected identity before querying, so a lost query can be recovered. + user = command(["id", "-un"]).strip() + receipt = {"task_id": config["task_id"], "created_at": now(), "identity": { + "JobId": job, "JobName": job_name, "Comment": comment, "WorkDir": str(run_dir), + "Account": ACCOUNT, "Partition": PARTITION, "UserId": f"{user}({os.getuid()})"}} + write(run_dir / "allocation.json", receipt) + need(result.returncode == 0, "Slurm returned a failure after granting an allocation; reconcile receipt") + return receipt + + +def stop_allocation(receipt: dict, task_id: str) -> dict: + record = job_record(receipt["identity"]["JobId"]) + verify_identity(receipt, record, task_id) + if record["JobState"] not in TERMINAL: + command(["scancel", receipt["identity"]["JobId"]]) + # scancel success is a request, not a terminal-state observation. + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + active = command(["squeue", "--noheader", "--jobs=" + receipt["identity"]["JobId"], "--format=%T"]).strip() + if not active or all(state in TERMINAL for state in active.split()): + return {"status": "released", "job_id": receipt["identity"]["JobId"]} + time.sleep(1) + raise RuntimeError("Owned allocation has not reached terminal state after cancellation") + + +def drain_step(receipt: dict, task_id: str, run_dir: Path) -> dict: + binding_path = run_dir / "binding.json" + if not binding_path.exists(): + return {"status": "not_observed", "reason": "payload did not write a step binding"} + binding = read(binding_path) + job, step = receipt["identity"]["JobId"], binding["step_id"] + need(binding["job_id"] == job and re.fullmatch(r"[0-9]+", step), "Step binding differs from owned allocation") + step_id = job + "." + step + def active(): + return step_id in command(["squeue", "--steps", "--noheader", "--jobs=" + job, "--format=%i"]).split() + if active(): + verify_identity(receipt, job_record(job), task_id) + # This exact step is ours; the parent of an attachment is never canceled. + command(["scancel", step_id]) + for _ in range(15): + if not active(): + break + time.sleep(1) + else: + raise RuntimeError("Owned Slurm step remains active; preserve evidence and reconcile") + return {"status": "ended", "step_id": step_id} + + +def inventory(root: Path, exclude: tuple[str, ...] = ()) -> dict[str, str]: + files = {} + for directory, dirs, names in os.walk(root, followlinks=False): + parent = Path(directory) + dirs[:] = [name for name in dirs if name != "__pycache__" and (parent / name).relative_to(root).as_posix() not in exclude] + need(not any((parent / name).is_symlink() for name in dirs), "Directory symlink in evidence or staged source") + for name in sorted(names): + path = parent / name + if name == "SHA256SUMS" or path.relative_to(root).as_posix() in exclude: + continue + need(not path.is_symlink() and path.is_file(), "Nonregular file in evidence or staged source: " + str(path)) + files[path.relative_to(root).as_posix()] = digest(path) + return dict(sorted(files.items())) + + +def collect(run_dir: Path, output: Path) -> None: + files = inventory(run_dir, CACHE_PATHS) + sums = "".join(f"{value} {path}\n" for path, value in files.items()) + (run_dir / "SHA256SUMS").write_text(sums) + output.mkdir(parents=True, exist_ok=True) + for name in files: + target = output / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(run_dir / name, target) + (output / "SHA256SUMS").write_text(sums) + need(inventory(output) == files, "Artifact collection hash mismatch") + + +def stage_package(source: Path, destination: Path) -> dict[str, str]: + selected = {p.relative_to(source).as_posix(): p for p in [*source.glob("*.py"), *(source / "evaluator").glob("*.py")]} + shared_power = source.parents[1] / "utils" / "aggregate_power.py" + if shared_power.is_file(): + selected["utils/aggregate_power.py"] = shared_power + expected = {name: digest(path) for name, path in selected.items()} + if destination.exists(): + need(inventory(destination) == expected, "Staged source differs from this GitHub commit") + else: + destination.mkdir(parents=True) + for name, path in selected.items(): + target = destination / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, target) + return expected + + +def step_argv(config: dict, receipt: dict, record: dict, run_dir: Path, package: Path) -> list[str]: + request = config["resources"] + return ["srun", "--jobid=" + receipt["identity"]["JobId"], "--nodelist=" + record["NodeList"], + "--exclusive", "--exact", "--nodes=1", "--ntasks=1", "--immediate=30", "--kill-on-bad-exit=1", + "--cpus-per-task=" + str(request["cpus"]), "--cpu-bind=verbose,cores", + "--gpus-per-task=" + str(request["gpus"]), "--gpu-bind=verbose,per_task:" + str(request["gpus"]), + "--mem=" + str(request["memory_gb"]) + "G", "--time=" + str(request["minutes"] - 5), + "--chdir=" + str(run_dir), "--export=PATH,PYTHONDONTWRITEBYTECODE,TZ,LC_ALL", + "python3", str(package / "ci.py"), "--enter", str(run_dir)] + + +def run_step(argv: list[str], log: Path, seconds: float) -> int: + process = None + def cancelled(signum, frame): + raise InterruptedError("CI canceled the owned Slurm step") + old = {sig: signal.signal(sig, cancelled) for sig in (signal.SIGINT, signal.SIGTERM)} + try: + with log.open("w") as stream: + process = subprocess.Popen(argv, stdout=stream, stderr=subprocess.STDOUT, env=environment(), start_new_session=True) + return process.wait(timeout=seconds) + finally: + for sig, handler in old.items(): + signal.signal(sig, handler) + if process is not None and process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=120) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + + +@contextmanager +def task_lock(path: Path) -> Iterator[None]: + with path.open("a") as handle: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + + +def prepared_spec(config: dict) -> dict: + runtime = config["runtime"] + need(Path(runtime["rootfs"]).is_dir() and Path(runtime["ready_marker"]).is_file(), "Existing persistent runtime or readiness record missing; no allocation requested") + need(digest(runtime["entry"]) == runtime["entry_sha256"], "Persistent entry script changed") + need(digest(config["spec"]["path"]) == config["spec"]["sha256"], "Prepared GPU specification changed") + spec = read(config["spec"]["path"]) + # Slurm assigns physical devices later. Validate the rest without a GPU call. + spec["gpu_uuids"] = [f"GPU-00000000-0000-0000-0000-{i:012d}" for i in range(config["resources"]["gpus"])] + from evaluator.mvp_gpu_job import validate_gpu_job + spec = validate_gpu_job(spec) + if config["mode"] == "serving-smoke": + from evaluator.mvp_serving_smoke import validate_spec + validate_spec(spec) + need(spec["authorization"]["compute_approved"] and spec["authorization"]["model_license_reviewed"] + and spec["authorization"]["approval_reference"].strip(), "Prepared spec must record compute and model approval") + need(spec["limits"]["job_seconds"] + 600 <= config["resources"]["minutes"] * 60, + "Allocation must leave ten minutes beyond supervisor budget for step entry/report/cleanup") + # Cheap inventory checks happen before salloc; full pinned source/weight + # hashing remains in the existing supervisor immediately before execution. + for role in ("baseline", "candidate"): + need(host_path(config, spec[role]["source"]).is_dir(), "Prepared runtime source missing: " + role) + model = host_path(config, spec["model"]["path"]) + for item in spec["model"]["files"]: + path = model / item["path"] + need(path.is_file() and path.stat().st_size == item["size_bytes"], + "Prepared model file missing or wrong size; stage weights before allocating: " + item["path"]) + return spec + + +def launch(config: dict, output: Path) -> int: + config = validate_config(config) + run_id = os.environ.get("GITHUB_RUN_ID", "") + attempt = os.environ.get("GITHUB_RUN_ATTEMPT", "") + sha = os.environ.get("H3_SOURCE_SHA", "") + need(re.fullmatch(r"[0-9]+", run_id) and re.fullmatch(r"[0-9]+", attempt), "GitHub run ID and attempt required") + need(re.fullmatch(r"[0-9a-f]{40}", sha), "Exact H3_SOURCE_SHA required") + source = Path(__file__).resolve().parent + need(command(["git", "-C", str(source), "rev-parse", "HEAD"]).strip() == sha, "Checkout differs from admitted source SHA") + need(not command(["git", "-C", str(source), "status", "--porcelain"]).strip(), "Harness checkout must be clean and committed") + spec = prepared_spec(config) + workspace = Path(config["workspace"]["host"]) + need(workspace.is_dir(), "Persistent workspace missing") + results = workspace / "results" / config["task_id"] + control = workspace / "campaigns" / config["task_id"] / "control" + results.mkdir(parents=True, exist_ok=True) + control.mkdir(parents=True, exist_ok=True) + run_dir = results / f"github-{run_id}-{attempt}" + reserved_gpus = config["resources"]["gpus"] if config["mode"] == "serving-smoke" else 8 + state = {"schema_version": 1, "task_id": config["task_id"], "run_id": run_id, "run_attempt": attempt, + "source_sha": sha, "started_at": now(), "phase": "preparing", "mode": config["mode"], + "ci_accepted": False, "release_qualified": False, "persistent_output": str(run_dir), + "excluded_cache_paths": list(CACHE_PATHS), + "ci": {"repository": os.environ.get("GITHUB_REPOSITORY"), + "workflow_ref": os.environ.get("GITHUB_WORKFLOW_REF"), + "workflow_sha": os.environ.get("GITHUB_WORKFLOW_SHA"), + "actor": os.environ.get("GITHUB_ACTOR"), + "triggering_actor": os.environ.get("GITHUB_TRIGGERING_ACTOR"), + "run_url": f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{os.environ.get('GITHUB_REPOSITORY', '')}/actions/runs/{run_id}"}, + "resources": {"requested": config["resources"], "new_allocation_gpus": reserved_gpus, + "new_allocation_gpu_hours_cap": reserved_gpus * config["resources"]["minutes"] / 60}} + with task_lock(control / "ci.lock"): + run_dir.mkdir(exist_ok=False) + write(run_dir / "ci.json", state) + receipt = None + reused = False + code = 2 + try: + shutil.copyfile(config["runtime"]["entry"], run_dir / "runtime-entry.sh") + shutil.copyfile(config["runtime"]["ready_marker"], run_dir / "runtime-readiness.record") + package = workspace / "campaigns" / config["task_id"] / "packages" / sha + package_files = stage_package(source, package) + decision = recover(config, results) + write(run_dir / "recovery.json", decision) + need(decision["action"] != "wait", "A task-owned allocation is pending or suspended; no duplicate submitted") + if decision["action"] == "reuse": + receipt, reused = decision["receipt"], True + write(run_dir / "allocation.json", receipt) + else: + receipt = allocate(config, run_dir) + record = job_record(receipt["identity"]["JobId"]) + verify_identity(receipt, record, config["task_id"]) + need(record["JobState"] == "RUNNING", "Owned allocation is not RUNNING") + need(capacity(record, config["resources"]) is None, "Allocation cannot serve bounded step") + if config["mode"] == "serving-smoke": + tres = dict(item.split("=", 1) for item in record["AllocTRES"].split(",")) + need(int(tres.get("gres/gpu", "0")) == reserved_gpus, "Serving allocation GPU count exceeds the declared budget") + need(digest(config["runtime"]["entry"]) == config["runtime"]["entry_sha256"], "Entry changed after preflight") + state.update(phase="starting", allocation_reused=reused, allocation=receipt, slurm_job=record) + write(run_dir / "ci.json", state) + # Any concurrent active measured step means cooperative sharing. + active_steps = command(["squeue", "--steps", "--noheader", "--jobs=" + record["JobId"], "--format=%i|%N"]) + write(run_dir / "context.json", {"config": config, "spec": spec, "allocation": receipt, + "node": record["NodeList"], "active_steps": active_steps, "source_sha": sha, + "exclusive_node": record.get("OverSubscribe") == "NO" and "gres/gpu=8" in record.get("AllocTRES", "").split(","), + "package_files": package_files, "run_id": f"github-{run_id}-{attempt}"}) + argv = step_argv(config, receipt, record, run_dir, package) + write(run_dir / "step-command.json", argv) + code = run_step(argv, run_dir / "srun.log", config["resources"]["minutes"] * 60 - 300 + 60) + inside = read(run_dir / "step-result.json") + need(code == inside["exit_code"], "Slurm exit and workload receipt differ") + state.update(phase="complete" if code == 0 else "failed", **inside) + except (Exception, KeyboardInterrupt) as error: + state.update(phase="failed", error=str(error), exit_code=2) + code = 2 + finally: + if receipt is None and (run_dir / "allocation.json").is_file(): + receipt = read(run_dir / "allocation.json") + try: + if receipt is not None: + state["step_cleanup"] = drain_step(receipt, config["task_id"], run_dir) + except Exception as error: + state.update(phase="failed", step_cleanup_error=str(error), ci_accepted=False) + code = 2 + try: + if receipt is not None and not reused: + state["allocation_cleanup"] = stop_allocation(receipt, config["task_id"]) + elif reused: + state["allocation_cleanup"] = {"status": "retained", "reason": "attached step does not own the parent allocation"} + except Exception as error: + state.update(phase="failed", cleanup_error=str(error), ci_accepted=False) + code = 2 + state.update(finished_at=now(), exit_code=code) + write(run_dir / "ci.json", state) + links = ("ci.json", "runtime-entry.sh", "runtime-readiness.record", "allocation.json", "recovery.json", "binding.json", "context.json", "step-result.json", + "gpu/spec.json", "gpu/gpu-job.json", "gpu/baseline/run.json", "gpu/candidate/run.json", + "gpu/comparison.json", "report/index.html") + if config["mode"] == "serving-smoke": + links += ("serving-smoke.json",) + links += tuple(f"gpu/c{concurrency}/{path}" for concurrency in (1, 2, 4) + for path in ("spec.json", "gpu-job.json", "baseline/run.json", "power.json")) + write(run_dir / "manifest.json", {"schema_version": 1, "task_id": config["task_id"], + "git_commit": sha, "ci": state["ci"], "run_id": run_id, "run_attempt": attempt, + "slurm_allocation": receipt, "runtime": config["runtime"], "prepared_spec": config["spec"], + "workload_plan": spec.get("plan"), "mode": config["mode"], "resources": state["resources"], "exit_code": code, + "evidence": {path: digest(run_dir / path) for path in links if (run_dir / path).is_file()}, + "artifact_checksums": "SHA256SUMS", "excluded_persistent_caches": list(CACHE_PATHS)}) + collect(run_dir, output) + return code + + +def enter(run_dir: Path) -> None: + """First command in the allocated step: retain identity before Enroot/CUDA.""" + context = read(run_dir / "context.json") + config = validate_config(context["config"]) + job, step = os.environ.get("SLURM_JOB_ID"), os.environ.get("SLURM_STEP_ID", "") + need(job == context["allocation"]["identity"]["JobId"] and re.fullmatch(r"[0-9]+", step) + and os.environ.get("SLURMD_NODENAME") == context["node"], "Wrong Slurm step assignment") + write(run_dir / "binding.json", {"job_id": job, "step_id": step, "node": context["node"], + "cpu_affinity": sorted(os.sched_getaffinity(0)), "observed_at": now(), "phase": "entering_runtime"}) + need(digest(config["runtime"]["entry"]) == config["runtime"]["entry_sha256"], "Entry changed on compute node") + argv = ["/bin/bash", config["runtime"]["entry"], config["runtime"]["python"], + str(mapped(config, Path(__file__).parent) / "ci.py"), "--inside", str(mapped(config, run_dir))] + os.execv(argv[0], argv) + + +def workload_complete(verified: dict) -> bool: + runs, comparison = verified["runs"], verified["comparison"] + return (all(run["summary"]["valid"] == run["summary"]["scheduled"] > 0 for run in runs.values()) + and bool(comparison["slots"]) + and all(slot[role]["status"] == "succeeded" and (slot[role].get("media") or {}).get("valid") is True + and not slot[role].get("analysis_error") + for slot in comparison["slots"] for role in ("baseline", "candidate")) + and all(check["status"] == "pass" for check in comparison["checks"] + if check["name"] in {"baseline.warmup", "candidate.warmup"})) + + +def smoke_exit(verified: dict, receipt: dict, mode: str) -> int: + if mode == "regression": + if receipt.get("regression_status") == "fail": + return 1 + return 0 if receipt.get("ci_accepted") is True else 2 + # Raw verification establishes identity, timing and cleanup. A smoke tests + # execution and fresh media validity independently of regression thresholds. + return 0 if workload_complete(verified) else 1 + + +def inside(run_dir: Path) -> int: + context = read(run_dir / "context.json") + config = validate_config(context["config"]) + expected = context["allocation"]["identity"]["JobId"] + step = os.environ.get("SLURM_STEP_ID", "") + result = {"exit_code": 2, "measurement_status": "incomplete", "regression_status": "inconclusive", "ci_accepted": False, "release_qualified": False} + try: + need(os.environ.get("SLURM_JOB_ID") == expected and re.fullmatch(r"[0-9]+", step) + and os.environ.get("SLURMD_NODENAME") == context["node"] + and os.environ.get("SLURM_PROCID") == "0" and os.environ.get("SLURM_NTASKS") == "1", + "Payload is not the exact single-node Slurm task") + need(inventory(Path(__file__).parent) == context["package_files"], "Staged harness bytes changed") + devices = cuda_devices() + assigned = os.environ.get("H3_ASSIGNED_GPU_UUIDS", "").split(",") + need(set(devices) == set(assigned), "CUDA-visible UUIDs differ from the Slurm global GPU assignment") + cpus = sorted(os.sched_getaffinity(0)) + binding = read(run_dir / "binding.json") + need(binding["job_id"] == expected and binding["step_id"] == step and binding["cpu_affinity"] == cpus, + "Container changed its assigned step identity or CPU binding") + need(len(devices) == config["resources"]["gpus"] and len(set(devices)) == len(devices), "Slurm step CUDA device count/UUIDs differ") + need(len(cpus) >= config["resources"]["cpus"], "Bound step CPU set is too small") + write(run_dir / "binding.json", {"job_id": expected, "step_id": step, "node": context["node"], + "gpu_uuids": devices, "cpu_affinity": cpus, + "slurm": {key: os.environ.get(key) for key in ("CUDA_VISIBLE_DEVICES", "H3_ORIGINAL_CUDA_VISIBLE_DEVICES", "SLURM_JOB_GPUS", "SLURM_STEP_GPUS", "SLURM_CPU_BIND", "SLURM_CPUS_PER_TASK")}, "observed_at": now()}) + spec = context["spec"] + spec["gpu_uuids"], spec["job_id"] = devices, context["run_id"] + active = [line for line in context["active_steps"].splitlines() if not re.match(r"[0-9]+\.(batch|extern)\|", line)] + spec["allocation"] = {"mode": "dedicated_ci" if not active and context["exclusive_node"] else "cooperative_shared", "label": f"Slurm {expected}.{step} on {context['node']}"} + if config["mode"] == "serving-smoke": + from evaluator.mvp_serving_smoke import run_matrix + matrix = run_matrix(spec, run_dir) + complete = matrix["status"] == "complete" + result.update(exit_code=0 if complete else 1, smoke_completed=complete, + measurement_status="complete" if complete else "incomplete", + serving_summary="serving-smoke.json") + else: + from evaluator.mvp_gpu_job import run_gpu_job + from evaluator.mvp_gpu_evidence import verify_measurement_job + receipt = run_gpu_job(spec, run_dir / "gpu") + result.update({key: receipt[key] for key in ("measurement_status", "regression_status", "ci_accepted", "release_qualified")}) + verified = verify_measurement_job(run_dir / "gpu", deadline=time.monotonic() + 120) + result["exit_code"] = smoke_exit(verified, receipt, config["mode"]) + result["smoke_completed"] = workload_complete(verified) + except (Exception, KeyboardInterrupt) as error: + result.update(exit_code=2, error=str(error), smoke_completed=False, ci_accepted=False) + finally: + if (run_dir / "gpu" / "gpu-job.json").is_file(): + try: + from evaluator.mvp_gpu_report import write_gpu_report + write_gpu_report(run_dir / "gpu", run_dir / "report") + except Exception as error: + result.update(exit_code=2, report_error=str(error), ci_accepted=False) + write(run_dir / "step-result.json", result) + return result["exit_code"] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--inside", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--enter", type=Path, help=argparse.SUPPRESS) + args = parser.parse_args() + if args.enter: + enter(args.enter) + return 2 + if args.inside: + return inside(args.inside) + need(args.config is not None and args.output is not None, "--config and --output required") + args.output.mkdir(parents=True, exist_ok=False) + try: + return launch(read(args.config), args.output) + except (Exception, KeyboardInterrupt) as error: + write(args.output / "adapter-error.json", {"error": str(error), "exit_code": 2, "recorded_at": now(), "ci_accepted": False}) + print(str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experimental/video-generation/evaluator/__init__.py b/experimental/video-generation/evaluator/__init__.py new file mode 100644 index 0000000000..b05d37bae3 --- /dev/null +++ b/experimental/video-generation/evaluator/__init__.py @@ -0,0 +1 @@ +"""H3 video generation, measurement, comparison, and artifact reports.""" diff --git a/experimental/video-generation/evaluator/cli.py b/experimental/video-generation/evaluator/cli.py new file mode 100644 index 0000000000..11536f1208 --- /dev/null +++ b/experimental/video-generation/evaluator/cli.py @@ -0,0 +1,217 @@ +"""Controlled H3 CI, full-stream comparison, and portable evidence reports.""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import stat +import sys +from pathlib import Path +from typing import Any + + +def _load_mvp_object(path: Path) -> dict[str, Any]: + """Read a bounded regular configuration file, never a FIFO or device.""" + limit = 4 * 1024 * 1024 + flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0) + with os.fdopen(os.open(path, flags), "rb") as stream: + metadata = os.fstat(stream.fileno()) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"MVP configuration must be a regular file: {path}") + if metadata.st_size > limit: + raise ValueError(f"MVP configuration exceeds 4 MiB: {path}") + data = stream.read(limit + 1) + if len(data) > limit: + raise ValueError(f"MVP configuration exceeds 4 MiB while being read: {path}") + + def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r} in {path}") + result[key] = value + return result + + def invalid_constant(value: str) -> None: + raise ValueError(f"non-finite JSON value {value!r} in {path}") + + result = json.loads( + data, + object_pairs_hook=unique_object, + parse_constant=invalid_constant, + ) + if not isinstance(result, dict): + raise ValueError(f"expected a JSON object: {path}") + return result + + +def _mvp_command(arguments: argparse.Namespace) -> int: + try: + if arguments.command == "gpu-manifest": + from .mvp_gpu_manifest import write_gpu_manifest + + result = write_gpu_manifest( + arguments.kind, arguments.directory, arguments.output, + model_revision=arguments.model_revision, timeout_seconds=arguments.timeout_seconds, + ) + print(json.dumps({key: value for key, value in result.items() if key != "files"}, + indent=2, sort_keys=True, allow_nan=False)) + return 0 + + if arguments.command == "gpu-job": + from .mvp_gpu_job import preview_gpu_job, run_gpu_job + + spec = _load_mvp_object(arguments.spec) + if not arguments.execute: + result = preview_gpu_job(spec) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + return 0 + if arguments.output is None: + raise ValueError("gpu-job --execute requires --output") + result = run_gpu_job(spec, arguments.output) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + if (result.get("ci_accepted") is True and result.get("status") == "complete" + and result.get("measurement_status") == "complete" + and result.get("regression_status") == "pass" and result.get("cleanup_status") == "clean"): + return 0 + if (result.get("regression_status") == "fail" or result.get("status") in {"failed", "aborted"} + or result.get("cleanup_status") == "failed"): + return 1 + return 2 + + if arguments.command == "gpu-report": + from .mvp_gpu_report import write_gpu_report + + result = write_gpu_report(arguments.job, arguments.output) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + # Successfully rendering an incomplete/failed run is not acceptance + # of that run. The GPU job's separate exit code remains authoritative. + return 0 + + if arguments.command == "run": + from .mvp_runner import preview_plan, run_plan + + plan = _load_mvp_object(arguments.plan) + from .mvp_serving import settings + serving = settings(arguments.serving_concurrency, arguments.delivery_deadline_seconds) + if not arguments.execute: + result = preview_plan(plan, runtime=arguments.runtime) + if serving: + result["serving"] = serving + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + return 0 + required = ("endpoint", "runtime_revision", "hardware_label", "model_revision", "output") + missing = [name.replace("_", "-") for name in required if not getattr(arguments, name)] + if missing: + raise ValueError("--execute also requires " + ", ".join("--" + name for name in missing)) + def interrupt_run(_signum, _frame): + raise KeyboardInterrupt + + previous_sigterm = signal.signal(signal.SIGTERM, interrupt_run) + try: + result = run_plan( + plan, + arguments.output, + endpoint=arguments.endpoint, + runtime=arguments.runtime, + runtime_revision=arguments.runtime_revision, + hardware_label=arguments.hardware_label, + model_revision=arguments.model_revision, + timeout_seconds=arguments.timeout_seconds, + api_key_env=arguments.api_key_env, + serving_concurrency=arguments.serving_concurrency, + delivery_deadline_seconds=arguments.delivery_deadline_seconds, + ) + finally: + signal.signal(signal.SIGTERM, previous_sigterm) + receipt = { + key: result.get(key) + for key in ("run_id", "evidence_kind", "status", "summary") + } + receipt["run_json"] = str((arguments.output / "run.json").resolve()) + print(json.dumps(receipt, indent=2, sort_keys=True, allow_nan=False)) + return 0 if result.get("status") == "complete" else 1 + + if arguments.command == "compare": + from .mvp_compare import compare_runs + from .mvp_report import write_report + + result = compare_runs( + arguments.baseline, arguments.candidate, + policy=_load_mvp_object(arguments.policy), + ) + if arguments.report is not None: + write_report(result, arguments.report) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + return {"pass": 0, "fail": 1, "inconclusive": 2}[result["overall_status"]] + + if arguments.command == "inspect-media": + from .mvp_media import analyze_media + + expected = _load_mvp_object(arguments.expected) if arguments.expected else None + result = analyze_media(arguments.media, expected=expected) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + return 0 if result["valid"] else 1 + except (ValueError, OSError, RuntimeError, ImportError) as exc: + message = str(exc) + if isinstance(exc, ImportError): + message += "; install media dependencies with: uv sync" + print(json.dumps({"status": "error", "error": message}, allow_nan=False), file=sys.stderr) + return 2 + raise AssertionError(arguments.command) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="vgbench") + subparsers = parser.add_subparsers(dest="command", required=True) + + gpu_manifest = subparsers.add_parser("gpu-manifest", help="hash staged runtime/model files; no GPU work or downloads") + gpu_manifest.add_argument("kind", choices=("runtime", "model")) + gpu_manifest.add_argument("directory", type=Path) + gpu_manifest.add_argument("--model-revision", help="immutable model revision, required for a model inventory") + gpu_manifest.add_argument("--timeout-seconds", type=float, default=600) + gpu_manifest.add_argument("--output", required=True, type=Path, help="new manifest file outside the inventoried tree") + + gpu_job = subparsers.add_parser("gpu-job", help="preview a bounded controlled GPU job; --execute requires authorization") + gpu_job.add_argument("spec", type=Path) + gpu_job.add_argument("--execute", action="store_true") + gpu_job.add_argument("--output", type=Path, help="new evidence directory; no overwrites") + + gpu_report = subparsers.add_parser("gpu-report", help="render recorded GPU job evidence, including incomplete runs") + gpu_report.add_argument("job", type=Path) + gpu_report.add_argument("--output", required=True, type=Path, help="new report directory") + + run = subparsers.add_parser("run", help="preview an H3 plan; --execute explicitly submits it") + run.add_argument("plan", type=Path) + run.add_argument("--runtime", choices=("sglang", "vllm-omni"), default="sglang") + run.add_argument("--execute", action="store_true", help="submit real requests to your H3 server") + run.add_argument("--endpoint", help="self-hosted server base URL, without /v1/videos") + run.add_argument("--runtime-revision", help="operator-declared exact server code/container revision") + run.add_argument("--hardware-label", help="operator-declared hardware, precision, and topology label") + run.add_argument("--model-revision", help="operator-declared exact checkpoint revision matching the plan") + run.add_argument("--output", type=Path, help="new output directory; existing paths are never overwritten") + run.add_argument("--timeout-seconds", type=float, default=3600) + run.add_argument("--api-key-env", help="name of an environment variable containing a bearer token") + run.add_argument("--serving-concurrency", type=int, help="opt into closed-loop delivery load with 1-32 concurrent requests; validation is separate") + run.add_argument("--delivery-deadline-seconds", type=float, help="optional submit-to-downloaded-media deadline for technical goodput; not an attempt timeout") + + compare = subparsers.add_parser("compare", help="compare paired run bundles and return a CI exit code") + compare.add_argument("baseline", type=Path) + compare.add_argument("candidate", type=Path) + compare.add_argument("--policy", required=True, type=Path) + compare.add_argument("--report", type=Path, help="new portable HTML report plus a local media-assets directory") + + inspect = subparsers.add_parser("inspect-media", help="fully decode a file and measure technical media integrity") + inspect.add_argument("media", type=Path) + inspect.add_argument("--expected", type=Path, help="optional JSON media expectations") + return parser + + +def main(argv: list[str] | None = None) -> int: + return _mvp_command(_parser().parse_args(argv)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experimental/video-generation/evaluator/mvp_compare.py b/experimental/video-generation/evaluator/mvp_compare.py new file mode 100644 index 0000000000..107f187d37 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_compare.py @@ -0,0 +1,741 @@ +"""Auditable, paired comparison of the small H3 execution bundles. + +The MVP measures implementation fidelity, not which generative model is better. +Threshold decisions are deliberately separate from release qualification. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import statistics +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit, urlunsplit + + +_POLICY_NUMBERS = { + "max_latency_increase_fraction": (0.0, None), + "min_video_psnr_db": (0.0, None), + "min_audio_spectral_cosine": (-1.0, 1.0), + "max_audio_rms_ratio_error": (0.0, None), +} + + +def analyze_media(path: Path, expected: dict | None = None) -> dict: + """Load the optional media backend only when a comparison needs it.""" + from evaluator.mvp_media import analyze_media as implementation + + return implementation(path, expected=expected) + + +def compare_media(baseline: Path, candidate: Path) -> dict: + from evaluator.mvp_media import compare_media as implementation + + return implementation(baseline, candidate) + + +def _finite(value: Any, *, minimum: float | None = None) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and (minimum is None or value >= minimum) + ) + + +def _canonical(value: Any) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).encode("utf-8") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _reject_nonfinite(value: str) -> None: + raise ValueError(f"non-finite JSON numeric literal is not allowed: {value}") + + +def _unique_keys(pairs: list[tuple[str, Any]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _text(mapping: dict, field: str, label: str) -> str: + value = mapping.get(field) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{label}.{field} must be a nonempty string") + return value + + +def _policy(policy: dict) -> dict: + if not isinstance(policy, dict): + raise ValueError("comparison policy must be an object") + _text(policy, "policy_id", "policy") + if policy.get("calibration_status") not in { + "uncalibrated", "fixture_control", "operator_calibrated" + }: + raise ValueError("policy.calibration_status must explicitly describe calibration") + for name, (minimum, maximum) in _POLICY_NUMBERS.items(): + value = policy.get(name) + if not _finite(value, minimum=minimum) or ( + maximum is not None and value > maximum + ): + raise ValueError(f"policy.{name} must be an explicit, finite value in range") + # Reject non-JSON extras as well; retain any declared calibration provenance. + return json.loads(_canonical(policy)) + + +def _planned_slots(plan: dict) -> dict[str, dict]: + for field in ("plan_id", "model_id", "model_revision"): + _text(plan, field, "plan") + cases = plan.get("cases") + repetitions = plan.get("repetitions") + if not isinstance(cases, list) or not cases: + raise ValueError("plan.cases must be a nonempty list") + if not isinstance(repetitions, int) or isinstance(repetitions, bool) or repetitions < 1: + raise ValueError("plan.repetitions must be a positive integer") + warmups = plan.get("warmup_runs") + if not isinstance(warmups, int) or isinstance(warmups, bool) or warmups < 0: + raise ValueError("plan.warmup_runs must be an explicit nonnegative integer") + if len(cases) * repetitions + warmups > 10000: + raise ValueError("plan exceeds 10000 total slots") + generation = plan.get("generation") + if not isinstance(generation, dict): + raise ValueError("plan.generation must declare expected media properties") + for field, maximum in (("width", 8192), ("height", 8192), ("frame_count", 10000)): + value = generation.get(field) + if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= maximum: + raise ValueError(f"plan.generation.{field} must be a positive bounded integer") + for field, maximum in (("fps", 240), ("duration_seconds", 3600)): + value = generation.get(field) + if not _finite(value, minimum=0.000001) or value > maximum: + raise ValueError(f"plan.generation.{field} must be a positive bounded number") + if "audio_required" in generation and not isinstance(generation["audio_required"], bool): + raise ValueError("plan.generation.audio_required must be a boolean") + if generation.get("audio_required") is not False: + for field, maximum in (("audio_sample_rate_hz", 192000), ("audio_channels", 8)): + value = generation.get(field) + if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= maximum: + raise ValueError(f"plan.generation.{field} must declare a positive bounded integer") + ids = [] + for case in cases: + if not isinstance(case, dict): + raise ValueError("plan cases must be objects") + ids.append(_text(case, "case_id", "case")) + _text(case, "prompt", "case") + if not isinstance(case.get("seed"), int) or isinstance(case.get("seed"), bool): + raise ValueError("each case must declare an integer seed") + for field in ("requires_motion", "requires_sound"): + if not isinstance(case.get(field), bool): + raise ValueError(f"case.{field} must be an explicit boolean") + if len(set(ids)) != len(ids): + raise ValueError("plan contains duplicate case_id values") + return { + f"measurement-r{repetition:03d}-c{index:03d}": { + **case, + "repetition": repetition, + } + for repetition in range(1, repetitions + 1) + for index, case in enumerate(cases, start=1) + } + + +def _artifact(run_dir: Path, record: dict) -> Path | None: + raw_path, digest = record.get("artifact_path"), record.get("sha256") + if raw_path is None: + if digest is not None: + raise ValueError(f"{record['slot_id']}: hash without an artifact") + if record.get("status") == "succeeded": + raise ValueError(f"{record['slot_id']}: successful record has no artifact") + return None + if not isinstance(raw_path, str) or not raw_path or "\\" in raw_path: + raise ValueError(f"{record['slot_id']}: artifact_path must be a relative path") + relative = Path(raw_path) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError(f"{record['slot_id']}: artifact path escapes the run directory") + try: + path = (run_dir / relative).resolve(strict=True) + except OSError as error: + raise ValueError(f"{record['slot_id']}: missing artifact: {raw_path}") from error + if not path.is_relative_to(run_dir) or not path.is_file(): + raise ValueError(f"{record['slot_id']}: artifact is not a file inside its run directory") + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + or _sha256(path) != digest + ): + raise ValueError(f"{record['slot_id']}: artifact SHA256 mismatch") + return path + + +def _validate_timing_boundaries(record: dict, evidence_kind: str) -> None: + """Validate the optional, nested client timings without inventing old data.""" + fields = ("submit_to_terminal_seconds", "submit_to_media_seconds", "media_validation_seconds") + if not any(field in record for field in fields): + return # Earlier bundles did not instrument these boundaries. + slot_id = record["slot_id"] + values = [record.get(field) for field in fields] + for field, value in zip(fields, values): + if value is not None and not _finite(value, minimum=0): + raise ValueError(f"{slot_id}: {field} must be a finite nonnegative timing or null") + if not any(value is not None for value in values): + if record["status"] == "succeeded" and evidence_kind != "imported_media": + raise ValueError(f"{slot_id}: successful instrumented record is missing client timing boundaries") + return + if evidence_kind == "imported_media" or record.get("attempted") is False: + raise ValueError(f"{slot_id}: imported or not-started media cannot have client timing boundaries") + latency = record.get("latency_seconds") + if not _finite(latency, minimum=0): + raise ValueError(f"{slot_id}: client timing boundaries require a finite nonnegative total latency") + terminal, media, validation = values + if record["status"] == "succeeded" and any(value is None for value in values): + raise ValueError(f"{slot_id}: successful instrumented record is missing client timing boundaries") + if (media is not None and terminal is None) or (validation is not None and media is None): + raise ValueError(f"{slot_id}: client timing boundaries are missing an earlier stage") + tolerance = max(1e-6, latency * 1e-6) + if any(value is not None and value > latency + tolerance for value in values): + raise ValueError(f"{slot_id}: client timing boundary exceeds total latency") + if terminal is not None and media is not None and terminal > media + tolerance: + raise ValueError(f"{slot_id}: terminal status timing follows completed media download") + if media is not None and validation is not None and media + validation > latency + tolerance: + raise ValueError(f"{slot_id}: download plus validation time exceeds total latency") + + +def _load_run(directory: Path) -> tuple[dict, dict[str, dict]]: + directory = Path(directory).resolve(strict=True) + path = directory / "run.json" + if not directory.is_dir() or not path.is_file(): + raise ValueError(f"not an MVP run directory: {directory}") + with path.open(encoding="utf-8") as stream: + run = json.load(stream, parse_constant=_reject_nonfinite, object_pairs_hook=_unique_keys) + if not isinstance(run, dict): + raise ValueError("run.json must be an object") + if run.get("bundle_version") != "0.1.0" or run.get("bundle_type") != "mvp_run": + raise ValueError("unsupported MVP run bundle type/version") + if run.get("status") not in {"complete", "partial", "failed"} or not run.get("finished_at"): + raise ValueError("run bundle is not finalized") + try: + started = datetime.fromisoformat(_text(run, "started_at", "run")) + finished = datetime.fromisoformat(_text(run, "finished_at", "run")) + if started.tzinfo is None or finished.tzinfo is None or finished < started: + raise ValueError("invalid run timestamps") + except (TypeError, ValueError) as error: + raise ValueError("finalized run requires ordered timezone-aware started_at/finished_at timestamps") from error + for field in ("run_id", "plan_id", "plan_sha256"): + _text(run, field, "run") + if run.get("evidence_kind") not in {"operator_endpoint", "live_h3", "fixture", "imported_media"}: + raise ValueError("run must declare its actual evidence_kind") + plan = run.get("plan") + if not isinstance(plan, dict) or hashlib.sha256(_canonical(plan)).hexdigest() != run["plan_sha256"]: + raise ValueError("run plan SHA256 does not match its canonical plan content") + if plan.get("plan_id") != run["plan_id"]: + raise ValueError("run.plan_id does not match its embedded plan") + configuration = run.get("configuration") + if not isinstance(configuration, dict): + raise ValueError("run.configuration must be an object") + for field in ("model_id", "model_revision", "runtime", "runtime_revision", "hardware_label"): + _text(configuration, field, "configuration") + for field in ("model_id", "model_revision"): + if configuration[field] != plan.get(field): + raise ValueError(f"configuration.{field} does not match the frozen plan") + unsigned_configuration = {key: value for key, value in configuration.items() if key != "configuration_sha256"} + configuration_digest = hashlib.sha256(_canonical(unsigned_configuration)).hexdigest() + if not (run.get("configuration_sha256") or configuration.get("configuration_sha256")): + raise ValueError("run must declare its configuration SHA256") + for declared in (run.get("configuration_sha256"), configuration.get("configuration_sha256")): + if declared is not None and declared != configuration_digest: + raise ValueError("configuration SHA256 does not match its content") + measurement = run.get("measurement") + if not isinstance(measurement, dict): + raise ValueError("run.measurement must be an object") + serving = configuration.get("serving") + boundary = "not_measured_imported_media" if run["evidence_kind"] == "imported_media" else ("submit_to_downloaded_media" if serving else "submit_to_validated_media") + if measurement.get("boundary") != boundary: + raise ValueError(f"MVP {run['evidence_kind']} comparison requires {boundary} timing boundary") + if serving: + from .mvp_serving import validate_window + if run["evidence_kind"] == "imported_media": + raise ValueError("imported media cannot establish serving load") + elif type(measurement.get("concurrency")) is not int or measurement["concurrency"] != 1: + raise ValueError("MVP comparison requires serial concurrency=1 measurements") + planned = _planned_slots(plan) + planned_warmups = { + f"warmup-{index:03d}": {**plan["cases"][(index - 1) % len(plan["cases"])], "repetition": 0} + for index in range(1, plan["warmup_runs"] + 1) + } + records = run.get("records") + if not isinstance(records, list): + raise ValueError("run.records must be a list") + measurements: dict[str, dict] = {} + seen = set() + warmups_succeeded = True + for record in records: + if not isinstance(record, dict): + raise ValueError("run records must be objects") + slot_id = _text(record, "slot_id", "record") + if slot_id in seen: + raise ValueError(f"duplicate slot_id: {slot_id}") + seen.add(slot_id) + if record.get("phase") not in {"warmup", "measurement"}: + raise ValueError(f"{slot_id}: unknown measurement phase") + if record.get("status") not in {"succeeded", "failed"}: + raise ValueError(f"{slot_id}: unknown execution status") + if "attempted" in record and not isinstance(record["attempted"], bool): + raise ValueError(f"{slot_id}: attempted must be a boolean") + if record.get("attempted") is False and ( + record["status"] != "failed" or record.get("artifact_path") is not None + or record.get("sha256") is not None or record.get("media") is not None + or record.get("latency_seconds") != 0 + ): + raise ValueError(f"{slot_id}: a not-started slot cannot contain a successful attempt or measured artifact") + _validate_timing_boundaries(record, run["evidence_kind"]) + # Resolve and hash even warmups and failed partial artifacts; none are trusted. + verified_path = _artifact(directory, record) + if record["phase"] == "warmup": + if slot_id not in planned_warmups: + raise ValueError(f"unexpected warmup slot: {slot_id}") + for field in ("case_id", "prompt", "seed", "repetition"): + if _canonical(record.get(field)) != _canonical(planned_warmups[slot_id].get(field)): + raise ValueError(f"{slot_id}: {field} differs from planned warmup") + if measurements: + raise ValueError("warmup slots must precede measurement slots") + warmup_valid = False + if verified_path is not None and record["status"] == "succeeded": + try: + warmup_media = analyze_media(verified_path, expected=_expected(plan, planned_warmups[slot_id])) + warmup_valid = warmup_media.get("valid") is True + except Exception: + warmup_valid = False + warmups_succeeded = warmups_succeeded and warmup_valid + continue + if slot_id not in planned: + raise ValueError(f"unexpected measurement slot: {slot_id}") + case = planned[slot_id] + for field in ("case_id", "prompt", "seed", "repetition"): + if _canonical(record.get(field)) != _canonical(case.get(field)): + raise ValueError(f"{slot_id}: {field} differs from the frozen plan") + measurements[slot_id] = {**record, "_verified_path": verified_path} + missing = set(planned) - set(measurements) + if missing: + raise ValueError(f"missing measurement slots (failures must be retained): {', '.join(sorted(missing))}") + missing_warmups = set(planned_warmups) - seen + if missing_warmups: + raise ValueError(f"missing warmup slots: {', '.join(sorted(missing_warmups))}") + if [record["slot_id"] for record in records] != list(planned_warmups) + list(planned): + raise ValueError("run records do not follow the frozen execution order") + if serving: + validate_window(run) + if run["evidence_kind"] != "imported_media" and not serving: + attempted_seconds = sum( + record["latency_seconds"] for record in measurements.values() + if record.get("attempted") is not False and _finite(record.get("latency_seconds"), minimum=0) + ) + wall = measurement.get("wall_seconds") + tolerance = max(1e-6, attempted_seconds * 1e-6) + if _finite(wall, minimum=0.000001) and wall + tolerance < attempted_seconds: + raise ValueError("measured wall time is shorter than summed serial attempted latencies") + run["_directory"] = directory + run["_bundle_sha256"] = _sha256(path) + run["_warmups_succeeded"] = warmups_succeeded + return run, measurements + + +def _expected(plan: dict, case: dict) -> dict: + generation = dict(plan.get("generation", {})) + frames, fps = generation.get("frame_count"), generation.get("fps") + if _finite(frames, minimum=1) and _finite(fps, minimum=0.000001): + generation["duration_seconds"] = frames / fps + generation["audio_required"] = bool( + generation.get("audio_required", generation.get("audio_sample_rate_hz")) + ) + for field in ("requires_motion", "requires_sound"): + generation[field] = bool(case.get(field, False)) + return generation + + +def _observation(record: dict, expected: dict, *, timing_measured: bool = True, latency_field: str = "latency_seconds") -> dict: + path = record["_verified_path"] + analysis, analysis_error = None, None + if path is not None: + try: + analysis = analyze_media(path, expected=expected) + except Exception as error: + # An evaluator failure is not evidence that the model failed a metric. + analysis_error = f"{type(error).__name__}: {error}" + if analysis and analysis.get("sha256") not in {None, record.get("sha256")}: + raise ValueError(f"{record['slot_id']}: artifact changed between hash verification and media analysis") + latency = record.get(latency_field) if timing_measured and record.get("attempted") is not False else None + return { + "status": record["status"], + "attempted": record.get("attempted", True), + "artifact_path": str(path) if path else None, + "sha256": record.get("sha256"), + "latency_seconds": latency if _finite(latency, minimum=0) else None, + "latency_boundary": "submit_to_downloaded_media" if latency_field == "submit_to_media_seconds" else "submit_to_validated_media", + "media": analysis, + "error": record.get("error"), + "analysis_error": analysis_error, + } + + +def _check(name: str, status: str, reason: str, **values: Any) -> dict: + return {"name": name, "status": status, "reason": reason, **values} + + +def _outcome(checks: list[dict]) -> str: + if any(check["status"] == "fail" for check in checks): + return "fail" + if any(check["status"] == "inconclusive" for check in checks): + return "inconclusive" + return "pass" + + +def _valid(observation: dict) -> bool: + return observation["status"] == "succeeded" and (observation.get("media") or {}).get("valid") is True + + +def _media_check(label: str, observation: dict) -> dict: + if observation["status"] != "succeeded": + return _check( + f"{label}.technical_success", "fail" if label == "candidate" else "inconclusive", + observation.get("error") or "generation did not succeed; retained in denominator", + ) + if observation["analysis_error"]: + return _check(f"{label}.media_validity", "inconclusive", observation["analysis_error"]) + media = observation.get("media") or {} + if media.get("valid") is not True: + reasons = [ + str(item.get("detail") or item.get("name")) + for item in media.get("checks", []) + if item.get("status") in {"failed", "fail"} + ] + return _check( + f"{label}.media_validity", "fail" if label == "candidate" else "inconclusive", + "; ".join(reasons) or "media did not pass fresh validation", + ) + return _check(f"{label}.media_validity", "pass", "fresh decode and requested media checks passed") + + +def _threshold(name: str, value: Any, threshold: float, *, minimum: bool, unit: str) -> dict: + if not _finite(value): + return _check(name, "inconclusive", "metric missing, undefined, or non-finite; not imputed", observed=None, threshold=threshold, unit=unit) + passed = value >= threshold if minimum else value <= threshold + return _check(name, "pass" if passed else "fail", "within declared threshold" if passed else "outside declared threshold", observed=value, threshold=threshold, unit=unit) + + +def _json_safe(value: Any) -> Any: + """Preserve missing metric states without emitting invalid JSON NaN/Infinity.""" + if isinstance(value, float) and not math.isfinite(value): + return None + if isinstance(value, dict): + return {key: _json_safe(item) for key, item in value.items()} + if isinstance(value, (tuple, list)): + return [_json_safe(item) for item in value] + return value + + +def _fidelity(baseline: dict, candidate: dict, policy: dict) -> tuple[dict, list[dict], list[str]]: + try: + result = compare_media(Path(baseline["artifact_path"]), Path(candidate["artifact_path"])) + except Exception as error: + return {}, [_check("fidelity.evaluator", "inconclusive", f"{type(error).__name__}: {error}")], [] + metrics = result.get("metrics", {}) + notes = list(result.get("notes", [])) + for diagnostic in result.get("checks", []): + if diagnostic.get("status") in {"failed", "fail"}: + notes.append( + f"{diagnostic.get('name', 'media comparison')}: {diagnostic.get('detail', '')} " + f"Observed {_json_safe(diagnostic.get('observed'))}; expected {_json_safe(diagnostic.get('expected'))}." + ) + checks = [_check( + "fidelity.compatible_media", "pass" if result.get("compatible") is True else "fail", + "same frame/audio geometry and time alignment" if result.get("compatible") is True + else "streams are not compatible for a no-resize/no-truncation fidelity comparison", + )] + if result.get("compatible") is not True: + return _json_safe(metrics), checks, notes + if metrics.get("video_identical") is True: + checks.append(_check( + "fidelity.video_psnr", "pass", "decoded video frames are identical; zero MSE makes finite PSNR undefined", + observed=None, exact_match=True, threshold=policy["min_video_psnr_db"], unit="dB", + )) + else: + checks.append(_threshold("fidelity.video_psnr", metrics.get("video_psnr_db"), policy["min_video_psnr_db"], minimum=True, unit="dB")) + audio_present = any((row.get("media") or {}).get("audio", {}).get("present") is True for row in (baseline, candidate)) + if not audio_present: + checks.extend([ + _check("fidelity.audio_spectral_cosine", "not_applicable", "neither stream contains audio"), + _check("fidelity.audio_rms_ratio_error", "not_applicable", "neither stream contains audio"), + ]) + else: + # Worst-channel checks prevent a healthy channel hiding a collapsed channel. + cosine_channels = metrics.get("audio_spectral_cosine_channels") + if cosine_channels: + cosine = min(cosine_channels) if all(_finite(value) for value in cosine_channels) else None + else: + cosine = metrics.get("audio_spectral_cosine") + checks.append(_threshold("fidelity.audio_spectral_cosine", cosine, policy["min_audio_spectral_cosine"], minimum=True, unit="cosine")) + ratios = metrics.get("audio_rms_ratio_channels") + if ratios: + rms_error = max(abs(value - 1.0) for value in ratios) if all(_finite(value, minimum=0) for value in ratios) else None + else: + ratio = metrics.get("audio_rms_ratio") + rms_error = abs(ratio - 1.0) if _finite(ratio, minimum=0) else None + metrics["audio_worst_channel_rms_ratio_error"] = rms_error + checks.append(_threshold("fidelity.audio_rms_ratio_error", rms_error, policy["max_audio_rms_ratio_error"], minimum=False, unit="absolute_ratio_error")) + return _json_safe(metrics), checks, notes + + +def _summary(run: dict, observations: list[dict]) -> dict: + valid = [row for row in observations if _valid(row)] + valid_latencies = [row["latency_seconds"] for row in valid if _finite(row["latency_seconds"], minimum=0.000001)] + attempted_latencies = [row["latency_seconds"] for row in observations if row["attempted"] is not False and _finite(row["latency_seconds"], minimum=0)] + wall = run["measurement"].get("wall_seconds") if run["evidence_kind"] != "imported_media" else None + if not _finite(wall, minimum=0.000001): + wall = None + return { + "scheduled": len(observations), + "completed": sum(row["status"] == "succeeded" for row in observations), + "valid": len(valid), + "failed": len(observations) - len(valid), + "failed_attempts": sum(row["status"] == "failed" and row["attempted"] is not False for row in observations), + "not_started": sum(row["attempted"] is False for row in observations), + "invalid_completed": sum(row["status"] == "succeeded" and not _valid(row) for row in observations), + "invalid_completed_interpretation": "completed but not verified valid; includes evaluator-unavailable observations", + "evaluator_unavailable": sum(bool(row.get("analysis_error")) for row in observations), + "known_invalid_completed": sum(row["status"] == "succeeded" and not row.get("analysis_error") and (row.get("media") or {}).get("valid") is False for row in observations), + "technical_success_rate": len(valid) / len(observations), + "verified_technical_success_fraction": len(valid) / len(observations), + "technical_success_rate_interpretation": "verified-valid fraction of scheduled slots, not an estimate of model success probability when the evaluator is unavailable", + "latency_median_seconds": statistics.median(valid_latencies) if valid_latencies else None, + "latency_measured_count": len(valid_latencies), + "latency_population": "valid measurement slots only; failures are retained separately", + "attempt_latency_median_seconds": statistics.median(attempted_latencies) if attempted_latencies else None, + "wall_seconds": wall, + "valid_clips_per_second": len(valid) / wall if wall is not None else None, + "throughput_population": "all scheduled measurement slots; measured wall time includes failed attempts", + "summary_recomputed_from_verified_artifacts": True, + } + + +def _configuration(configuration: dict) -> dict: + result = dict(configuration) + # The report does not need credentials, endpoint query parameters, or fragments. + endpoint = result.get("endpoint") + if isinstance(endpoint, str): + try: + parts = urlsplit(endpoint) + hostname = parts.hostname or "" + if ":" in hostname: + hostname = f"[{hostname}]" + authority = f"{hostname}:{parts.port}" if parts.port else hostname + result["endpoint"] = urlunsplit((parts.scheme, authority, parts.path, "", "")) + except ValueError: + result["endpoint"] = "[invalid endpoint redacted]" + return result + + +def _active_media_code() -> dict: + from evaluator import mvp_media + + return { + "implementation_version": mvp_media.IMPLEMENTATION_VERSION, + "source_sha256": _sha256(Path(mvp_media.__file__)), + } + + +def _performance_differences(baseline: dict, candidate: dict, slots: list[dict]) -> list[str]: + """Compare measured client work as well as operator-declared server class.""" + differences = [] + left, right = baseline["configuration"], candidate["configuration"] + if left.get("serving") or right.get("serving"): + differences.append("serving delivery measurements are descriptive; the existing serial regression policy is not calibrated for serving load") + if _canonical(left.get("serving")) != _canonical(right.get("serving")): + differences.append("serving load or delivery deadline differs") + live = bool({"operator_endpoint", "live_h3"} & {baseline["evidence_kind"], candidate["evidence_kind"]}) + for field in ("hardware_label", "runtime"): + if left[field] != right[field]: + differences.append(f"{field} differs") + for field in ("limits", "client_environment", "client_source_sha256", "measurement_semantics", "media_evaluator"): + a, b = left.get(field), right.get(field) + absent_left, absent_right = a in (None, "", {}), b in (None, "", {}) + if absent_left and absent_right and not live: + continue + if absent_left or absent_right: + differences.append(f"client {field} is missing") + elif _canonical(a) != _canonical(b): + differences.append(f"client {field} differs") + # Re-analysis versions must agree with the analyzer that was timed. Otherwise + # a changed client evaluator could be mistaken for a server speed regression. + declared_pins = [(label, run["configuration"].get("media_evaluator")) for label, run in (("baseline", baseline), ("candidate", candidate))] + if any(isinstance(pin, dict) and pin for _, pin in declared_pins): + code = _active_media_code() + fields = ("implementation_version", "source_sha256", "pyav_version", "numpy_version", "ffmpeg_libraries") + for label, pin in declared_pins: + if not isinstance(pin, dict) or not all(pin.get(field) for field in fields): + differences.append(f"{label} media evaluator pin is incomplete") + continue + if any(pin[field] != code[field] for field in code): + differences.append(f"{label} media evaluator code differs from fresh analysis") + for slot in slots: + implementation = (slot[label].get("media") or {}).get("implementation", {}) + actual = { + "implementation_version": implementation.get("version"), + "pyav_version": implementation.get("pyav_version"), + "numpy_version": implementation.get("numpy_version"), + "ffmpeg_libraries": implementation.get("ffmpeg_libraries"), + } + if any(_canonical(actual[field]) != _canonical(pin[field]) for field in actual): + differences.append(f"{label} media evaluator versions are unverified or differ from fresh analysis") + break + return list(dict.fromkeys(differences)) + + +def compare_runs(baseline_dir: Path, candidate_dir: Path, *, policy: dict) -> dict: + """Verify bundles and compare matched measurement slots under explicit policy. + + Malformed, incomplete, or tampered bundles raise ValueError. Valid bundles can + produce a fail or inconclusive decision. An MVP threshold pass is never a + release qualification or a claim about generative model quality. + """ + policy = _policy(policy) + baseline_run, baseline_records = _load_run(baseline_dir) + candidate_run, candidate_records = _load_run(candidate_dir) + if {"operator_endpoint", "live_h3"} & {baseline_run["evidence_kind"], candidate_run["evidence_kind"]} and ( + baseline_run["_directory"] == candidate_run["_directory"] or baseline_run["run_id"] == candidate_run["run_id"] + ): + raise ValueError("live comparison requires distinct baseline and candidate executions") + if baseline_run["plan_sha256"] != candidate_run["plan_sha256"]: + raise ValueError("baseline and candidate must use the identical frozen plan SHA256") + for field in ("model_id", "model_revision"): + if baseline_run["configuration"][field] != candidate_run["configuration"][field]: + raise ValueError(f"implementation fidelity requires identical {field}") + if set(baseline_records) != set(candidate_records): + raise ValueError("baseline and candidate measurement slots must match") + evidence_kind = baseline_run["evidence_kind"] if baseline_run["evidence_kind"] == candidate_run["evidence_kind"] else "mixed" + imported_only = evidence_kind == "imported_media" + plan = baseline_run["plan"] + planned = _planned_slots(plan) + slots = [] + for slot_id, case in planned.items(): + baseline = _observation(baseline_records[slot_id], _expected(plan, case), timing_measured=baseline_run["evidence_kind"] != "imported_media", latency_field="submit_to_media_seconds" if baseline_run["configuration"].get("serving") else "latency_seconds") + candidate = _observation(candidate_records[slot_id], _expected(plan, case), timing_measured=candidate_run["evidence_kind"] != "imported_media", latency_field="submit_to_media_seconds" if candidate_run["configuration"].get("serving") else "latency_seconds") + checks = [_media_check("baseline", baseline), _media_check("candidate", candidate)] + metrics, notes = {}, [] + if _valid(baseline) and _valid(candidate): + metrics, fidelity_checks, notes = _fidelity(baseline, candidate, policy) + checks.extend(fidelity_checks) + else: + checks.append(_check("fidelity.available_pair", "inconclusive", "requires valid baseline and candidate media; failed slots are retained")) + for label, observation in (("baseline", baseline), ("candidate", candidate)): + if not imported_only and _valid(observation) and not _finite(observation["latency_seconds"], minimum=0.000001): + checks.append(_check(f"{label}.latency", "inconclusive", "valid generation is missing a finite, positive measured latency")) + left, right = baseline["latency_seconds"], candidate["latency_seconds"] + metrics["latency_increase_fraction"] = right / left - 1 if _valid(baseline) and _valid(candidate) and _finite(left, minimum=0.000001) and _finite(right, minimum=0.000001) else None + slots.append({ + "slot_id": slot_id, "case_id": case["case_id"], "prompt": case["prompt"], + "seed": case["seed"], "repetition": case["repetition"], + "status": _outcome(checks), "baseline": baseline, "candidate": candidate, + "metrics": metrics, "checks": checks, "notes": notes, + }) + summaries = { + "baseline": _summary(baseline_run, [slot["baseline"] for slot in slots]), + "candidate": _summary(candidate_run, [slot["candidate"] for slot in slots]), + } + checks = [] + if evidence_kind == "mixed": + checks.append(_check("evidence.same_kind", "inconclusive", "baseline and candidate have different evidence kinds")) + different = _performance_differences(baseline_run, candidate_run, slots) + performance_mode = ( + "not_measured_imported_media" if imported_only + else "descriptive_only" if different or evidence_kind == "mixed" + else "same_configuration_class_regression" + ) + baseline_median = summaries["baseline"]["latency_median_seconds"] + candidate_median = summaries["candidate"]["latency_median_seconds"] + increase = candidate_median / baseline_median - 1 if _finite(baseline_median, minimum=0.000001) and _finite(candidate_median, minimum=0.000001) else None + complete_latency = all(summary["latency_measured_count"] == len(slots) for summary in summaries.values()) + if imported_only: + increase = None + checks.append(_check("performance.median_latency", "not_applicable", "imported media has no measured generation latency; this is a media-fidelity-only comparison")) + elif performance_mode == "descriptive_only": + checks.append(_check("performance.median_latency", "descriptive", "latency is descriptive, not a regression gate: " + ("; ".join(different) if different else "imported/mixed evidence"), observed=increase, threshold=policy["max_latency_increase_fraction"], unit="fraction")) + elif not complete_latency: + checks.append(_check("performance.median_latency", "inconclusive", "cannot gate a survivor-only or missing-latency population", observed=increase, threshold=policy["max_latency_increase_fraction"], unit="fraction")) + else: + checks.append(_threshold("performance.median_latency", increase, policy["max_latency_increase_fraction"], minimum=False, unit="fraction")) + for label in ("baseline", "candidate"): + if imported_only: + checks.append(_check(f"{label}.wall_time", "not_applicable", "no inference wall time or throughput was measured for imported media")) + elif summaries[label]["wall_seconds"] is None: + checks.append(_check(f"{label}.wall_time", "inconclusive", "missing finite measured wall time; throughput is not inferred from summed request latency")) + for label, run in (("baseline", baseline_run), ("candidate", candidate_run)): + if not run["_warmups_succeeded"]: + checks.append(_check(f"{label}.warmup", "inconclusive", "a planned warmup failed or produced invalid media; warmed performance is not established")) + all_checks = checks + [check for slot in slots for check in slot["checks"]] + result = { + "bundle_version": "0.1.0", "bundle_type": "mvp_comparison", + "created_at": datetime.now(timezone.utc).isoformat(), + "plan_id": baseline_run["plan_id"], "plan_sha256": baseline_run["plan_sha256"], "plan": plan, + "evidence_kind": evidence_kind, + "comparison_scope": "media_fidelity_only" if imported_only else "runtime_regression", + "overall_status": _outcome(all_checks), + "release_qualified": False, + "release_qualification_reason": "MVP threshold decisions are not a calibrated, independently verified release qualification.", + "policy": policy, + "measurement": { + "boundary": baseline_run["measurement"]["boundary"] if baseline_run["measurement"]["boundary"] == candidate_run["measurement"]["boundary"] else "mixed_incomparable_boundaries", + "concurrency": baseline_run["measurement"]["concurrency"] if baseline_run["measurement"]["concurrency"] == candidate_run["measurement"]["concurrency"] else None, + "performance_mode": performance_mode, + "performance_comparability_limitations": different, + "latency_increase_fraction": increase, + "hardware_and_model_identity": "operator-declared; not independently attested", + "statistical_claim": "point estimates only; no confidence or significance claim", + "timing_evidence": { + "baseline": baseline_run["measurement"].get("timing_evidence", "not separately declared"), + "candidate": candidate_run["measurement"].get("timing_evidence", "not separately declared"), + }, + }, + "checks": checks, + "slots": slots, + "summary": { + "measurement_slots": len(slots), + "passed_slots": sum(slot["status"] == "pass" for slot in slots), + "failed_slots": sum(slot["status"] == "fail" for slot in slots), + "inconclusive_slots": sum(slot["status"] == "inconclusive" for slot in slots), + "matched_valid_pairs": sum(_valid(slot["baseline"]) and _valid(slot["candidate"]) for slot in slots), + "warmups_excluded": True, + "failed_slots_retained": True, + }, + "limitations": [ + "Frame PSNR and audio similarity measure same-request implementation fidelity, not generative video/audio quality.", + "Fixed seeds do not guarantee matching output across different inference implementations; a fidelity failure requires investigation.", + "Hardware, runtime revision, and model identity are supplied by the operator, not independently verified by this harness.", + "Timing covers client submission through downloaded, validated media; it is not GPU-only kernel latency.", + "Recorded valid-clips throughput is descriptive; serial or closed-loop delivery measurements do not establish sustainable serving capacity.", + "Thresholds require domain calibration; no human evaluation, statistical significance, or release certification is claimed.", + "No memory, energy, cost, prompt-following, physics, or perceptual-quality results are inferred from these fidelity checks.", + ], + } + for label, run in (("baseline", baseline_run), ("candidate", candidate_run)): + result[label] = { + "run_id": run["run_id"], "evidence_kind": run["evidence_kind"], + "configuration": _configuration(run["configuration"]), + "run_bundle_sha256": run["_bundle_sha256"], "summary": summaries[label], + "provenance": run.get("provenance", {}), + } + return _json_safe(result) diff --git a/experimental/video-generation/evaluator/mvp_gpu_evidence.py b/experimental/video-generation/evaluator/mvp_gpu_evidence.py new file mode 100644 index 0000000000..c0af7c8245 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_gpu_evidence.py @@ -0,0 +1,382 @@ +"""Bounded, nonrecursive integrity checks for trusted-runner GPU evidence. + +No models, media decoders, subprocesses, network calls, or live GPUs are used. +Media bytes are hashed and rebound to the recorded full-stream comparison. This +does not turn unsigned operator artifacts into independent hardware attestation. +""" + +from __future__ import annotations + +import json +import math +import os +import stat +from datetime import datetime +from pathlib import Path + +from .mvp_runner import _slots, canonical_json_bytes + + +def _finite(value, *, positive=False): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and (value > 0 if positive else value >= 0) + + +def _equal(left, right): + return canonical_json_bytes(left) == canonical_json_bytes(right) + + +def _date(value): + if not isinstance(value, str): + raise ValueError("evidence timestamp is missing") + timestamp = datetime.fromisoformat(value) + if timestamp.tzinfo is None: + raise ValueError("evidence timestamp must include its timezone") + return timestamp + + +def _file(root: Path, raw, *, required=None) -> Path: + if not isinstance(raw, str) or not raw or Path(raw).is_absolute() or ".." in Path(raw).parts or "\\" in raw: + raise ValueError("evidence path is not relative and contained") + if required is not None and raw != required: + raise ValueError("evidence path differs from supervisor-owned artifact layout") + path = (root / raw).resolve(strict=True) + if not path.is_relative_to(root) or not path.is_file(): + raise ValueError("evidence file escapes its job directory") + return path + + +def _timing(record): + fields = ("submit_to_terminal_seconds", "submit_to_media_seconds", "media_validation_seconds", "latency_seconds") + if not all(_finite(record.get(key)) for key in fields) or not _finite(record["latency_seconds"], positive=True): + raise ValueError("valid measured media lacks real, finite recorded timing components") + terminal, delivered, validation, end = [record[key] for key in fields] + if terminal > delivered or delivered + validation > end + max(1e-6, end * 1e-6): + raise ValueError("recorded timing components are not ordered within their measured boundary") + + +def verify_measurement_job(directory: Path, *, deadline: float, require_success: bool = False, serving_smoke: bool = False) -> dict: + """Verify one complete controlled job; never follow its calibration references.""" + from . import mvp_gpu_job as gpu + + directory = Path(directory).resolve(strict=True) + gpu._check_deadline(deadline) + spec = gpu.validate_gpu_job(gpu._read(_file(directory, "spec.json"))) + receipt = gpu._read(_file(directory, "gpu-job.json")) + bundle_type = "controlled_serving_smoke" if serving_smoke else "controlled_gpu_job" + if serving_smoke and not spec.get("serving"): + raise ValueError("single-runtime smoke requires an explicit serving load") + if receipt.get("schema_version") != gpu.VERSION or receipt.get("bundle_type") != bundle_type or receipt.get("spec_sha256") != gpu._digest(spec): + raise ValueError("GPU job/spec identity is not verified") + if receipt.get("job_id") != spec["job_id"] or receipt.get("plan_sha256") != gpu._digest(spec["plan"]): + raise ValueError("GPU job workload identity mismatch") + if receipt.get("evidence_kind") != "controlled_h3_gpu" or receipt.get("status") != "complete" or receipt.get("measurement_status") != "complete" or receipt.get("failures"): + raise ValueError("GPU job has incomplete, fixture, or failed supervision evidence") + if receipt.get("cleanup_status") != "clean": + raise ValueError("GPU job did not complete clean owned-resource teardown") + started, finished = _date(receipt.get("started_at")), _date(receipt.get("finished_at")) + if finished < started: + raise ValueError("GPU job timestamps are reversed") + execution_id = receipt.get("execution_id") + if not isinstance(execution_id, str) or not execution_id: + raise ValueError("GPU job lacks its independent execution identity") + model = receipt.get("model_identity", {}) + if model.get("revision") != spec["model"]["revision"] or model.get("manifest_sha256") != gpu._digest(spec["model"]["files"]): + raise ValueError("GPU job checkpoint manifest is not bound to the specification") + labels = ("baseline",) if serving_smoke else ("baseline", "candidate") + if set(receipt.get("roles", {})) != set(labels): + raise ValueError("the exact supervised role receipts are required") + runs, identities, run_ids, nonces = {}, {}, set(), set() + expected_slots = _slots(spec["plan"]) + scheduled = sum(slot["phase"] == "measurement" for slot in expected_slots) + for label in labels: + gpu._check_deadline(deadline) + role = receipt["roles"][label] + if role.get("status") != "complete" or role.get("client_exit_code") not in {0, 1}: + raise ValueError("role client did not finalize its entire workload") + if role.get("cleanup", {}).get("status") != "clean" or role["cleanup"].get("idle_after") is not True or role.get("client_cleanup", {}).get("status") != "clean": + raise ValueError("role/client resource cleanup is not verified clean") + before, after = role.get("gpu_before", {}), role["cleanup"].get("gpu_after", {}) + for idle in (before, after): + if sorted(item.get("uuid") for item in idle.get("gpus", [])) != sorted(spec["gpu_uuids"]) or not gpu._idle(idle, spec["limits"]["max_idle_memory_mib"]): + raise ValueError("GPU idle observations do not match the selected devices") + process = role.get("process_identity", {}) + if not all(isinstance(process.get(field), int) and not isinstance(process[field], bool) and process[field] > 0 for field in ("pid", "pgid", "session_id", "start_ticks")) or process["pid"] != process["pgid"] or process["pid"] != process["session_id"]: + raise ValueError("owned runtime session identity is missing or invalid") + nonce = process.get("launch_nonce") + if not isinstance(nonce, str) or not nonce or nonce in nonces: + raise ValueError("runtime boot identities are not distinct") + nonces.add(nonce) + identity = role.get("source_identity", {}) + if any(identity.get(field) != spec[label][field] for field in ("revision", "source_sha256")) or not identity.get("python_sha256") or not identity.get("packages", {}).get("torch"): + raise ValueError("observed runtime source/environment is not pinned") + identities[label] = identity + run_path = _file(directory, role.get("run_path"), required=f"{label}/run.json") + if gpu._hash(run_path, deadline) != role.get("run_sha256"): + raise ValueError("role run bundle hash mismatch") + run = gpu._read(run_path) + if run.get("bundle_type") != "mvp_run" or run.get("bundle_version") != "0.1.0" or run.get("evidence_kind") not in {"live_h3", "operator_endpoint"}: + raise ValueError("role contains fixture/imported or unsupported run evidence") + if run.get("status") not in {"complete", "partial", "failed"} or _date(run.get("finished_at")) < _date(run.get("started_at")): + raise ValueError("role run is not finalized") + run_id = run.get("run_id") + if not isinstance(run_id, str) or not run_id or run_id in run_ids: + raise ValueError("run execution IDs are missing or reused") + run_ids.add(run_id) + if run.get("plan_sha256") != gpu._digest(spec["plan"]) or not _equal(run.get("plan"), spec["plan"]): + raise ValueError("run workload differs from supervised frozen plan") + config = run.get("configuration", {}) + declared = config.get("configuration_sha256") or run.get("configuration_sha256") + unsigned = {key: value for key, value in config.items() if key != "configuration_sha256"} + if declared != gpu._digest(unsigned): + raise ValueError("run configuration hash mismatch") + if config.get("runtime") != "sglang" or config.get("runtime_revision") != identity["revision"] or config.get("model_id") != spec["plan"]["model_id"] or config.get("model_revision") != spec["model"]["revision"]: + raise ValueError("run configuration differs from observed runtime/model identity") + records = run.get("records", []) + if not isinstance(records, list) or len(records) != len(expected_slots): + raise ValueError("run must retain every planned warmup and measured outcome") + valid_measured = 0 + attempted_seconds = 0.0 + for record, slot in zip(records, expected_slots): + gpu._check_deadline(deadline) + if any(not _equal(record.get(field), slot[field]) for field in ("slot_id", "case_id", "prompt", "seed", "repetition", "phase")): + raise ValueError("run slot identity/order differs from the frozen schedule") + if record.get("status") not in {"succeeded", "failed"} or not isinstance(record.get("attempted"), bool): + raise ValueError("run outcome or attempted flag is missing") + if record.get("artifact_path") is not None: + artifact = _file(run_path.parent, record["artifact_path"]) + if gpu._hash(artifact, deadline) != record.get("sha256"): + raise ValueError("raw model media bytes differ from their recorded hash") + elif record.get("sha256") or record.get("status") == "succeeded": + raise ValueError("successful/hashed record is missing its media artifact") + valid = record.get("status") == "succeeded" and record.get("media", {}).get("valid") is True + if record["phase"] == "warmup" and not valid: + raise ValueError("planned warmup failed; measurement is not qualified") + if valid: + _timing(record) + if record["phase"] == "measurement": + valid_measured += valid + if record["attempted"]: + if not _finite(record.get("latency_seconds")): + raise ValueError("attempted record lacks finite measured client time") + attempted_seconds += record["latency_seconds"] + if (require_success or label == "baseline") and not valid: + raise ValueError("same-build calibration or baseline requires every measured slot valid") + summary = run.get("summary", {}) + if summary.get("scheduled") != scheduled or summary.get("valid") != valid_measured or summary.get("failed") != scheduled - valid_measured: + raise ValueError("run failure/success denominator does not match raw outcomes") + measurement = run.get("measurement", {}) + wall = measurement.get("wall_seconds") + if not _equal(config.get("serving"), spec.get("serving")): + raise ValueError("client serving load differs from supervised specification") + if config.get("serving"): + from .mvp_serving import summarize, validate_window + validate_window(run) + if not _equal(run.get("serving"), summarize(run)): + raise ValueError("serving summary differs from raw request records") + elif measurement.get("boundary") != "submit_to_validated_media" or measurement.get("concurrency") != 1 or not _finite(wall, positive=True) or wall + max(1e-6, attempted_seconds * 1e-6) < attempted_seconds: + raise ValueError("run timing boundary or serial wall duration is invalid") + run["_verified_run_sha256"] = role["run_sha256"] + runs[label] = run + telemetry_path = _file(directory, role.get("telemetry_path"), required=f"supervisor/{label}/telemetry.jsonl") + if telemetry_path.stat().st_size > 128 * 1024 * 1024 or gpu._hash(telemetry_path, deadline) != role.get("telemetry_sha256"): + raise ValueError("raw telemetry is oversized or has a mismatched hash") + samples = [] + previous_time = None + descriptor = os.open(telemetry_path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("raw telemetry must be a regular file") + while True: + gpu._check_deadline(deadline) + line = stream.readline(1024 * 1024 + 1) + if not line: + break + if len(line) > 1024 * 1024 or not line.endswith(b"\n") or len(samples) >= 100000: + raise ValueError("raw telemetry has a truncated or oversized sample") + sample = json.loads(line) + stamp = sample.get("monotonic_seconds") + if not _finite(stamp) or (previous_time is not None and stamp <= previous_time) or sample.get("phase") not in {"startup", "measurement", "cleanup"}: + raise ValueError("raw telemetry sample order/phase is invalid") + previous_time = stamp + devices = sample.get("gpus", []) + if sorted(item.get("uuid") for item in devices) != sorted(spec["gpu_uuids"]): + raise ValueError("raw telemetry device inventory differs from selected UUIDs") + if any(not _finite(item.get("memory_used_mib")) or not _finite(item.get("memory_total_mib"), positive=True) or not _finite(item.get("utilization_percent")) for item in devices): + raise ValueError("raw telemetry contains invalid required measurements") + if sample.get("unowned_compute_apps"): + raise ValueError("raw telemetry records unowned/invisible GPU compute") + original = {(app.get("gpu_uuid"), app.get("pid")) for app in sample.get("compute_apps", [])} + observed = sample.get("owned_compute_apps", []) + if original != {(app.get("gpu_uuid"), app.get("pid")) for app in observed}: + raise ValueError("raw GPU compute inventory is not fully attributed") + for app in observed: + observed_process = app.get("process_identity", {}) + if app.get("gpu_uuid") not in spec["gpu_uuids"] or observed_process.get("pid") != app.get("pid") or any(observed_process.get(field) != process[field] for field in ("pgid", "session_id")) or not _finite(observed_process.get("start_ticks"), positive=True) or observed_process["start_ticks"] < process["start_ticks"]: + raise ValueError("raw GPU compute process is outside the observed owned session") + samples.append(sample) + declared_summary = role.get("telemetry_summary", {}) + window_start = declared_summary.get("measurement_window_start_monotonic_seconds") + window_end = declared_summary.get("measurement_window_end_monotonic_seconds") + if not _finite(window_start) or not _finite(window_end) or window_end - window_start + 1e-6 < wall: + raise ValueError("telemetry window does not cover the measured client workload") + recomputed = gpu.summarize_gpu_samples(samples, spec["gpu_uuids"], spec["limits"]["telemetry_interval_seconds"], + window_start=window_start, window_end=window_end, + command_seconds=spec["limits"]["command_seconds"]) + if not recomputed["qualified"] or not _equal(recomputed, role.get("telemetry_summary")): + raise ValueError("telemetry summary does not match qualified raw observations") + verified = {"directory": directory, "spec": spec, "receipt": receipt, "runs": runs, "identities": identities, "run_ids": run_ids, + "nonces": nonces, "started": started, "finished": finished, "execution_id": execution_id} + if serving_smoke: + if receipt.get("comparison_path") is not None or receipt.get("ci_accepted") is not False or receipt.get("release_qualified") is not False: + raise ValueError("single-runtime smoke cannot claim paired or calibrated acceptance") + return {**verified, "comparison": None} + compared_path = _file(directory, receipt.get("comparison_path"), required="comparison.json") + if gpu._hash(compared_path, deadline) != receipt.get("comparison_sha256"): + raise ValueError("comparison artifact hash mismatch") + compared = gpu._read(compared_path) + if compared.get("bundle_type") != "mvp_comparison" or compared.get("evidence_kind") not in {"operator_endpoint", "live_h3"} or compared.get("plan_sha256") != gpu._digest(spec["plan"]) or not _equal(compared.get("policy"), spec["policy"]): + raise ValueError("comparison evidence/workload/policy is not bound to this GPU job") + slots = compared.get("slots", []) + if len(slots) != scheduled: + raise ValueError("comparison does not retain every planned measurement pair") + for label in ("baseline", "candidate"): + bound = compared.get(label, {}) + if bound.get("run_bundle_sha256") != runs[label]["_verified_run_sha256"] or bound.get("run_id") != runs[label]["run_id"]: + raise ValueError("comparison references different baseline/candidate executions") + measured_records = [record for record in runs[label]["records"] if record["phase"] == "measurement"] + for slot, record in zip(slots, measured_records): + observation = slot.get(label, {}) + if slot.get("slot_id") != record["slot_id"] or observation.get("sha256") != record.get("sha256") or observation.get("status") != record["status"]: + raise ValueError("comparison media/outcomes differ from the measured run bundle") + checks = compared.get("checks", []) + [check for slot in slots for check in slot.get("checks", [])] + if not checks: + raise ValueError("comparison contains no checks") + outcome = "fail" if any(check.get("status") == "fail" for check in checks) else "inconclusive" if any(check.get("status") == "inconclusive" for check in checks) else "pass" + if compared.get("overall_status") != outcome: + raise ValueError("comparison decision does not match its checks") + return {**verified, "comparison": compared} + + +def verify_calibration(spec: dict, current: dict, *, deadline: float) -> tuple[bool, str]: + """Verify 2–4 prior same-build jobs without following nested references.""" + from . import mvp_gpu_job as gpu + + if spec.get("serving"): + return False, "serving load is descriptive; serial calibration cannot qualify concurrent delivery metrics" + if spec["policy"]["calibration_status"] != "operator_calibrated": + return False, "policy is not calibrated; measurements are useful but CI acceptance is inconclusive" + references = spec["policy"].get("calibration_evidence") + if not isinstance(references, list) or not 2 <= len(references) <= 4: + return False, "calibrated CI requires 2–4 hash-pinned independent same-build GPU jobs" + seen_jobs = {current["execution_id"]} + seen_runs = set(current["run_ids"]) + seen_nonces = set(current["nonces"]) + intervals = [] + for reference in references: + gpu._check_deadline(deadline) + if not isinstance(reference, dict) or set(reference) != {"job_path", "sha256"}: + raise ValueError("calibration reference must identify one prior full job directory receipt") + path = Path(reference["job_path"]) + if not isinstance(reference["sha256"], str) or not gpu._SHA.fullmatch(reference["sha256"]): + raise ValueError("calibration reference must pin a SHA256") + portable = current["directory"] / "calibration" / reference["sha256"] / "gpu-job.json" + if not path.is_absolute() or path.name != "gpu-job.json": + raise ValueError("calibration reference requires an absolute prior GPU receipt path") + if portable.exists(): + path = _file(current["directory"], portable.relative_to(current["directory"]).as_posix()) + if gpu._hash(path, deadline) != reference["sha256"]: + raise ValueError("calibration receipt path or hash did not verify") + prior = verify_measurement_job(path.parent, deadline=deadline, require_success=True) + frozen = prior["spec"] + if prior["execution_id"] in seen_jobs or prior["run_ids"] & seen_runs or prior["nonces"] & seen_nonces: + raise ValueError("calibration repeats an execution, run, or runtime boot identity") + seen_jobs.add(prior["execution_id"]) + seen_runs.update(prior["run_ids"]) + seen_nonces.update(prior["nonces"]) + if prior["finished"] >= current["started"]: + raise ValueError("calibration must predate the independent candidate job") + if any(not (prior["finished"] <= start or end <= prior["started"]) for start, end in intervals): + raise ValueError("calibration jobs overlap on the same GPUs") + intervals.append((prior["started"], prior["finished"])) + for field in ("plan", "server", "gpu_uuids"): + if not _equal(frozen[field], spec[field]): + raise ValueError(f"calibration {field} differs from current controlled cell") + if frozen["model"]["revision"] != spec["model"]["revision"] or not _equal(frozen["model"]["files"], spec["model"]["files"]): + raise ValueError("calibration uses a different checkpoint manifest") + for label in ("baseline", "candidate"): + identity = prior["identities"][label] + baseline = current["identities"]["baseline"] + for field in ("revision", "source_sha256", "python_sha256", "python_version", "packages"): + if not _equal(identity.get(field), baseline.get(field)): + raise ValueError("calibration is not the current baseline build and dependency environment") + for field in ("client_source_sha256", "client_environment", "media_evaluator", "measurement_semantics", "limits"): + if not _equal(prior["runs"][label]["configuration"].get(field), current["runs"]["baseline"]["configuration"].get(field)): + raise ValueError("calibration client timing/evaluator environment differs") + observed = prior["receipt"]["roles"][label]["telemetry_summary"]["gpu_identity"] + if not _equal(observed, current["receipt"]["roles"]["baseline"]["telemetry_summary"]["gpu_identity"]): + raise ValueError("calibration observed hardware or driver identity differs") + return True, "eligibility verified against independent prior same-build raw artifacts; threshold choice remains operator-declared, not statistical certification" + + +def preflight_calibration(spec: dict, directory: Path, *, started_at: str, + execution_id: str, deadline: float) -> dict: + """Reject known-invalid calibration before allocating any current GPUs. + + This checks prior raw evidence and every cell property known from the frozen + specification. Runtime-package/driver observations are still compared again + after the current job; preflight cannot predict a later environment drift. + """ + from . import mvp_gpu_job as gpu + + if spec["policy"]["calibration_status"] != "operator_calibrated": + return {"status": "not_required", "performed_before_gpu_lease": True, + "reason": "policy does not claim calibrated CI acceptance"} + references = spec["policy"].get("calibration_evidence") + if not isinstance(references, list) or not 2 <= len(references) <= 4: + raise ValueError("calibration preflight requires 2–4 hash-pinned complete prior jobs") + directory = Path(directory).resolve(strict=True) + started = _date(started_at) + jobs, runs, nonces, intervals, hashes = {execution_id}, set(), set(), [], [] + for reference in references: + gpu._check_deadline(deadline) + if not isinstance(reference, dict) or set(reference) != {"job_path", "sha256"}: + raise ValueError("calibration preflight requires explicit receipt path/hash pairs") + digest = reference["sha256"] + if not isinstance(digest, str) or not gpu._SHA.fullmatch(digest): + raise ValueError("calibration preflight reference lacks a valid SHA256") + raw = reference["job_path"] + if not isinstance(raw, str): + raise ValueError("calibration preflight reference path must be a string") + path = Path(raw) + if not path.is_absolute() or path.name != "gpu-job.json": + raise ValueError("calibration preflight needs an absolute prior GPU receipt path") + portable = directory / "calibration" / digest / "gpu-job.json" + if portable.exists(): + path = _file(directory, portable.relative_to(directory).as_posix()) + if gpu._hash(path, deadline) != digest: + raise ValueError("calibration preflight receipt hash does not match") + prior = verify_measurement_job(path.parent, deadline=deadline, require_success=True) + if prior["execution_id"] in jobs or prior["run_ids"] & runs or prior["nonces"] & nonces: + raise ValueError("calibration preflight found repeated job/run/boot identities") + jobs.add(prior["execution_id"]) + runs.update(prior["run_ids"]) + nonces.update(prior["nonces"]) + if prior["finished"] >= started: + raise ValueError("calibration preflight requires prior jobs completed before this job") + if any(not (prior["finished"] <= start or end <= prior["started"]) for start, end in intervals): + raise ValueError("calibration preflight jobs overlap on the same GPUs") + intervals.append((prior["started"], prior["finished"])) + previous = prior["spec"] + for field in ("plan", "server", "gpu_uuids"): + if not _equal(previous[field], spec[field]): + raise ValueError(f"calibration preflight {field} differs from the frozen current cell") + if previous["model"]["revision"] != spec["model"]["revision"] or not _equal(previous["model"]["files"], spec["model"]["files"]): + raise ValueError("calibration preflight checkpoint manifest differs from the current cell") + for label in ("baseline", "candidate"): + if any(prior["identities"][label].get(field) != spec["baseline"][field] for field in ("revision", "source_sha256")): + raise ValueError("calibration preflight does not match the current baseline source pins") + hashes.append(digest) + return {"status": "passed", "performed_before_gpu_lease": True, + "reference_count": len(references), "verified_receipt_sha256": hashes, + "verified_prior_execution_ids": sorted(jobs - {execution_id}), + "reason": "prior raw evidence and known frozen cell verified before current GPU allocation; observed runtime/driver equivalence is checked after measurement"} diff --git a/experimental/video-generation/evaluator/mvp_gpu_job.py b/experimental/video-generation/evaluator/mvp_gpu_job.py new file mode 100644 index 0000000000..3afbfdc906 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_gpu_job.py @@ -0,0 +1,1389 @@ +"""Bounded, locally supervised H3 GPU measurements on a trusted Linux runner. + +This is NOT a sandbox for untrusted runtime code. The operator must provision +the source trees, environments, weights, and (for CI acceptance) a dedicated +allocation. UUID file locks are cooperative, not a GPU scheduler. No container, +foreign PID, port owner, or GPU is ever killed/reset by this module. + +The supervisor retains each child session leader until its process group has +drained. That unreaped leader reserves the group ID; PID/start-time/session +checks precede every group signal. Escaped or invisible GPU processes are a +cleanup/attribution failure, never permission to kill an unrelated process. +""" + +from __future__ import annotations + +import csv +import hashlib +import http.client +import io +import json +import math +import os +import platform +import re +import signal +import socket +import stat +import subprocess +import subprocess +import sys +import threading +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .mvp_runner import canonical_json_bytes, preview_plan, validate_plan + + +VERSION = "0.1.0" +_ROLES = ("baseline", "candidate") +_SHA = re.compile(r"[0-9a-f]{64}") +_REV = re.compile(r"[0-9a-f]{40}") +_GPU = re.compile(r"GPU-[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") +_LAUNCH = """import os +from evaluator.mvp_gpu_job import cuda_devices +if cuda_devices() != os.environ["VGBENCH_GPU_UUIDS"].split(","): + raise RuntimeError("Runtime CUDA device UUIDs differ from assigned GPUs") +from sglang.cli.main import main +main() +""" +_IDENTITY = ( + "import importlib.util,importlib.metadata,json,os,platform;" + "s=importlib.util.find_spec('sglang');" + "print(json.dumps({'python_version':platform.python_version()," + "'sglang_module':s.origin if s else None," + "'cpu_native_thread_limits':{name:os.environ.get(name) for name in " + "('OMP_NUM_THREADS','OPENBLAS_NUM_THREADS','MKL_NUM_THREADS','NUMEXPR_NUM_THREADS')}," + "'compilation_worker_limit':os.environ.get('MAX_JOBS')," + "'packages':{d.metadata['Name'].lower():d.version for d in importlib.metadata.distributions()}}))" +) + + +class JobCancelled(RuntimeError): + """A signal or whole-job watchdog cancelled further work.""" + + +def _digest(value: Any) -> str: + return hashlib.sha256(canonical_json_bytes(value)).hexdigest() + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _check_deadline(deadline: float | None) -> None: + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError("supervised operation exceeded its deadline") + + +def _hash(path: Path, deadline: float | None = None, cancelled: threading.Event | None = None) -> str: + digest = hashlib.sha256() + _check_deadline(deadline) + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("hash input must be a regular file, not a FIFO/device/socket") + while True: + _check_deadline(deadline) + if cancelled is not None and cancelled.is_set(): + raise JobCancelled("cancelled while verifying pinned files") + block = stream.read(1024 * 1024) + if not block: + return digest.hexdigest() + digest.update(block) + + +def _write(path: Path, value: Any) -> None: + temporary = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp") + with temporary.open("xb") as stream: + stream.write(canonical_json_bytes(value)) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +def _read(path: Path, limit: int = 32 * 1024 * 1024) -> dict: + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode): + raise ValueError("JSON evidence must be a regular file, not a FIFO/device/socket") + if info.st_size > limit: + raise ValueError("JSON evidence exceeds the bounded input size") + data = stream.read(limit + 1) + if len(data) > limit: + raise ValueError("JSON evidence grew beyond the bounded input size") + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON evidence key") + result[key] = value + return result + + def nonfinite(value): + raise ValueError("nonfinite JSON evidence") + + result = json.loads(data, object_pairs_hook=unique, parse_constant=nonfinite) + if not isinstance(result, dict): + raise ValueError("JSON evidence must be an object") + return result + + +def _keys(value: Any, expected: set[str], label: str, optional: set[str] | None = None) -> None: + if not isinstance(value, dict) or set(value) - expected - (optional or set()) or expected - set(value): + raise ValueError(f"{label} requires exactly the supported explicit fields") + + +def _absolute(value: Any, label: str) -> None: + if not isinstance(value, str) or not value or "\x00" in value or not Path(value).is_absolute() or ".." in Path(value).parts: + raise ValueError(f"{label} must be an absolute, traversal-free path") + if Path(value) == Path("/"): + raise ValueError(f"{label} cannot be the filesystem root") + + +def _number(value: Any, label: str, minimum: float, maximum: float, integer: bool = False) -> None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or not minimum <= value <= maximum or (integer and not isinstance(value, int)): + raise ValueError(f"{label} must be a finite {'integer' if integer else 'number'} in [{minimum}, {maximum}]") + + +def validate_gpu_job(spec: dict) -> dict: + """Pure validation: no filesystem probes, network, GPU calls, or execution.""" + frozen = json.loads(canonical_json_bytes(spec)) + _keys(frozen, {"schema_version", "job_id", "authorization", "allocation", "gpu_uuids", "port", "lock_directory", "baseline", "candidate", "model", "server", "plan", "policy", "limits"}, "GPU job", optional={"serving"}) + if "serving" in frozen: + from .mvp_serving import settings + load = frozen["serving"] + _keys(load, {"concurrency"}, "serving", optional={"mode", "delivery_deadline_seconds"}) + if load.get("mode", "closed_loop") != "closed_loop": + raise ValueError("only closed_loop serving load is supported") + if load["concurrency"] is None: + raise ValueError("serving requires explicit concurrency") + frozen["serving"] = settings(load["concurrency"], load.get("delivery_deadline_seconds")) + if frozen["schema_version"] != VERSION: + raise ValueError("unsupported GPU job schema_version") + if not isinstance(frozen["job_id"], str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,99}", frozen["job_id"]): + raise ValueError("job_id must be a bounded safe identifier") + authorization = frozen["authorization"] + _keys(authorization, {"compute_approved", "model_license_reviewed", "approval_reference"}, "authorization") + if any(not isinstance(authorization[field], bool) for field in ("compute_approved", "model_license_reviewed")) or not isinstance(authorization["approval_reference"], str) or len(authorization["approval_reference"]) > 2000: + raise ValueError("authorization requires explicit booleans and a bounded approval-reference string") + allocation = frozen["allocation"] + _keys(allocation, {"mode", "label"}, "allocation") + if allocation["mode"] not in {"cooperative_shared", "dedicated_ci"} or not isinstance(allocation["label"], str) or not allocation["label"].strip() or len(allocation["label"]) > 500: + raise ValueError("allocation requires an explicit operator-declared supported mode and label") + devices = frozen["gpu_uuids"] + if not isinstance(devices, list) or not 1 <= len(devices) <= 8 or any(not isinstance(item, str) or not _GPU.fullmatch(item) for item in devices) or len(set(devices)) != len(devices): + raise ValueError("gpu_uuids must contain 1–8 distinct full NVIDIA GPU UUIDs; MIG is unsupported") + _number(frozen["port"], "port", 1024, 65535, True) + _absolute(frozen["lock_directory"], "lock_directory") + for role in _ROLES: + runtime = frozen[role] + _keys(runtime, {"python", "source", "revision", "source_sha256"}, role) + _absolute(runtime["python"], f"{role}.python") + _absolute(runtime["source"], f"{role}.source") + if not isinstance(runtime["revision"], str) or not _REV.fullmatch(runtime["revision"]): + raise ValueError(f"{role}.revision must be an immutable lowercase commit") + if not isinstance(runtime["source_sha256"], str) or not _SHA.fullmatch(runtime["source_sha256"]): + raise ValueError(f"{role}.source_sha256 must freeze the tracked source-file manifest") + model = frozen["model"] + _keys(model, {"path", "revision", "files"}, "model") + _absolute(model["path"], "model.path") + frozen["plan"] = validate_plan(frozen["plan"]) + if model["revision"] != frozen["plan"]["model_revision"]: + raise ValueError("staged model revision must match the frozen plan") + files = model["files"] + if not isinstance(files, list) or not files or len(files) > 20000: + raise ValueError("model.files requires a nonempty bounded complete file manifest") + names = set() + for entry in files: + _keys(entry, {"path", "size_bytes", "sha256"}, "model file") + path = entry["path"] + if not isinstance(path, str) or not path or "\\" in path or "\x00" in path or Path(path).is_absolute() or ".." in Path(path).parts or path in names: + raise ValueError("model file paths must be unique relative traversal-free names") + names.add(path) + _number(entry["size_bytes"], "model file size", 0, 2**50, True) + if not isinstance(entry["sha256"], str) or not _SHA.fullmatch(entry["sha256"]): + raise ValueError("each model file requires its SHA256") + if not any(name.endswith((".safetensors", ".pt", ".bin")) for name in names): + raise ValueError("model manifest contains no weight files") + frozen["model"]["files"] = sorted(files, key=lambda item: item["path"]) + server = frozen["server"] + _keys(server, {"ulysses_degree", "tp_size", "encoder_parallel", "performance_mode"}, "server", + optional={"dit_cpu_offload", "layerwise_offload"}) + _number(server["ulysses_degree"], "server.ulysses_degree", 1, len(devices), True) + _number(server["tp_size"], "server.tp_size", 1, len(devices), True) + if len(devices) % server["ulysses_degree"] or len(devices) % server["tp_size"]: + raise ValueError("server requires dividing Ulysses and tensor-parallel degrees") + if server["encoder_parallel"] not in {"auto", "fold", "replicate"} or server["performance_mode"] not in {"manual", "speed", "memory"}: + raise ValueError("unsupported explicit encoder parallelism or performance mode") + if "layerwise_offload" in server: + # This is one documented lossless placement, not a general argument + # passthrough. Its transfer from 2x5090 to 2xH200 remains experimental. + if "dit_cpu_offload" in server: + raise ValueError("layerwise offload must not also set coarse dit_cpu_offload") + layerwise = server["layerwise_offload"] + _keys(layerwise, {"components", "prefetch_size", "resident_layers"}, "server.layerwise_offload") + _number(layerwise["prefetch_size"], "layerwise prefetch_size", 1, 1, True) + _number(layerwise["resident_layers"], "layerwise resident_layers", 20, 20, True) + if (layerwise["components"] != ["dit", "text_encoder", "vae"] + or len(devices) != 2 or server["tp_size"] != 2 or server["ulysses_degree"] != 1 + or server["encoder_parallel"] != "auto" or server["performance_mode"] != "memory"): + raise ValueError("only the explicit two-GPU TP2/Ulysses1 lossless layerwise profile is supported") + elif not isinstance(server.get("dit_cpu_offload"), bool): + raise ValueError("server requires an explicit coarse offload boolean or supported layerwise profile") + limits = frozen["limits"] + ranges = { + "job_seconds": (5, 86400), "startup_seconds": (0.1, 43200), + "request_seconds": (0.1, 86400), "cleanup_seconds": (0.1, 120), + "telemetry_interval_seconds": (0.1, 60), "command_seconds": (0.1, 60), + "max_idle_memory_mib": (0, 5000), + } + _keys(limits, set(ranges), "limits") + for name, (minimum, maximum) in ranges.items(): + _number(limits[name], f"limits.{name}", minimum, maximum) + if limits["job_seconds"] <= 2 * limits["cleanup_seconds"] or limits["startup_seconds"] >= limits["job_seconds"] or limits["request_seconds"] >= limits["job_seconds"]: + raise ValueError("whole-job budget must leave explicit cleanup time and bound each phase") + if limits["job_seconds"] / limits["telemetry_interval_seconds"] > 100000: + raise ValueError("telemetry plan exceeds 100000 samples") + if frozen["plan"]["warmup_runs"] < 1: + raise ValueError("controlled GPU measurements require a separately recorded warmup") + from .mvp_compare import _policy + frozen["policy"] = _policy(frozen["policy"]) + if frozen.get("serving") and frozen["policy"]["calibration_status"] == "operator_calibrated": + raise ValueError("serving load requires an uncalibrated policy; serial calibration cannot qualify concurrent delivery metrics") + memory_gate = frozen["policy"].get("max_memory_increase_fraction") + if memory_gate is not None: + _number(memory_gate, "max_memory_increase_fraction", 0, 100) + return frozen + + +def _server_argv(spec: dict, role: str) -> list[str]: + args = [spec[role]["python"], "-c", _LAUNCH, "serve", "--model-type", "diffusion", + "--model-path", spec["model"]["path"], "--model-id", spec["plan"]["model_id"], + "--revision", spec["model"]["revision"], "--model-variant", "fl2va", + "--num-gpus", str(len(spec["gpu_uuids"])), "--ulysses-degree", str(spec["server"]["ulysses_degree"]), + "--tp-size", str(spec["server"]["tp_size"]), "--encoder-parallel", spec["server"]["encoder_parallel"], + "--performance-mode", spec["server"]["performance_mode"], + "--host", "127.0.0.1", "--port", str(spec["port"]), "--enable-torch-compile", "false"] + if "layerwise_offload" in spec["server"]: + layerwise = spec["server"]["layerwise_offload"] + args.extend(["--layerwise-offload-components", ",".join(layerwise["components"]), + "--dit-offload-prefetch-size", str(layerwise["prefetch_size"]), + "--dit-layerwise-resident-layers", str(layerwise["resident_layers"])]) + else: + args.extend(["--dit-cpu-offload", str(spec["server"]["dit_cpu_offload"]).lower()]) + return args + + +def preview_gpu_job(spec: dict) -> dict: + frozen = validate_gpu_job(spec) + return { + "evidence_kind": "gpu_job_preview_no_execution", "spec_sha256": _digest(frozen), + "job_id": frozen["job_id"], "gpu_uuids": frozen["gpu_uuids"], + "authorization": frozen["authorization"], "authorization_verification": "operator assertion; not proof of model-license rights", + "allocation": frozen["allocation"], "allocation_verification": "operator-declared prerequisite, not scheduler attestation", + "commands": {role: _server_argv(frozen, role) for role in _ROLES}, + "workload": preview_plan(frozen["plan"]), "limits": frozen["limits"], + "sequence": ["verify pinned files", "acquire UUID locks", "verify idle", "baseline startup/warmup/measure/cleanup", "candidate startup/warmup/measure/cleanup", "compare"], + "warnings": ["Trusted Linux runner and visible GPU-process PIDs are required.", "No GPU lock can enforce exclusive access against non-cooperating processes.", "No GPU work, model download, server startup, or CI acceptance occurs in preview."], + } + + +def _command(argv: list[str], *, timeout: float, cwd: Path | None = None, env: dict | None = None) -> bytes: + # Only bounded read-only utilities and a trusted, fixed Python identity probe. + result = subprocess.run(argv, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=timeout, check=False) + if result.returncode or len(result.stdout) > 8 * 1024 * 1024: + raise RuntimeError(f"bounded identity/telemetry command failed: {Path(argv[0]).name}") + return result.stdout + + +def source_file_manifest(source: Path, *, timeout: float = 10, deadline: float | None = None, cancelled: threading.Event | None = None) -> dict: + """Generated build metadata is part of the pinned runtime identity.""" + source = Path(source).resolve(strict=True) + if not source.is_dir(): + raise ValueError("runtime source is not a directory") + revision = _command(["git", "-C", str(source), "rev-parse", "HEAD"], timeout=timeout).decode().strip() + dirty = _command(["git", "-C", str(source), "status", "--porcelain", "--untracked-files=all"], timeout=timeout) + if dirty: + raise ValueError("runtime source must be a clean committed checkout without untracked files") + raw = _command(["git", "-C", str(source), "ls-files", "-z"], timeout=timeout) + names = sorted(name.decode("utf-8") for name in raw.split(b"\x00") if name) + if not names or len(names) > 100000: + raise ValueError("runtime tracked-file inventory is empty or oversized") + # setuptools-scm generates this ignored module, and SGLang imports it at + # runtime. Pin its bytes while continuing to reject other ignored Python. + generated = "python/sglang/_version.py" + if generated not in names and ((source / generated).exists() or (source / generated).is_symlink()): + names = sorted([*names, generated]) + entries = [] + for name in names: + _check_deadline(deadline) + target = (source / name).resolve(strict=True) + if not target.is_relative_to(source) or not target.is_file(): + raise ValueError("runtime tracked file escapes checkout or is not a regular file") + entries.append({"path": name, "size_bytes": target.stat().st_size, "sha256": _hash(target, deadline, cancelled)}) + # Ignored source files can shadow installed modules despite a clean git tree. + package = source / "python" / "sglang" + if not (package / "cli" / "main.py").is_file(): + raise ValueError("source does not contain the supported SGLang CLI layout") + tracked = set(names) + if any(path.relative_to(source).as_posix() not in tracked for path in package.rglob("*.py")): + raise ValueError("untracked/ignored Python files exist inside the runtime package") + return {"source": str(source), "revision": revision, "source_sha256": _digest(entries), "files": entries} + + +def _model_manifest(spec: dict, deadline: float, cancelled: threading.Event | None = None) -> dict: + root = Path(spec["model"]["path"]).resolve(strict=True) + allowed_blob_root = None + if root.parent.name == "snapshots" and root.name == spec["model"]["revision"]: + blobs = root.parent.parent / "blobs" + if blobs.is_symlink(): + raise ValueError("HF blob directory cannot itself be a symlink") + if blobs.exists(): + allowed_blob_root = blobs.resolve(strict=True) + if allowed_blob_root != blobs or not allowed_blob_root.is_dir(): + raise ValueError("HF blob directory is not the canonical same-cache directory") + entries = spec["model"]["files"] + actual = set() + for directory, directories, files in os.walk(root, followlinks=False): + _check_deadline(deadline) + if any((Path(directory) / name).is_symlink() for name in directories): + raise ValueError("model directory symlinks are unsupported") + actual.update((Path(directory) / name).relative_to(root).as_posix() for name in files) + if actual != {entry["path"] for entry in entries}: + raise ValueError("staged model file inventory differs from the complete frozen manifest") + for entry in entries: + target = (root / entry["path"]).resolve(strict=True) + if not target.is_file() or not (target.is_relative_to(root) or (allowed_blob_root and target.is_relative_to(allowed_blob_root))): + raise ValueError("model file escapes the snapshot and its validated HF blob directory") + if target.stat().st_size != entry["size_bytes"] or _hash(target, deadline, cancelled) != entry["sha256"]: + raise ValueError("staged model file hash or size differs from frozen manifest") + return {"path": str(root), "revision": spec["model"]["revision"], "manifest_sha256": _digest(entries), + "verified_files": len(entries), "total_bytes": sum(item["size_bytes"] for item in entries), + "verification": "complete file inventory and SHA256 before launch; not hardware attestation"} + + +def cuda_devices() -> list[str]: + # Resolve the current mask through the driver without creating a CUDA context; + # nvidia-smi ordinals alone do not establish CUDA device identity. + import ctypes + cuda = ctypes.CDLL("libcuda.so.1") + def ok(code): + if code != 0: + raise RuntimeError("CUDA driver inventory failed: " + str(code)) + ok(cuda.cuInit(0)) + count = ctypes.c_int() + ok(cuda.cuDeviceGetCount(ctypes.byref(count))) + values = [] + for ordinal in range(count.value): + device, raw = ctypes.c_int(), (ctypes.c_ubyte * 16)() + ok(cuda.cuDeviceGet(ctypes.byref(device), ordinal)) + ok(cuda.cuDeviceGetUuid(ctypes.byref(raw), device)) + values.append("GPU-" + str(uuid.UUID(bytes=bytes(raw)))) + return values + + +def _runtime_env(source: str, gpu_uuids: list[str], nonce: str, cache: Path) -> dict[str, str]: + # Do not inherit authentication, PYTHONPATH, remote endpoints, LD_PRELOAD, or + # performance overrides. Never overwrite HOME/CODEX_HOME or user caches. + result = {key: os.environ[key] for key in ("PATH", "HOME", "LANG", "LC_ALL", "TMPDIR") if key in os.environ} + result.update({"PYTHONPATH": os.pathsep.join((str(Path(source) / "python"), str(Path(__file__).resolve().parent.parent))), "PYTHONNOUSERSITE": "1", "PYTHONDONTWRITEBYTECODE": "1", + "CUDA_VISIBLE_DEVICES": ",".join(gpu_uuids), "CUDA_DEVICE_ORDER": "PCI_BUS_ID", + # Host core discovery ignores container CPU/PID budgets in some native libraries. + # Keep the same explicit import/runtime thread policy for both arms and clients. + "OMP_NUM_THREADS": "1", "OPENBLAS_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "NUMEXPR_NUM_THREADS": "1", + "MAX_JOBS": "2", + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "HF_HUB_DISABLE_TELEMETRY": "1", + "VGBENCH_LAUNCH_NONCE": nonce, "XDG_CACHE_HOME": str(cache), "TORCHINDUCTOR_CACHE_DIR": str(cache / "torchinductor"), + # SGLang's native JIT/FlashInfer defaults do not follow XDG. + "SGLANG_CACHE_DIR": str(cache / "sglang"), "SGLANG_JIT_CACHE_DIR": str(cache / "sglang" / "jit"), + "FLASHINFER_WORKSPACE_BASE": str(cache / "flashinfer"), + "HF_HOME": str(cache / "huggingface"), "HF_HUB_CACHE": str(cache / "huggingface" / "hub"), + "TRITON_CACHE_DIR": str(cache / "triton"), "CUDA_CACHE_PATH": str(cache / "cuda")}) + return result + + +def _source_identity(spec: dict, role: str, env: dict, deadline: float, cancelled: threading.Event | None = None) -> dict: + declared = spec[role] + observed = source_file_manifest(Path(declared["source"]), timeout=spec["limits"]["command_seconds"], deadline=deadline, cancelled=cancelled) + if any(observed[field] != declared[field] for field in ("revision", "source_sha256")): + raise ValueError(f"{role} observed source revision/tree differs from pinned specification") + executable = Path(declared["python"]).resolve(strict=True) + if not executable.is_file() or not os.access(executable, os.X_OK): + raise ValueError("runtime Python executable is unavailable") + data = _command([declared["python"], "-c", _IDENTITY], cwd=Path(declared["source"]), env=env, + timeout=min(spec["limits"]["command_seconds"], max(0.01, deadline - time.monotonic()))) + identity = json.loads(data) + module = Path(identity.get("sglang_module") or "").resolve(strict=True) + expected_module = (Path(declared["source"]) / "python" / "sglang" / "__init__.py").resolve(strict=True) + if module != expected_module or not identity.get("packages", {}).get("torch"): + raise ValueError("runtime identity probe did not resolve pinned source with an installed torch dependency") + return {key: value for key, value in observed.items() if key != "files"} | identity | { + "python": declared["python"], "python_sha256": _hash(executable, deadline), + "environment_control": "allowlisted supervisor environment; no inherited runtime overrides", + } + + +def _proc_identity(pid: int) -> dict | None: + try: + content = Path(f"/proc/{pid}/stat").read_text() + # comm can itself contain spaces and parentheses; fields after its last + # ')' begin with state (field 3), not process name tokens. + fields = content[content.rfind(")") + 2:].split() + return {"pid": pid, "state": fields[0], "ppid": int(fields[1]), "pgid": int(fields[2]), + "session_id": int(fields[3]), "start_ticks": int(fields[19])} + except (FileNotFoundError, ProcessLookupError): + return None + + +def _diagnostic_proc_file(pid: int, name: str) -> str: + """Bounded reads of only the two non-command-line diagnostic proc files.""" + if name not in {"status", "cgroup"}: + raise ValueError("unsupported process diagnostic file") + with Path(f"/proc/{pid}/{name}").open(encoding="utf-8") as stream: + value = stream.read(65537) + if len(value) > 65536: + raise ValueError("process diagnostic file exceeds size limit") + return value + + +def _unowned_process_diagnostic(pid: int) -> dict: + """Best-effort attribution evidence only; never establishes ownership. + + PID disappearance, reuse and permission failures are retained explicitly. + No command line, environment, process name or exception text is recorded. + UID/cgroup observations are not atomic with stat; the second stat detects + some lifetime races but cannot prove provenance for an already exited PID. + """ + def failure(error: Exception) -> dict: + status = ("missing" if isinstance(error, (FileNotFoundError, ProcessLookupError)) + else "permission_denied" if isinstance(error, PermissionError) + else "unavailable") + return {"status": status, "error_type": type(error).__name__} + + def identity() -> dict: + try: + value = _proc_identity(pid) + return ({"status": "observed", **{key: value[key] for key in + ("pid", "ppid", "pgid", "session_id", "start_ticks", "state")}} + if value is not None else {"status": "missing"}) + except Exception as error: + return failure(error) + + result = {"pid": pid, "observed_at": _now(), "diagnostic_only": True, + "ownership_established": False, "identity": identity()} + try: + status = _diagnostic_proc_file(pid, "status") + match = re.search(r"^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s*$", status, re.MULTILINE) + if match is None: + raise ValueError("UID fields unavailable") + result["uid"] = {"status": "observed", **dict(zip( + ("real", "effective", "saved", "filesystem"), map(int, match.groups())))} + except Exception as error: + result["uid"] = failure(error) + try: + entries = [] + for line in _diagnostic_proc_file(pid, "cgroup").splitlines(): + fields = line.split(":", 2) + if len(fields) != 3 or not fields[0].isdigit() or not fields[2].startswith("/"): + raise ValueError("cgroup membership unavailable") + entries.append({"hierarchy_id": int(fields[0]), "controllers": fields[1], "path": fields[2]}) + if not entries: + raise ValueError("cgroup membership unavailable") + result["cgroup"] = {"status": "observed", "entries": entries} + except Exception as error: + result["cgroup"] = failure(error) + result["identity_after"] = identity() + before, after = result["identity"], result["identity_after"] + if before["status"] == after["status"] == "observed": + result["lifetime_check"] = ("same_start_ticks" if before["start_ticks"] == after["start_ticks"] + else "pid_reused_during_reads") + elif before["status"] == "observed" and after["status"] == "missing": + result["lifetime_check"] = "disappeared_during_reads" + else: + result["lifetime_check"] = "unverified" + return result + + +def _group_members(pgid: int) -> list[dict]: + result = [] + for name in os.listdir("/proc"): + if name.isdigit(): + identity = _proc_identity(int(name)) + if identity and identity["pgid"] == pgid and identity["state"] != "Z": + result.append(identity) + return result + + +class OwnedProcess: + """Own a new child session; never reap its leader before group cleanup.""" + + def __init__(self, argv: list[str], *, cwd: Path, env: dict, stdout: Path, stderr: Path, nonce: str): + self._mutex = threading.Lock() + self._closed = False + self.returncode = None + self.receipt = None + self.log_paths = (stdout, stderr) + with stdout.open("xb") as out, stderr.open("xb") as err: + self.process = subprocess.Popen(argv, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=out, stderr=err, start_new_session=True, close_fds=True) + identity = _proc_identity(self.process.pid) + if not identity or identity["ppid"] != os.getpid() or identity["pgid"] != self.process.pid or identity["session_id"] != self.process.pid: + # Popen's exact child PID may be terminated, but never an unverified + # process group. This branch cannot authorize any foreign PID. + self.process.terminate() + self.process.wait(timeout=1) + raise RuntimeError("could not establish ownership of child session") + self.identity = {**identity, "launch_nonce": nonce} + + def running(self) -> bool: + current = _proc_identity(self.identity["pid"]) + return bool(current and current["start_ticks"] == self.identity["start_ticks"] and current["state"] != "Z") + + def check_output_budget(self): + if any(path.stat().st_size > 128 * 1024 * 1024 for path in self.log_paths): + raise RuntimeError("owned process output exceeded the 128 MiB per-stream safety limit") + + def owns(self, pid: int) -> bool: + return self.observe(pid) is not None + + def observe(self, pid: int) -> dict | None: + current = _proc_identity(pid) + return current if current and current["pgid"] == self.identity["pgid"] and current["session_id"] == self.identity["session_id"] else None + + def _signal(self, sig: int) -> None: + current = _proc_identity(self.identity["pid"]) + if not current or any(current[field] != self.identity[field] for field in ("pid", "pgid", "session_id", "start_ticks")): + raise RuntimeError("refusing process-group signal: leader identity no longer matches") + members = _group_members(self.identity["pgid"]) + if any(item["session_id"] != self.identity["session_id"] for item in members): + raise RuntimeError("refusing process-group signal: unexpected session membership") + if members: + os.killpg(self.identity["pgid"], sig) + + def close(self, timeout: float | None = None, *, deadline: float | None = None) -> dict: + end = deadline if deadline is not None else time.monotonic() + float(timeout) + acquired = self._mutex.acquire(timeout=max(0, end - time.monotonic())) + if not acquired: + return {"status": "failed", "remaining_owned_pids": None, "reason": "shared cleanup deadline expired while another owner cleanup was active"} + try: + if self._closed: + return self.receipt + remaining_seconds = max(0, end - time.monotonic()) + self._signal(signal.SIGTERM if remaining_seconds > 0 else signal.SIGKILL) + grace = time.monotonic() + remaining_seconds / 2 + while _group_members(self.identity["pgid"]) and time.monotonic() < grace: + time.sleep(min(0.05, max(0, grace - time.monotonic()))) + if _group_members(self.identity["pgid"]): + self._signal(signal.SIGKILL) + while _group_members(self.identity["pgid"]) and time.monotonic() < end: + time.sleep(min(0.05, max(0, end - time.monotonic()))) + remaining = [item["pid"] for item in _group_members(self.identity["pgid"])] + self.receipt = {"status": "failed" if remaining else "clean", "remaining_owned_pids": remaining, + "scope": "exact verified child session/process group only"} + if not remaining: + self.returncode = self.process.wait(timeout=max(0.001, end - time.monotonic())) + self._closed = True + return self.receipt + finally: + self._mutex.release() + + +class _Supervisor: + def __init__(self, limits: dict): + self.total_deadline = time.monotonic() + limits["job_seconds"] + self.deadline = self.total_deadline - 2 * limits["cleanup_seconds"] + self.cleanup_seconds = limits["cleanup_seconds"] + self.cancelled = threading.Event() + self.finished = threading.Event() + self.reason = None + self.processes: list[OwnedProcess] = [] + self.lock = threading.Lock() + self.previous = {} + self.watchdog = threading.Thread(target=self._watch, daemon=True) + + def __enter__(self): + if threading.current_thread() is not threading.main_thread(): + raise RuntimeError("GPU supervisor must run on the main Python thread for signal handling") + for sig in (signal.SIGINT, signal.SIGTERM): + self.previous[sig] = signal.getsignal(sig) + signal.signal(sig, self._on_signal) + self.watchdog.start() + return self + + def _on_signal(self, sig, frame): + self.reason = f"received signal {sig}" + self.cancelled.set() + + def _watch(self): + while not self.finished.wait(0.1): + if time.monotonic() >= self.deadline: + self.reason = self.reason or "whole-job work deadline exceeded; cleanup reserve entered" + self.cancelled.set() + if self.cancelled.is_set(): + with self.lock: + processes = list(self.processes) + for process in reversed(processes): + try: + process.close(deadline=self.total_deadline) + except (OSError, RuntimeError, subprocess.TimeoutExpired): + pass # Main-thread cleanup records failures and quarantines. + return + + def check(self): + if self.cancelled.is_set(): + raise JobCancelled(self.reason or "job cancelled") + _check_deadline(self.deadline) + + def spawn(self, argv: list[str], **kwargs) -> OwnedProcess: + with self.lock: + self.check() + process = OwnedProcess(argv, **kwargs) + self.processes.append(process) + return process + + def __exit__(self, exc_type, exc, tb): + self.finished.set() + self.watchdog.join(timeout=0.2) + for process in reversed(self.processes): + try: + process.close(deadline=min(self.total_deadline, time.monotonic() + self.cleanup_seconds)) + except (OSError, RuntimeError, subprocess.TimeoutExpired): + pass + for sig, previous in self.previous.items(): + signal.signal(sig, previous) + + +class GpuLease: + def __init__(self, directory: Path, gpu_uuids: list[str], job_id: str): + self.directory = directory + self.gpus = sorted(gpu_uuids) + self.job_id = job_id + self.handles = [] + + def __enter__(self): + import fcntl + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + info = self.directory.lstat() + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o022: + raise ValueError("lock directory must be owned by this UID and not group/world writable") + try: + for device in self.gpus: + if (self.directory / (device + ".blocked.json")).exists(): + raise RuntimeError("GPU lease is quarantined after unresolved cleanup; operator inspection required") + descriptor = os.open(self.directory / (device + ".lock"), os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + handle = os.fdopen(descriptor, "r+b") + self.handles.append(handle) + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return self + except BaseException: + self.__exit__(None, None, None) + raise + + def quarantine(self, reason: str): + for device in self.gpus: + path = self.directory / (device + ".blocked.json") + # Do not replace a quarantine another operator already recorded. + with path.open("xb") as stream: + stream.write(canonical_json_bytes({"job_id": self.job_id, "at": _now(), "reason": reason})) + + def __exit__(self, *_): + import fcntl + for handle in reversed(self.handles): + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + handle.close() + self.handles.clear() + + +class GpuProbe: + def __init__(self, devices: list[str], timeout: float): + self.devices = devices + self.timeout = timeout + + def power_configuration(self, *, deadline: float | None = None) -> dict: + fields = "uuid,power.limit,enforced.power.limit,power.default_limit,power.max_limit" + result = {"observed_at": _now(), "query": fields, "status": "unavailable", "gpus": []} + try: + _check_deadline(deadline) + timeout = self.timeout if deadline is None else min(self.timeout, max(0.001, deadline - time.monotonic())) + raw = _command(["nvidia-smi", "--query-gpu=" + fields, "--format=csv,noheader,nounits", + "--id=" + ",".join(self.devices)], timeout=timeout).decode() + result["raw"] = raw + for row in csv.reader(io.StringIO(raw)): + if len(row) != 5: + raise ValueError("incomplete power configuration") + device = {"uuid": row[0].strip()} + for key, value in zip(("configured_limit_w", "enforced_limit_w", "default_limit_w", "maximum_limit_w"), row[1:]): + try: + watts = float(value) + except ValueError: + watts = None + device[key] = watts if watts is not None and math.isfinite(watts) and watts > 0 else None + result["gpus"].append(device) + if sorted(d["uuid"] for d in result["gpus"]) != sorted(self.devices): + raise ValueError("power configuration GPU inventory mismatch") + result["status"] = "recorded" + except (RuntimeError, ValueError, OSError, subprocess.TimeoutExpired, TimeoutError) as error: + result.update(error=str(error), gpus=[]) + return result + + def snapshot(self, *, deadline: float | None = None) -> dict: + def budget(): + _check_deadline(deadline) + return self.timeout if deadline is None else min(self.timeout, max(0.001, deadline - time.monotonic())) + fields = "index,uuid,name,memory.total,memory.used,utilization.gpu,driver_version,power.draw,temperature.gpu,mig.mode.current" + query_start = time.monotonic() + query_utc = _now() + raw = _command(["nvidia-smi", "--query-gpu=" + fields, "--format=csv,noheader,nounits", "--id=" + ",".join(self.devices)], timeout=budget()) + query_end = time.monotonic() + rows = list(csv.reader(io.StringIO(raw.decode()))) + gpus = [] + for row in rows: + row = [item.strip() for item in row] + if len(row) != 10 or row[1] not in self.devices or row[9] not in {"Disabled", "[N/A]", "N/A", "[Not Supported]"}: + raise RuntimeError("GPU inventory is incomplete, changed, or MIG-enabled") + device = {"index": int(row[0]), "uuid": row[1], "name": row[2], "memory_total_mib": float(row[3]), + "memory_used_mib": float(row[4]), "utilization_percent": float(row[5]), "driver_version": row[6], "mig_mode": row[9]} + for position, name in ((7, "power_watts"), (8, "temperature_celsius")): + try: + device[name] = float(row[position]) + except ValueError: + device[name] = None + if any(not math.isfinite(device[key]) or device[key] < 0 for key in ("memory_total_mib", "memory_used_mib", "utilization_percent")): + raise RuntimeError("required GPU telemetry is not finite") + gpus.append(device) + if sorted(item["uuid"] for item in gpus) != sorted(self.devices): + raise RuntimeError("observed GPU UUID inventory does not match leased devices") + raw = _command(["nvidia-smi", "--query-compute-apps=gpu_uuid,pid,used_memory", "--format=csv,noheader,nounits", "--id=" + ",".join(self.devices)], timeout=budget()) + apps = [] + for row in csv.reader(io.StringIO(raw.decode())): + row = [item.strip() for item in row] + if len(row) != 3 or row[0] not in self.devices or not row[1].isdigit(): + raise RuntimeError("GPU compute process inventory is unavailable or malformed") + try: + memory = float(row[2]) + except ValueError: + memory = None + apps.append({"gpu_uuid": row[0], "pid": int(row[1]), "memory_used_mib": memory}) + return {"at": _now(), "monotonic_seconds": time.monotonic(), "gpus": gpus, "compute_apps": apps, + "power_query": {"start_utc": query_utc, "start_monotonic_seconds": query_start, + "end_monotonic_seconds": query_end, "field": "power.draw"}} + + +def _idle(snapshot: dict, maximum: float) -> bool: + return not snapshot["compute_apps"] and all(item["memory_used_mib"] <= maximum for item in snapshot["gpus"]) + + +def summarize_gpu_samples(samples: list[dict], devices: list[str], interval: float, errors: list[str] | None = None, + *, window_start: float | None = None, window_end: float | None = None, command_seconds: float = 1) -> dict: + """Pure telemetry aggregation shared by live recording and saved verification.""" + errors = errors or [] + measurement = [sample for sample in samples if sample["phase"] == "measurement"] + peaks = {device: max((item["memory_used_mib"] for sample in samples for item in sample["gpus"] if item["uuid"] == device), default=None) for device in devices} + measured_peaks = {device: max((item["memory_used_mib"] for sample in measurement for item in sample["gpus"] if item["uuid"] == device), default=None) for device in devices} + owned = {device: sum(any(app["gpu_uuid"] == device for app in sample["owned_compute_apps"]) for sample in measurement) for device in devices} + intervals = [right["monotonic_seconds"] - left["monotonic_seconds"] for left, right in zip(measurement, measurement[1:])] + max_gap = max(interval * 3, interval + 2 * command_seconds + 0.1) + window_valid = window_start is not None and window_end is not None and window_end > window_start + coverage = bool(window_valid and measurement and measurement[0]["monotonic_seconds"] - window_start <= max_gap + and window_end - measurement[-1]["monotonic_seconds"] <= max_gap + and all(0 < gap <= max_gap for gap in intervals)) + return {"sample_count": len(samples), "measurement_sample_count": len(measurement), "errors": errors, + "gpu_identity": [{key: item[key] for key in ("uuid", "index", "name", "memory_total_mib", "driver_version", "mig_mode")} for item in samples[0]["gpus"]] if samples else [], + "observed_memory_peak_mib_by_gpu": peaks, "measurement_observed_memory_peak_mib_by_gpu": measured_peaks, + "observed_owned_compute_by_gpu": owned, "requested_interval_seconds": interval, + "maximum_observed_sample_gap_seconds": max(intervals) if intervals else None, + "maximum_allowed_sample_gap_seconds": max_gap, "cadence_coverage_qualified": coverage, + "measurement_window_start_monotonic_seconds": window_start, "measurement_window_end_monotonic_seconds": window_end, + "memory_semantics": "maximum observed device-used VRAM samples during this role, not exact framework allocator peaks", + "measurement_window": "client workload including separately recorded warmup requests; not GPU kernel timing", + "qualified": bool(len(measurement) >= 2 and not errors and all(owned.values()) and coverage)} + + +class _Sampler: + def __init__(self, probe: GpuProbe, owner: OwnedProcess, path: Path, interval: float, *, initial_sample: dict | None = None): + self.probe, self.owner, self.path, self.interval = probe, owner, path, interval + self.phase = "startup" + self.window_start = None + self.window_end = None + self.done = threading.Event() + self.failed = threading.Event() + self.samples = [] + self.errors = [] + self.thread = threading.Thread(target=self._loop, daemon=True) + self.path.touch(exist_ok=False) + if initial_sample is not None: + if initial_sample.get("compute_apps") != []: + raise ValueError("Prelaunch sample must have an empty compute inventory") + sample = {**initial_sample, "phase": "startup", "owned_compute_apps": [], "unowned_compute_apps": [], + "observation": "verified_idle_before_runtime_launch"} + self.samples.append(sample) + self.path.write_bytes(canonical_json_bytes(sample) + b"\n") + + def _loop(self): + while not self.done.is_set(): + try: + sample = self.probe.snapshot() + if self.done.is_set(): + return # Never append evidence after a timed-out stop/hash. + sample["phase"] = self.phase + sample["owned_compute_apps"] = [] + sample["unowned_compute_apps"] = [] + for app in sample["compute_apps"]: + observation_status = "not_owned" + try: + observed = self.owner.observe(app["pid"]) + except PermissionError: + observed = None + observation_status = "permission_denied" + if observed is None: + sample["unowned_compute_apps"].append({ + **app, "ownership_observation": observation_status, + "process_diagnostic": _unowned_process_diagnostic(app["pid"]), + }) + else: + sample["owned_compute_apps"].append({**app, "process_identity": observed}) + with self.path.open("ab") as stream: + stream.write(canonical_json_bytes(sample) + b"\n") + stream.flush() + self.samples.append(sample) + if sample["unowned_compute_apps"]: + raise RuntimeError("foreign or PID-namespace-invisible GPU process detected; no ownership established") + except Exception as error: + if self.done.is_set(): + return + self.errors.append(str(error) if isinstance(error, RuntimeError) else type(error).__name__) + self.failed.set() + return + self.done.wait(self.interval) + + def start(self): + self.thread.start() + + def begin_measurement(self): + self.window_start = time.monotonic() + self.phase = "measurement" + + def end_measurement(self): + self.window_end = time.monotonic() + self.phase = "cleanup" + + def bracket_completion(self, deadline: float) -> bool: + """Retain the next ordinary sample before ending the owned runtime.""" + boundary = time.monotonic() + while time.monotonic() < deadline and not self.failed.is_set(): + if self.samples and self.samples[-1]["monotonic_seconds"] >= boundary: + return True + time.sleep(min(0.05, max(0, deadline - time.monotonic()))) + return False + + def stop(self, *, deadline: float): + self.done.set() + self.thread.join(timeout=max(0, deadline - time.monotonic())) + if self.thread.is_alive(): + self.errors.append("telemetry worker did not terminate within bounded command deadline") + self.failed.set() + + def summary(self) -> dict: + return summarize_gpu_samples(self.samples, self.probe.devices, self.interval, self.errors, + window_start=self.window_start, window_end=self.window_end, command_seconds=self.probe.timeout) + + +def _port_available(port: int): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", port)) + + +def _health(port: int, timeout: float) -> bool: + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=timeout) + try: + connection.request("GET", "/health") + response = connection.getresponse() + ready = response.status == 200 + response.close() + return ready + except (OSError, http.client.HTTPException): + return False + finally: + connection.close() + + +def _owned_listener(owner: OwnedProcess, port: int) -> bool: + # Linux socket inodes bind the responding loopback listener to the launched + # process group; a foreign server racing for the port is never benchmarked. + inodes = set() + for row in Path("/proc/net/tcp").read_text().splitlines()[1:]: + fields = row.split() + if len(fields) > 9 and fields[1] == f"0100007F:{port:04X}" and fields[3] == "0A": + inodes.add(fields[9]) + for member in _group_members(owner.identity["pgid"]): + try: + for link in Path(f"/proc/{member['pid']}/fd").iterdir(): + try: + target = os.readlink(link) + except (FileNotFoundError, PermissionError): + continue + if target.startswith("socket:[") and target[8:-1] in inodes: + return True + except (FileNotFoundError, PermissionError): + continue + return False + + +class _AttemptMonitor: + """Preempt native decoder hangs using the client's durable attempt ledger.""" + + def __init__(self, journal: Path, seconds: float, concurrency: int = 1): + self.journal, self.seconds = journal, seconds + self.offset = 0 + self.concurrency = concurrency + self.active = {} + self.transport = set() + self.waiting = set() + self.seen = set() + + def check(self): + try: + with self.journal.open("rb") as stream: + stream.seek(self.offset) + while True: + line = stream.readline(1024 * 1024 + 1) + if not line: + break + if len(line) > 1024 * 1024: + raise RuntimeError("attempt ledger record exceeds bounded monitor size") + if not line.endswith(b"\n"): + break + self.offset = stream.tell() + event = json.loads(line) + slot = event.get("slot_id") + if event.get("event") == "attempt_started": + if not isinstance(slot, str) or slot in self.seen or len(self.transport) >= self.concurrency: + raise RuntimeError("client attempts exceed declared concurrency or reuse a slot") + self.seen.add(slot) + self.transport.add(slot) + self.active[slot] = time.monotonic() + elif event.get("event") == "transport_finished": + if slot not in self.transport: + raise RuntimeError("transport completion does not match active slot") + self.transport.remove(slot) + self.active.pop(slot) + self.waiting.add(slot) + elif event.get("event") == "validation_started": + if slot not in self.waiting: + raise RuntimeError("validation does not follow completed transport") + self.waiting.remove(slot) + self.active[slot] = time.monotonic() + elif event.get("event") == "attempt_finished": + slot = event.get("record", {}).get("slot_id") + if slot not in self.active and slot not in self.waiting: + raise RuntimeError("attempt ledger completion does not match active slot") + self.active.pop(slot, None) + self.transport.discard(slot) + self.waiting.discard(slot) + except FileNotFoundError: + pass + if any(time.monotonic() - start >= self.seconds for start in self.active.values()): + raise TimeoutError("hard supervised per-attempt deadline exceeded; stopping owned runtime and client") + + +def _wait_client(process: OwnedProcess, owner: OwnedProcess | None, sampler: _Sampler | None, supervisor: _Supervisor, deadline: float, monitor: _AttemptMonitor | None = None): + while process.running(): + supervisor.check() + process.check_output_budget() + _check_deadline(deadline) + if owner is not None and not owner.running(): + raise RuntimeError("owned runtime exited during measurement") + if owner is not None: + owner.check_output_budget() + if sampler is not None and sampler.failed.is_set(): + raise RuntimeError("GPU telemetry/ownership failed during measurement") + if monitor is not None: + monitor.check() + time.sleep(0.05) + cleanup = process.close(supervisor.cleanup_seconds) + if cleanup["status"] != "clean": + raise RuntimeError("owned client process group failed to drain") + return process.returncode + + +def _cleanup_role(owner: OwnedProcess, sampler: _Sampler, probe: GpuProbe, limits: dict, *, deadline: float | None = None) -> dict: + end = deadline if deadline is not None else time.monotonic() + limits["cleanup_seconds"] + sampler.end_measurement() + sampler.done.set() + receipt = owner.close(deadline=end) + sampler.stop(deadline=end) + idle_after = False + snapshot = None + observed = {(app["gpu_uuid"], app["pid"]): app["process_identity"] + for sample in sampler.samples for app in sample.get("owned_compute_apps", [])} + waited_for = set() + while time.monotonic() < end: + snapshot = probe.snapshot(deadline=end) + if _idle(snapshot, limits["max_idle_memory_mib"]): + idle_after = True + break + # NVML can retain exited ranks while the driver releases their memory. + # Wait only for previously attributed identities, never unknown/reused PIDs. + foreign = False + for app in snapshot["compute_apps"]: + expected = observed.get((app["gpu_uuid"], app["pid"])) + current = _proc_identity(app["pid"]) + if expected is None or (current is not None and any(current[key] != expected[key] for key in ("pid", "pgid", "session_id", "start_ticks"))): + foreign = True + break + waited_for.add(app["pid"]) + if foreign: + break + time.sleep(min(0.2, max(0, end - time.monotonic()))) + return {**receipt, "status": "clean" if receipt["status"] == "clean" and idle_after else "failed", + "idle_after": idle_after, "gpu_after": snapshot, "waited_for_driver_pids": sorted(waited_for), + "reason": "owned group drained and leased devices idle" if idle_after else "GPU resources not verified idle after exact-owned cleanup"} + + +def _role(spec: dict, label: str, directory: Path, supervisor: _Supervisor, probe: GpuProbe, receipt: dict) -> dict: + metadata = directory / "supervisor" / label + metadata.mkdir(parents=True) + cache = metadata / "cache" + cache.mkdir() + nonce = uuid.uuid4().hex + env = _runtime_env(spec[label]["source"], spec["gpu_uuids"], nonce, cache) + role = receipt["roles"][label] = {"status": "preflight", "source_identity": None, "cleanup": {"status": "not_started"}, "run_path": f"{label}/run.json"} + _write(directory / "gpu-job.json", receipt) + identity = _source_identity(spec, label, env, supervisor.deadline, supervisor.cancelled) + role["source_identity"] = identity + snapshot = probe.snapshot() + role["gpu_before"] = snapshot + if not _idle(snapshot, spec["limits"]["max_idle_memory_mib"]): + raise RuntimeError("leased GPUs have existing compute/memory use; no runtime started") + role["power_configuration_before"] = probe.power_configuration(deadline=supervisor.deadline) + # This SGLang revision expects NVML ordinals. The child checks the CUDA + # driver's UUID order before importing SGLang; NVML/CUDA order may differ. + indices = {device["uuid"]: device["index"] for device in snapshot["gpus"]} + selected = [indices[device] for device in spec["gpu_uuids"]] + if len(set(selected)) != len(selected) or any(type(index) is not int or index < 0 for index in selected): + raise RuntimeError("GPU inventory has invalid or duplicate runtime indices") + env["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, selected)) + env["VGBENCH_GPU_UUIDS"] = ",".join(spec["gpu_uuids"]) + role["runtime_gpu_binding"] = {"cuda_visible_devices": env["CUDA_VISIBLE_DEVICES"], "expected_gpu_uuids": spec["gpu_uuids"], + "verification": "child CUDA driver UUID check before SGLang import"} + _port_available(spec["port"]) + owner = None + sampler = None + client = None + try: + boot_started = time.monotonic() + role["startup_timing_window"] = {"start_monotonic_seconds": boot_started, "start_utc": _now(), + "end_monotonic_seconds": None} + owner = supervisor.spawn(_server_argv(spec, label), cwd=metadata, env=env, + stdout=metadata / "runtime.stdout.log", stderr=metadata / "runtime.stderr.log", nonce=nonce) + role.update(status="starting", process_identity=owner.identity, launch_argv=_server_argv(spec, label), started_at=_now()) + _write(directory / "gpu-job.json", receipt) + sampler = _Sampler(probe, owner, metadata / "telemetry.jsonl", spec["limits"]["telemetry_interval_seconds"], + initial_sample=snapshot) + sampler.start() + ready_deadline = min(supervisor.deadline, time.monotonic() + spec["limits"]["startup_seconds"]) + while True: + supervisor.check() + _check_deadline(ready_deadline) + if not owner.running() or sampler.failed.is_set(): + raise RuntimeError("runtime exited or GPU ownership/telemetry failed before readiness") + owner.check_output_budget() + if _owned_listener(owner, spec["port"]) and _health(spec["port"], min(1, max(0.01, ready_deadline - time.monotonic()))): + break + time.sleep(0.1) + role["startup_seconds"] = time.monotonic() - boot_started + role["startup_timing_window"]["end_monotonic_seconds"] = boot_started + role["startup_seconds"] + role["status"] = "measuring" + sampler.begin_measurement() + client_env = _runtime_env("", [], nonce, cache) + client_env["PYTHONPATH"] = str(Path(__file__).resolve().parent.parent) + hardware = ",".join(spec["gpu_uuids"]) + argv = [sys.executable, "-m", "evaluator.cli", "run", str(directory / "plan.json"), "--runtime", "sglang", + "--endpoint", f"http://127.0.0.1:{spec['port']}", "--runtime-revision", spec[label]["revision"], + "--hardware-label", hardware, "--model-revision", spec["model"]["revision"], + "--timeout-seconds", str(spec["limits"]["request_seconds"]), "--output", str(directory / label), "--execute"] + if spec.get("serving"): + argv.extend(["--serving-concurrency", str(spec["serving"]["concurrency"])]) + if spec["serving"]["delivery_deadline_seconds"] is not None: + argv.extend(["--delivery-deadline-seconds", str(spec["serving"]["delivery_deadline_seconds"])]) + client = supervisor.spawn(argv, cwd=directory, env=client_env, stdout=metadata / "client.stdout.json", + stderr=metadata / "client.stderr.log", nonce=nonce) + role["client_process_identity"] = client.identity + _write(directory / "gpu-job.json", receipt) + slot_count = len(spec["plan"]["cases"]) * spec["plan"]["repetitions"] + spec["plan"]["warmup_runs"] + phase_count = 2 if spec.get("serving") else 1 + client_deadline = min(supervisor.deadline, time.monotonic() + phase_count * slot_count * spec["limits"]["request_seconds"]) + code = _wait_client(client, owner, sampler, supervisor, client_deadline, + _AttemptMonitor(directory / label / "events.jsonl", spec["limits"]["request_seconds"], spec.get("serving", {}).get("concurrency", 1))) + role["client_exit_code"] = code + role["power_completion_bracket_recorded"] = sampler.bracket_completion( + min(supervisor.deadline, time.monotonic() + 3 * spec["limits"]["telemetry_interval_seconds"])) + run_path = directory / label / "run.json" + if run_path.is_file(): + run = _read(run_path) + role["run_sha256"] = _hash(run_path) + role["run_summary"] = run.get("summary") + if run.get("evidence_kind") not in {"operator_endpoint", "live_h3"} or run.get("plan_sha256") != _digest(spec["plan"]): + raise RuntimeError("supervised client returned non-endpoint or wrong-plan evidence") + role["run_status"] = run.get("status") + finalized = bool(run.get("finished_at") and run.get("status") in {"complete", "partial", "failed"}) + role["status"] = "complete" if code in {0, 1} and finalized else "failed" + else: + role["status"] = "failed" + if role["status"] != "complete": + raise RuntimeError("supervised client did not finalize the scheduled workload") + finally: + cleanup_end = min(supervisor.total_deadline, time.monotonic() + spec["limits"]["cleanup_seconds"]) + if client is not None: + try: + role["client_cleanup"] = client.close(deadline=cleanup_end) + except Exception as error: + role["client_cleanup"] = {"status": "failed", "reason": type(error).__name__} + if owner is not None and sampler is not None: + try: + role["cleanup"] = _cleanup_role(owner, sampler, probe, spec["limits"], deadline=cleanup_end) + except Exception as error: + role["cleanup"] = {"status": "failed", "idle_after": False, "reason": f"cleanup verification failed: {type(error).__name__}"} + role["telemetry_summary"] = sampler.summary() + role["telemetry_path"] = (metadata / "telemetry.jsonl").relative_to(directory).as_posix() + role["telemetry_sha256"] = _hash(metadata / "telemetry.jsonl") + elif owner is not None: + role["cleanup"] = owner.close(deadline=cleanup_end) + role["cleanup"].update(status="failed", idle_after=False, reason="telemetry did not start; GPU idle unverified") + role["finished_at"] = _now() + role["power_configuration_after"] = probe.power_configuration(deadline=supervisor.total_deadline) + partial = directory / label / "run.json" + if partial.is_file(): + role["run_sha256"] = _hash(partial, supervisor.total_deadline) + role["run_summary"] = _read(partial).get("summary") + _write(directory / "gpu-job.json", receipt) + if role["cleanup"]["status"] != "clean": + raise RuntimeError("owned runtime cleanup did not establish idle GPUs") + if role.get("client_cleanup", {}).get("status") != "clean": + raise RuntimeError("owned benchmark client cleanup did not qualify") + if not role["telemetry_summary"]["qualified"]: + raise RuntimeError("measurement lacks complete telemetry and visible owned compute on every selected GPU") + if label == "baseline" and role.get("run_status") != "complete": + raise RuntimeError("baseline workload contains failed/invalid slots; candidate was not started") + # Files are verified again after execution; changed source cannot qualify. + after = source_file_manifest(Path(spec[label]["source"]), timeout=spec["limits"]["command_seconds"], deadline=supervisor.deadline, cancelled=supervisor.cancelled) + if after["source_sha256"] != role["source_identity"]["source_sha256"] or after["revision"] != role["source_identity"]["revision"]: + raise RuntimeError("runtime source changed while executing") + return role + + +def _calibration(spec: dict, current: dict | None = None, *, deadline: float | None = None) -> tuple[bool, str]: + if spec["policy"]["calibration_status"] != "operator_calibrated": + return False, "policy is not calibrated; measured hardware results remain useful but regression acceptance is inconclusive" + if current is None or deadline is None: + return False, "calibrated CI requires independently verified current and prior raw GPU evidence" + from .mvp_gpu_evidence import verify_calibration + return verify_calibration(spec, current, deadline=deadline) + + +def _gate(spec: dict, receipt: dict, comparison: dict | None, *, verified_evidence: dict | None = None, deadline: float | None = None) -> dict: + reasons = [] + if verified_evidence is None: + reasons.append("raw run/media/telemetry/comparison evidence has not been verified") + if receipt.get("status") != "complete" or receipt.get("failures"): + reasons.append("GPU supervision did not finish without infrastructure failures") + measured = receipt.get("measurement_status") == "complete" + if not measured: + reasons.append("controlled GPU measurement is incomplete") + if receipt.get("evidence_kind") != "controlled_h3_gpu": + reasons.append("receipt does not contain controlled GPU execution evidence") + if comparison is None: + reasons.append("verified paired comparison is unavailable") + elif comparison.get("measurement", {}).get("performance_mode") != "same_configuration_class_regression": + reasons.append("performance comparison is descriptive or incomparable, not a passing CI gate") + if comparison is not None and (comparison.get("evidence_kind") not in {"operator_endpoint", "live_h3"} or comparison.get("plan_sha256") != _digest(spec["plan"])): + reasons.append("comparison evidence is fixture/imported/mixed or does not match the frozen workload") + roles = [receipt.get("roles", {}).get(role, {}) for role in _ROLES] + for label, role in zip(_ROLES, roles): + if role.get("cleanup", {}).get("status") != "clean" or not role.get("telemetry_summary", {}).get("qualified"): + reasons.append("cleanup or observed GPU telemetry requirements did not qualify") + if not role.get("source_identity") or not role.get("process_identity"): + reasons.append("observed runtime source or owned process identity is missing") + elif any(role["source_identity"].get(field) != spec[label][field] for field in ("revision", "source_sha256")): + reasons.append("observed source identity does not match pinned role") + gpu_ids = [item.get("uuid") for item in role.get("telemetry_summary", {}).get("gpu_identity", [])] + if sorted(gpu_ids) != sorted(spec["gpu_uuids"]): + reasons.append("observed GPUs do not match selected UUIDs") + if all(role.get("source_identity") for role in roles): + for field in ("python_sha256", "python_version", "packages"): + left, right = roles[0]["source_identity"].get(field), roles[1]["source_identity"].get(field) + if field == "packages": + left = {key: value for key, value in (left or {}).items() if key != "sglang"} + right = {key: value for key, value in (right or {}).items() if key != "sglang"} + if left != right: + reasons.append(f"baseline/candidate runtime dependency {field} differs") + if roles[0].get("telemetry_summary", {}).get("gpu_identity") != roles[1].get("telemetry_summary", {}).get("gpu_identity"): + reasons.append("observed hardware/driver identity differs") + try: + calibrated, calibration_reason = _calibration(spec, verified_evidence, deadline=deadline) + except Exception as error: + calibrated = False + detail = str(error) if isinstance(error, (ValueError, RuntimeError, TimeoutError)) else type(error).__name__ + calibration_reason = f"calibration evidence unavailable or invalid: {detail}" + if not calibrated: + reasons.append(calibration_reason) + if spec["allocation"]["mode"] != "dedicated_ci": + reasons.append("cooperative shared-node allocation is not dedicated CI isolation") + memory_change = None + if all(role.get("telemetry_summary", {}).get("qualified") for role in roles): + left = roles[0]["telemetry_summary"]["observed_memory_peak_mib_by_gpu"] + right = roles[1]["telemetry_summary"]["observed_memory_peak_mib_by_gpu"] + if all(isinstance(left.get(device), (int, float)) and left[device] > 0 and isinstance(right.get(device), (int, float)) and right[device] >= 0 for device in spec["gpu_uuids"]): + memory_change = max(right[device] / left[device] - 1 for device in spec["gpu_uuids"]) + memory_threshold = spec["policy"].get("max_memory_increase_fraction") + memory_failed = memory_change is not None and memory_threshold is not None and memory_change > memory_threshold + if memory_threshold is None or memory_change is None: + reasons.append("an explicit sampled-memory regression gate with complete observations is required for CI acceptance") + detected = bool(memory_failed or (comparison and comparison.get("overall_status") == "fail")) + result = "fail" if detected else ("inconclusive" if reasons or not comparison or comparison.get("overall_status") != "pass" else "pass") + return {"regression_status": result, "ci_accepted": result == "pass", "release_qualified": False, + "acceptance_reasons": list(dict.fromkeys(reasons)), "calibration": {"verified": calibrated, "reason": calibration_reason}, + "memory_increase_fraction": memory_change, "memory_threshold": memory_threshold, + "memory_gate_semantics": "worst per-GPU relative change in observed sampled device-used VRAM maxima; not exact allocator peaks", + "memory_gate_status": "fail" if memory_failed else ("pass" if memory_change is not None and memory_threshold is not None else "inconclusive")} + + +def _require_linux(): + if platform.system() != "Linux" or not Path("/proc/self/stat").is_file(): + raise RuntimeError("controlled GPU execution requires Linux /proc ownership checks; preview is cross-platform") + + +def run_gpu_job(spec: dict, output_dir: Path, *, serving_smoke: bool = False) -> dict: + """Launch owned H3 sessions, or one serving smoke session; Linux-only. + + Calling this function is execution authorization. CLI callers must put an + explicit --execute barrier in front of it. No provisioning/download occurs. + The result's measurement_status, regression_status, and ci_accepted are + intentionally independent; an uncalibrated hardware measurement can finish + successfully without claiming that CI acceptance or release was earned. + Calibrated policies undergo prior-evidence/cell verification before weight + hashing or any GPU lease/probe. Observed environment equivalence and evidence + integrity are checked again after measurement; later drift can still reject + acceptance after compute was consumed. + """ + spec = validate_gpu_job(spec) + if serving_smoke and not spec.get("serving"): + raise ValueError("single-runtime smoke requires an explicit serving load") + approval = spec["authorization"] + if not approval["compute_approved"] or not approval["model_license_reviewed"] or not approval["approval_reference"].strip(): + raise ValueError("GPU execution requires explicit compute approval and model-license review with an approval reference; no work started") + _require_linux() + directory = Path(output_dir).absolute() + if directory.exists() or directory.is_symlink(): + raise FileExistsError("GPU evidence output must be a new directory") + directory.mkdir(parents=True, exist_ok=False) + _write(directory / "spec.json", spec) + _write(directory / "plan.json", spec["plan"]) + _write(directory / "policy.json", spec["policy"]) + receipt = {"schema_version": VERSION, "bundle_type": "controlled_serving_smoke" if serving_smoke else "controlled_gpu_job", "job_id": spec["job_id"], + "execution_id": uuid.uuid4().hex, + "evidence_kind": "no_gpu_measurement", "status": "running", "measurement_status": "incomplete", + "regression_status": "inconclusive", "ci_accepted": False, "release_qualified": False, + "spec_sha256": _digest(spec), "plan_sha256": _digest(spec["plan"]), "started_at": _now(), "finished_at": None, + "allocation": spec["allocation"], "allocation_verification": "operator-declared prerequisite; UUID locks and process checks are cooperative, not scheduler attestation", + "authorization": spec["authorization"], "authorization_verification": "operator assertion, not independent legal or allocation verification", + "roles": {}, "failures": [], "comparison_path": None, "cleanup_status": "not_started", + "supervisor_source_sha256": _hash(Path(__file__)), "trust_boundary": "trusted runtime code and operator-provisioned runner; not a hostile-code sandbox"} + _write(directory / "gpu-job.json", receipt) + comparison = None + verification_deadline = time.monotonic() + spec["limits"]["job_seconds"] + try: + with _Supervisor(spec["limits"]) as supervisor: + verification_deadline = supervisor.deadline + from .mvp_gpu_evidence import preflight_calibration + try: + receipt["calibration_preflight"] = preflight_calibration( + spec, directory, started_at=receipt["started_at"], + execution_id=receipt["execution_id"], deadline=supervisor.deadline, + ) + except Exception as error: + detail = str(error) if isinstance(error, (ValueError, RuntimeError, TimeoutError)) else type(error).__name__ + receipt["calibration_preflight"] = {"status": "failed", "performed_before_gpu_lease": True, "reason": detail} + _write(directory / "gpu-job.json", receipt) + raise + _write(directory / "gpu-job.json", receipt) + receipt["model_identity"] = _model_manifest(spec, supervisor.deadline, supervisor.cancelled) + supervisor.check() + probe = GpuProbe(spec["gpu_uuids"], spec["limits"]["command_seconds"]) + with GpuLease(Path(spec["lock_directory"]), spec["gpu_uuids"], spec["job_id"]) as lease: + try: + for role in (("baseline",) if serving_smoke else _ROLES): + supervisor.check() + _role(spec, role, directory, supervisor, probe, receipt) + receipt["evidence_kind"] = "controlled_h3_gpu" + _write(directory / "gpu-job.json", receipt) + # Model mutation during measurement invalidates the pinned identity. + if _model_manifest(spec, supervisor.deadline, supervisor.cancelled) != receipt["model_identity"]: + raise RuntimeError("staged model identity changed during measurement") + receipt.update(measurement_status="complete", evidence_kind="controlled_h3_gpu", cleanup_status="clean") + _write(directory / "gpu-job.json", receipt) + finally: + if any(role.get("cleanup", {}).get("status") == "failed" for role in receipt["roles"].values()): + receipt["cleanup_status"] = "failed" + lease.quarantine("GPU idle or exact-owned cleanup was not established; inspect before clearing quarantine") + elif receipt["roles"] and all(role.get("cleanup", {}).get("status") == "clean" for role in receipt["roles"].values()): + receipt["cleanup_status"] = "clean" + if serving_smoke: + receipt["status"] = "complete" + else: + # Native decode/comparison is also a supervised child with the same + # global work deadline, so a wedged codec cannot retain GPU jobs. + metadata = directory / "supervisor" + client_env = _runtime_env("", [], uuid.uuid4().hex, metadata / "compare-cache") + client_env["PYTHONPATH"] = str(Path(__file__).resolve().parent.parent) + compare = supervisor.spawn([sys.executable, "-m", "evaluator.cli", "compare", str(directory / "baseline"), str(directory / "candidate"), "--policy", str(directory / "policy.json")], + cwd=directory, env=client_env, stdout=directory / "comparison.json", stderr=metadata / "compare.stderr.log", nonce=uuid.uuid4().hex) + code = _wait_client(compare, None, None, supervisor, supervisor.deadline) + if code not in {0, 1, 2}: + raise RuntimeError("supervised comparison process exited abnormally") + comparison = _read(directory / "comparison.json") + if comparison.get("bundle_type") != "mvp_comparison": + raise RuntimeError("supervised comparison did not produce its validated contract") + receipt.update(comparison_path="comparison.json", comparison_sha256=_hash(directory / "comparison.json"), status="complete") + except (Exception, KeyboardInterrupt) as error: + receipt["status"] = "aborted" if isinstance(error, (JobCancelled, KeyboardInterrupt, TimeoutError)) else "failed" + receipt["failures"].append(str(error) if isinstance(error, (ValueError, RuntimeError, TimeoutError)) else type(error).__name__) + receipt["finished_at"] = _now() + _write(directory / "gpu-job.json", receipt) + verified = None + try: + if receipt["status"] == "complete": + from .mvp_gpu_evidence import verify_measurement_job + verified = verify_measurement_job(directory, deadline=verification_deadline, serving_smoke=serving_smoke) + if serving_smoke: + receipt["measurement_verified"] = verified is not None + else: + receipt.update(_gate(spec, receipt, comparison, verified_evidence=verified, deadline=verification_deadline)) + except Exception as error: + receipt.update(regression_status="inconclusive", ci_accepted=False, + acceptance_reasons=[f"acceptance evidence could not be verified: {type(error).__name__}"]) + receipt["finished_at"] = _now() + _write(directory / "gpu-job.json", receipt) + return receipt + + +def evaluate_gpu_job(jobdir: Path, *, verification_timeout_seconds: float = 60) -> dict: + """Recheck saved hashes and fail-closed acceptance without starting GPU work. + + These hashes provide integrity within the trusted-runner threat model, not + cryptographic proof that an arbitrary supplied directory came from a GPU. + """ + directory = Path(jobdir).resolve(strict=True) + _number(verification_timeout_seconds, "verification_timeout_seconds", 0.01, 600) + deadline = time.monotonic() + verification_timeout_seconds + from .mvp_gpu_evidence import verify_measurement_job + verified = verify_measurement_job(directory, deadline=deadline) + receipt, spec, comparison = verified["receipt"], verified["spec"], verified["comparison"] + return {**receipt, **_gate(spec, receipt, comparison, verified_evidence=verified, deadline=deadline)} diff --git a/experimental/video-generation/evaluator/mvp_gpu_manifest.py b/experimental/video-generation/evaluator/mvp_gpu_manifest.py new file mode 100644 index 0000000000..4fe256fa78 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_gpu_manifest.py @@ -0,0 +1,161 @@ +"""Read-only file inventory preparation for a controlled GPU job. + +This module never imports a model, starts a server, downloads files, or probes a +GPU. Hashing a large staged snapshot can still cause substantial disk I/O. The +manifest establishes file identity, not model permission or runtime correctness. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .mvp_gpu_job import source_file_manifest +from .mvp_runner import canonical_json_bytes + + +def _deadline_check(deadline: float) -> None: + if time.monotonic() >= deadline: + raise TimeoutError("file inventory exceeded its read-only preparation deadline") + + +@contextmanager +def _open_regular(path: Path): + """Resolve the already-approved absolute path without following new links.""" + descriptors = [] + try: + current = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + descriptors.append(current) + for part in path.parts[1:-1]: + current = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=current) + descriptors.append(current) + descriptor = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=current) + with os.fdopen(descriptor, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("model inventory accepts only regular files") + yield stream + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def _model_inventory(root: Path, revision: str, deadline: float) -> dict[str, Any]: + blob_root = None + if root.parent.name == "snapshots" and root.name == revision: + possible = root.parent.parent / "blobs" + if possible.is_symlink(): + raise ValueError("HF blob directory must not be a symlink outside its cache") + if possible.is_dir(): + blob_root = possible.resolve(strict=True) + + names: list[str] = [] + + def walk_error(error: OSError) -> None: + raise error + + for directory, directories, files in os.walk(root, followlinks=False, onerror=walk_error): + _deadline_check(deadline) + if any((Path(directory) / name).is_symlink() for name in directories): + raise ValueError("model inventory does not follow directory symlinks") + names.extend((Path(directory) / name).relative_to(root).as_posix() for name in files) + if len(names) > 20000: + raise ValueError("model inventory exceeds 20000 files") + if not names or not any(name.endswith((".safetensors", ".pt", ".bin")) for name in names): + raise ValueError("staged model directory contains no supported weight files") + + entries = [] + for name in sorted(names): + _deadline_check(deadline) + if "\\" in name or "\x00" in name: + raise ValueError("model file name is not supported by the GPU job contract") + target = (root / name).resolve(strict=True) + if not (target.is_relative_to(root) or (blob_root and target.is_relative_to(blob_root))): + raise ValueError("model file escapes the staged snapshot and its HF blob directory") + before = target.stat() + if not stat.S_ISREG(before.st_mode): + raise ValueError("model inventory accepts only regular files") + digest = hashlib.sha256() + count = 0 + with _open_regular(target) as stream: + opened = os.fstat(stream.fileno()) + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + opened.st_dev, opened.st_ino, opened.st_size, opened.st_mtime_ns + ): + raise ValueError("model file changed before its inventory read") + while True: + _deadline_check(deadline) + chunk = stream.read(1024 * 1024) + if not chunk: + break + count += len(chunk) + digest.update(chunk) + after = os.fstat(stream.fileno()) + final_path = target.lstat() + if count != before.st_size or (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns + ) or (final_path.st_dev, final_path.st_ino) != (after.st_dev, after.st_ino): + raise ValueError("model file changed while preparing its inventory") + entries.append({"path": name, "size_bytes": count, "sha256": digest.hexdigest()}) + return { + "path": str(root), "revision": revision, "files": entries, + "manifest_sha256": hashlib.sha256(canonical_json_bytes(entries)).hexdigest(), + "total_bytes": sum(item["size_bytes"] for item in entries), + } + + +def build_gpu_manifest(kind: str, directory: Path, *, model_revision: str | None = None, + timeout_seconds: float = 600) -> dict[str, Any]: + """Hash an operator-selected staged tree; inputs are never modified.""" + if kind not in {"runtime", "model"}: + raise ValueError("manifest kind must be runtime or model") + if (isinstance(timeout_seconds, bool) or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(timeout_seconds) or not 0.1 <= timeout_seconds <= 7200): + raise ValueError("manifest timeout must be finite and between 0.1 and 7200 seconds") + if kind == "model" and (not isinstance(model_revision, str) or not re.fullmatch(r"[0-9a-f]{40}", model_revision)): + raise ValueError("model inventory requires an explicit immutable --model-revision") + if kind == "runtime" and model_revision is not None: + raise ValueError("--model-revision applies only to model inventories") + deadline = time.monotonic() + timeout_seconds + root = Path(directory).resolve(strict=True) + if not root.is_dir() or root == Path(root.anchor): + raise ValueError("inventory path must name a specific staged directory, not the filesystem root") + if kind == "runtime": + inventory = source_file_manifest(root, timeout=min(10, timeout_seconds), deadline=deadline) + else: + inventory = _model_inventory(root, model_revision, deadline) + _deadline_check(deadline) + return { + "schema_version": "0.1.0", "bundle_type": "gpu_preparation_manifest", + "evidence_kind": "staged_files_only_no_gpu_execution", "kind": kind, + "created_at": datetime.now(timezone.utc).isoformat(), **inventory, + } + + +def write_gpu_manifest(kind: str, directory: Path, output: Path, *, model_revision: str | None = None, + timeout_seconds: float = 600) -> dict[str, Any]: + """Write a new manifest outside the frozen input; refuse existing targets.""" + output = Path(output).absolute() + if output.exists() or output.is_symlink(): + raise FileExistsError("manifest output must be a new file") + source = Path(directory).resolve(strict=True) + resolved_output = output.resolve(strict=False) + if resolved_output.is_relative_to(source): + raise ValueError("write the manifest outside the frozen input directory") + for component in (output.parent, *output.parent.parents): + if component.is_symlink(): + raise ValueError("manifest output parents cannot be symlinks") + result = build_gpu_manifest(kind, source, model_revision=model_revision, timeout_seconds=timeout_seconds) + # Creating a new, exclusive file never replaces existing evidence. + with output.open("x", encoding="utf-8") as stream: + json.dump(result, stream, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + stream.write("\n") + return result diff --git a/experimental/video-generation/evaluator/mvp_gpu_report.py b/experimental/video-generation/evaluator/mvp_gpu_report.py new file mode 100644 index 0000000000..9b759efc61 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_gpu_report.py @@ -0,0 +1,804 @@ +"""Portable, read-only dashboard for controlled GPU job artifacts. + +This viewer never starts a job, invents a run, or imports demonstration media. +It rechecks file hashes and recomputes denominators from the frozen schedule, +but it does not rerun the decoder or independently attest to the supervisor. +Only a bound ``controlled_h3_gpu`` receipt permits GPU timing presentation. +""" + +from __future__ import annotations + +import hashlib +import html +import json +import math +import os +import stat +import statistics +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Iterator + + +MAX_JSON_BYTES = 8 * 1024 * 1024 +MAX_MEDIA_BYTES = 512 * 1024 * 1024 +MAX_TOTAL_MEDIA_BYTES = 2 * 1024 * 1024 * 1024 +MAX_TELEMETRY_BYTES = 16 * 1024 * 1024 +MAX_SLOTS = 10000 +MAX_DISPLAY_SLOTS = 256 +_ROLES = ("baseline", "candidate") +_MEDIA_SUFFIXES = {".mp4", ".webm", ".mov", ".mkv", ".avi", ".wav", ".mp3", ".m4a", ".ogg", ".flac", ".aac"} +_AUDIO_SUFFIXES = {".wav", ".mp3", ".m4a", ".ogg", ".flac", ".aac"} + + +class _InvalidEvidence(ValueError): + pass + + +def _finite(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 + + +def _text(value: Any, limit: int = 2000) -> str: + if value is None: + return "Not recorded" + if isinstance(value, (dict, list)): + return "Invalid field type" + return str(value)[:limit] + + +def _escape(value: Any) -> str: + return html.escape(_text(value, 32000), quote=True) + + +def _canonical(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode() + + +def _is_digest(value: Any) -> bool: + return isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value) + + +def _issue(issues: list[dict], context: str, message: str) -> None: + if len(issues) < 256: + issues.append({"context": _text(context, 300), "message": _text(message)}) + + +def _relative(value: Any) -> PurePosixPath: + if not isinstance(value, str) or not value or "\\" in value or "\x00" in value: + raise _InvalidEvidence("Expected a nonempty relative POSIX path") + path = PurePosixPath(value) + if path.is_absolute() or any(part in {"", ".", ".."} for part in value.split("/")): + raise _InvalidEvidence("Absolute paths and path traversal are not permitted") + return path + + +def _no_symlink_parents(path: Path) -> None: + for component in (path, *path.parents): + if component.is_symlink(): + raise ValueError("Job and report paths must not contain symlinks") + + +class _Tree: + """Open each input component relative to an fd, never following symlinks.""" + + def __init__(self, root: Path): + self.root = root + + @contextmanager + def open(self, relative: str, maximum: int) -> Iterator[Any]: + parts = _relative(relative).parts + descriptors = [] + try: + current = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + descriptors.append(current) + for part in parts[:-1]: + current = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=current) + descriptors.append(current) + descriptor = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=current) + with os.fdopen(descriptor, "rb") as stream: + metadata = os.fstat(stream.fileno()) + if not stat.S_ISREG(metadata.st_mode): + raise _InvalidEvidence("Input must be a regular file") + if metadata.st_size > maximum: + raise _InvalidEvidence("Input exceeds the viewer byte limit") + yield stream + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def _pairs(pairs: list[tuple[str, Any]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise _InvalidEvidence("Duplicate JSON keys are not permitted") + result[key] = value + return result + + +def _reject_constant(_: str) -> None: + raise _InvalidEvidence("Non-finite JSON values are not permitted") + + +def _document(tree: _Tree, relative: str, issues: list[dict], *, expected: Any = None, + canonical_hash: bool = False) -> tuple[dict | None, str | None]: + try: + with tree.open(relative, MAX_JSON_BYTES) as stream: + data = stream.read(MAX_JSON_BYTES + 1) + if len(data) > MAX_JSON_BYTES: + raise _InvalidEvidence("JSON exceeds the viewer byte limit") + value = json.loads(data, object_pairs_hook=_pairs, parse_constant=_reject_constant) + if not isinstance(value, dict): + raise _InvalidEvidence("JSON document must be an object") + canonical = _canonical(value) # also rejects overflow-to-infinity numbers + digest = hashlib.sha256(canonical if canonical_hash else data).hexdigest() + if expected is not None and (not _is_digest(expected) or expected != digest): + raise _InvalidEvidence("Document SHA256 does not match its receipt") + return value, digest + except FileNotFoundError: + return None, None + except (OSError, ValueError, TypeError, RecursionError, OverflowError) as error: + message = str(error) if isinstance(error, _InvalidEvidence) else "Unreadable, unsafe, or malformed JSON document" + _issue(issues, relative, message) + return None, None + + +def _planned(plan: Any, issues: list[dict], context: str) -> list[dict]: + if not isinstance(plan, dict): + return [] + cases, repetitions, warmups = plan.get("cases"), plan.get("repetitions"), plan.get("warmup_runs") + if (not isinstance(cases, list) or not cases or type(repetitions) is not int or repetitions < 1 + or type(warmups) is not int or warmups < 0 or len(cases) * repetitions + warmups > MAX_SLOTS): + _issue(issues, context, "Frozen schedule is missing or exceeds the supported slot limit") + return [] + if any(not isinstance(case, dict) or not isinstance(case.get("case_id"), str) + or not isinstance(case.get("prompt"), str) or type(case.get("seed")) is not int for case in cases): + _issue(issues, context, "Frozen cases require case_id, prompt, and integer seed") + return [] + if len({case["case_id"] for case in cases}) != len(cases): + _issue(issues, context, "Frozen case identifiers are not unique") + return [] + result = [] + for index in range(1, warmups + 1): + result.append({**cases[(index - 1) % len(cases)], "phase": "warmup", "repetition": 0, "slot_id": f"warmup-{index:03d}"}) + for repetition in range(1, repetitions + 1): + for index, case in enumerate(cases, 1): + result.append({**case, "phase": "measurement", "repetition": repetition, "slot_id": f"measurement-r{repetition:03d}-c{index:03d}"}) + return result + + +def _copy_media(tree: _Tree, relative: str, digest: Any, assets: Path, budget: list[int]) -> str: + if not _is_digest(digest): + raise _InvalidEvidence("Media requires a lowercase SHA256 digest") + suffix = _relative(relative).suffix.lower() + if suffix not in _MEDIA_SUFFIXES: + raise _InvalidEvidence("Unsupported media extension; not embedded") + target = assets / (digest + suffix) + created = False + destination = None + try: + with tree.open(relative, MAX_MEDIA_BYTES) as source: + size = os.fstat(source.fileno()).st_size + if budget[0] + size > MAX_TOTAL_MEDIA_BYTES: + raise _InvalidEvidence("Total media exceeds the viewer byte limit") + budget[0] += size + if target.exists() or target.is_symlink(): + descriptor = os.open(target, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as existing: + existing_metadata = os.fstat(existing.fileno()) + if not stat.S_ISREG(existing_metadata.st_mode) or existing_metadata.st_size > MAX_MEDIA_BYTES: + raise _InvalidEvidence("Existing report media is not a safe regular file") + existing_hash = hashlib.sha256() + existing_bytes = 0 + while chunk := existing.read(1024 * 1024): + existing_bytes += len(chunk) + if existing_bytes > MAX_MEDIA_BYTES: + raise _InvalidEvidence("Existing report media exceeds the byte limit") + existing_hash.update(chunk) + if existing_hash.hexdigest() != digest: + raise _InvalidEvidence("Existing content-addressed report asset has different bytes") + else: + destination = target.open("xb") + created = True + hasher, count = hashlib.sha256(), 0 + while chunk := source.read(1024 * 1024): + count += len(chunk) + if count > size or count > MAX_MEDIA_BYTES: + raise _InvalidEvidence("Media changed size while being read") + hasher.update(chunk) + if destination: + destination.write(chunk) + if count != size or hasher.hexdigest() != digest: + raise _InvalidEvidence("Media SHA256 mismatch or file changed during export") + except Exception: + if destination: + destination.close() + destination = None + if created: + target.unlink(missing_ok=True) + raise + finally: + if destination: + destination.close() + return "assets/" + target.name + + +def _selected(mapping: Any, fields: tuple[str, ...]) -> dict: + if not isinstance(mapping, dict): + return {} + return {field: mapping[field] for field in fields if field in mapping and isinstance(mapping[field], (str, bool, int, float, type(None)))} + + +def _check_data(check: dict) -> dict: + result = _selected(check, ("name", "status", "unit", "reason")) + for field in ("observed", "threshold"): + value = check.get(field) + if isinstance(value, (dict, list)): + result[field] = json.dumps(value, ensure_ascii=False, allow_nan=False)[:2000] + else: + result[field] = value + return result + + +def _telemetry(value: Any) -> dict: + if not isinstance(value, dict): + value = {} + peaks = value.get("observed_memory_peak_mib_by_gpu", {}) + identities = value.get("gpu_identity", []) + return { + "sample_count": value.get("sample_count") if type(value.get("sample_count")) is int and value["sample_count"] >= 0 else None, + "measurement_sample_count": value.get("measurement_sample_count") if type(value.get("measurement_sample_count")) is int and value["measurement_sample_count"] >= 0 else None, + "gpu_identity": [_selected(gpu, ("uuid", "index", "name", "memory_total_mib", "driver_version")) for gpu in identities[:64] if isinstance(gpu, dict)] if isinstance(identities, list) else [], + "observed_memory_peak_mib_by_gpu": {str(key)[:200]: item for key, item in list(peaks.items())[:64] if _finite(item)} if isinstance(peaks, dict) else {}, + "errors": [_text(error) for error in value.get("errors", [])[:64]] if isinstance(value.get("errors"), list) else [], + "observed_owned_compute_by_gpu": {str(key)[:200]: count for key, count in list(value.get("observed_owned_compute_by_gpu", {}).items())[:64] if type(count) is int and count >= 0} if isinstance(value.get("observed_owned_compute_by_gpu"), dict) else {}, + } + + +def _verify_telemetry(tree: _Tree, metadata: dict, issues: list[dict], label: str, selected: Any) -> dict: + relative, expected = metadata.get("telemetry_path"), metadata.get("telemetry_sha256") + result = {"file_sha256_verified": False, "samples_consistent": False, "selected_owned_compute_observed": False} + if relative is None and expected is None: + return result + try: + if not _is_digest(expected): + raise _InvalidEvidence("Telemetry lacks a valid receipt hash") + with tree.open(relative, MAX_TELEMETRY_BYTES) as stream: + data = stream.read(MAX_TELEMETRY_BYTES + 1) + if len(data) > MAX_TELEMETRY_BYTES: + raise _InvalidEvidence("Telemetry exceeds the viewer byte limit") + if hashlib.sha256(data).hexdigest() != expected: + raise _InvalidEvidence("Telemetry SHA256 does not match its receipt") + result["file_sha256_verified"] = True + if not isinstance(selected, list) or not selected or len(selected) > 64 or any(not isinstance(device, str) for device in selected) or len(set(selected)) != len(selected): + raise _InvalidEvidence("Selected GPU UUIDs are missing or invalid") + peaks, owned = {device: None for device in selected}, dict.fromkeys(selected, 0) + sample_count = measurement_count = 0 + identities = None + previous = None + no_foreign = True + for line in data.splitlines(): + if not line or len(line) > 1024 * 1024: + raise _InvalidEvidence("Empty or oversized telemetry sample") + sample = json.loads(line, object_pairs_hook=_pairs, parse_constant=_reject_constant) + _canonical(sample) + if not isinstance(sample, dict) or sample.get("phase") not in {"startup", "measurement", "cleanup"}: + raise _InvalidEvidence("Telemetry sample has an invalid phase or shape") + monotonic = sample.get("monotonic_seconds") + if not _finite(monotonic) or previous is not None and monotonic < previous: + raise _InvalidEvidence("Telemetry sampling timestamps are invalid") + previous = monotonic + gpus = sample.get("gpus") + if not isinstance(gpus, list) or any(not isinstance(gpu, dict) for gpu in gpus) or sorted(gpu.get("uuid", "") for gpu in gpus) != sorted(selected): + raise _InvalidEvidence("Telemetry device inventory differs from the selected UUIDs") + current = [_selected(gpu, ("uuid", "index", "name", "memory_total_mib", "driver_version")) for gpu in gpus] + if identities is None: + identities = current + elif identities != current: + raise _InvalidEvidence("Observed GPU identity changed during the job") + for gpu in gpus: + memory, total = gpu.get("memory_used_mib"), gpu.get("memory_total_mib") + if not _finite(memory) or not _finite(total) or memory > total: + raise _InvalidEvidence("Telemetry memory sample is invalid") + peaks[gpu["uuid"]] = max(memory, peaks[gpu["uuid"]] or 0) + apps, owned_apps, foreign = sample.get("compute_apps"), sample.get("owned_compute_apps"), sample.get("unowned_compute_apps") + if any(not isinstance(items, list) for items in (apps, owned_apps, foreign)): + raise _InvalidEvidence("Telemetry lacks explicit compute ownership observations") + if any(not isinstance(app, dict) or app.get("gpu_uuid") not in selected or type(app.get("pid")) is not int or app["pid"] < 2 for app in apps + owned_apps + foreign): + raise _InvalidEvidence("Telemetry compute process observations are malformed") + # Attribution adds identity/diagnostic fields to the raw NVML rows. + raw_by_id = {(app["gpu_uuid"], app["pid"]): app for app in apps} + partition = owned_apps + foreign + partition_by_id = {(app["gpu_uuid"], app["pid"]): app for app in partition} + if (len(raw_by_id) != len(apps) or len(partition_by_id) != len(partition) + or raw_by_id.keys() != partition_by_id.keys() + or any(raw_by_id[key].get("memory_used_mib") != partition_by_id[key].get("memory_used_mib") for key in raw_by_id)): + raise _InvalidEvidence("Telemetry compute ownership partition is inconsistent") + no_foreign = no_foreign and not foreign + sample_count += 1 + if sample["phase"] == "measurement": + measurement_count += 1 + for device in selected: + owned[device] += any(app["gpu_uuid"] == device for app in owned_apps) + summary = _telemetry(metadata.get("telemetry_summary")) + if (summary["sample_count"] != sample_count or summary["measurement_sample_count"] != measurement_count + or summary["gpu_identity"] != (identities or []) or summary["observed_memory_peak_mib_by_gpu"] != {key: value for key, value in peaks.items() if value is not None} + or summary["observed_owned_compute_by_gpu"] != owned): + raise _InvalidEvidence("Telemetry summary does not match its hash-verified samples") + result["samples_consistent"] = True + result["selected_owned_compute_observed"] = bool(measurement_count >= 2 and no_foreign and not summary["errors"] and all(owned.values())) + return result + except (OSError, ValueError, TypeError, RecursionError, OverflowError) as error: + message = str(error) if isinstance(error, _InvalidEvidence) else "Telemetry file is missing or unsafe" + _issue(issues, label + ".telemetry", message) + return result + + +def _provenance_reasons(spec: dict, label: str, metadata: dict, configuration: dict, telemetry: dict) -> list[str]: + declared, observed = spec.get(label), metadata.get("source_identity") + reasons = [] + if (not isinstance(declared, dict) or not isinstance(observed, dict) + or any(not isinstance(declared.get(field), str) or observed.get(field) != declared[field] for field in ("source", "revision", "source_sha256", "python")) + or not _is_digest(observed.get("source_sha256")) or not _is_digest(observed.get("python_sha256")) + or not PurePosixPath(declared["source"]).is_absolute() or not PurePosixPath(declared["python"]).is_absolute() + or len(declared["revision"]) != 40 or any(c not in "0123456789abcdef" for c in declared["revision"]) + or configuration.get("runtime_revision") != declared.get("revision") + or observed.get("sglang_module") != str(PurePosixPath(declared.get("source", "")) / "python" / "sglang" / "__init__.py") + or not isinstance(observed.get("packages"), dict) or not observed["packages"].get("torch")): + reasons.append("Observed source/Python identity does not match the pinned runtime specification") + process = metadata.get("process_identity") + if (not isinstance(process, dict) or any(type(process.get(field)) is not int or process[field] < 1 for field in ("pid", "pgid", "session_id", "start_ticks")) + or process.get("pid", 0) < 2 or process.get("pgid") != process.get("pid") or process.get("session_id") != process.get("pid") + or not isinstance(process.get("launch_nonce"), str) or len(process["launch_nonce"]) != 32 or any(c not in "0123456789abcdef" for c in process["launch_nonce"])): + reasons.append("A plausible owned process/session receipt is unavailable") + if not telemetry.get("file_sha256_verified") or not telemetry.get("samples_consistent") or not telemetry.get("selected_owned_compute_observed"): + reasons.append("Hash-verified measurement samples do not establish owned compute on every selected GPU") + return reasons + + +def _finalized(run: dict) -> bool: + if run.get("status") not in {"complete", "partial", "failed"}: + return False + try: + started = datetime.fromisoformat(run.get("started_at", "")) + finished = datetime.fromisoformat(run.get("finished_at", "")) + return started.tzinfo is not None and finished.tzinfo is not None and finished >= started + except (TypeError, ValueError): + return False + + +def _cleanup(value: Any) -> dict: + result = _selected(value, ("status", "reason", "idle_after")) + if isinstance(value, dict) and isinstance(value.get("remaining_owned_pids"), list): + result["remaining_owned_pids"] = [pid for pid in value["remaining_owned_pids"][:256] if type(pid) is int] + return result + + +def _role(tree: _Tree, label: str, metadata: Any, spec: dict, controlled: bool, + assets: Path, budget: list[int], issues: list[dict]) -> dict: + metadata = metadata if isinstance(metadata, dict) else {} + source = metadata.get("source_identity", {}) + identity = _selected(source, ("source", "revision", "source_sha256", "python", "python_sha256", "python_version", "sglang_module")) + if isinstance(source, dict) and isinstance(source.get("packages"), dict): + identity["packages"] = {str(key)[:100]: _text(value, 200) for key, value in list(source["packages"].items())[:64]} + result = { + "status": _text(metadata.get("status", "not_started")), "source_identity": identity, + "process_identity": _selected(metadata.get("process_identity"), ("pid", "pgid", "start_ticks", "session_id")), + "telemetry": _telemetry(metadata.get("telemetry_summary")), "cleanup": _cleanup(metadata.get("cleanup")), + "configuration": {}, "observations": [], "warmups": [], "summary": {}, "run_sha256": None, + "plan_sha256": None, "evidence_kind": "missing", "gpu_timing_presented": False, + } + result["telemetry"].update(_verify_telemetry(tree, metadata, issues, label, spec.get("gpu_uuids"))) + relative = metadata.get("run_path", f"{label}/run.json") + run, digest = _document(tree, relative, issues, expected=metadata.get("run_sha256")) + if run is None and metadata.get("run_sha256") is not None: + _issue(issues, label, "Receipt references a run bundle that is missing or invalid") + if result["status"] == "complete": + result["status"] = "incomplete" + plan = spec.get("plan") if isinstance(spec.get("plan"), dict) else (run or {}).get("plan") + schedule = _planned(plan, issues, label + ".plan") + bound = run is not None and _is_digest(metadata.get("run_sha256")) and digest == metadata["run_sha256"] + eligible = False + records = {} + if run: + result.update(run_sha256=digest, evidence_kind=_text(run.get("evidence_kind")), run_id=_text(run.get("run_id")), + run_status=_text(run.get("status")), started_at=_text(run.get("started_at")), finished_at=run.get("finished_at")) + if run.get("bundle_type") != "mvp_run" or run.get("bundle_version") != "0.1.0": + _issue(issues, label, "Unsupported run bundle type or version") + run = None + elif not isinstance(run.get("plan"), dict) or hashlib.sha256(_canonical(run["plan"])).hexdigest() != run.get("plan_sha256"): + _issue(issues, label, "Run plan hash mismatch") + run = None + elif plan != run.get("plan"): + _issue(issues, label, "Run differs from the job's frozen workload") + run = None + if run: + result["plan_sha256"] = run["plan_sha256"] + configuration = run.get("configuration", {}) + result["configuration"] = _selected(configuration, ("model_id", "model_revision", "runtime", "runtime_revision", "hardware_label", "identity_verification", "client_source_sha256")) + if not isinstance(configuration, dict): + configuration = {} + unsigned = {key: value for key, value in configuration.items() if key != "configuration_sha256"} + declared = [value for value in (run.get("configuration_sha256"), configuration.get("configuration_sha256")) if value is not None] + config_hash = hashlib.sha256(_canonical(unsigned)).hexdigest() + config_valid = bool(declared) and all(_is_digest(value) and value == config_hash for value in declared) + if not config_valid: + _issue(issues, label, "Run configuration hash is absent or mismatched") + if not bound: + _issue(issues, label, "Run is not hash-bound to a finalized supervisor receipt; GPU timings withheld") + measurement = run.get("measurement", {}) + if not isinstance(measurement, dict): + measurement = {} + timing_valid = measurement.get("boundary") == "submit_to_validated_media" and type(measurement.get("concurrency")) is int and measurement["concurrency"] == 1 + if configuration.get("serving"): + from .mvp_serving import validate_window + try: + validate_window(run) + timing_valid = configuration["serving"] == spec.get("serving") + except (ValueError, KeyError, TypeError): + timing_valid = False + _issue(issues, label, "Invalid serving measurement window; timing withheld") + eligible = bool(controlled and bound and config_valid and timing_valid and run.get("evidence_kind") in {"operator_endpoint", "live_h3"}) + if eligible and not _finalized(run): + _issue(issues, label, "Run is not finalized with ordered timezone-aware timestamps; GPU timing withheld") + eligible = False + if eligible: + reasons = _provenance_reasons(spec, label, metadata, configuration, result["telemetry"]) + for reason in reasons: + _issue(issues, label, reason + "; GPU timing withheld") + eligible = not reasons + result["gpu_timing_presented"] = eligible + raw_records = run.get("records") + if not isinstance(raw_records, list) or len(raw_records) > MAX_SLOTS: + _issue(issues, label, "Run records are malformed or exceed the slot limit") + else: + expected_ids = {slot["slot_id"] for slot in schedule} + for record in raw_records: + slot_id = record.get("slot_id") if isinstance(record, dict) else None + if not isinstance(slot_id, str) or slot_id not in expected_ids or slot_id in records: + _issue(issues, label, "Unexpected, duplicate, or malformed run slot; not counted as valid") + eligible = False + result["gpu_timing_presented"] = False + continue + records[slot_id] = record + try: + from .mvp_compare import _validate_timing_boundaries + _validate_timing_boundaries(record, run["evidence_kind"]) + except (ValueError, KeyError, TypeError): + _issue(issues, label + "." + slot_id, "Invalid nested client timing boundaries; GPU timing withheld") + eligible = False + result["gpu_timing_presented"] = False + for slot in schedule: + observation = {key: slot[key] for key in ("slot_id", "case_id", "prompt", "seed", "phase", "repetition")} + observation.update(status="not_recorded", attempted=None, valid=False, artifact_path=None, sha256=None, + latency_seconds=None, submit_to_terminal_seconds=None, submit_to_media_seconds=None, + media_validation_seconds=None, error=None, media={}) + record = records.get(slot["slot_id"]) + if record is not None: + attempted = record.get("attempted", True if result["evidence_kind"] in {"fixture", "imported_media"} else None) + if any(record.get(field) != slot[field] for field in ("case_id", "prompt", "seed", "phase", "repetition")): + observation.update(status="invalid_record", error="Record does not match its frozen slot") + _issue(issues, label + "." + slot["slot_id"], observation["error"]) + elif record.get("status") not in {"succeeded", "failed"} or type(attempted) is not bool: + observation.update(status="invalid_record", error="Record status or attempted flag is invalid") + _issue(issues, label + "." + slot["slot_id"], observation["error"]) + else: + observation.update(status=record["status"] if attempted else "not_started", attempted=attempted, error=_text(record["error"]) if record.get("error") else None) + if eligible and attempted: + for field in ("latency_seconds", "submit_to_terminal_seconds", "submit_to_media_seconds", "media_validation_seconds"): + observation[field] = record.get(field) if _finite(record.get(field)) else None + media = record.get("media") if isinstance(record.get("media"), dict) else {} + observation["media"] = { + "video": _selected(media.get("video"), ("width", "height", "frame_count", "fps", "duration_seconds")), + "audio": _selected(media.get("audio"), ("present", "sample_rate_hz", "channels", "sample_count", "duration_seconds")), + } + if record.get("artifact_path") is not None: + try: + parent = _relative(relative).parent + path = parent / _relative(record["artifact_path"]) + observation["artifact_path"] = _copy_media(tree, path.as_posix(), record.get("sha256"), assets, budget) + observation["sha256"] = record["sha256"] + observation["valid"] = record["status"] == "succeeded" and attempted and media.get("valid") is True + if media.get("sha256") is not None and media["sha256"] != record["sha256"]: + observation["valid"] = False + raise _InvalidEvidence("Saved decoder hash differs from the media hash") + except (OSError, ValueError) as error: + message = str(error) if isinstance(error, _InvalidEvidence) else "Missing or unsafe media artifact" + observation.update(artifact_path=None, valid=False, error=message) + _issue(issues, label + "." + slot["slot_id"], message) + if observation["status"] == "succeeded" and not observation["valid"]: + observation["status"] = "unverified_media" + result["warmups" if slot["phase"] == "warmup" else "observations"].append(observation) + rows = result["observations"] + valid = [row for row in rows if row["valid"]] + latencies = [row["latency_seconds"] for row in valid if _finite(row["latency_seconds"])] + measurement = run.get("measurement", {}) if run else {} + wall = measurement.get("wall_seconds") if isinstance(measurement, dict) else None + attempted_sum = sum(row["latency_seconds"] for row in rows if _finite(row["latency_seconds"])) + serving = run.get("configuration", {}).get("serving") if run else None + wall_valid = bool(eligible and _finite(wall) and wall > 0 and (serving or wall + 1e-6 >= attempted_sum) and run and run.get("finished_at")) + if eligible and not serving and _finite(wall) and wall + 1e-6 < attempted_sum: + _issue(issues, label, "Measured wall time is shorter than summed serial request times; throughput withheld") + result["summary"] = { + "scheduled": len(rows) if schedule else None, "recorded": sum(row["status"] != "not_recorded" for row in rows), + "valid": len(valid), "failed_attempts": sum(row["status"] == "failed" for row in rows), + "not_started": sum(row["status"] == "not_started" for row in rows), + "not_recorded": sum(row["status"] == "not_recorded" for row in rows), + "unverified_media": sum(row["status"] in {"unverified_media", "invalid_record"} for row in rows), + "latency_median_seconds": statistics.median(latencies) if latencies else None, "latency_count": len(latencies), + "wall_seconds": wall if wall_valid else None, "valid_clips_per_second": len(valid) / wall if wall_valid else None, + "warmups_scheduled": len(result["warmups"]), "warmups_valid": sum(row["valid"] for row in result["warmups"]), + "client_ready_p90_seconds": None, "deadline_goodput_clips_per_second": None, + } + if eligible and serving: + from .mvp_serving import summarize + stats = summarize(run) + result["serving"] = stats + # Rejected/hash-mismatched media cannot contribute to report goodput. + if len(valid) == stats["client_ready_latency_seconds"]["valid_clip_count"]: + result["summary"].update(client_ready_p90_seconds=stats["client_ready_latency_seconds"]["p90"], + deadline_goodput_clips_per_second=stats["deadline_goodput_clips_per_second"]) + return result + + +def _number(value: Any, digits: int = 3, unit: str = "") -> str: + return f"{value:,.{digits}f}{unit}" if _finite(value) else "Not measured" + + +def _badge(value: Any) -> str: + label = _text(value).replace("_", " ") + style = "good" if value in {"clean", "complete", "succeeded", "pass"} else "bad" if value in {"failed", "fail", "invalid", "invalid_record", "unverified_media", "aborted"} else "pending" + return f'{_escape(label)}' + + +def _kv(values: list[tuple[str, Any]]) -> str: + return '
' + ''.join(f'
{_escape(name)}
{_escape(value)}
' for name, value in values) + '
' + + +def _player(label: str, row: dict | None) -> str: + row = row or {} + path = row.get("artifact_path") + if path: + tag = "audio" if PurePosixPath(path).suffix in _AUDIO_SUFFIXES else "video" + player = f'<{tag} controls preload="metadata" src="{_escape(path)}">' + else: + player = f'
No generated artifact available

{_escape(row.get("error") or "This slot has not produced a verified, exportable media file.")}

' + return (f'
{_escape(label.title())}{_badge(row.get("status", "not_recorded"))}
{player}' + f'
Submit → validated media: {_escape(_number(row.get("latency_seconds"), unit=" s"))}' + f'
Terminal observed: {_escape(_number(row.get("submit_to_terminal_seconds"), unit=" s"))} · Download complete: {_escape(_number(row.get("submit_to_media_seconds"), unit=" s"))}' + f'
Client validation: {_escape(_number(row.get("media_validation_seconds"), unit=" s"))}
') + + +_CSS = """ +:root{color-scheme:light;--ink:#18323b;--muted:#58717b;--line:#d7e3e7;--teal:#08796e;--soft:#f2f7f8;--amber:#806017;--red:#a03c48}*{box-sizing:border-box}body{margin:0;background:var(--soft);color:var(--ink);font:14px/1.6 -apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif}a{color:var(--teal)}.top{background:#132e38;color:#fff;padding:14px max(22px,calc((100vw - 1232px)/2));font-size:12px;letter-spacing:.05em}.top span{color:#a1c5ca;margin-left:18px}main{max-width:1280px;margin:auto;padding:32px 24px 64px}header{display:flex;justify-content:space-between;gap:22px;align-items:center}h1{font-size:38px;line-height:1.1;letter-spacing:-.04em;margin:10px 0 14px}h2{font-size:22px;letter-spacing:-.025em;margin:0 0 12px}h3{margin:0 0 10px;font-size:17px}p{margin:0 0 12px}.eyebrow{font-size:11px;font-weight:750;text-transform:uppercase;letter-spacing:.13em;color:var(--teal)}.muted,small{color:var(--muted)}.subtitle{max-width:760px;color:var(--muted)}.download{padding:10px 15px;border:1px solid #b5cbce;border-radius:7px;background:#fff;text-decoration:none;white-space:nowrap;font-size:12px;font-weight:700}.notice{border:1px solid #d9c485;border-left:4px solid #b08b29;background:#fffbec;border-radius:8px;padding:17px 20px;margin:20px 0}.notice p{margin:3px 0 0;font-size:12px}.notice.live{background:#edf8f4;border-color:#a0cbbd;border-left-color:var(--teal)}.notice.error{background:#fff0f1;border-color:#dbafb5;border-left-color:var(--red)}.badge{display:inline-block;font-size:10px;letter-spacing:.04em;text-transform:uppercase;font-weight:750;padding:4px 8px;border-radius:5px;background:#e9eff1;white-space:nowrap}.good{color:#136b4c;background:#e2f2e9}.pending{color:var(--amber);background:#fff0c9}.bad{color:var(--red);background:#f9e3e7}.statusbar{display:flex;gap:12px;align-items:center;margin:20px 0}.cards{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:13px;margin:21px 0 30px}.card,.panel{border:1px solid var(--line);border-radius:10px;padding:20px;background:#fff;min-width:0}.label{font-size:11px;font-weight:650;color:var(--muted)}.value{font-size:27px;letter-spacing:-.035em;font-weight:750;line-height:1.2;margin:10px 0}.detail{font-size:11px;color:var(--muted)}section{margin-top:30px}.section-head{display:flex;justify-content:space-between;align-items:baseline;gap:20px;margin-bottom:12px}.section-head h2{margin:0}.section-head small{font-size:11px}.grid2{display:grid;grid-template-columns:1fr 1fr;gap:18px}.scroll{overflow-x:auto}table{width:100%;border-collapse:collapse;text-align:left;font-size:12px}td,th{padding:11px 10px;border-bottom:1px solid var(--line);vertical-align:top}th{font-size:10px;font-weight:650;color:var(--muted);text-transform:uppercase;letter-spacing:.035em}tr:last-child td{border-bottom:0}.mono,dd{font:11px/1.7 ui-monospace,SFMono-Regular,Consolas,monospace;overflow-wrap:anywhere}dl{display:grid;grid-template-columns:130px minmax(0,1fr);gap:9px 15px;margin:0}dt{font-size:11px;color:var(--muted)}dd{margin:0}.case{background:#fff;border:1px solid var(--line);border-radius:11px;margin-bottom:18px;overflow:hidden}.case-top{padding:18px 22px;border-bottom:1px solid var(--line)}.case-top h3{margin:0 0 4px}.case-top p{margin:9px 0 0;font-size:13px}.media-grid{display:grid;grid-template-columns:1fr 1fr;gap:20px;padding:20px 22px}figure{margin:0;min-width:0}.figure-head{display:flex;justify-content:space-between;align-items:center;margin-bottom:9px;font-size:12px}video{display:block;background:#0c1c24;border-radius:7px;aspect-ratio:16/9;width:100%;object-fit:contain}audio{width:100%;margin:48px 0}.empty-media{background:#f3f6f8;min-height:180px;aspect-ratio:16/9;border:1px dashed #bdcfd5;border-radius:7px;display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center;padding:24px}.empty-media p{font-size:12px;color:var(--muted);max-width:330px;margin:8px 0 0}figcaption{font-size:11px;color:var(--muted);margin-top:10px}.checks{padding:0 22px 18px}.notes{padding-left:20px;font-size:12px;color:var(--muted)}.notes li+li{margin-top:8px}summary{cursor:pointer;font-weight:650;font-size:12px}details{margin-top:13px}footer{border-top:1px solid var(--line);padding-top:18px;margin-top:38px;color:var(--muted);font-size:11px}.tight{margin-top:14px}.empty-state{padding:40px;text-align:center;background:#fff;border:1px dashed #b7cbd0;border-radius:10px}.empty-state h3{font-size:20px}.empty-state p{max-width:650px;margin:auto;color:var(--muted)}@media(max-width:920px){.cards{grid-template-columns:1fr 1fr}.grid2{grid-template-columns:1fr}header{display:block}.download{display:inline-block;margin:6px 0}.section-head{display:block}}@media(max-width:620px){main{padding:23px 12px 40px}h1{font-size:31px}.card,.panel{padding:15px}.value{font-size:23px}.media-grid{grid-template-columns:1fr;padding:15px}.case-top{padding:16px}.statusbar{flex-wrap:wrap}dl{grid-template-columns:100px minmax(0,1fr)}.top span{display:none}.section-head small{display:block}.empty-state{padding:28px 18px}}@media print{.top{background:white;color:#18323b}body{background:white}.download{display:none}.case{break-inside:avoid}main{padding:10px}} +""" + + +def _checks(checks: list[dict]) -> str: + if not checks: + return '

No comparison checks are available. Missing checks are not passes.

' + rows = ''.join(f'{_escape(check.get("name"))}{_badge(check.get("status", "inconclusive"))}{_escape(check.get("observed"))}{_escape(check.get("threshold"))}{_escape(check.get("unit"))}{_escape(check.get("reason"))}' for check in checks) + return '
' + rows + '
Recorded checkStatusObservedThresholdUnitInterpretation
' + + +def _pair_metrics(pair: dict) -> str: + if not pair: + return "" + metrics = pair.get("metrics", {}) + psnr = "Exact decoded match" if metrics.get("video_identical") is True else _number(metrics.get("video_psnr_db"), unit=" dB") + rows = [("Video PSNR", psnr, "RGB fidelity; not visual quality"), + ("Video MAE", _number(metrics.get("video_mae"), 6), "Normalized RGB, 0–1"), + ("Audio spectral cosine", _number(metrics.get("audio_spectral_cosine"), 5), "Spectral fidelity; not perceptual quality"), + ("Audio RMS ratio", _number(metrics.get("audio_rms_ratio"), 5), "Candidate / baseline; channels checked separately")] + table = ''.join(f'{_escape(name)}{_escape(value)}{_escape(note)}' for name, value, note in rows) + return '
Recorded paired fidelity metrics and checks
' + table + '
MetricValueMeaning
' + _checks(pair.get("checks", [])) + '
' + + +def _render(report: dict) -> str: + roles = report["roles"] + left, right = roles["baseline"], roles["candidate"] + b, c = left["summary"], right["summary"] + gpu_timing = any(role["gpu_timing_presented"] for role in roles.values()) + fixture = any(role["evidence_kind"] in {"fixture", "imported_media"} for role in roles.values()) + notice = "Controlled GPU job artifacts" if gpu_timing else "No verified GPU timing to present" + note = "Source, process, and GPU identities are recorded by the supervisor, not independently attested. Media hashes were checked during export." if gpu_timing else "No model inference is performed by this viewer. Empty, incomplete, or unbound artifacts do not establish an H3 benchmark result." + if fixture: + notice, note = "Harness-only / imported evidence — not an H3 GPU result", "At least one run is a fixture or imported-media bundle. Its timing is withheld; any displayed media is explicitly non-live evidence." + same_gpu = report["same_gpu_uuid_set"] + pairing = "Same GPU UUIDs" if same_gpu is True else "Different GPU UUIDs" if same_gpu is False else "GPU identity missing" + peaks = right["telemetry"]["observed_memory_peak_mib_by_gpu"] + peak = max(peaks.values()) if peaks and right["gpu_timing_presented"] else None + cards = [ + ("Candidate median end-to-end", _number(c["latency_median_seconds"], unit=" s"), f'Baseline {_number(b["latency_median_seconds"], unit=" s")} · {c["latency_count"]} valid timed slots'), + ("Candidate runner-valid + hash checked", f'{c["valid"]} / {c["scheduled"]}' if c["scheduled"] is not None else "Not scheduled", "All scheduled measurement slots stay in the denominator"), + ("Candidate sampled device memory maximum", _number(peak, 0, " MiB"), "Highest observed device sample, NOT exact peak or model-only VRAM"), + ("Hardware comparison", pairing, "Matching device identities; a regression claim still needs complete comparable runs" if same_gpu is True else "Cross-GPU results are descriptive, not a same-device regression gate"), + ] + card_html = ''.join(f'
{_escape(label)}
{_escape(value)}
{_escape(detail)}
' for label, value, detail in cards) + rows = [ + ("Scheduled measurement slots", "scheduled", False), ("Runner-valid, hash-checked media", "valid", False), + ("Failed attempts", "failed_attempts", False), ("Explicitly not started", "not_started", False), + ("Not recorded / pending", "not_recorded", False), ("Invalid or unverified media", "unverified_media", False), + ("Scheduled warmups (not measurements)", "warmups_scheduled", False), ("Valid warmups", "warmups_valid", False), + ("Valid timed population", "latency_count", False), ("Median submit → validated media (s)", "latency_median_seconds", True), + ("Recorded measurement wall time (s)", "wall_seconds", True), ("Valid clips / measured wall second", "valid_clips_per_second", True), + ("Serving P90 submit → downloaded media (s)", "client_ready_p90_seconds", True), + ("Serving valid clips meeting the delivery deadline / s", "deadline_goodput_clips_per_second", True), + ] + summary_rows = ''.join(f'{_escape(label)}{_escape(_number(b[key]) if number else b[key])}{_escape(_number(c[key]) if number else c[key])}' for label, key, number in rows) + summary_table = '
' + summary_rows + '
Metric and populationBaselineCandidate

Warmups are recorded separately and excluded from latency and throughput. Medians condition on valid timed clips; missing or failed attempts are not converted to zero latency.

' + role_panels = [] + for name, role in roles.items(): + config, source, telemetry, cleanup = role["configuration"], role["source_identity"], role["telemetry"], role["cleanup"] + gpu_names = "; ".join(f'{gpu.get("name", "unknown")} · {gpu.get("uuid", "missing UUID")}' for gpu in telemetry["gpu_identity"]) or "Not recorded" + fields = [("Evidence kind", role["evidence_kind"]), ("Model", config.get("model_id")), ("Model revision", config.get("model_revision")), + ("Runtime", config.get("runtime")), ("Runtime revision", config.get("runtime_revision")), ("Observed source revision", source.get("revision")), + ("Source fingerprint", source.get("source_sha256")), ("GPU identities", gpu_names), ("Telemetry samples", telemetry["sample_count"]), + ("Measurement samples", telemetry["measurement_sample_count"]), ("Telemetry file hash", "Verified" if telemetry["file_sha256_verified"] else "Not verified"), + ("Observed owned compute", "; ".join(f"{gpu}: {count} samples" for gpu, count in telemetry["observed_owned_compute_by_gpu"].items()) or "Not recorded"), + ("Sampled device maxima", "; ".join(f"{gpu}: {value:,.0f} MiB" for gpu, value in telemetry["observed_memory_peak_mib_by_gpu"].items()) or "Not measured"), + ("Run SHA256", role["run_sha256"]), ("Cleanup", cleanup.get("status")), ("Idle after cleanup", cleanup.get("idle_after")), + ("Remaining owned PIDs", ", ".join(map(str, cleanup.get("remaining_owned_pids", []))) if "remaining_owned_pids" in cleanup else None), + ("Cleanup note", cleanup.get("reason"))] + extra = [("Source directory", source.get("source")), ("Python", source.get("python")), ("Python version", source.get("python_version")), ("Python SHA256", source.get("python_sha256")), ("SGLang module", source.get("sglang_module"))] + extra += [("Package: " + name, version) for name, version in source.get("packages", {}).items()] + extra += [("Process: " + key, value) for key, value in role["process_identity"].items()] + telemetry_errors = ''.join(f'
  • {_escape(error)}
    • ' for error in telemetry["errors"]) + role_panels.append(f'

    {name.title()}

    {_badge(role["status"])}
    {_kv(fields)}
    Runtime and process provenance{_kv(extra)}
    ' + (f'
      {telemetry_errors}
    ' if telemetry_errors else '') + '
    ') + indexed = {name: {row["slot_id"]: row for row in role["observations"]} for name, role in roles.items()} + slot_ids = list(dict.fromkeys(list(indexed["baseline"]) + list(indexed["candidate"]))) + cases = [] + for slot_id in slot_ids[:MAX_DISPLAY_SLOTS]: + slot = indexed["baseline"].get(slot_id) or indexed["candidate"][slot_id] + cases.append(f'

    {_escape(slot["case_id"])}

    {_escape(slot_id)} · seed {_escape(slot["seed"])} · repetition {_escape(slot["repetition"])}

    {_escape(slot["prompt"])}

    {_player("baseline", indexed["baseline"].get(slot_id))}{_player("candidate", indexed["candidate"].get(slot_id))}
    {_pair_metrics(report["slot_comparisons"].get(slot_id, {}))}
    ') + if not cases: + cases = ['

    Ready for actual run artifacts

    No frozen measurement slots are available yet. This page deliberately contains no sample video, invented latency, or placeholder benchmark result.

    '] + truncated = f'

    Showing the first {MAX_DISPLAY_SLOTS} of {len(slot_ids)} scheduled slots in frozen order. Every slot remains in the evidence JSON and summary denominators.

    ' if len(slot_ids) > MAX_DISPLAY_SLOTS else '' + failures = report["failures"] + [f'{issue["context"]}: {issue["message"]}' for issue in report["issues"]] + problems = '' if failures else '' + policy = report["policy"] + policy_fields = [("Policy", policy.get("policy_id")), ("Calibration", policy.get("calibration_status", "Not calibrated")), ("Allocation", report["allocation"].get("mode")), ("Allocation label", report["allocation"].get("label")), ("Same frozen workload", report["same_workload"]), ("Job started", report.get("started_at")), ("Job finished", report.get("finished_at")), ("Spec SHA256", report.get("spec_sha256"))] + policy_fields += [("Max latency increase (fraction)", policy.get("max_latency_increase_fraction")), ("Min video PSNR (dB)", policy.get("min_video_psnr_db")), + ("Min spectral cosine", policy.get("min_audio_spectral_cosine")), ("Max RMS ratio error", policy.get("max_audio_rms_ratio_error")), + ("Max sampled-memory increase (fraction)", policy.get("max_memory_increase_fraction"))] + authorization = report["authorization"] + policy_fields += [("Compute approved (operator assertion)", authorization.get("compute_approved")), ("License reviewed (operator assertion)", authorization.get("model_license_reviewed")), ("Approval reference", authorization.get("approval_reference"))] + gate = report["recorded_gate"] + gate_fields = [("Recorded regression gate", gate.get("regression_status")), ("CI accepted (supervisor reported)", gate.get("ci_accepted")), ("Sampled-memory gate", gate.get("memory_gate_status")), + ("Sampled-memory change (fraction)", gate.get("memory_increase_fraction")), ("Sampled-memory threshold (fraction)", gate.get("memory_threshold"))] + acceptance_notes = ''.join(f'
  • {_escape(reason)}
    • ' for reason in report["acceptance_reasons"]) + warmup_rows = ''.join(f'{name.title()}{_escape(row["slot_id"])}{_badge(row["status"])}{_escape(row["error"] or ("Runner-valid, hash checked" if row["valid"] else "Not verified valid"))}' for name, role in roles.items() for row in role["warmups"]) + warmup_ledger = '
    Warmup ledger (excluded from measurements)
    ' + warmup_rows + '
    ArmSlotStatusEvidence
    ' if warmup_rows else '' + return ('' + '' + f'H3 controlled GPU benchmark · {_escape(report["job_id"])}' + '
    VIDEO GENERATION BENCHMARKArtifact-driven · local · script-free
    Controlled GPU execution

    H3 benchmark evidence

    A fixed-workload view of generated video, native audio, request timing, observed GPU memory, and cleanup. Evidence comes only from this job directory.

    Download evidence JSON ↗
    ' + f'
    {_badge(report["status"])}{_escape(report["job_id"])}Recorded CI verdict: {_badge(report["ci_status"])}
    ' + f'' + '

    The viewer checks artifact integrity; it does not independently qualify CI or certify a release. Uncalibrated thresholds, shared-host allocation, missing evidence, and same-build repeatability are not a demonstrated performance improvement.

    ' + f'{problems}
    {card_html}

    Generated media, side by side

    Video and native audio use the original exported bytes
    {truncated}{"".join(cases)}
    ' + f'

    All-slot accounting

    Recorded media validity is not a prompt-quality score
    {summary_table}{warmup_ledger}
    ' + '

    What the timing means

    All timing values are seconds. Submit → validated media includes server queueing and generation, status polling, download, and client-side decoding/validation. Terminal observed is poll-observed completion, not exact GPU completion. Download complete includes transfer; client validation is CPU-side analysis. These are not GPU kernel latency or time-to-first-frame measurements.

    Throughput divides valid clips by the recorded measurement wall time. Serial mode includes validation; closed-loop serving mode ends at the final delivery or transport failure and does not wait for local validation. Serving P90 uses technically valid downloaded clips and requires at least 10 samples; the sample floor is not statistical qualification. Deadline goodput counts technically valid clips delivered within the declared deadline. Neither mode establishes sustainable capacity. Device VRAM is periodically sampled; sampled maxima can miss a true peak and are not process-isolated allocations.

    ' + f'

    Identity, resources, and cleanup

    Supervisor-recorded provenance · not independent attestation
    {"".join(role_panels)}
    ' + f'

    Recorded comparison checks

    {_checks(report["checks"])}

    Check status: {_escape(report["comparison_status"])}. Checks are read from the hash-bound comparison; this viewer does not rerun the decoder, calibrate thresholds, or make statistical significance claims.

    ' + f'

    Recorded supervisor acceptance

    {_kv(gate_fields)}
      {acceptance_notes}

    Acceptance is reported, not independently re-evaluated by this viewer. External calibration references are not opened. No release qualification is claimed.

    ' + f'

    Workload and policy

    {_kv(policy_fields)}

    Approval and license-review fields are operator assertions, not proof of legal rights or scheduler isolation. Same-GPU runtime regression requires the same observed device UUIDs and frozen request cell. Cross-chip performance is a separate descriptive comparison. Neither fidelity checks nor timing establish aesthetics, physics, prompt following, lip synchronization, or human preference.

    ' + f'
    Exported {_escape(report["created_at"])}. Share index.html, evidence.json, and assets/ together. No remote resources, scripts, automatic playback, or background job execution.
    ') + + +def write_gpu_report(jobdir: Path, outputdir: Path) -> dict: + """Export a safe artifact-backed dashboard into a *new* directory. + + Missing/malformed job artifacts produce a clearly incomplete/error page. + Unsafe root/output paths and an existing output directory are rejected. + The return value is the sanitized JSON evidence also written to disk. + """ + root, output = Path(jobdir).absolute(), Path(outputdir).absolute() + _no_symlink_parents(root) + _no_symlink_parents(output) + if not root.is_dir(): + raise ValueError("jobdir must be an existing directory") + if output.exists(): + raise FileExistsError("Report output directory already exists") + tree, issues = _Tree(root), [] + receipt, _ = _document(tree, "gpu-job.json", issues) + if receipt and (receipt.get("bundle_type") != "controlled_gpu_job" or receipt.get("schema_version") != "0.1.0"): + _issue(issues, "gpu-job.json", "Unsupported controlled GPU job type or version") + receipt = None + receipt = receipt or {} + spec, spec_digest = _document(tree, "spec.json", issues, expected=receipt.get("spec_sha256"), canonical_hash=True) + if receipt and spec is None: + _issue(issues, "spec.json", "Job specification is missing or invalid") + spec = spec or {} + output.mkdir(parents=True, exist_ok=False) + assets = output / "assets" + assets.mkdir() + metadata = receipt.get("roles") if isinstance(receipt.get("roles"), dict) else {} + controlled = bool(receipt.get("evidence_kind") == "controlled_h3_gpu" and spec and _is_digest(receipt.get("spec_sha256")) and spec_digest == receipt["spec_sha256"]) + budget = [0] + roles = {label: _role(tree, label, metadata.get(label), spec, controlled, assets, budget, issues) for label in _ROLES} + comparison = None + if receipt.get("comparison_path") is not None: + comparison, _ = _document(tree, receipt["comparison_path"], issues, expected=receipt.get("comparison_sha256")) + if comparison is None: + _issue(issues, "comparison", "Receipt references a comparison that is missing or invalid") + if comparison: + matches = (comparison.get("bundle_type") == "mvp_comparison" and _is_digest(receipt.get("comparison_sha256")) + and all(isinstance(comparison.get(label), dict) and roles[label]["run_sha256"] is not None + and comparison[label].get("run_bundle_sha256") == roles[label]["run_sha256"] for label in _ROLES)) + if not matches: + _issue(issues, "comparison", "Comparison is not bound to both observed run bundles") + comparison = None + comparison = comparison or {} + checks = comparison.get("checks", []) + checks = [_check_data(check) for check in checks[:1000] if isinstance(check, dict)] if isinstance(checks, list) else [] + paired_slots = {} + comparison_slots = comparison.get("slots", []) + if isinstance(comparison_slots, list): + observed = {label: {row["slot_id"]: row for row in roles[label]["observations"]} for label in _ROLES} + for slot in comparison_slots[:MAX_SLOTS]: + if not isinstance(slot, dict) or not isinstance(slot.get("slot_id"), str): + continue + identifier = slot["slot_id"] + if not all(identifier in observed[label] and isinstance(slot.get(label), dict) and slot[label].get("sha256") == observed[label][identifier]["sha256"] for label in _ROLES): + _issue(issues, "comparison", "Paired slot does not match the exported media hashes") + continue + slot_checks = slot.get("checks", []) + paired_slots[identifier] = { + "metrics": _selected(slot.get("metrics"), ("video_psnr_db", "video_mae", "video_identical", "audio_spectral_cosine", "audio_rms_ratio", "latency_increase_fraction")), + "checks": [_check_data(check) for check in slot_checks[:256] if isinstance(check, dict)] if isinstance(slot_checks, list) else [], + } + gpu_sets = [{gpu["uuid"] for gpu in roles[label]["telemetry"]["gpu_identity"] if isinstance(gpu.get("uuid"), str)} if roles[label]["telemetry"]["samples_consistent"] else set() for label in _ROLES] + plans = [roles[label]["plan_sha256"] for label in _ROLES] + policy = comparison.get("policy", spec.get("policy", {})) + policy = policy if isinstance(policy, dict) else {} + allocation = spec.get("allocation") if isinstance(spec.get("allocation"), dict) else {} + ci_status = receipt.get("regression_status", "inconclusive") + if ci_status not in {"pass", "fail", "inconclusive"}: + ci_status = "inconclusive" + if ci_status == "pass" and (receipt.get("status") != "complete" or receipt.get("failures") != [] + or receipt.get("cleanup_status") != "clean" or not _finalized(receipt) + or issues or receipt.get("ci_accepted") is not True or receipt.get("measurement_status") != "complete" + or comparison.get("overall_status") != "pass" or policy.get("calibration_status") != "operator_calibrated" + or allocation.get("mode") != "dedicated_ci" or not all(role["gpu_timing_presented"] for role in roles.values()) + or any(role["cleanup"].get("status") != "clean" or role["cleanup"].get("idle_after") is not True for role in roles.values())): + ci_status = "inconclusive" + _issue(issues, "CI verdict", "A recorded pass is not shown as passing because completed execution, cleanup, integrity, provenance, allocation, or calibration evidence is incomplete") + status = receipt.get("status", "not_started") + if status not in {"complete", "failed", "aborted", "running", "preflight", "not_started"}: + _issue(issues, "gpu-job.json", "Unknown job state") + status = "invalid" + if issues and status in {"complete", "not_started"}: + status = "invalid" if not receipt else "incomplete" + if status == "complete" and (receipt.get("measurement_status") != "complete" or any(role["summary"]["scheduled"] is None or role["summary"]["not_recorded"] for role in roles.values())): + status = "incomplete" + report = { + "schema_version": "0.1.0", "bundle_type": "controlled_gpu_report", "created_at": datetime.now(timezone.utc).isoformat(), + "job_id": _text(spec.get("job_id", receipt.get("job_id", "No job receipt"))), "status": status, + "supervisor_evidence_kind": _text(receipt.get("evidence_kind", "no_gpu_measurement")), + "measurement_status": _text(receipt.get("measurement_status", "incomplete")), + "ci_status": ci_status, "ci_accepted": ci_status == "pass", "release_qualified": False, + "ci_interpretation": "Supervisor-reported CI outcome, guarded against incomplete local artifact integrity; not independent CI qualification", + "supervisor_claimed_ci_accepted": receipt.get("ci_accepted") if isinstance(receipt.get("ci_accepted"), bool) else None, + "started_at": _text(receipt.get("started_at")), "finished_at": _text(receipt.get("finished_at")), + "spec_sha256": spec_digest, "roles": roles, "same_gpu_uuid_set": gpu_sets[0] == gpu_sets[1] if all(gpu_sets) else None, + "same_workload": plans[0] == plans[1] if all(plans) else None, + "allocation": _selected(spec.get("allocation"), ("mode", "label")), + "authorization": _selected(spec.get("authorization"), ("compute_approved", "model_license_reviewed", "approval_reference")), + "model_identity": _selected(receipt.get("model_identity"), ("path", "revision", "manifest_sha256", "verified_files", "total_bytes")), + "policy": _selected(policy, ("policy_id", "calibration_status", "max_latency_increase_fraction", "min_video_psnr_db", "min_audio_spectral_cosine", "max_audio_rms_ratio_error", "max_memory_increase_fraction")), + "recorded_gate": _selected(receipt, ("regression_status", "ci_accepted", "memory_gate_status", "memory_increase_fraction", "memory_threshold")), + "acceptance_reasons": [_text(reason) for reason in receipt.get("acceptance_reasons", [])[:128]] if isinstance(receipt.get("acceptance_reasons"), list) else [], + "comparison_status": _text(comparison.get("overall_status", "inconclusive")), "checks": checks, "slot_comparisons": paired_slots, + "failures": [_text(item) for item in receipt.get("failures", [])[:256]] if isinstance(receipt.get("failures"), list) else [], + "issues": issues, "verification": "Media SHA256 checked; recorded decoder results displayed, not recomputed; supervisor identity is not independently attested", + "report": {"html": "index.html", "json": "evidence.json", "assets": "assets", "scripts": False}, + } + with (output / "evidence.json").open("x", encoding="utf-8") as stream: + json.dump(report, stream, indent=2, ensure_ascii=False, allow_nan=False) + stream.write("\n") + with (output / "index.html").open("x", encoding="utf-8") as stream: + stream.write(_render(report)) + return report diff --git a/experimental/video-generation/evaluator/mvp_media.py b/experimental/video-generation/evaluator/mvp_media.py new file mode 100644 index 0000000000..b9e778a4c9 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_media.py @@ -0,0 +1,627 @@ +"""Bounded-memory, full-stream media integrity and paired signal measurements. + +These are decoded-signal measurements, not generative quality, semantic accuracy, +or lip-sync metrics. No resizing, resampling, channel mixing, or time shifting is +performed. PyAV and NumPy are optional and imported only when a measurement runs. +""" + +from __future__ import annotations + +import hashlib +import math +import time +from itertools import zip_longest +from pathlib import Path +from typing import Any + + +IMPLEMENTATION_VERSION = "1.0.0" +SILENCE_AMPLITUDE = 1e-4 # -80 dBFS sample threshold, also used for RMS presence. +CLIPPING_AMPLITUDE = 1.0 - 1.0 / 32768.0 +FROZEN_MAE = 1.0 / 1024.0 +SPECTRAL_WINDOW_SAMPLES = 1024 + + +def _libraries() -> tuple[Any, Any]: + try: + import av + import numpy as np + except ImportError as exc: + raise RuntimeError("Media measurements require the 'mvp' extra: uv sync --extra mvp") from exc + return av, np + + +def _check(name: str, passed: bool | None, observed: Any, expected: Any, detail: str) -> dict: + return { + "name": name, + "status": "not_applicable" if passed is None else "passed" if passed else "failed", + "observed": observed, + "expected": expected, + "detail": detail, + } + + +def _deadline_check(deadline: float | None) -> None: + if deadline is not None and time.monotonic() > deadline: + raise TimeoutError("media analysis exceeded its declared timeout_seconds") + + +def _time(frame: Any) -> float | None: + if frame.pts is None or frame.time_base is None: + return None + return float(frame.pts * frame.time_base) + + +def _positive_number(value: Any) -> float | None: + if value is None: + return None + number = float(value) + return number if math.isfinite(number) and number > 0 else None + + +def _validate_expected(expected: dict) -> None: + integer_keys = {"width", "height", "frame_count", "audio_sample_rate_hz", "audio_channels"} + positive_keys = integer_keys | {"fps", "duration_seconds", "timeout_seconds"} + nonnegative_keys = {"duration_tolerance_seconds", "max_av_start_skew_seconds", "max_av_end_skew_seconds", "max_frozen_fraction"} + for key in (positive_keys | nonnegative_keys) & expected.keys(): + value = expected[key] + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ValueError(f"expected.{key} must be a finite number, got {value!r}") + outside_domain = value <= 0 if key in positive_keys else value < 0 + if outside_domain: + raise ValueError(f"expected.{key} is outside its nonnegative/positive domain") + if key in integer_keys and int(value) != value: + raise ValueError(f"expected.{key} must be an integer") + if key == "max_frozen_fraction" and value > 1: + raise ValueError("expected.max_frozen_fraction must be <=1") + for key in {"audio_required", "requires_motion", "requires_sound"} & expected.keys(): + if not isinstance(expected[key], bool): + raise ValueError(f"expected.{key} must be a boolean") + + +def _audio_array(frame: Any, np: Any) -> Any: + """Preserve channel order/rate; only unpack and normalize decoded PCM.""" + values = frame.to_ndarray() + channels = len(frame.layout.channels) + if frame.format.is_planar: + values = values.reshape(channels, frame.samples) + else: + values = values.reshape(frame.samples, channels).T + kind, bits = values.dtype.kind, values.dtype.itemsize * 8 + normalized = values.astype(np.float64) + if kind == "u": + midpoint = float(2 ** (bits - 1)) + normalized = (normalized - midpoint) / midpoint + elif kind == "i": + normalized /= float(2 ** (bits - 1)) + elif kind != "f": + raise ValueError(f"unsupported decoded audio sample type: {values.dtype}") + if not np.isfinite(normalized).all(): + raise ValueError("decoded audio contains non-finite samples") + return normalized + + +class _Timing: + def __init__(self) -> None: + self.first: float | None = None + self.last: float | None = None + self.last_duration: float | None = None + self.missing = 0 + self.nonincreasing = 0 + self.min_step: float | None = None + self.max_step: float | None = None + self.tick = 0.0 + self.hash = hashlib.sha256() + + def add(self, frame: Any, duration: float | None) -> None: + timestamp = _time(frame) + self.tick = max(self.tick, float(frame.time_base or 0)) + self.hash.update(f"{frame.pts}@{frame.time_base};{duration};".encode()) + if timestamp is None: + self.missing += 1 + return + if self.first is None: + self.first = timestamp + if self.last is not None: + step = timestamp - self.last + if step <= 0: + self.nonincreasing += 1 + else: + self.min_step = step if self.min_step is None else min(self.min_step, step) + self.max_step = step if self.max_step is None else max(self.max_step, step) + self.last, self.last_duration = timestamp, duration + + +def _video_analysis(container: Any, stream: Any, np: Any, deadline: float | None) -> dict: + timing = _Timing() + count = blank = black = white = duplicates = frozen = corrupt = 0 + previous = None + first_shape = None + geometry_stable = True + adjacent_mae_sum = 0.0 + nominal_fps = _positive_number(stream.average_rate) + last_duration_source = None + pixel_formats: set[str] = set() + for frame in container.decode(stream): + _deadline_check(deadline) + pixels = frame.to_ndarray(format="rgb24") + pixel_formats.add(frame.format.name) + shape = pixels.shape + if first_shape is None: + first_shape = shape + geometry_stable = geometry_stable and shape == first_shape + count += 1 + corrupt += int(bool(getattr(frame, "is_corrupt", False))) + frame_duration = _positive_number(getattr(frame, "duration", None)) + duration = frame_duration * float(frame.time_base) if frame_duration and frame.time_base else None + last_duration_source = "decoded_frame_duration" if duration else None + timing.add(frame, duration) + maximum, minimum = int(pixels.max()), int(pixels.min()) + black += int(maximum <= 3) + white += int(minimum >= 252) + blank += int(maximum <= 3 or minimum >= 252) + if previous is not None and previous.shape == shape: + exact = bool(np.array_equal(previous, pixels)) + duplicates += int(exact) + absolute_sum = 0.0 + for row in range(0, shape[0], 64): + difference = pixels[row : row + 64].astype(np.int16) - previous[row : row + 64] + absolute_sum += float(np.abs(difference).sum(dtype=np.float64)) + mae = absolute_sum / (pixels.size * 255.0) + adjacent_mae_sum += mae + frozen += int(mae <= FROZEN_MAE) + previous = pixels + tail_duration = timing.last_duration + if tail_duration is None and timing.min_step is not None and timing.max_step is not None: + # A timestamp does not encode the final frame's display duration. Explicitly + # label the estimate instead of substituting container duration silently. + tail_duration = (timing.max_step + timing.min_step) / 2.0 + last_duration_source = "observed_frame_interval_estimate" + if tail_duration is None and nominal_fps is not None: + tail_duration = 1.0 / nominal_fps + last_duration_source = "nominal_rate_tail_estimate" + end = timing.last + tail_duration if timing.last is not None and tail_duration else None + duration = end - timing.first if end is not None and timing.first is not None else None + fps = None + fps_source = None + if count > 1 and timing.first is not None and timing.last is not None and timing.last > timing.first: + fps = (count - 1) / (timing.last - timing.first) + fps_source = "decoded_frame_timestamps" + elif count == 1 and tail_duration: + fps, fps_source = 1.0 / tail_duration, last_duration_source + return { + "present": True, + "codec": stream.codec_context.name, + "pixel_formats": sorted(pixel_formats), + "width": first_shape[1] if first_shape else None, + "height": first_shape[0] if first_shape else None, + "geometry_stable": geometry_stable, + "frame_count": count, + "fps": fps, + "fps_source": fps_source, + "nominal_fps": nominal_fps, + "duration_seconds": duration, + "duration_source": last_duration_source, + "start_time_seconds": timing.first, + "end_time_seconds": end, + "time_base_seconds": timing.tick, + "missing_timestamps": timing.missing, + "nonincreasing_timestamps": timing.nonincreasing, + "min_frame_interval_seconds": timing.min_step, + "max_frame_interval_seconds": timing.max_step, + "timestamp_sha256": timing.hash.hexdigest(), + "corrupt_frame_count": corrupt, + "blank_fraction": blank / count if count else None, + "black_fraction": black / count if count else None, + "white_fraction": white / count if count else None, + "duplicate_fraction": duplicates / (count - 1) if count > 1 else None, + "frozen_fraction": frozen / (count - 1) if count > 1 else None, + "mean_adjacent_frame_mae": adjacent_mae_sum / (count - 1) if count > 1 else None, + } + + +def _audio_analysis(container: Any, stream: Any, np: Any, deadline: float | None) -> dict: + timing = _Timing() + sample_count = frame_count = corrupt = 0 + rate = channels = None + names: list[str] = [] + sum_squares = sums = peaks = silent = clipped = cross = None + stable = True + previous_end = None + max_gap = 0.0 + for frame in container.decode(stream): + _deadline_check(deadline) + values = _audio_array(frame, np) + this_rate, this_channels = frame.sample_rate, values.shape[0] + if not this_rate or frame.samples <= 0: + raise ValueError("audio frame has no samples or sample rate") + if rate is None: + rate, channels = this_rate, this_channels + names = [channel.name for channel in frame.layout.channels] + sum_squares = np.zeros(channels, dtype=np.float64) + sums = np.zeros(channels, dtype=np.float64) + peaks = np.zeros(channels, dtype=np.float64) + silent = np.zeros(channels, dtype=np.int64) + clipped = np.zeros(channels, dtype=np.int64) + cross = np.zeros((channels, channels), dtype=np.float64) + if this_rate != rate or this_channels != channels or names != [channel.name for channel in frame.layout.channels]: + stable = False + raise ValueError("audio sample rate or channel layout changes inside the stream") + timestamp = _time(frame) + if timestamp is not None and previous_end is not None: + max_gap = max(max_gap, abs(timestamp - previous_end)) + duration = frame.samples / rate + previous_end = timestamp + duration if timestamp is not None else None + timing.add(frame, duration) + frame_count += 1 + sample_count += frame.samples + corrupt += int(bool(getattr(frame, "is_corrupt", False))) + absolute = np.abs(values) + sums += values.sum(axis=1) + sum_squares += (values * values).sum(axis=1) + peaks = np.maximum(peaks, absolute.max(axis=1)) + silent += (absolute <= SILENCE_AMPLITUDE).sum(axis=1) + clipped += (absolute >= CLIPPING_AMPLITUDE).sum(axis=1) + cross += values @ values.T + if not np.isfinite(sum_squares).all() or not np.isfinite(sums).all() or not np.isfinite(cross).all(): + raise ValueError("decoded audio overflows finite signal-statistic accumulation") + rms = np.sqrt(sum_squares / sample_count) if sample_count else np.array([]) + identical_pairs = [] + if sample_count: + for left in range(channels): + for right in range(left + 1, channels): + residual = max(0.0, float(sum_squares[left] + sum_squares[right] - 2 * cross[left, right])) + if residual / sample_count <= 1e-16: + identical_pairs.append([left, right]) + end = timing.last + timing.last_duration if timing.last is not None and timing.last_duration else None + return { + "present": True, + "codec": stream.codec_context.name, + "sample_rate_hz": rate, + "channels": channels, + "channel_names": names, + "format_stable": stable, + "frame_count": frame_count, + "sample_count": sample_count, + "duration_seconds": sample_count / rate if rate else None, + "timeline_duration_seconds": end - timing.first if end is not None and timing.first is not None else None, + "start_time_seconds": timing.first, + "end_time_seconds": end, + "time_base_seconds": timing.tick, + "missing_timestamps": timing.missing, + "nonincreasing_timestamps": timing.nonincreasing, + "max_timestamp_gap_seconds": max_gap, + "timestamp_sha256": timing.hash.hexdigest(), + "corrupt_frame_count": corrupt, + "rms_channels": rms.tolist(), + "rms_dbfs_channels": [20 * math.log10(float(value)) if value > 0 else None for value in rms], + "peak_channels": peaks.tolist() if sample_count else [], + "dc_offset_channels": (sums / sample_count).tolist() if sample_count else [], + "clipping_fraction_channels": (clipped / sample_count).tolist() if sample_count else [], + "silence_fraction_channels": (silent / sample_count).tolist() if sample_count else [], + "silent_channels": [index for index, value in enumerate(rms) if value <= SILENCE_AMPLITUDE], + "identical_channel_pairs": identical_pairs, + } + + +def analyze_media(path: Path, expected: dict | None = None) -> dict: + """Decode a local clip fully and validate only explicitly supplied expectations. + + ``duration_seconds`` is expected *media* duration, not the provider's rounded + request duration. A caller must resolve those differences explicitly. Motion + and sound presence are technical proxies only. A timeout is checked between + native decode calls, not a hard interrupt of one native FFmpeg operation. + """ + av, np = _libraries() + expected = dict(expected or {}) + path = Path(path).expanduser().resolve() + deadline = None + report: dict[str, Any] = { + "path": str(path), "sha256": None, "byte_size": None, "decode_ok": False, + "video": {"present": False}, "audio": {"present": False}, + "checks": [], "valid": False, "errors": [], "metrics": {}, + "implementation": { + "name": "vgbench.cpu_media", "version": IMPLEMENTATION_VERSION, + "pyav_version": av.__version__, "numpy_version": np.__version__, + "ffmpeg_libraries": {key: ".".join(map(str, value)) for key, value in av.library_versions.items()}, + "coverage": "all decoded video frames and all decoded audio samples", + "video_comparison_color_space": "decoded RGB24; no spatial resizing", + "silence_amplitude_threshold": SILENCE_AMPLITUDE, + "clipping_amplitude_threshold": CLIPPING_AMPLITUDE, + "frozen_adjacent_rgb_mae_threshold": FROZEN_MAE, + "blank_definition": "all decoded RGB samples <=3 or all >=252 on the 0..255 scale", + "timestamp_policy": "use decoded PTS; final video duration fallback is explicitly labeled", + "timeout_policy": "monotonic checks between hash chunks and decoded frames; not a native-call interrupt", + "non_claims": ["generative quality", "semantic accuracy", "perceptual fidelity", "lip synchronization"], + }, + } + try: + _validate_expected(expected) + timeout = expected.get("timeout_seconds") + deadline = time.monotonic() + timeout if timeout else None + if not path.is_file(): + raise ValueError("media path is not a regular file") + initial = path.stat() + report["byte_size"] = initial.st_size + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + _deadline_check(deadline) + digest.update(chunk) + report["sha256"] = digest.hexdigest() + with av.open(str(path), mode="r") as container: + video_streams = list(container.streams.video) + audio_streams = list(container.streams.audio) + if len(video_streams) != 1 or len(audio_streams) > 1: + raise ValueError("MVP requires exactly one video stream and at most one audio stream; ambiguous tracks are not silently selected") + report["video"] = _video_analysis(container, video_streams[0], np, deadline) + has_audio = bool(audio_streams) + # Reopen, not seek: video decoding consumed the demuxer to EOF. + if has_audio: + with av.open(str(path), mode="r") as container: + report["audio"] = _audio_analysis(container, container.streams.audio[0], np, deadline) + final = path.stat() + if (initial.st_size, initial.st_mtime_ns) != (final.st_size, final.st_mtime_ns): + raise ValueError("media file changed while it was being analyzed") + report["decode_ok"] = True + except Exception as exc: + report["errors"].append(f"{type(exc).__name__}: {exc}") + # Invalid expectations must never reach numerical checks or serialize + # non-finite caller values into a report that looks like a valid run. + if report["byte_size"] is None: + report["checks"].append(_check("media.preflight", False, report["errors"][-1], "valid contract and local file", "No media measurement was accepted.")) + return report + + checks = report["checks"] + video, audio = report["video"], report["audio"] + checks.append(_check("media.decode", report["decode_ok"], report["decode_ok"], True, "Full-stream local decoding completed without an exception.")) + checks.append(_check("video.present", video.get("frame_count", 0) > 0, video.get("frame_count", 0), ">0", "At least one video frame must be decoded.")) + if video.get("frame_count", 0): + checks.append(_check("video.geometry_stable", video["geometry_stable"], video["geometry_stable"], True, "All decoded frames retain the original geometry.")) + timestamp_ok = video["missing_timestamps"] == 0 and video["nonincreasing_timestamps"] == 0 + checks.append(_check("video.timestamps", timestamp_ok, {"missing": video["missing_timestamps"], "nonincreasing": video["nonincreasing_timestamps"]}, {"missing": 0, "nonincreasing": 0}, "Every decoded frame has strictly increasing PTS.")) + checks.append(_check("video.corrupt_frames", video["corrupt_frame_count"] == 0, video["corrupt_frame_count"], 0, "Decoder corruption flags are not ignored.")) + for name, key in (("width", "width"), ("height", "height"), ("frame_count", "frame_count")): + if key in expected: + checks.append(_check(f"video.{name}", video.get(name) == expected[key], video.get(name), expected[key], "Exact expected decoded media contract.")) + if "fps" in expected: + target = float(expected["fps"]) + actual = video.get("fps") + tolerance = max(0.001, abs(target) * 0.001) + checks.append(_check("video.fps", actual is not None and abs(actual - target) <= tolerance, actual, {"value": target, "absolute_tolerance": tolerance}, "FPS is derived from decoded frame timestamps; the tolerance covers container timestamp quantization.")) + if video.get("frame_count", 0) > 1: + interval = 1.0 / target + cadence_tolerance = max(video.get("time_base_seconds", 0), interval * 0.001) + 1e-9 + intervals = [video.get("min_frame_interval_seconds"), video.get("max_frame_interval_seconds")] + cadence_ok = all(value is not None and abs(value - interval) <= cadence_tolerance for value in intervals) + checks.append(_check("video.cadence", cadence_ok, {"min_seconds": intervals[0], "max_seconds": intervals[1]}, {"interval_seconds": interval, "absolute_tolerance_seconds": cadence_tolerance}, "When a fixed FPS is requested, average FPS alone must not conceal internal gaps or bursty timestamps.")) + if "duration_seconds" in expected: + target = float(expected["duration_seconds"]) + tolerance = float(expected.get("duration_tolerance_seconds", 1.0 / video["fps"] if video.get("fps") else 0.05)) + actual = video.get("duration_seconds") + checks.append(_check("video.duration_seconds", actual is not None and tolerance >= 0 and abs(actual - target) <= tolerance + 1e-9, actual, {"value": target, "absolute_tolerance": tolerance}, "Caller supplies expected media duration, with any request-to-output rounding already resolved.")) + if "audio_required" in expected: + required = bool(expected["audio_required"]) + checks.append(_check("audio.required", bool(audio.get("present")) if required else None, audio.get("present", False), required, "False means audio is optional, not prohibited.")) + if audio.get("present"): + checks.append(_check("audio.samples", audio.get("sample_count", 0) > 0, audio.get("sample_count", 0), ">0", "An advertised audio stream must contain decoded samples.")) + timestamp_ok = audio["missing_timestamps"] == 0 and audio["nonincreasing_timestamps"] == 0 + checks.append(_check("audio.timestamps", timestamp_ok, {"missing": audio["missing_timestamps"], "nonincreasing": audio["nonincreasing_timestamps"]}, {"missing": 0, "nonincreasing": 0}, "Every audio frame has strictly increasing PTS.")) + tolerance = max(audio.get("time_base_seconds", 0), 1.0 / audio["sample_rate_hz"]) + 1e-9 if audio.get("sample_rate_hz") else 0 + checks.append(_check("audio.continuity", audio["max_timestamp_gap_seconds"] <= tolerance, audio["max_timestamp_gap_seconds"], {"maximum_seconds": tolerance}, "PCM sample continuity allows one container timestamp tick, without filling gaps or removing overlaps.")) + checks.append(_check("audio.corrupt_frames", audio["corrupt_frame_count"] == 0, audio["corrupt_frame_count"], 0, "Decoder corruption flags are not ignored.")) + for key, field in (("audio_sample_rate_hz", "sample_rate_hz"), ("audio_channels", "channels")): + if key in expected: + checks.append(_check(f"audio.{field}", audio.get(field) == expected[key], audio.get(field), expected[key], "Original decoded audio contract; no resampling or channel conversion.")) + if expected.get("requires_motion"): + actual = video.get("mean_adjacent_frame_mae") + checks.append(_check("video.motion_presence", actual is not None and actual > FROZEN_MAE, actual, {"greater_than": FROZEN_MAE}, "Adjacent RGB change is a technical motion-presence proxy, not action understanding or physical correctness.")) + if "max_frozen_fraction" in expected: + actual = video.get("frozen_fraction") + target = expected["max_frozen_fraction"] + checks.append(_check("video.frozen_fraction", actual is not None and actual <= target, actual, {"maximum": target}, "Explicit caller-supplied limit on near-identical adjacent-frame transitions; no universal freeze threshold is assumed.")) + if expected.get("requires_sound"): + actual = max(audio.get("rms_channels", []) or [0.0]) + checks.append(_check("audio.sound_presence", actual > SILENCE_AMPLITUDE, actual, {"greater_than": SILENCE_AMPLITUDE}, "At least one channel has non-silent RMS; intentionally silent channels are reported separately.")) + metrics = report["metrics"] + for key in ("blank_fraction", "duplicate_fraction", "frozen_fraction", "mean_adjacent_frame_mae"): + metrics[f"video_{key}"] = video.get(key) + metrics["audio_rms_channels"] = audio.get("rms_channels", []) + metrics["audio_silent_channel_count"] = len(audio.get("silent_channels", [])) + for boundary in ("start", "end"): + video_time, audio_time = video.get(f"{boundary}_time_seconds"), audio.get(f"{boundary}_time_seconds") + metrics[f"av_{boundary}_skew_seconds"] = audio_time - video_time if video_time is not None and audio_time is not None else None + expectation_key = f"max_av_{boundary}_skew_seconds" + if expectation_key in expected: + actual = metrics[f"av_{boundary}_skew_seconds"] + checks.append(_check(f"av.{boundary}_skew_seconds", actual is not None and abs(actual) <= expected[expectation_key] + 1e-9, actual, {"maximum_absolute_seconds": expected[expectation_key]}, "Explicit caller-supplied media-boundary limit, not semantic or lip synchronization.")) + report["valid"] = report["decode_ok"] and all(check["status"] != "failed" for check in checks) + return report + + +def _decoded_video(path: Path, av: Any): + with av.open(str(path), mode="r") as container: + yield from container.decode(container.streams.video[0]) + + +def _decoded_audio(path: Path, av: Any, np: Any): + with av.open(str(path), mode="r") as container: + for frame in container.decode(container.streams.audio[0]): + yield _audio_array(frame, np), _time(frame), float(frame.time_base or 0) + + +def _compare_audio(baseline: Path, candidate: Path, av: Any, np: Any, channels: int, rate: int) -> dict: + """Compare all samples despite harmless differences in codec frame chunking.""" + left_iter, right_iter = iter(_decoded_audio(baseline, av, np)), iter(_decoded_audio(candidate, av, np)) + left = right = None + left_offset = right_offset = 0 + absolute = np.zeros(channels) + squared_error = np.zeros(channels) + energy_left = np.zeros(channels) + energy_right = np.zeros(channels) + spectral_dot = np.zeros(channels) + spectral_left = np.zeros(channels) + spectral_right = np.zeros(channels) + pending_left = np.empty((channels, 0)) + pending_right = np.empty((channels, 0)) + window = np.hanning(SPECTRAL_WINDOW_SAMPLES) + count = windows = 0 + max_timestamp_delta = 0.0 + + def spectrum(a: Any, b: Any) -> None: + nonlocal windows + if a.shape[1] < SPECTRAL_WINDOW_SAMPLES: + padding = ((0, 0), (0, SPECTRAL_WINDOW_SAMPLES - a.shape[1])) + a, b = np.pad(a, padding), np.pad(b, padding) + fa = np.abs(np.fft.rfft(a * window, axis=1)) + fb = np.abs(np.fft.rfft(b * window, axis=1)) + spectral_dot[:] += (fa * fb).sum(axis=1) + spectral_left[:] += (fa * fa).sum(axis=1) + spectral_right[:] += (fb * fb).sum(axis=1) + windows += 1 + + while True: + if left is None or left_offset == left[0].shape[1]: + left, left_offset = next(left_iter, None), 0 + if right is None or right_offset == right[0].shape[1]: + right, right_offset = next(right_iter, None), 0 + if left is None or right is None: + if left is not None or right is not None: + raise ValueError("audio sample counts differ; no truncation is allowed") + break + if left[1] is None or right[1] is None: + raise ValueError("audio timestamps are unavailable") + delta = abs((left[1] + left_offset / rate) - (right[1] + right_offset / rate)) + max_timestamp_delta = max(max_timestamp_delta, delta) + if delta > max(left[2], right[2], 1.0 / rate) + 1e-9: + raise ValueError("paired audio samples are not timestamp-aligned") + length = min(left[0].shape[1] - left_offset, right[0].shape[1] - right_offset, 65536) + a = left[0][:, left_offset : left_offset + length] + b = right[0][:, right_offset : right_offset + length] + difference = a - b + absolute += np.abs(difference).sum(axis=1) + squared_error += (difference * difference).sum(axis=1) + energy_left += (a * a).sum(axis=1) + energy_right += (b * b).sum(axis=1) + count += length + left_offset += length + right_offset += length + pending_left = np.concatenate((pending_left, a), axis=1) + pending_right = np.concatenate((pending_right, b), axis=1) + cursor = 0 + while pending_left.shape[1] - cursor >= SPECTRAL_WINDOW_SAMPLES: + spectrum(pending_left[:, cursor : cursor + SPECTRAL_WINDOW_SAMPLES], pending_right[:, cursor : cursor + SPECTRAL_WINDOW_SAMPLES]) + cursor += SPECTRAL_WINDOW_SAMPLES + pending_left, pending_right = pending_left[:, cursor:].copy(), pending_right[:, cursor:].copy() + if pending_left.shape[1]: + spectrum(pending_left, pending_right) + if count == 0: + raise ValueError("paired audio contains no samples") + ratios = [math.sqrt(float(b / a)) if a > 0 else None for a, b in zip(energy_left, energy_right)] + cosines = [min(1.0, max(0.0, float(dot / math.sqrt(a * b)))) if a > 0 and b > 0 else None for dot, a, b in zip(spectral_dot, spectral_left, spectral_right)] + return { + "audio_channel_count": channels, + "audio_compared_samples_per_channel": count, + "audio_sample_coverage_fraction": 1.0, + "audio_max_alignment_delta_seconds": max_timestamp_delta, + "audio_waveform_mae": float(absolute.sum() / (count * channels)), + "audio_waveform_mae_channels": (absolute / count).tolist(), + "audio_waveform_rmse_channels": np.sqrt(squared_error / count).tolist(), + "audio_identical": bool(np.all(squared_error == 0)), + "audio_rms_ratio": math.sqrt(float(energy_right.sum() / energy_left.sum())) if energy_left.sum() > 0 else None, + "audio_rms_ratio_channels": ratios, + "audio_spectral_cosine": min(cosines) if all(value is not None for value in cosines) else None, + "audio_spectral_cosine_channels": cosines, + "audio_spectral_windows": windows, + "audio_newly_silent_channels": [index for index, (a, b) in enumerate(zip(energy_left, energy_right)) if math.sqrt(float(a / count)) > SILENCE_AMPLITUDE and math.sqrt(float(b / count)) <= SILENCE_AMPLITUDE], + } + + +def compare_media(baseline: Path, candidate: Path) -> dict: + """Measure full-stream decoded fidelity only when geometry/timing agree. + + The caller chooses calibrated decision thresholds. Compatible means the + measurements share shape and timing, not that the candidate is good quality. + Any incompatibility leaves *all* quality-comparison metrics unavailable. + """ + av, np = _libraries() + baseline, candidate = Path(baseline), Path(candidate) + result: dict[str, Any] = { + "compatible": False, + "metrics": { + "video_mae": None, "video_psnr_db": None, "video_identical": None, + "video_compared_frames": 0, "video_total_frames": 0, "video_sample_coverage_fraction": 0.0, + "audio_waveform_mae": None, "audio_spectral_cosine": None, "audio_rms_ratio": None, + "audio_rms_ratio_channels": [], "audio_spectral_cosine_channels": [], "audio_waveform_mae_channels": [], + "audio_channel_count": 0, + }, + "checks": [], + "notes": [ + "Technical paired fidelity only: not semantic accuracy, generative quality, perceptual quality, or lip sync.", + "All decoded RGB pixels/frames and PCM samples are compared; no resizing, resampling, mixing, trimming, or best-offset alignment.", + "Video MAE uses RGB values divided by 255. PSNR uses peak=1 and full-stream MSE; exact identity gives null PSNR plus video_identical=true.", + "Audio spectra use aligned non-overlapping 1024-sample Hann windows, zero-padding only the final partial window. Channel cosine is pooled over all window magnitudes; aggregate is the worst channel, or null if any channel has zero spectral energy.", + "Audio RMS ratios are candidate/baseline. A zero baseline RMS has undefined ratio, reported null, never infinity.", + "Timestamp alignment permits at most one native container tick (or one audio sample); the actual maximum delta is reported.", + ], + } + left, right = analyze_media(baseline), analyze_media(candidate) + checks = result["checks"] + checks.append(_check("pair.valid_media", left["valid"] and right["valid"], {"baseline": left["valid"], "candidate": right["valid"]}, {"baseline": True, "candidate": True}, "Both original files must decode fully with valid media timing.")) + if not left["valid"] or not right["valid"]: + result["notes"].extend(left["errors"] + right["errors"]) + return result + for field in ("width", "height", "frame_count"): + a, b = left["video"][field], right["video"][field] + checks.append(_check(f"pair.video.{field}", a == b, {"baseline": a, "candidate": b}, "equal", "Original dimensions and complete decoded frame counts must match.")) + video_tick = max(left["video"]["time_base_seconds"], right["video"]["time_base_seconds"], 1e-9) + for field in ("start_time_seconds", "end_time_seconds"): + a, b = left["video"][field], right["video"][field] + checks.append(_check(f"pair.video.{field}", a is not None and b is not None and abs(a - b) <= video_tick + 1e-9, {"baseline": a, "candidate": b}, {"maximum_delta_seconds": video_tick}, "No timestamp offset or duration trimming is allowed.")) + left_audio, right_audio = left["audio"]["present"], right["audio"]["present"] + checks.append(_check("pair.audio.presence", left_audio == right_audio if left_audio or right_audio else None, {"baseline": left_audio, "candidate": right_audio}, "equal", "Both clips may omit audio; an audio stream may not disappear in only one clip.")) + if left_audio and right_audio: + for field in ("sample_rate_hz", "channels", "channel_names", "sample_count"): + a, b = left["audio"][field], right["audio"][field] + checks.append(_check(f"pair.audio.{field}", a == b, {"baseline": a, "candidate": b}, "equal", "PCM channel order, sample rate, and complete sample counts must match.")) + if any(check["status"] == "failed" for check in checks): + return result + metrics = dict(result["metrics"]) + try: + count = samples = 0 + absolute_sum = squared_sum = max_alignment_delta = 0.0 + for a, b in zip_longest(_decoded_video(baseline, av), _decoded_video(candidate, av)): + if a is None or b is None: + raise ValueError("video frame counts changed during comparison") + delta = abs(_time(a) - _time(b)) + max_alignment_delta = max(max_alignment_delta, delta) + if delta > video_tick + 1e-9: + raise ValueError("video frame timestamps do not align; frame-index matching alone is insufficient") + x, y = a.to_ndarray(format="rgb24"), b.to_ndarray(format="rgb24") + if x.shape != y.shape: + raise ValueError("video geometry changed during comparison") + for row in range(0, x.shape[0], 64): + difference = x[row : row + 64].astype(np.float64) - y[row : row + 64] + absolute_sum += float(np.abs(difference).sum()) + squared_sum += float((difference * difference).sum()) + samples += x.size + count += 1 + mse = squared_sum / (samples * 255.0 * 255.0) + metrics.update({ + "video_mae": absolute_sum / (samples * 255.0), + "video_psnr_db": -10.0 * math.log10(mse) if mse > 0 else None, + "video_identical": squared_sum == 0, + "video_compared_frames": count, + "video_total_frames": left["video"]["frame_count"], + "video_sample_coverage_fraction": 1.0, + "video_max_alignment_delta_seconds": max_alignment_delta, + }) + if left_audio and right_audio: + metrics.update(_compare_audio(baseline, candidate, av, np, left["audio"]["channels"], left["audio"]["sample_rate_hz"])) + checks.append(_check("pair.full_stream_alignment", True, True, True, "Every compared frame/sample met temporal compatibility; all streams were exhausted.")) + result["metrics"] = metrics + result["compatible"] = True + except Exception as exc: + checks.append(_check("pair.full_stream_alignment", False, str(exc), "complete aligned streams", "Partial comparison metrics are discarded on any decoding, shape, or timing failure.")) + result["notes"].append(f"{type(exc).__name__}: {exc}") + return result diff --git a/experimental/video-generation/evaluator/mvp_power.py b/experimental/video-generation/evaluator/mvp_power.py new file mode 100644 index 0000000000..9926367dd6 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_power.py @@ -0,0 +1,340 @@ +"""Validate H3 GPU-board power without turning missing samples into energy.""" + +from __future__ import annotations + +import csv +import math +from datetime import datetime +from decimal import Decimal +from pathlib import Path +from statistics import median +from tempfile import TemporaryDirectory + +from utils.aggregate_power import integrate_power + + +_CLOCK_TOLERANCE_SECONDS = 0.1 +_LEGACY_JOURNAL_TOLERANCE_SECONDS = 0.25 + + +def _number(value: object) -> bool: + try: + return type(value) in (int, float) and math.isfinite(value) + except OverflowError: + return False + + +def _utc(value: object) -> float: + if not isinstance(value, str): + raise ValueError("missing UTC timestamp") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("timestamp has no timezone") + return parsed.timestamp() + + +def _ownership(sample: dict, owner: dict, devices: list[str]) -> list[str]: + """Require the retained raw process inventory to partition into owned PIDs.""" + raw = sample.get("compute_apps") + owned = sample.get("owned_compute_apps") + if not isinstance(raw, list) or not isinstance(owned, list) or sample.get("unowned_compute_apps") != []: + return ["gpu_ownership_unverified"] + keys = ("gpu_uuid", "pid", "memory_used_mib") + try: + raw_rows = [tuple(app.get(key) for key in keys) for app in raw] + owned_rows = [tuple(app.get(key) for key in keys) for app in owned] + if len(set(raw_rows)) != len(raw_rows) or set(raw_rows) != set(owned_rows) or len(raw_rows) != len(owned_rows): + return ["gpu_ownership_partition_mismatch"] + for app in owned: + identity = app.get("process_identity", {}) + if (app.get("gpu_uuid") not in devices or type(app.get("pid")) is not int + or app["pid"] <= 0 or identity.get("pid") != app["pid"] + or not owner.get("pgid") or identity.get("pgid") != owner["pgid"] + or not owner.get("session_id") or identity.get("session_id") != owner["session_id"] + or not _number(identity.get("start_ticks")) or identity["start_ticks"] <= 0): + return ["gpu_process_identity_mismatch"] + except (AttributeError, TypeError): + return ["gpu_ownership_unverified"] + return [] + + +def _timing(record: dict, events: list[dict], offset: float | None, spread: float | None) -> dict: + """Prefer explicit monotonic boundaries; recover only tightly agreeing journals.""" + result = {"start_monotonic_seconds": None, "end_monotonic_seconds": None, + "timing_source": None, "timing_uncertainty_seconds": None, "invalid_reasons": []} + if record.get("attempted") is not True: + result["invalid_reasons"].append("request_not_attempted") + return result + duration = record.get("submit_to_terminal_seconds") + latency = record.get("latency_seconds") + if not _number(duration) or not _number(latency) or not 0 < duration <= latency: + result["invalid_reasons"].append("invalid_request_duration") + return result + explicit = record.get("timing_window") + if explicit is not None: + result["timing_source"] = "recorded_monotonic_submit_to_terminal" + try: + start, terminal, end = (explicit[key] for key in ( + "start_monotonic_seconds", "terminal_monotonic_seconds", "end_monotonic_seconds")) + if (not all(_number(value) for value in (start, terminal, end)) or not start < terminal <= end + or abs((terminal - start) - duration) > _CLOCK_TOLERANCE_SECONDS + or abs((end - start) - latency) > _CLOCK_TOLERANCE_SECONDS + or offset is None or abs(_utc(explicit["start_utc"]) - offset - start) > _CLOCK_TOLERANCE_SECONDS): + raise ValueError("inconsistent timing") + result.update(start_monotonic_seconds=start, end_monotonic_seconds=terminal, + timing_uncertainty_seconds=0.0) + except (KeyError, TypeError, ValueError): + result["invalid_reasons"].append("invalid_recorded_timing_window") + return result + result["timing_source"] = "legacy_utc_journal_mapped_to_monotonic" + starts = [event for event in events if event.get("event") == "attempt_started" and event.get("slot_id") == record.get("slot_id")] + finishes = [event for event in events if event.get("event") == "attempt_finished" + and isinstance(event.get("record"), dict) and event["record"].get("slot_id") == record.get("slot_id")] + try: + if len(starts) != 1 or len(finishes) != 1 or finishes[0].get("record") != record or offset is None: + raise ValueError("missing or conflicting journal") + start_utc, finish_utc = _utc(starts[0]["at"]), _utc(finishes[0]["at"]) + slack = finish_utc - start_utc - latency + if not 0 <= slack <= _LEGACY_JOURNAL_TOLERANCE_SECONDS: + raise ValueError("journal does not agree with monotonic duration") + # The completed journal follows the timed request. Its residual bounds + # the possible boundary overhead; it is not exact kernel timing. + start = finish_utc - offset - latency + result.update(start_monotonic_seconds=start, end_monotonic_seconds=start + duration, + timing_uncertainty_seconds=slack + (spread or 0.0)) + except (KeyError, TypeError, ValueError): + result["invalid_reasons"].append("legacy_timing_unverifiable") + return result + + +def _startup(role: dict, offset: float | None, spread: float | None) -> dict: + result = {"phase": "startup", "slot_id": None, "attempted": 0, "completed": 0, "valid_clips": 0, + "start_monotonic_seconds": None, "end_monotonic_seconds": None, + "timing_source": None, "timing_uncertainty_seconds": None, "invalid_reasons": []} + explicit = role.get("startup_timing_window") + try: + duration = role.get("startup_seconds") + if not _number(duration) or duration <= 0 or offset is None: + raise ValueError("missing startup time") + if explicit is not None: + start, end = explicit["start_monotonic_seconds"], explicit["end_monotonic_seconds"] + result["timing_source"] = "recorded_monotonic_startup" + uncertainty = 0.0 + if abs(_utc(explicit["start_utc"]) - offset - start) > _CLOCK_TOLERANCE_SECONDS: + raise ValueError("startup clock disagreement") + else: + start = _utc(role.get("started_at")) - offset + end = role.get("telemetry_summary", {}).get("measurement_window_start_monotonic_seconds") + result["timing_source"] = "legacy_role_start_to_sampler_measurement_start" + uncertainty = _CLOCK_TOLERANCE_SECONDS + (spread or 0.0) + if not _number(start) or not _number(end) or end <= start or abs(end - start - duration) > _CLOCK_TOLERANCE_SECONDS: + raise ValueError("startup duration disagreement") + result.update(start_monotonic_seconds=start, end_monotonic_seconds=end, + timing_uncertainty_seconds=uncertainty) + except (KeyError, TypeError, ValueError): + result["invalid_reasons"].append("startup_timing_unverifiable") + return result + + +def _summarize_windows(windows: list[dict], devices: list[str]) -> dict: + result = {"status": "not_requested" if not windows else "invalid", "valid": False, + "invalid_reasons": sorted({reason for window in windows for reason in window["invalid_reasons"]}), + "window_count": len(windows), "valid_window_count": sum(window["valid"] for window in windows), + "attempted": sum(window["attempted"] for window in windows), + "completed": sum(window["completed"] for window in windows), + "valid_clips": sum(window["valid_clips"] for window in windows), + "duration_seconds": None, "per_gpu": None, "aggregate": None} + if not windows or not all(window["valid"] for window in windows): + return result + duration = sum(window["duration_seconds"] for window in windows) + per_gpu = {device: {"energy_j": sum(window["per_gpu"][device]["energy_j"] for window in windows), + "observed_peak_power_w": max(window["per_gpu"][device]["observed_peak_power_w"] for window in windows)} + for device in devices} + for value in per_gpu.values(): + value["avg_power_w"] = value["energy_j"] / duration + energy = sum(value["energy_j"] for value in per_gpu.values()) + if (not _number(duration) or duration <= 0 or not _number(energy) + or any(not _number(value[key]) or value[key] < 0 for value in per_gpu.values() for key in ("energy_j", "avg_power_w"))): + result["invalid_reasons"].append("nonfinite_phase_power_integration") + return result + result.update(status="valid", valid=True, duration_seconds=duration, per_gpu=per_gpu, + aggregate={"energy_j": energy, "avg_power_w": energy / duration, + "observed_peak_power_w": max(window["aggregate"]["observed_peak_power_w"] for window in windows), + "joules_per_valid_clip": energy / result["valid_clips"] if result["valid_clips"] else None}) + return result + + +def analyze_power(role: dict, run: dict, samples: list[dict], events: list[dict], gpu_uuids: list[str], *, interval_seconds: float) -> dict: + """Return phase-scoped, reproducible GPU-board power with invalid values withheld. + + Request energy covers submit through observed provider terminal state, not + client download/decoding or exact GPU kernel execution. Failed/invalid clips + retain their observed generation energy in the phase numerator; only valid + clips enter the denominator. Legacy UTC recovery requires a stable sampled + clock offset within 100 ms and journal/latency agreement within 250 ms. + """ + global_reasons = [] + if not gpu_uuids or any(not isinstance(device, str) or not device for device in gpu_uuids) or len(set(gpu_uuids)) != len(gpu_uuids): + global_reasons.append("invalid_gpu_inventory") + if not _number(interval_seconds) or interval_seconds <= 0: + global_reasons.append("invalid_sampling_interval") + max_gap = 3 * interval_seconds if not global_reasons else 0.0 + offsets, series = [], [] + previous = None + for sample in samples: + stamp = sample.get("monotonic_seconds") + reasons = [] + try: + if not _number(stamp) or (previous is not None and stamp <= previous): + raise ValueError("sample timestamps are not increasing") + offsets.append(_utc(sample.get("at")) - stamp) + previous = stamp + except (TypeError, ValueError): + global_reasons.append("invalid_or_nonmonotonic_sample_time") + devices = sample.get("gpus", []) + powers = {device: None for device in gpu_uuids} + if not isinstance(devices, list) or any(not isinstance(device, dict) for device in devices): + devices = [] + if sorted(str(device.get("uuid")) for device in devices) != sorted(gpu_uuids): + reasons.append("sample_gpu_inventory_mismatch") + for device in devices: + if device.get("uuid") in powers: + value = device.get("power_watts") + powers[device["uuid"]] = value if _number(value) and value >= 0 else None + if any(value is None for value in powers.values()): + reasons.append("invalid_power_sample") + aggregate = sum(powers.values()) if not reasons else None + if aggregate is not None and not _number(aggregate): + reasons.append("nonfinite_aggregate_power_sample") + query = sample.get("power_query") + if query is not None: + try: + begin, finish = query["start_monotonic_seconds"], query["end_monotonic_seconds"] + if (not all(_number(value) for value in (begin, finish, stamp)) or not begin <= finish <= stamp + or abs((_utc(query["start_utc"]) - begin) - (_utc(sample["at"]) - stamp)) > _CLOCK_TOLERANCE_SECONDS): + raise ValueError("power acquisition bracket is inconsistent") + except (KeyError, TypeError, ValueError): + reasons.append("invalid_power_acquisition_window") + reasons.extend(_ownership(sample, role.get("process_identity", {}), gpu_uuids)) + series.append({"at": sample.get("at"), "monotonic_seconds": stamp if _number(stamp) else None, + "per_gpu_watts": powers, "aggregate_watts": aggregate if not reasons else None, + "valid": not reasons, "invalid_reasons": reasons}) + spread = max(offsets) - min(offsets) if offsets else None + offset = median(offsets) if offsets else None + if spread is None or spread > _CLOCK_TOLERANCE_SECONDS: + global_reasons.append("sample_clock_offset_unstable_or_missing") + offset = None + windows = [_startup(role, offset, spread)] + seen_slots = set() + for record in run.get("records", []): + phase, slot = record.get("phase"), record.get("slot_id") + if phase not in ("warmup", "measurement") or not isinstance(slot, str) or slot in seen_slots: + global_reasons.append("invalid_or_duplicate_record_slot") + continue + seen_slots.add(slot) + windows.append({"phase": phase, "slot_id": slot, "case_id": record.get("case_id"), + "attempted": int(record.get("attempted") is True), + "completed": int(record.get("status") == "succeeded"), + "valid_clips": int(record.get("status") == "succeeded" and isinstance(record.get("media"), dict) and record["media"].get("valid") is True), + **_timing(record, events, offset, spread)}) + if run.get("configuration", {}).get("serving"): + from .mvp_serving import validate_window + try: + validate_window(run) + except (KeyError, TypeError, ValueError): + global_reasons.append("invalid_serving_measurement_window") + measured = [window for window in windows if window["phase"] == "measurement"] + if measured: + starts = [w["start_monotonic_seconds"] for w in measured] + ends = [w["end_monotonic_seconds"] for w in measured] + # Per-request board power cannot be attributed under concurrency. + combined = {"phase": "measurement", "slot_id": None, + "request_slot_ids": [w["slot_id"] for w in measured], + "timing_source": "serving_first_submit_to_last_observed_terminal", + "timing_uncertainty_seconds": max((w["timing_uncertainty_seconds"] or 0 for w in measured)), + "start_monotonic_seconds": min(starts) if all(_number(v) for v in starts) else None, + "end_monotonic_seconds": max(ends) if all(_number(v) for v in ends) else None, + "invalid_reasons": sorted({reason for w in measured for reason in w["invalid_reasons"]}), + **{key: sum(w[key] for w in measured) for key in ("attempted", "completed", "valid_clips")}} + windows = [w for w in windows if w["phase"] != "measurement"] + [combined] + ordered = sorted((window for window in windows if window["start_monotonic_seconds"] is not None), key=lambda window: window["start_monotonic_seconds"]) + for left, right in zip(ordered, ordered[1:]): + if left["end_monotonic_seconds"] > right["start_monotonic_seconds"]: + left["invalid_reasons"].append("overlapping_phase_windows") + right["invalid_reasons"].append("overlapping_phase_windows") + with TemporaryDirectory(prefix="h3-power-") as directory: + path = Path(directory) / "power.csv" + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.writer(stream) + writer.writerow(["timestamp", "gpu_id", "power.draw [W]"]) + for sample in series: + for device, power in sample["per_gpu_watts"].items(): + # The shared CSV parser accepts decimal notation, not + # exponents. Preserve float values rather than parsing 6e2 as 6. + stamp = sample["monotonic_seconds"] + writer.writerow([format(Decimal(str(stamp)), "f") if stamp is not None else "", + device, format(Decimal(str(power)), "f") if power is not None else "N/A"]) + for window in windows: + reasons = window["invalid_reasons"] + global_reasons + start, end = window["start_monotonic_seconds"], window["end_monotonic_seconds"] + window.update(valid=False, status="invalid", duration_seconds=None, per_gpu=None, aggregate=None, coverage=None) + if start is not None and end is not None and end > start and not global_reasons: + integrated = integrate_power(path, start_unix=start, end_unix=end, + expected_num_gpus=len(gpu_uuids), max_sample_gap_s=max_gap) + reasons.extend(integrated.invalid_reasons) + if integrated.power_valid and any(not _number(value) or value < 0 for value in ( + integrated.total_gpu_energy_j, integrated.avg_total_gpu_power_w, integrated.avg_power_w, + *integrated.per_gpu_energy_j.values())): + reasons.append("nonfinite_or_negative_power_integration") + supporting = [sample for sample in series if start - max_gap <= sample["monotonic_seconds"] <= end + max_gap] + for sample in supporting: + reasons.extend(sample["invalid_reasons"]) + within = [sample for sample in series if start <= sample["monotonic_seconds"] <= end] + if not within: + reasons.append("no_in_window_power_samples") + if window["phase"] != "startup": + owned = {app.get("gpu_uuid") for sample in samples if start <= sample["monotonic_seconds"] <= end + for app in sample.get("owned_compute_apps", [])} + if not set(gpu_uuids).issubset(owned): + reasons.append("owned_compute_not_observed_on_every_gpu") + covered = sum(max(0.0, min(end, right["monotonic_seconds"]) - max(start, left["monotonic_seconds"])) + for left, right in zip(series, series[1:]) + if left["valid"] and right["valid"] and right["monotonic_seconds"] - left["monotonic_seconds"] <= max_gap) + window["duration_seconds"] = end - start + window["coverage"] = {"sample_count": len(within), "covered_duration_seconds": covered, + "coverage_fraction": min(1.0, covered / (end - start)), + "per_gpu_sample_counts": integrated.per_gpu_sample_counts, + "per_gpu_max_sample_gap_seconds": integrated.per_gpu_max_sample_gap_s, + "maximum_allowed_sample_gap_seconds": max_gap, + "bracketed": "benchmark_window_not_bracketed" not in integrated.invalid_reasons} + if not reasons: + energy = integrated.total_gpu_energy_j + window.update(valid=True, status="valid", + per_gpu={device: {"energy_j": integrated.per_gpu_energy_j[device], + "avg_power_w": integrated.per_gpu_energy_j[device] / (end - start), + "observed_peak_power_w": max(sample["per_gpu_watts"][device] for sample in within)} for device in gpu_uuids}, + aggregate={"energy_j": energy, "avg_power_w": integrated.avg_total_gpu_power_w, + "observed_peak_power_w": max(sample["aggregate_watts"] for sample in within), + "joules_per_valid_clip": energy / window["valid_clips"] if window["valid_clips"] else None}) + window["invalid_reasons"] = sorted(set(reasons)) + phases = {phase: _summarize_windows([window for window in windows if window["phase"] == phase], gpu_uuids) + for phase in ("startup", "warmup", "measurement")} + requested = [phase for phase in phases.values() if phase["window_count"]] + valid = bool(requested) and all(phase["valid"] for phase in requested) + return {"schema_version": "1.0.0", "valid": valid, + "status": "valid" if valid else ("partial" if any(phase["valid"] for phase in requested) else "invalid"), + "invalid_reasons": sorted({reason for phase in phases.values() for reason in phase["invalid_reasons"]}), + "semantics": {"scope": "selected_gpu_boards_including_memory; excludes_host_and_unselected_gpus", + "power_unit": "W", "energy_unit": "J", "time_unit": "s", + "integration": "per_device_trapezoidal_with_linear_boundary_interpolation", + "generation_window": ("first_submit_to_last_observed_provider_terminal; includes_intervening_idle_download_and_validation_time; concurrent_board_energy_integrated_once" + if run.get("configuration", {}).get("serving") else + "submit_to_observed_provider_terminal; excludes_client_download_and_decode"), + "peak": "maximum_observed_sensor_sample_in_window; not_instantaneous_electrical_peak", + "energy_per_valid_clip": "sum_generation_energy_including_failed_or_invalid_completed_attempts_divided_by_technically_valid_clips", + "sensor": "nvidia-smi power.draw; H200 NVML trailing_one_second_average; phase_edges_have_sensor_averaging_uncertainty", + "clock_agreement_limit_seconds": _CLOCK_TOLERANCE_SECONDS, + "legacy_journal_agreement_limit_seconds": _LEGACY_JOURNAL_TOLERANCE_SECONDS}, + "clock_alignment": {"utc_minus_monotonic_seconds": offset, "observed_offset_spread_seconds": spread}, + "gpu_uuids": gpu_uuids, "requested_interval_seconds": interval_seconds, + "sample_series": series, "windows": windows, "phases": phases} diff --git a/experimental/video-generation/evaluator/mvp_report.py b/experimental/video-generation/evaluator/mvp_report.py new file mode 100644 index 0000000000..0d832dc60c --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_report.py @@ -0,0 +1,365 @@ +"""Portable, script-free presentation of a verified MVP comparison.""" + +from __future__ import annotations + +import copy +import hashlib +import html +import json +import math +import os +from contextlib import ExitStack +from pathlib import Path +from typing import Any +from urllib.parse import quote + + +_CSS = """ +:root{color-scheme:light;--ink:#132b37;--muted:#526875;--line:#d7e2e5;--paper:#fff;--wash:#f3f7f8;--accent:#086b66;--pass:#086b48;--fail:#a7313e;--warn:#805b11} +*{box-sizing:border-box}body{margin:0;background:var(--wash);font:15px/1.6 -apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;color:var(--ink)} +a{color:var(--accent)}main{max-width:1280px;margin:auto;padding:42px 36px 70px}.eyebrow{font-size:11px;letter-spacing:.15em;text-transform:uppercase;font-weight:750;color:var(--muted)} +header{display:flex;justify-content:space-between;align-items:start;gap:24px;margin-bottom:22px}h1{font-size:clamp(28px,4vw,44px);letter-spacing:-.035em;line-height:1.1;margin:10px 0 16px}h2{font-size:23px;letter-spacing:-.02em;margin:0 0 12px}h3{font-size:18px;margin:0 0 10px}p{margin:0 0 12px}.sub{color:var(--muted);max-width:850px} +.download{white-space:nowrap;padding:10px 15px;border:1px solid #adc5ca;border-radius:8px;background:white;text-decoration:none;font-weight:650;font-size:13px;margin-top:22px} +.banner{padding:18px 21px;border-radius:10px;border:1px solid #d5b968;background:#fff8df;margin:0 0 20px}.banner.live{border-color:#93c8c4;background:#edf9f6}.banner strong{display:block;font-size:17px}.banner p{font-size:13px;margin:5px 0 0} +.decision{display:flex;gap:15px;align-items:center;margin:24px 0}.decision h2{margin:0}.badge{display:inline-block;border-radius:6px;padding:3px 9px;background:#edf3f5;font-size:11px;font-weight:750;text-transform:uppercase;letter-spacing:.045em;white-space:nowrap}.badge.pass{color:var(--pass);background:#e1f4e9}.badge.fail{color:var(--fail);background:#fbe9eb}.badge.inconclusive{color:var(--warn);background:#fff0c8}.badge.descriptive,.badge.not_applicable{color:var(--muted);background:#e8eff2} +.cards{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px;margin-bottom:27px}.card,.panel{border:1px solid var(--line);border-radius:11px;background:var(--paper);padding:20px}.card .label{font-size:12px;color:var(--muted)}.card .value{font-size:28px;font-weight:730;letter-spacing:-.03em;margin:7px 0 5px;overflow-wrap:anywhere}.card .detail{font-size:12px;color:var(--muted)} +.section{margin-top:30px}.section-head{display:flex;justify-content:space-between;gap:20px;align-items:center;margin-bottom:14px}.section-head h2{margin:0}.section-head p{font-size:12px;margin:0;color:var(--muted)} +.grid2{display:grid;grid-template-columns:1fr 1fr;gap:18px}.kv{display:grid;grid-template-columns:145px minmax(0,1fr);gap:8px 16px;font-size:12px}.kv dt{color:var(--muted)}.kv dd{margin:0;overflow-wrap:anywhere}.mono,code{font:12px/1.6 ui-monospace,SFMono-Regular,Consolas,monospace;overflow-wrap:anywhere}.pin{font-size:11px} +.table-wrap{overflow-x:auto}table{width:100%;border-collapse:collapse;font-size:12px;text-align:left}th{color:var(--muted);font-weight:600;font-size:11px;letter-spacing:.02em}th,td{padding:11px 10px;border-bottom:1px solid var(--line);vertical-align:top}td:first-child,th:first-child{padding-left:0}tbody tr:last-child td{border-bottom:0}.number{font-variant-numeric:tabular-nums;white-space:nowrap}.reason{color:var(--muted)} +.case{background:white;border:1px solid var(--line);border-radius:12px;margin-bottom:22px;overflow:hidden}.case-heading{padding:21px 23px 17px;border-bottom:1px solid var(--line)}.case-title{display:flex;justify-content:space-between;gap:20px;align-items:center}.case-title h3{margin:0}.case-meta{font-size:11px;color:var(--muted);margin:6px 0 10px}.prompt{font-size:14px;margin:0;overflow-wrap:anywhere}.media-grid{display:grid;grid-template-columns:1fr 1fr;gap:18px;padding:20px 23px 10px}figure{margin:0;min-width:0}.media-label{font-size:11px;font-weight:750;letter-spacing:.08em;text-transform:uppercase;margin-bottom:8px;color:var(--muted)}video{display:block;width:100%;aspect-ratio:16/9;object-fit:contain;background:#0c151e;border-radius:7px}.no-media{aspect-ratio:16/9;display:flex;flex-direction:column;justify-content:center;align-items:center;padding:20px;background:#f8eff0;border:1px dashed #e1b9bd;border-radius:7px;text-align:center;overflow-wrap:anywhere}.no-media p{font-size:12px;max-width:390px}.no-media strong{margin-bottom:8px}figcaption{font-size:11px;color:var(--muted);margin:8px 0;overflow-wrap:anywhere}.case-body{padding:5px 23px 20px}.case-body details{margin-top:12px;padding-top:10px;border-top:1px solid var(--line)}summary{cursor:pointer;font-size:12px;font-weight:650}.notes{font-size:12px;color:var(--muted);margin:12px 0 0;padding-left:19px}.notes li+li{margin-top:7px}.foot{margin-top:35px;padding-top:18px;border-top:1px solid var(--line);font-size:11px;color:var(--muted)} +@media(max-width:900px){main{padding:26px 20px 45px}.cards{grid-template-columns:1fr 1fr}.grid2{grid-template-columns:1fr}.kv{grid-template-columns:130px minmax(0,1fr)}} +@media(max-width:600px){main{padding:20px 12px 35px}header{display:block}.download{display:inline-block;margin:0 0 8px}.cards{gap:8px}.card{padding:14px}.card .value{font-size:24px}.media-grid{grid-template-columns:1fr;padding:15px}.case-heading{padding:17px 15px}.case-body{padding:5px 15px 15px}.section-head{display:block}.decision{align-items:start;flex-wrap:wrap}.kv{grid-template-columns:110px minmax(0,1fr)}} +@media print{body{background:white}main{max-width:none;padding:10px}.download{display:none}.case{break-inside:avoid}details{display:block}.banner{print-color-adjust:exact}.cards{grid-template-columns:repeat(4,1fr)}} +""" + + +def _escape(value: Any) -> str: + return html.escape(str(value), quote=True) + + +def _number(value: Any, digits: int = 3) -> str: + if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value): + return "not measured" + return f"{value:,.{digits}f}" + + +def _percent(value: Any, *, signed: bool = False) -> str: + if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value): + return "not measured" + return f"{value:+.1%}" if signed else f"{value:.1%}" + + +def _badge(status: Any) -> str: + status = str(status) + css = status if status in {"pass", "fail", "inconclusive", "descriptive", "not_applicable"} else "inconclusive" + return f'{_escape(status.replace("_", " "))}' + + +def _digest(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def _copy_artifact(source: Path, digest: str, assets: Path) -> Path: + if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): + raise ValueError("report artifact requires a valid SHA256") + source = source.resolve(strict=True) + if not source.is_file() or _digest(source) != digest: + raise ValueError("report source artifact changed after comparison") + suffix = source.suffix.lower() + if suffix not in {".mp4", ".webm", ".mov", ".mkv", ".avi"}: + suffix = ".media" + target = assets / f"{digest}{suffix}" + if target.exists() or target.is_symlink(): + if target.is_symlink() or not target.is_file() or _digest(target) != digest: + raise ValueError("existing content-addressed report asset has different content") + return target + # Exclusive creation prevents replacement of a preexisting report asset. + try: + with source.open("rb") as origin, target.open("xb") as destination: + hasher = hashlib.sha256() + for chunk in iter(lambda: origin.read(1024 * 1024), b""): + destination.write(chunk) + hasher.update(chunk) + if hasher.hexdigest() != digest: + target.unlink() + raise ValueError("report source changed while copying") + except FileExistsError: + if target.is_symlink() or _digest(target) != digest: + raise ValueError("report asset collision") + return target + + +def _check_table(checks: list[dict]) -> str: + rows = [] + for check in checks: + observed = check.get("observed") + threshold = check.get("threshold") + if check.get("exact_match"): + observed_text = "exact match" + elif observed is None: + observed_text = "—" + elif isinstance(observed, (int, float)): + observed_text = _number(observed, 4) + else: + observed_text = _escape(observed) + threshold_text = _number(threshold, 4) if threshold is not None else "—" + rows.append( + f'{_escape(check.get("name", ""))}' + f'{_badge(check.get("status", "inconclusive"))}' + f'{observed_text}{threshold_text}' + f'{_escape(check.get("reason", ""))}' + ) + return '
    ' + "".join(rows) + "
    CheckDecisionObservedThresholdExplanation
    " + + +def _transformation_label(value: Any) -> str: + if not isinstance(value, dict): + return str(value or "not recorded")[:300] + labels = { + "none_original_bytes_preserved": "Original bytes preserved", + "exact_file_copy": "Exact file copy", + "deliberate_audio_mute": "Deliberately muted audio", + } + parts = [labels.get(value.get("kind"), str(value.get("kind") or "Declared transformation").replace("_", " "))] + for key, label in ( + ("compressed_video_bitexact", "compressed video bit-exact"), + ("decoded_video_identical", "decoded video identical"), + ("audio_sample_count_preserved", "audio sample count preserved"), + ): + if key in value: + parts.append(f"{label}: {'yes' if value[key] is True else 'no' if value[key] is False else 'unverified'}") + return "; ".join(parts) + + +def _configuration(label: str, run: dict) -> str: + configuration = run.get("configuration", {}) + provenance = run.get("provenance") or {} + fields = [ + ("Run", run.get("run_id", "not recorded")), + ("Hardware", configuration.get("hardware_label", "not recorded")), + ("Runtime", configuration.get("runtime", "not recorded")), + ("Runtime revision", configuration.get("runtime_revision", "not recorded")), + ("Model", configuration.get("model_id", "not recorded")), + ("Model revision", configuration.get("model_revision", "not recorded")), + ("Identity", configuration.get("identity_verification", "operator-declared")), + ("Run SHA256", run.get("run_bundle_sha256", "not recorded")), + ] + if provenance: + fields.extend([ + ("Source", provenance.get("source_url") or provenance.get("source_filename", "not recorded")), + ("Source SHA256", provenance.get("source_sha256", "not recorded")), + ("Attribution", provenance.get("attribution_status", "unverified")), + ("Transformation", _transformation_label(provenance.get("transformation"))), + ("Contract origin", provenance.get("contract_origin", "not recorded")), + ]) + body = "".join(f'
    {_escape(name)}
    {_escape(value)}
    ' for name, value in fields) + return f'

    {_escape(label)}

    {body}
    ' + + +def _media(label: str, observation: dict) -> str: + path = observation.get("artifact_path") + media = observation.get("media") or {} + video = media.get("video") or {} + audio = media.get("audio") or {} + if path: + body = f'' + else: + reason = observation.get("error") or observation.get("analysis_error") or "No media artifact was produced." + body = f'
    No playable artifact

    {_escape(reason)}

    ' + latency = observation.get("latency_seconds") + boundary = 'downloaded media' if observation.get('latency_boundary') == 'submit_to_downloaded_media' else 'validated media' + details = [f'{_number(latency)} s submit → {boundary}' if latency is not None else 'generation timing not measured'] + if video.get("width") and video.get("height"): + details.append(f'{video["width"]}×{video["height"]} · {video.get("frame_count", "?")} frames') + if audio.get("present"): + details.append(f'{audio.get("channels", "?")} audio channels · {audio.get("sample_rate_hz", "?")} Hz') + elif media: + details.append("no audio stream") + caption = " · ".join(_escape(part) for part in details) + return f'
    {_escape(label)}
    {body}
    {caption}
    ' + + +def _case(slot: dict, *, imported: bool = False) -> str: + metrics = slot.get("metrics", {}) + psnr = "exact decoded match" if metrics.get("video_identical") is True else _number(metrics.get("video_psnr_db")) + " dB" + pairs = [ + ("Video PSNR", psnr, "same-request RGB fidelity, not generative quality"), + ("Video mean absolute error", _number(metrics.get("video_mae"), 6), "normalized RGB, 0–1"), + ("Audio spectral cosine", _number(metrics.get("audio_spectral_cosine"), 5), "spectral similarity; not semantic or perceptual quality"), + ("Audio RMS ratio", _number(metrics.get("audio_rms_ratio"), 5), "candidate ÷ baseline; threshold uses worst channel"), + ("Request latency change", _percent(metrics.get("latency_increase_fraction"), signed=True), "paired request; run-level gate uses median population"), + ] + if imported: + pairs[-1] = ("Generation latency", "Not applicable", "Imported media; no inference timing was measured") + metric_rows = "".join(f'{_escape(label)}{_escape(value)}{_escape(note)}' for label, value, note in pairs) + notes = "".join(f'
  • {_escape(note)}
    • ' for note in slot.get("notes", [])) + note_list = f'
      {notes}
    ' if notes else "" + opened = " open" if slot.get("status") != "pass" else "" + seed_label = "pairing ID seed (not a generation seed)" if imported else "seed" + return ( + '
    ' + f'

    {_escape(slot.get("case_id", "Case"))}

    {_badge(slot.get("status", "inconclusive"))}
    ' + f'
    {_escape(slot.get("slot_id", ""))} · {seed_label} {_escape(slot.get("seed", ""))} · repetition {_escape(slot.get("repetition", ""))}
    ' + f'

    {_escape(slot.get("prompt", ""))}

    ' + f'
    {_media("Baseline", slot.get("baseline", {}))}{_media("Candidate", slot.get("candidate", {}))}
    ' + f'
    {metric_rows}
    MetricObservedInterpretation
    ' + f'Per-slot checks and failure reasons{_check_table(slot.get("checks", []))}{note_list}
    ' + ) + + +def _render(comparison: dict, json_name: str) -> str: + evidence = comparison.get("evidence_kind", "unknown") + imported = comparison.get("comparison_scope") == "media_fidelity_only" + title = "H3 sample media comparison" if imported else "H3 runtime comparison" + titles = { + "fixture": ("Synthetic fixture evidence — not an H3 result", "These test clips and fixture timings exercise the harness. They do not measure MiniMax model performance or quality."), + "operator_endpoint": ("Endpoint results — H3 execution is not independently verified", "Clips were collected from a configured endpoint. Only a separate controlled-GPU job receipt can establish observed runtime, model files, GPU use, and resource cleanup."), + "live_h3": ("Legacy endpoint results — H3 execution is not independently verified", "This older bundle used the label live_h3, but its model, hardware, and runtime identities were only operator-declared. The label is not proof that H3 ran."), + "imported_media": ("Imported-media evidence — no H3 inference run or timing measurement", "These source clips demonstrate media validation and paired fidelity only. Model attribution is operator-supplied; decoded media properties are a post-hoc contract, not proof of prompt compliance."), + "mixed": ("Mixed evidence — this comparison is inconclusive", "The baseline and candidate use different evidence kinds. Do not interpret this as a controlled live-model benchmark."), + } + evidence_title, evidence_note = titles.get(evidence, ("Unverified evidence", "The report does not establish where these artifacts were generated.")) + status = comparison.get("overall_status", "inconclusive") + decision = {"pass": "Declared checks passed", "fail": "Regression checks failed", "inconclusive": "Comparison needs more evidence"}.get(status, "Comparison needs more evidence") + baseline = comparison.get("baseline", {}) + candidate = comparison.get("candidate", {}) + left, right = baseline.get("summary", {}), candidate.get("summary", {}) + summary = comparison.get("summary", {}) + measurement = comparison.get("measurement", {}) + policy = comparison.get("policy", {}) + if imported and status == "fail": + transformation = (candidate.get("provenance") or {}).get("transformation") or {} + decision = "Controlled audio defect detected" if isinstance(transformation, dict) and transformation.get("kind") == "deliberate_audio_mute" else "Media checks failed" + mode = measurement.get("performance_mode", "not recorded").replace("_", " ") + cards = [ + ("Candidate median latency", f'{_number(right.get("latency_median_seconds"))} s', f'Baseline {_number(left.get("latency_median_seconds"))} s · {_percent(measurement.get("latency_increase_fraction"), signed=True)}'), + ("Candidate verified-valid clips", f'{right.get("valid", "?")} / {right.get("scheduled", "?")}', f'{_percent(right.get("verified_technical_success_fraction", right.get("technical_success_rate")))} verified yield · failures stay in denominator'), + ("Matched valid media pairs", str(summary.get("matched_valid_pairs", "?")), f'{summary.get("measurement_slots", "?")} scheduled measurement slots · warmups excluded'), + ("Candidate throughput", f'{_number(right.get("valid_clips_per_second"), 4)}', 'valid clips / recorded measurement wall second · not saturation capacity'), + ] + if imported: + cards = [ + ("Generation latency", "Not measured", "Imported clips do not establish model speed"), + ("Candidate valid media", f'{right.get("valid", "?")} / {right.get("scheduled", "?")}', "Full-stream decode and declared media contract"), + ("Matched valid media pairs", str(summary.get("matched_valid_pairs", "?")), "Same-source paired fidelity; not a model-quality score"), + ("Generation throughput", "Not measured", "No inference job or hardware utilization was measured"), + ] + card_html = "".join(f'
    {_escape(label)}
    {_escape(value)}
    {_escape(detail)}
    ' for label, value, detail in cards) + unavailable = int(left.get("evaluator_unavailable", 0)) + int(right.get("evaluator_unavailable", 0)) + unavailable_note = ( + f'' + if unavailable else "" + ) + policy_rows = [ + ("Policy", policy.get("policy_id")), + ("Calibration", policy.get("calibration_status")), + ("Max median latency increase", _percent(policy.get("max_latency_increase_fraction"))), + ("Min video PSNR", _number(policy.get("min_video_psnr_db")) + " dB (exact matches pass separately)"), + ("Min audio spectral cosine", _number(policy.get("min_audio_spectral_cosine"), 5)), + ("Max channel RMS ratio error", _number(policy.get("max_audio_rms_ratio_error"), 5)), + ] + policy_body = "".join(f'
    {_escape(label)}
    {_escape(value)}
    ' for label, value in policy_rows) + measurement_rows = [ + ("Plan", comparison.get("plan_id")), + ("Plan SHA256", comparison.get("plan_sha256")), + ("Timing boundary", measurement.get("boundary")), + ("Concurrency", measurement.get("concurrency")), + ("Performance use", mode), + ("Timing evidence", measurement.get("timing_evidence", "see evidence kind and source run")), + ("Statistical claim", measurement.get("statistical_claim")), + ] + measurement_body = "".join(f'
    {_escape(label)}
    {_escape(value)}
    ' for label, value in measurement_rows) + limitations = "".join(f'
  • {_escape(note)}
    • ' for note in comparison.get("limitations", [])) + cases = "".join(_case(slot, imported=imported) for slot in comparison.get("slots", [])) + banner_class = "banner live" if evidence == "live_h3" else "banner" + return ( + '' + '' + f'{title} · evidence report' + f'
    Video generation benchmark · execution MVP
    ' + f'

    {title}

    ' + ('A paired, artifact-backed view of an imported source clip and controlled transformations. Generation performance is not measured.' if imported else 'A paired, artifact-backed view of media validity, implementation fidelity, and recorded end-to-end performance.') + '

    ' + f'Download evidence JSON ↗
    ' + f'' + f'
    {_badge(status)}

    {_escape(decision)}

    ' + f'

    Policy calibration: {_escape(policy.get("calibration_status", "unknown"))}. No release qualification is claimed. Performance mode: {_escape(mode)}.

    ' + f'
    {card_html}
    {unavailable_note}' + f'

    Side-by-side evidence

    Use each player’s controls to inspect video and native audio

    {cases}
    ' + '

    Configurations and pins

    Supplied by the operator · artifacts checked against SHA256

    ' + f'
    {_configuration("Baseline", baseline)}{_configuration("Candidate", candidate)}
    ' + f'

    Run-level decisions

    Missing values are not treated as zero

    {_check_table(comparison.get("checks", []))}
    ' + '

    Declared policy and measurement

    ' + f'

    Explicit thresholds

    {policy_body}

    Measurement boundary

    {measurement_body}
    ' + f'

    What this report does not claim

      {limitations}
    ' + f'
    Generated {_escape(comparison.get("created_at", ""))}. Report version {_escape(comparison.get("bundle_version", ""))}. ' + 'This report is read-only and contains no scripts or remote dependencies. Share the HTML, its comparison JSON, and the adjacent assets folder together.
    ' + '
    ' + ) + + +def write_report(comparison: dict, output_path: Path) -> None: + """Write HTML, downloadable JSON, and immutable content-addressed media. + + Artifact hashes are checked again immediately before copying. Existing assets + are reused only when their content matches. Existing HTML/JSON, including + symlinks, is never overwritten; exclusive creation also guards racing writes. + """ + if comparison.get("bundle_type") != "mvp_comparison": + raise ValueError("write_report requires an MVP comparison") + output_path = Path(output_path).absolute() + if output_path.suffix.lower() not in {".html", ".htm"}: + raise ValueError("report output_path must have an .html or .htm extension") + json_path = output_path.with_suffix(".comparison.json") + for target in (output_path, json_path): + if target.exists() or target.is_symlink(): + raise FileExistsError(f"report output already exists: {target}") + assets = output_path.parent / f"{output_path.stem}_assets" + if assets.is_symlink(): + raise ValueError("report assets directory must not be a symlink") + output_path.parent.mkdir(parents=True, exist_ok=True) + created: list[tuple[Path, tuple[int, int]]] = [] + try: + # Reserve both files before making assets. A post-preflight collision must + # not clobber either file or produce an apparently complete report. + with ExitStack() as stack: + streams = {} + for target in (output_path, json_path): + stream = stack.enter_context(target.open("x", encoding="utf-8")) + metadata = os.fstat(stream.fileno()) + created.append((target, (metadata.st_dev, metadata.st_ino))) + streams[target] = stream + assets.mkdir(exist_ok=True) + portable = copy.deepcopy(comparison) + for slot in portable.get("slots", []): + for label in ("baseline", "candidate"): + observation = slot.get(label, {}) + source = observation.get("artifact_path") + if source: + asset = _copy_artifact(Path(source), str(observation.get("sha256", "")), assets) + observation["artifact_path"] = asset.relative_to(output_path.parent).as_posix() + observation["artifact_path_base"] = "report_directory" + if isinstance(observation.get("media"), dict) and "path" in observation["media"]: + observation["media"]["path"] = observation["artifact_path"] + for label in ("baseline", "candidate"): + provenance = portable.get(label, {}).get("provenance") + if isinstance(provenance, dict) and provenance.get("source_path"): + provenance.setdefault("source_filename", Path(str(provenance["source_path"])).name) + del provenance["source_path"] + portable["report"] = {"html": output_path.name, "portable_assets": assets.name, "scripts": False} + payload = json.dumps(portable, indent=2, ensure_ascii=False, allow_nan=False) + "\n" + document = _render(portable, json_path.name) + streams[json_path].write(payload) + streams[output_path].write(document) + except Exception: + for target, identity in created: + try: + metadata = target.stat(follow_symlinks=False) + if (metadata.st_dev, metadata.st_ino) == identity: + target.unlink() + except FileNotFoundError: + pass + raise diff --git a/experimental/video-generation/evaluator/mvp_result.py b/experimental/video-generation/evaluator/mvp_result.py new file mode 100644 index 0000000000..4e21f051d0 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_result.py @@ -0,0 +1,362 @@ +"""Versioned frontend export of a verified H3 CI artifact; never runs inference.""" + +from __future__ import annotations + +import hashlib +import html +import math +import json +import re +import time +from datetime import datetime, timezone +from pathlib import Path +from html.parser import HTMLParser +from urllib.parse import unquote, urlsplit + +from .mvp_gpu_evidence import verify_measurement_job +from .mvp_gpu_report import _Tree, _no_symlink_parents, _pairs, _relative, _reject_constant +from .mvp_power import analyze_power +from .mvp_runner import _summary + +VERSION = "1.0.0" +ROLES = ("baseline", "candidate") +DEFINITIONS = { + "latency": {"unit": "s", "window": "request submission through downloaded, technically validated media", "population": "valid measured clips only; excludes warmup and startup"}, + "valid_clips_per_second": {"unit": "clip/s", "window": "recorded measured block including failed attempts; serving mode ends at final delivery/transport failure and excludes subsequent local validation", "definition": "valid measured clips divided by measured-block wall seconds; not saturation throughput"}, + "serving": {"unit": "per-field: seconds, clip/s, video-second/s, fraction, count", "definition": "closed-loop submission through downloaded media; raw samples and percentile floors retained; deadline goodput requires technical validity, not calibrated perceptual fidelity"}, + "completion": {"unit": "clip", "definition": "scheduled, attempted, completed, valid, failed and not-started measured slots; warmup separate"}, + "gpu_memory": {"unit": "MiB", "window": "role or client workload including warmup, as labelled", "definition": "maximum observed device-used memory per selected GPU, not exact allocator peaks"}, + "gpu_power": {"unit": "W", "window": "startup, warmup, and measured submit-to-terminal generation windows separately", "definition": "timestamped GPU-board sensor watts; aggregate sums selected GPUs; average is integrated energy / covered window duration; peak is observed samples"}, + "gpu_energy": {"unit": "J", "window": "same separately bounded power windows", "definition": "trapezoidal integral of timestamped GPU-board watts; includes GPU idle board draw, excludes CPU/node energy"}, + "gpu_energy_per_valid_clip": {"unit": "J/clip", "window": "serial request windows or one serving interval from first submission to last observed terminal, including idle gaps", "definition": "integrated board energy / technically valid measured clips; concurrent requests are not summed; null for invalid coverage or zero valid clips"}, + "media_integrity": {"unit": "per-check units in original media record", "definition": "full-stream video/audio decode, geometry, timestamps, cadence, duration, motion and sound checks; not prompt adherence or human quality"}, + "paired_fidelity": {"unit": "PSNR dB, spectral cosine, absolute RMS ratio error", "definition": "aligned original decoded baseline/candidate outputs; exact video match has null finite PSNR and exact_match=true; not generative quality"}, + "hardware_tdp": {"unit": "W/GPU", "definition": "verified hardware specification, separate from configured/enforced limits and observed watts; generic H200 name does not identify form factor"}, +} + + +def _json(tree: _Tree, name: str) -> dict: + with tree.open(name, 8 * 1024 * 1024) as stream: + value = json.load(stream, object_pairs_hook=_pairs, parse_constant=_reject_constant) + if not isinstance(value, dict): + raise ValueError(f"{name}: expected a JSON object") + return value + + +def _hash(tree: _Tree, name: str) -> str: + with tree.open(name, 512 * 1024 * 1024) as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def _inventory(root: Path, tree: _Tree) -> tuple[dict, str | None]: + inventory = {} + for path in root.rglob("*"): + if path.is_symlink(): + raise ValueError("Artifact contains a symlink") + if path.is_file() and path != root / "SHA256SUMS": + relative = path.relative_to(root).as_posix() + inventory[relative] = _hash(tree, relative) + if len(inventory) > 10000: + raise ValueError("Artifact file inventory exceeds 10000 files") + if not (root / "SHA256SUMS").exists(): + return inventory, None + expected = {} + with tree.open("SHA256SUMS", 2 * 1024 * 1024) as stream: + for line in stream.read().decode().splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) + if not match: + raise ValueError("Malformed artifact checksum entry") + digest, name = match.groups() + _relative(name) + if name in expected: + raise ValueError("Duplicate artifact checksum path") + expected[name] = digest + if inventory != expected: + raise ValueError("Artifact checksum mismatch or incomplete file inventory") + return inventory, _hash(tree, "SHA256SUMS") + + +def _join(tree: _Tree, manifest: dict, ci: dict, binding: dict, spec: dict, inventory: dict, source_ci: dict | None, producer: dict) -> dict: + if manifest.get("schema_version") != 1 or ci.get("schema_version") != 1: + raise ValueError("Unsupported source manifest or CI version") + for path, digest in manifest.get("evidence", {}).items(): + if inventory.get(path) != digest: + raise ValueError(f"Manifest evidence hash mismatch: {path}") + if (manifest.get("git_commit") != ci.get("source_sha") or manifest.get("run_id") != ci.get("run_id") + or manifest.get("run_attempt") != ci.get("run_attempt") or manifest.get("ci") != ci.get("ci")): + raise ValueError("Execution Git/CI provenance differs between receipts") + job = str(binding.get("job_id")) + step = f"{job}.{binding.get('step_id')}" + allocation = _json(tree, "allocation.json") + if (manifest.get("slurm_allocation") != allocation or ci.get("allocation") != allocation + or allocation.get("identity", {}).get("JobId") != job + or ci.get("slurm_job", {}).get("JobId") != job + or ci.get("slurm_job", {}).get("NodeList") != binding.get("node") + or ci.get("step_cleanup") != {"status": "ended", "step_id": step} + or binding.get("gpu_uuids") != spec["gpu_uuids"] + or spec["allocation"].get("label") != f"Slurm {step} on {binding.get('node')}"): + raise ValueError("Slurm allocation/step/node/GPU provenance mismatch") + if ci.get("allocation_cleanup", {}).get("status") not in {"released", "retained"}: + raise ValueError("Allocation cleanup disposition is unavailable") + if manifest.get("workload_plan") != spec["plan"]: + raise ValueError("Outer workload plan differs from verified GPU plan") + source = {"git_commit": manifest["git_commit"], "run_id": manifest["run_id"], "run_attempt": manifest["run_attempt"], **manifest["ci"]} + if source_ci is not None: + if (str(source_ci.get("databaseId")) != str(source["run_id"]) + or str(source_ci.get("runAttempt")) != str(source["run_attempt"]) + or source_ci.get("headSha") != source["git_commit"] or source_ci.get("url") != source["run_url"]): + raise ValueError("Trusted GitHub metadata does not match source execution identity") + current = producer.get("ci", {}) + completed = source_ci.get("status") == "completed" and source_ci.get("conclusion") == "success" + same_run = (source_ci.get("status") == "in_progress" and source_ci.get("conclusion") is None + and producer.get("mode") == "same_run_export" and producer.get("git_commit") == source["git_commit"] + and all(str(current.get(field)) == str(source[field]) for field in ("run_id", "run_attempt", "repository"))) + if not (completed or same_run): + raise ValueError("Source workflow is neither successful nor the exact currently exporting run") + jobs = [job for job in source_ci.get("jobs", []) if "H3 video H200 smoke" in job.get("name", "")] + if len(jobs) != 1 or jobs[0].get("status") != "completed" or jobs[0].get("conclusion") != "success": + raise ValueError("Trusted source CI lacks one successful completed H3 workload job") + source["workflow_status_at_export"] = source_ci["status"] + source["workflow_conclusion_at_export"] = source_ci.get("conclusion") + source["ci_accepted"] = ci.get("ci_accepted", False) + source["external_ci_verification"] = "passed" if source_ci is not None else "not_supplied" + return source + + +def _report_references(tree: _Tree, inventory: dict) -> None: + class Links(HTMLParser): + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + for key, value in attrs: + if key not in {"src", "href"} or not value: + continue + link = urlsplit(value) + if link.scheme or link.netloc: + raise ValueError("Portable report contains an external media or asset reference") + if link.path: + relative = "report/" + str(_relative(unquote(link.path))) + if relative not in inventory: + raise ValueError(f"Portable report reference is missing: {relative}") + with tree.open("report/index.html", 8 * 1024 * 1024) as stream: + Links().feed(stream.read().decode()) + + +def _records(run: dict, role: str) -> list[dict]: + return [{key: record.get(key) for key in ("slot_id", "case_id", "seed", "repetition", "phase", "status", "attempted", "latency_seconds", "submit_to_terminal_seconds", "submit_to_media_seconds", "media_validation_seconds", "media", "error", "job_id", "outcome", "provider_status", "submit_to_accepted_seconds", "server_timings", "timing_window")} | { + "media_file": {"path": f"gpu/{role}/{record['artifact_path']}", "sha256": record["sha256"]} if record.get("artifact_path") else None, + } for record in run["records"]] + + +def _metrics(run: dict, role: dict) -> dict: + records = [record for record in run["records"] if record["phase"] == "measurement"] + valid = [record for record in records if record["status"] == "succeeded" and (record.get("media") or {}).get("valid") is True] + latencies = [record["latency_seconds"] for record in valid] + summary = _summary(run["records"], len(records), run["measurement"]["wall_seconds"]) + telemetry = role["telemetry_summary"] + return { + "status": "valid", "latency_seconds": {"values": latencies, "sample_count": summary["latency_samples"], + **{key: summary[f"latency_{key}_seconds"] for key in ("mean", "median", "min", "max", "sample_stddev")}}, + "valid_clips_per_second": summary["valid_clips_per_second"], "measurement": run["measurement"], + "serving": run.get("serving"), + "completion": {key: summary[key] for key in ("scheduled", "completed", "valid", "failed", "failed_attempts", "invalid_completed", "not_started")} | { + "attempted": sum(row["attempted"] for row in records), "technical_success_fraction": summary["technical_success_rate"]}, + "startup_seconds": role.get("startup_seconds"), + "gpu_memory": {"role_observed_peak_mib_by_gpu": telemetry["observed_memory_peak_mib_by_gpu"], + "client_including_warmup_observed_peak_mib_by_gpu": telemetry.get("measurement_observed_memory_peak_mib_by_gpu"), + "client_window": telemetry.get("measurement_window"), "start_monotonic_seconds": telemetry.get("measurement_window_start_monotonic_seconds"), + "end_monotonic_seconds": telemetry.get("measurement_window_end_monotonic_seconds"), "coverage_qualified": telemetry["qualified"]}, + } + + +def _power_limits(roles: dict, gpu_uuids: list[str]) -> dict: + """Retain contemporaneous endpoints without claiming continuous settings.""" + fields = ("configured_limit_w", "enforced_limit_w", "default_limit_w", "maximum_limit_w") + result = {"status": "unavailable", "watts_by_gpu": None, "by_role": {}, + "scope": "role prelaunch and postcleanup snapshots; no continuous stability claim", + "source": "gpu/gpu-job.json", "reason": "Not recorded in original execution; later probes cannot backfill historical settings"} + statuses = [] + for label, role in roles.items(): + snapshots = {} + for when in ("before", "after"): + observation = role.get(f"power_configuration_{when}") + value = {"status": "unavailable", "observed_at": None, "gpus": None, "reason": "not recorded"} + if observation is not None: + try: + stamp = datetime.fromisoformat(observation["observed_at"]) + boundary = datetime.fromisoformat(role["started_at" if when == "before" else "finished_at"]) + devices = observation["gpus"] + if (observation.get("status") != "recorded" or stamp.tzinfo is None or boundary.tzinfo is None + or (when == "before" and stamp > boundary) or (when == "after" and stamp < boundary) + or sorted(device.get("uuid") for device in devices) != sorted(gpu_uuids) + or any(field not in device or (device[field] is not None and (type(device[field]) not in (int, float) + or not math.isfinite(device[field]) or device[field] <= 0)) for device in devices for field in fields)): + raise ValueError("power-limit snapshot has unavailable fields or invalid time/device/value binding") + missing = any(device[field] is None for device in devices for field in fields) + value = {"status": "partial" if missing else "recorded", "observed_at": observation["observed_at"], + "gpus": [{"uuid": device["uuid"], **{field: device[field] for field in fields}} for device in devices], + "reason": "some limit readings unavailable" if missing else None} + except (KeyError, TypeError, ValueError) as error: + value["reason"] = str(error) + snapshots[when] = value + statuses.append(value["status"]) + before, after = snapshots["before"], snapshots["after"] + snapshots["same_observed_values"] = (sorted(before["gpus"], key=lambda item: item["uuid"]) == sorted(after["gpus"], key=lambda item: item["uuid"]) + if before["status"] == after["status"] == "recorded" else None) + result["by_role"][label] = snapshots + if any(status in {"recorded", "partial"} for status in statuses): + result.update(status="recorded" if all(status == "recorded" for status in statuses) else "partial", reason=None) + return result + + +def _hardware_profile(hardware: dict, profile: dict | None) -> None: + if profile is None: + return + if (profile.get("schema_version") != 1 or profile.get("observation_kind") != "read_only_inventory" + or sorted(profile.get("gpu_uuids", [])) != sorted(hardware["gpu_uuids"]) + or sorted(profile.get("slurm", {}).get("gpu_uuids", [])) != sorted(hardware["gpu_uuids"])): + raise ValueError("Supplemental hardware inventory does not match the measured physical GPU UUIDs") + hardware["later_hardware_observation"] = profile + tdp = profile.get("tdp", {}) + watts = tdp.get("watts_per_gpu") + if (tdp.get("status") == "verified" and type(watts) in (int, float) and math.isfinite(watts) and watts > 0 + and tdp.get("hardware_variant") and str(tdp.get("source_url", "")).startswith("https://") and tdp.get("evidence")): + hardware["tdp"] = tdp | {"applies_to": "same physical GPU UUIDs; specification identity, not historic configured limits"} + + +def _power_report(result: dict) -> str: + escape = lambda value: html.escape(str(value), quote=True) + rows = [] + for role, record in result["roles"].items(): + for phase, values in record["power"]["phases"].items(): + aggregate = values.get("aggregate") or {} + per_gpu = values.get("per_gpu") or {} + average = "; ".join(f"{uuid}: {gpu['avg_power_w']:.1f}" for uuid, gpu in per_gpu.items()) or "withheld" + fractions = [item["coverage"]["coverage_fraction"] for item in record["power"]["windows"] if item["phase"] == phase and item.get("coverage")] + coverage = f"{min(fractions):.1%} minimum" if fractions else "unavailable" + ratio = values.get("tdp_comparison", {}).get("aggregate_average_fraction_of_tdp") + tdp_fraction = f"{ratio:.1%}" if ratio is not None else "unavailable" + cells = [role, phase, values["status"], average, aggregate.get("avg_power_w"), tdp_fraction, + aggregate.get("observed_peak_power_w"), aggregate.get("joules_per_valid_clip"), + f"{values['valid_window_count']}/{values['window_count']}; {coverage}", ", ".join(values["invalid_reasons"]) or "none"] + rows.append("" + "".join("" + escape(f"{cell:.3f}" if isinstance(cell, float) else "withheld" if cell is None else cell) + "" for cell in cells) + "") + status = f"Export: {result['status']}; workload: {result['workload_status']}; regression: {result['regression_status']}" + return ("" + "H3 measured GPU power

    H3 measured GPU power

    " + escape(status) + + "

    Original video and fidelity report · Frontend result · " + "Baseline samples and coverage · Candidate samples and coverage

    " + "

    GPU-board power for the selected devices. Generation covers submission to observed provider completion; download and local media validation are excluded. " + "Startup and warmup are separate. A missing or unbracketed window withholds its power and energy. Peaks are observed sensor samples.

    " + "" + "" + + "".join(rows) + "
    RolePhaseValidityPer-GPU average WAggregate average WAverage / spec TDPObserved aggregate peak WJ / valid clipValid / total windowsWithholding reasons
    Hardware identity, TDP and separately observed power configuration
    "
+            + escape(json.dumps(result["hardware"], indent=2)) + "

    " + escape(" ".join(result["limitations"])) + "

    \n") + + +def write_result(root: Path, *, producer: dict, source_ci: dict | None = None, hardware_profile: dict | None = None) -> dict: + """Add a frontend manifest to a copied bundle. Save failure status, then raise. + + Call before refreshing SHA256SUMS. ``producer`` identifies this exporter, + while ``source_ci`` is independently obtained GitHub metadata for execution. + Existing result/power files are rejected rather than silently overwritten. + """ + root = Path(root).absolute() + _no_symlink_parents(root) + if (root / "result.json").exists() or (root / "power").exists() or (root / "power-report.html").exists(): + raise ValueError("Export requires a source bundle without result.json or power outputs") + result = {"schema_version": VERSION, "bundle_type": "h3_benchmark_result", "created_at": datetime.now(timezone.utc).isoformat(), + "producer": dict(producer), "status": "failed", "invalid_reasons": [], "workload_status": "unknown", + "regression_status": "inconclusive", "release_qualified": False, "definitions": DEFINITIONS, + "execution": None, "hardware": None, "workload": None, "policy": None, "roles": {}, "paired_fidelity": None, + "files": [], "report": None, "report_links": {}, "checksums": {"path": "SHA256SUMS", "scope": "final artifact files excluding SHA256SUMS; refreshed by publisher after export"}, + "limitations": ["Point estimates from the frozen A/A schedule; no significance or performance improvement claim.", + "Workload execution success is separate from regression calibration and release qualification.", + "GPU-board energy excludes CPU, DRAM, other allocated GPUs and whole-node power.", + "Matched LLM workload, topology, precision, warmup, phase boundaries, sampling and hardware limits are required before architectural comparisons."]} + error = None + try: + if not isinstance(producer, dict) or not re.fullmatch(r"[0-9a-f]{40}", producer.get("git_commit", "")): + raise ValueError("Exporter producer requires its exact Git commit") + result["producer"].update(exporter_source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + power_analyzer_source_sha256=hashlib.sha256(Path(analyze_power.__code__.co_filename).read_bytes()).hexdigest()) + tree = _Tree(root) + inventory, checksums_sha = _inventory(root, tree) + for label in ROLES: + for name in ("runtime.stdout.log", "runtime.stderr.log", "client.stderr.log", "client.stdout.json"): + if f"gpu/supervisor/{label}/{name}" not in inventory: + raise ValueError(f"Required {label} runtime/client log is missing: {name}") + manifest, ci, binding = [_json(tree, name) for name in ("manifest.json", "ci.json", "binding.json")] + verified = verify_measurement_job(root / "gpu", deadline=time.monotonic() + 120) + spec, receipt, comparison = verified["spec"], verified["receipt"], verified["comparison"] + source = _join(tree, manifest, ci, binding, spec, inventory, source_ci, producer) + result["execution"] = {"ci": source, "slurm": binding, "source_input_checksums_sha256": checksums_sha, + "source_manifest": {"path": "manifest.json", "sha256": inventory["manifest.json"]}, "model": receipt["model_identity"], + "runtime": {role: {key: receipt["roles"][role]["source_identity"].get(key) for key in ("revision", "source_sha256", "python_sha256", "python_version", "packages")} for role in ROLES}, + "supervisor_source_sha256": receipt.get("supervisor_source_sha256"), "cleanup_status": receipt["cleanup_status"]} + allocation = re.search(r"(?:^|,)gres/gpu=(\d+)(?:,|$)", ci["slurm_job"].get("AllocTRES", "")) + result["hardware"] = {"selected_gpu_count": len(spec["gpu_uuids"]), "reserved_gpu_count": int(allocation[1]) if allocation else None, + "gpu_uuids": spec["gpu_uuids"], "devices": receipt["roles"]["baseline"]["telemetry_summary"]["gpu_identity"], + "tdp": {"status": "unavailable", "watts_per_gpu": None, "reason": "No verified form-factor-specific TDP evidence in source execution"}, + "configured_power_limits": _power_limits(receipt["roles"], spec["gpu_uuids"])} + _hardware_profile(result["hardware"], hardware_profile) + result["workload"] = {"plan": spec["plan"], "plan_sha256": receipt["plan_sha256"], "server": spec["server"], "comparison": "same_revision_A/A" if spec["baseline"]["revision"] == spec["candidate"]["revision"] else "baseline_candidate"} + result["execution"]["deployment"] = { + "scope": "single_supervised_endpoint", "replica_count": 1, + "gpus_per_replica": len(spec["gpu_uuids"]), "server_configuration": spec["server"], + "serving": spec.get("serving"), "configured_batch_size": None, "observed_batch_sizes": None, + "full_deployment_cost_usd_per_hour": None, + } + result["policy"] = spec["policy"] + (root / "power").mkdir() + for label in ROLES: + role, run = receipt["roles"][label], verified["runs"][label] + with tree.open(f"gpu/{role['telemetry_path']}", 128 * 1024 * 1024) as stream: + samples = [json.loads(line, object_pairs_hook=_pairs, parse_constant=_reject_constant) for line in stream] + with tree.open(f"gpu/{label}/events.jsonl", 16 * 1024 * 1024) as stream: + events = [json.loads(line, object_pairs_hook=_pairs, parse_constant=_reject_constant) for line in stream] + power = analyze_power(role, run, samples, events, spec["gpu_uuids"], interval_seconds=spec["limits"]["telemetry_interval_seconds"]) + tdp = result["hardware"]["tdp"] + if tdp["status"] == "verified": + denominator = tdp["watts_per_gpu"] * len(spec["gpu_uuids"]) + for phase in power["phases"].values(): + aggregate = phase.get("aggregate") + phase["tdp_comparison"] = {"status": "valid" if aggregate else "withheld", "specification_watts_per_gpu": tdp["watts_per_gpu"], + "aggregate_average_fraction_of_tdp": aggregate["avg_power_w"] / denominator if aggregate else None, + "observed_aggregate_peak_fraction_of_tdp": aggregate["observed_peak_power_w"] / denominator if aggregate else None, + "interpretation": "descriptive ratios only; TDP is not a measured or configured power limit"} + power_name = f"power/{label}.json" + (root / power_name).write_text(json.dumps(power, indent=2, allow_nan=False) + "\n") + inventory[power_name] = _hash(tree, power_name) + result["roles"][label] = {"run_id": run["run_id"], "metrics": _metrics(run, role), "records": _records(run, label), + "power": {"path": power_name, "sha256": inventory[power_name], "status": power["status"], "schema_version": power["schema_version"], "phases": power["phases"], + "windows": [{key: window.get(key) for key in ("phase", "slot_id", "request_slot_ids", "coverage", "timing_source", "timing_uncertainty_seconds", "invalid_reasons")} for window in power["windows"]]}, + "raw_telemetry": {"path": f"gpu/{role['telemetry_path']}", "sha256": role["telemetry_sha256"]}, + "media_evaluator": run["configuration"].get("media_evaluator")} + result["paired_fidelity"] = {"source": "gpu/comparison.json", "summary": comparison.get("summary"), "checks": comparison["checks"], + "slots": [{key: slot.get(key) for key in ("slot_id", "case_id", "seed", "repetition", "status", "metrics", "checks")} for slot in comparison["slots"]]} + if "report/index.html" not in inventory: + raise ValueError("Portable report is missing") + _report_references(tree, inventory) + result["report"] = {"path": "report/index.html", "sha256": inventory["report/index.html"]} + result["files"] = [{"path": name, "sha256": digest} for name, digest in sorted(inventory.items())] + result["regression_status"] = receipt.get("regression_status", "inconclusive") + complete = all(run["summary"]["valid"] == run["summary"]["scheduled"] > 0 for run in verified["runs"].values()) + source_exit = (manifest.get("exit_code"), ci.get("exit_code"), _json(tree, "step-result.json").get("exit_code")) + result["workload_status"] = "passed" if complete and all(code == 0 for code in source_exit) else "failed" + if result["workload_status"] != "passed": + raise ValueError("Original workload or Slurm payload has unsuccessful exit status") + result["status"] = "complete" + (root / "power-report.html").write_text(_power_report(result)) + inventory["power-report.html"] = _hash(tree, "power-report.html") + result["report_links"] = {"original": result["report"], "power": {"path": "power-report.html", "sha256": inventory["power-report.html"]}} + result["files"] = [{"path": name, "sha256": digest} for name, digest in sorted(inventory.items())] + except (OSError, ValueError, KeyError, TypeError) as caught: + error = caught + result["invalid_reasons"].append(str(caught)) + result["status"] = "failed" + for role in result["roles"].values(): + role["metrics"] = {"status": "withheld", "reason": "Artifact export failed validation; consult original evidence"} + (root / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") + if error is not None: + raise ValueError(f"H3 result export failed: {error}") from error + return result diff --git a/experimental/video-generation/evaluator/mvp_runner.py b/experimental/video-generation/evaluator/mvp_runner.py new file mode 100644 index 0000000000..64424c2d2b --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_runner.py @@ -0,0 +1,781 @@ +"""Auditable serial and closed-loop serving client for an operator-managed H3 video endpoint. + +This module does not launch a server, download weights, verify server identity, +or contact MiniMax's paid API. ``preview_plan`` is network-free; ``run_plan`` +submits real requests to the explicitly supplied endpoint. Local HTTP fixtures +can exercise that transport, but are not evidence of H3 inference. + +Protocol implementations were inspected at SGLang 253020450290328e9deb307eece1e402fa17f35e +and vLLM-Omni eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c. They are deliberately +separate: SGLang accepts the H3 JSON contract, while vLLM-Omni expects multipart +form fields. Those source pins do not attest to a supplied server's revision. +""" + +from __future__ import annotations + +import hashlib +import http.client +import json +import math +import os +import platform +import re +import socket +import ssl +import statistics +import threading +import time +import urllib.parse +import uuid +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .mvp_serving import settings as serving_settings, summarize as serving_summary + + +MODEL_ID = "MiniMaxAI/MiniMax-H3" +RUNTIMES = {"sglang", "vllm-omni"} +MAX_JSON_BYTES = 1024 * 1024 +MAX_MEDIA_BYTES = 512 * 1024 * 1024 +MAX_SLOTS = 10000 +MAX_POLLS = 10000 +SOCKET_TIMEOUT_SECONDS = 30.0 +POLL_INTERVAL_SECONDS = 0.5 +_CHUNK_BYTES = 64 * 1024 +_EVENT_LOCK = threading.Lock() +_PROTOCOL_SOURCES = { + "sglang": { + "repository": "https://github.com/sgl-project/sglang", + "revision": "253020450290328e9deb307eece1e402fa17f35e", + "files": [ + "python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py", + "python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/video_adapter.py", + ], + }, + "vllm-omni": { + "repository": "https://github.com/vllm-project/vllm-omni", + "revision": "eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c", + "files": [ + "vllm_omni/entrypoints/openai/api_server.py", + "vllm_omni/entrypoints/openai/serving_video.py", + ], + }, +} + + +def canonical_json_bytes(value: Any) -> bytes: + """Stable hash representation, UTF-8, sorted keys, no trailing newline.""" + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).encode("utf-8") + + +def _digest(value: Any) -> str: + return hashlib.sha256(canonical_json_bytes(value)).hexdigest() + + +def _positive(value: Any, name: str, *, maximum: float, integer: bool = False) -> None: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not 0 < value <= maximum + or not math.isfinite(value) + or (integer and not isinstance(value, int)) + ): + raise ValueError(f"{name} must be a positive {'integer' if integer else 'number'} <= {maximum}") + + +def _text(value: Any, name: str, *, limit: int = 1000) -> None: + if not isinstance(value, str) or not value.strip() or len(value) > limit: + raise ValueError(f"{name} must be a nonempty string of at most {limit} characters") + + +def validate_plan(plan: dict[str, Any]) -> dict[str, Any]: + """Validate the supported frozen T2VA subset without making a request. + + This is the execution safety contract; the broader study/registry schemas + remain separate. Unknown root metadata is preserved in the plan digest. + """ + if not isinstance(plan, dict): + raise ValueError("plan must be an object") + try: + frozen = json.loads(canonical_json_bytes(plan)) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError("plan must contain finite JSON values") from exc + for field in ("plan_id", "model_id", "model_revision"): + _text(frozen.get(field), field) + if frozen["model_id"] != MODEL_ID: + raise ValueError(f"this narrow MVP supports only {MODEL_ID}") + if not re.fullmatch(r"[a-fA-F0-9]{40}", frozen["model_revision"]): + raise ValueError("model_revision must be an immutable 40-character commit") + generation = frozen.get("generation") + if not isinstance(generation, dict): + raise ValueError("generation must be an object with explicit controls") + allowed_controls = { + "duration_seconds", "aspect_ratio", "width", "height", "frame_count", "fps", + "audio_sample_rate_hz", "audio_channels", "num_inference_steps", "flow_shift", "audio_flow_shift", + } + if set(generation) - allowed_controls: + raise ValueError("unsupported generation controls must not be silently ignored") + for field, maximum in ( + ("width", 8192), ("height", 8192), ("frame_count", 10000), + ("fps", 240), ("audio_sample_rate_hz", 192000), + ("audio_channels", 8), ("num_inference_steps", 1000), + ): + _positive(generation.get(field), f"generation.{field}", maximum=maximum, integer=True) + for field, maximum in (("duration_seconds", 15), ("flow_shift", 1000), ("audio_flow_shift", 1000)): + _positive(generation.get(field), f"generation.{field}", maximum=maximum) + if generation["duration_seconds"] < 4: + raise ValueError("H3 duration_seconds must be between 4 and 15") + _text(generation.get("aspect_ratio"), "generation.aspect_ratio", limit=20) + if not re.fullmatch(r"[1-9][0-9]?:[1-9][0-9]?", generation["aspect_ratio"]): + raise ValueError("generation.aspect_ratio must be an explicit positive ratio") + if generation["fps"] != 24 or generation["audio_sample_rate_hz"] != 32000 or generation["audio_channels"] != 2: + raise ValueError("the H3 contract requires 24 fps and 32000 Hz stereo audio") + # H3 rounds delivery frames differently for these two requested durations. + audited_frames = {4: 107, 8: 192} + audited_cell = { + "aspect_ratio": "16:9", "width": 1344, "height": 768, + } + if (any(generation[key] != value for key, value in audited_cell.items()) + or audited_frames.get(generation["duration_seconds"]) != generation["frame_count"]): + raise ValueError("this MVP requires a 16:9, 1344x768 H3 cell: 4s/107 frames or 8s/192 frames") + cases = frozen.get("cases") + if not isinstance(cases, list) or not cases: + raise ValueError("cases must be a nonempty array") + ids: set[str] = set() + for case in cases: + if not isinstance(case, dict): + raise ValueError("every case must be an object") + if set(case) - {"case_id", "prompt", "seed", "requires_motion", "requires_sound"}: + raise ValueError("unsupported per-case controls must not be silently ignored") + _text(case.get("case_id"), "case_id", limit=200) + _text(case.get("prompt"), "prompt", limit=32000) + if case["case_id"] in ids: + raise ValueError("case_id must be unique") + ids.add(case["case_id"]) + seed = case.get("seed") + if isinstance(seed, bool) or not isinstance(seed, int) or not 0 <= seed < 2**63: + raise ValueError("case.seed must be an integer in [0, 2**63)") + for field in ("requires_motion", "requires_sound"): + if not isinstance(case.get(field), bool): + raise ValueError(f"case.{field} must be an explicit boolean") + _positive(frozen.get("repetitions"), "repetitions", maximum=MAX_SLOTS, integer=True) + warmups = frozen.get("warmup_runs") + if isinstance(warmups, bool) or not isinstance(warmups, int) or warmups < 0: + raise ValueError("warmup_runs must be an explicit nonnegative integer") + if len(cases) * frozen["repetitions"] + warmups > MAX_SLOTS: + raise ValueError(f"plan exceeds {MAX_SLOTS} total requests") + return frozen + + +def _slots(plan: dict[str, Any]) -> list[dict[str, Any]]: + slots = [] + for index in range(1, plan["warmup_runs"] + 1): + slots.append({**plan["cases"][(index - 1) % len(plan["cases"])], + "slot_id": f"warmup-{index:03d}", "phase": "warmup", "repetition": 0}) + for repetition in range(1, plan["repetitions"] + 1): + for index, case in enumerate(plan["cases"], 1): + slots.append({**case, "slot_id": f"measurement-r{repetition:03d}-c{index:03d}", + "phase": "measurement", "repetition": repetition}) + return slots + + +def _payload(plan: dict[str, Any], slot: dict[str, Any], runtime: str) -> dict[str, Any]: + generation = plan["generation"] + target = { + "duration_seconds": generation["duration_seconds"], + "aspect_ratio": generation["aspect_ratio"], + "short_edge": min(generation["width"], generation["height"]), + } + common = { + "model": plan["model_id"], "prompt": slot["prompt"], "seed": slot["seed"], + "num_inference_steps": generation["num_inference_steps"], + "flow_shift": generation["flow_shift"], + } + if runtime == "sglang": + # H3's SGLang admission explicitly rejects caller-set fps/num_frames; + # its target resolves both. Expected dimensions are verified on delivery. + return {**common, "task": "t2va", "conditions": [], "target": target, + "audio_flow_shift": generation["audio_flow_shift"]} + return { + **common, "width": generation["width"], "height": generation["height"], + "fps": generation["fps"], "num_frames": generation["frame_count"], + "num_outputs_per_prompt": 1, + "extra_params": {"task": "t2va", "target": target, + "audio_flow_shift": generation["audio_flow_shift"]}, + } + + +def preview_plan(plan: dict[str, Any], *, runtime: str = "sglang") -> dict[str, Any]: + """Return exact semantic payloads and counts, with no disk/network effects.""" + if runtime not in RUNTIMES: + raise ValueError("runtime must be sglang or vllm-omni") + frozen = validate_plan(plan) + return { + "evidence_kind": "request_preview_no_generation", "plan_id": frozen["plan_id"], + "plan_sha256": _digest(frozen), "runtime": runtime, + "measurement_count": len(frozen["cases"]) * frozen["repetitions"], + "warmup_count": frozen["warmup_runs"], + "total_requests": len(frozen["cases"]) * frozen["repetitions"] + frozen["warmup_runs"], + "slots": [{**slot, "request": _payload(frozen, slot, runtime)} for slot in _slots(frozen)], + } + + +class _RequestError(RuntimeError): + """A safe, locally composed message (never an HTTP response body).""" + + +def _endpoint(value: str) -> tuple[str, urllib.parse.SplitResult, str]: + try: + parts = urllib.parse.urlsplit(value) + port = parts.port + except (TypeError, ValueError) as exc: + raise ValueError("endpoint must be a valid absolute HTTP(S) base URL") from exc + if ( + parts.scheme not in {"http", "https"} or not parts.hostname + or parts.username is not None or parts.password is not None + or parts.query or parts.fragment or any(char.isspace() for char in value) + ): + raise ValueError("endpoint requires HTTP(S) and may not include credentials, query, or fragment") + if parts.hostname.lower() == "api.minimax.io" or parts.hostname.lower().endswith(".minimaxi.com"): + raise ValueError("the MVP runner supports operator-managed endpoints, not the paid hosted MiniMax API") + if port is not None and not 1 <= port <= 65535: + raise ValueError("endpoint port is invalid") + base = value.rstrip("/") + path = parts.path.rstrip("/") + api_path = path if path.endswith("/v1/videos") else path + ("/videos" if path.endswith("/v1") else "/v1/videos") + return base, parts, api_path + + +def _remaining(deadline: float) -> float: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("attempt deadline exceeded") + return remaining + + +def _multipart(payload: dict[str, Any]) -> tuple[bytes, str]: + boundary = "vgbench-" + uuid.uuid4().hex + chunks: list[bytes] = [] + for name, value in payload.items(): + text = json.dumps(value, ensure_ascii=False, allow_nan=False) if isinstance(value, (dict, list)) else str(value) + chunks.append((f"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n" + text + "\r\n").encode("utf-8")) + chunks.append(f"--{boundary}--\r\n".encode("ascii")) + return b"".join(chunks), f"multipart/form-data; boundary={boundary}" + + +def _open_response( + parts: urllib.parse.SplitResult, method: str, path: str, *, deadline: float, + credential: str | None, body: bytes | None = None, content_type: str | None = None, +) -> tuple[http.client.HTTPConnection, http.client.HTTPResponse, threading.Timer]: + """No redirect following, environment proxies, cookies, or server URLs. + + A deadline watchdog shuts down an established socket even when a peer + trickles response headers. DNS is bounded in a daemon resolver; its late + result is discarded before any request can be sent. + """ + result: list[Any] = [] + resolved = threading.Event() + + def resolve() -> None: + try: + result.append(socket.getaddrinfo(parts.hostname, parts.port or (443 if parts.scheme == "https" else 80), type=socket.SOCK_STREAM)) + except OSError: + result.append(None) + finally: + resolved.set() + + threading.Thread(target=resolve, daemon=True).start() + if not resolved.wait(min(SOCKET_TIMEOUT_SECONDS, _remaining(deadline))): + raise TimeoutError("endpoint DNS resolution timed out") + if not result[0]: + raise _RequestError("endpoint DNS resolution failed") + cls = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection + connection = cls(parts.hostname, parts.port, timeout=min(SOCKET_TIMEOUT_SECONDS, _remaining(deadline))) + connected_socket = None + for family, socktype, proto, _, address in result[0]: + candidate = socket.socket(family, socktype, proto) + try: + candidate.settimeout(min(SOCKET_TIMEOUT_SECONDS, _remaining(deadline))) + candidate.connect(address) + connected_socket = candidate + break + except OSError: + candidate.close() + if connected_socket is None: + raise _RequestError("endpoint connection failed") + connection.sock = connected_socket + + def interrupt_socket() -> None: + try: + connected_socket.shutdown(socket.SHUT_RDWR) + except OSError: + pass + connection.close() + + try: + timer = threading.Timer(_remaining(deadline), interrupt_socket) + except BaseException: + connection.close() + raise + timer.daemon = True + timer.start() + try: + if parts.scheme == "https": + # Use system trust roots and hostname verification, never disable TLS. + context = ssl.create_default_context() + context.minimum_version = ssl.TLSVersion.TLSv1_2 + connection.sock = context.wrap_socket(connected_socket, server_hostname=parts.hostname) + connected_socket = connection.sock + connection.sock.settimeout(min(SOCKET_TIMEOUT_SECONDS, _remaining(deadline))) + headers = {"Accept": "application/json, video/mp4", "User-Agent": "vgbench-mvp/0.1.0"} + if content_type: + headers["Content-Type"] = content_type + if credential: + headers["Authorization"] = "Bearer " + credential + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + if not 200 <= response.status < 300: + raise _RequestError(f"{method} received HTTP {response.status}; response body omitted") + return connection, response, timer + except BaseException: + timer.cancel() + connection.close() + raise + + +def _transfer( + parts: urllib.parse.SplitResult, method: str, path: str, *, deadline: float, + credential: str | None, body: bytes | None = None, content_type: str | None = None, + destination: Path | None = None, +) -> bytes | str: + connection, response, timer = _open_response( + parts, method, path, deadline=deadline, credential=credential, + body=body, content_type=content_type, + ) + limit = MAX_MEDIA_BYTES if destination else MAX_JSON_BYTES + sink = None + temporary = destination.with_suffix(".mp4.part") if destination else None + try: + length = response.getheader("Content-Length") + if length is not None and (not length.isdigit() or int(length) > limit): + raise _RequestError("response Content-Length is invalid or exceeds byte limit") + if response.getheader("Content-Encoding", "identity").lower() != "identity": + raise _RequestError("compressed HTTP responses are not accepted") + sink = temporary.open("xb") if temporary else None + hasher = hashlib.sha256() + chunks: list[bytes] = [] + size = 0 + while True: + if connection.sock: + connection.sock.settimeout(min(SOCKET_TIMEOUT_SECONDS, _remaining(deadline))) + _remaining(deadline) + chunk = response.read1(min(_CHUNK_BYTES, limit - size + 1)) + _remaining(deadline) + if not chunk: + break + size += len(chunk) + if size > limit: + raise _RequestError("response exceeds byte limit") + if sink: + sink.write(chunk) + hasher.update(chunk) + else: + chunks.append(chunk) + if length is not None and size != int(length): + raise _RequestError("response body is truncated") + if not size: + raise _RequestError("empty response body") + if sink: + sink.flush() + os.fsync(sink.fileno()) + sink.close() + sink = None + temporary.replace(destination) + return hasher.hexdigest() + return b"".join(chunks) + finally: + if sink: + sink.close() + timer.cancel() + response.close() + connection.close() + + +def _json_request(parts: urllib.parse.SplitResult, method: str, path: str, **kwargs: Any) -> dict[str, Any]: + data = _transfer(parts, method, path, **kwargs) + try: + value = json.loads(data) + except (ValueError, TypeError, UnicodeDecodeError) as exc: + raise _RequestError("endpoint returned invalid JSON") from exc + if not isinstance(value, dict): + raise _RequestError("endpoint returned a non-object JSON response") + return value + + +def _timestamp() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _write_json(path: Path, value: Any, *, exclusive: bool = False) -> None: + target = path if exclusive else path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp") + with target.open("xb") as handle: + handle.write(canonical_json_bytes(value)) + handle.flush() + os.fsync(handle.fileno()) + if not exclusive: + target.replace(path) + + +def _event(path: Path, event: str, **fields: Any) -> None: + with _EVENT_LOCK, path.open("ab") as handle: + handle.write(canonical_json_bytes({"event": event, "at": _timestamp(), **fields}) + b"\n") + handle.flush() + os.fsync(handle.fileno()) + + +def _safe_error(exc: BaseException) -> str: + if isinstance(exc, (TimeoutError, socket.timeout)): + return "attempt timed out; server completion may be unknown" + if isinstance(exc, _RequestError): + return str(exc) + if isinstance(exc, KeyboardInterrupt): + return "interrupted by operator; server completion may be unknown" + # Provider/body text, URL query strings, credentials, and arbitrary exception + # messages are deliberately excluded from durable traces. + return f"{type(exc).__name__}: local transport or media analysis failed; details omitted" + + +def _summary(records: list[dict[str, Any]], scheduled: int, wall: float) -> dict[str, Any]: + measured = [record for record in records if record["phase"] == "measurement"] + completed = [record for record in measured if record["status"] == "succeeded"] + valid = [record for record in completed if record["media"]["valid"] is True] + latencies = [record["latency_seconds"] for record in valid] + stage_medians = {} + for field in ("submit_to_terminal_seconds", "submit_to_media_seconds", "media_validation_seconds"): + values = [record.get(field) for record in valid] + # Never silently calculate a successful-subset timing from missing + # stages. The request denominator and missing measurements stay visible. + stage_medians[field.removesuffix("_seconds") + "_median_seconds"] = ( + statistics.median(values) if values and all(value is not None for value in values) else None + ) + return { + "scheduled": scheduled, "completed": len(completed), "valid": len(valid), + "failed": scheduled - len(valid), + "failed_attempts": sum(record["status"] == "failed" and record.get("attempted", False) for record in measured), + "not_started": sum(not record.get("attempted", False) for record in measured), + "invalid_completed": len(completed) - len(valid), + "technical_success_rate": len(valid) / scheduled, + "latency_samples": len(latencies), + "latency_median_seconds": statistics.median(latencies) if latencies else None, + "latency_mean_seconds": statistics.mean(latencies) if latencies else None, + "latency_min_seconds": min(latencies) if latencies else None, + "latency_max_seconds": max(latencies) if latencies else None, + "latency_sample_stddev_seconds": statistics.stdev(latencies) if len(latencies) > 1 else None, + **stage_medians, + "valid_clips_per_second": len(valid) / wall if wall > 0 else None, + } + + +def run_plan( + plan: dict[str, Any], output_dir: Path, *, endpoint: str, runtime: str = "sglang", + runtime_revision: str, hardware_label: str, model_revision: str, + timeout_seconds: float = 3600, api_key_env: str | None = None, + serving_concurrency: int | None = None, delivery_deadline_seconds: float | None = None, +) -> dict[str, Any]: + """Execute a serial or explicit closed-loop run, retaining every outcome. + + ``completed`` counts completed downloads/analyses; ``valid`` additionally + requires media-contract success. ``failed = scheduled - valid`` includes + invalid media and slots not started after uncertain remote completion. + Latency is conditional on valid clips and is never reported without the + all-scheduled technical-success denominator. No submission is retried. + + A client timeout does not cancel server work. To prevent such an unknown + job from violating the declared concurrency, remaining slots are not submitted. + Native media decoding observes a cooperative, not preemptive, deadline. + """ + frozen = validate_plan(plan) + serving = serving_settings(serving_concurrency, delivery_deadline_seconds) + if runtime not in RUNTIMES: + raise ValueError("runtime must be sglang or vllm-omni") + for field, value in (("runtime_revision", runtime_revision), ("hardware_label", hardware_label), ("model_revision", model_revision)): + _text(value, field) + if not re.fullmatch(r"[a-fA-F0-9]{40}", runtime_revision): + raise ValueError("runtime_revision must be an operator-declared immutable 40-character commit") + if model_revision != frozen["model_revision"]: + raise ValueError("operator model_revision does not match the frozen plan") + _positive(timeout_seconds, "timeout_seconds", maximum=86400) + safe_endpoint, parts, api_path = _endpoint(endpoint) + credential = None + if api_key_env: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", api_key_env): + raise ValueError("api_key_env must be an environment-variable name") + credential = os.environ.get(api_key_env) + if not credential or any(char in credential for char in "\r\n"): + raise ValueError("configured API-key environment variable is missing or invalid") + if parts.scheme != "https" and parts.hostname not in {"127.0.0.1", "::1", "localhost"}: + raise ValueError("authenticated non-loopback endpoints require HTTPS") + # Resolve dependency availability before generating anything or creating a + # run directory. The analyzer is imported lazily so preview is lightweight. + import av + import numpy + from .mvp_media import IMPLEMENTATION_VERSION, __file__ as media_source_file, analyze_media + + media_evaluator = { + "implementation_version": IMPLEMENTATION_VERSION, + "source_sha256": hashlib.sha256(Path(media_source_file).read_bytes()).hexdigest(), + "pyav_version": av.__version__, + "numpy_version": numpy.__version__, + "ffmpeg_libraries": { + name: ".".join(map(str, version)) for name, version in av.library_versions.items() + }, + } + + directory = Path(output_dir) + directory.mkdir(parents=True, exist_ok=False) + (directory / "artifacts").mkdir() + (directory / "requests").mkdir() + plan_sha256 = _digest(frozen) + _write_json(directory / "plan.json", frozen, exclusive=True) + configuration: dict[str, Any] = { + "runtime": runtime, "runtime_revision": runtime_revision, + "hardware_label": hardware_label, "model_id": frozen["model_id"], + "model_revision": model_revision, "endpoint": safe_endpoint, + "identity_verification": "operator_declared", "plan_sha256": plan_sha256, + "protocol_source": _PROTOCOL_SOURCES[runtime], + "client_source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "media_evaluator": media_evaluator, + "server_identity_caveat": "Model weights, runtime revision, hardware and server launch flags are not remotely verified.", + "client_environment": {"python": platform.python_version(), "system": platform.system(), "machine": platform.machine(), + "av": av.__version__, "numpy": numpy.__version__}, + "limits": {"attempt_timeout_seconds": timeout_seconds, "socket_timeout_seconds": SOCKET_TIMEOUT_SECONDS, + "max_json_bytes": MAX_JSON_BYTES, "max_media_bytes": MAX_MEDIA_BYTES, + "max_polls": MAX_POLLS, "poll_interval_seconds": POLL_INTERVAL_SECONDS}, + "hash_convention": "SHA256 of sorted-key compact UTF-8 JSON; configuration hash excludes configuration_sha256", + "measurement_semantics": { + "latency": "monotonic submit to downloaded and validated media, including polling, transfer and client analysis; not GPU kernel latency", + "submit_to_terminal_seconds": "monotonic submit to first observed terminal provider status; includes queueing, generation, encoding, HTTP and polling delay, not GPU kernel latency", + "submit_to_media_seconds": "monotonic submit to complete downloaded media; excludes local media validation", + "media_validation_seconds": "local full-stream media analysis wall time; not model inference", + "latency_population": "valid measured clips only; paired with all-scheduled technical success rate", + "throughput": "valid measured clips divided by serial measured-block wall time; not saturated server capacity", + "failed": "scheduled minus technically valid, including invalid completed media and unstarted slots", + "warmups": "recorded, excluded from measurement counts and measured wall time", + "deadline": "network watchdog plus cooperative media deadline; one native decoder call may overrun", + }, + } + if serving: + from . import mvp_serving + configuration["serving"] = serving + configuration["serving_source_sha256"] = hashlib.sha256(Path(mvp_serving.__file__).read_bytes()).hexdigest() + configuration["measurement_semantics"]["throughput"] = "valid clips / closed-loop delivery wall time, including failures and download; local media validation occurs after delivery" + configuration["measurement_semantics"]["deadline"] = "separate bounded transport and local validation phases; unknown remote completion stops new submissions" + configuration["configuration_sha256"] = _digest(configuration) + _write_json(directory / "configuration.json", configuration, exclusive=True) + slots = _slots(frozen) + scheduled = len(frozen["cases"]) * frozen["repetitions"] + run: dict[str, Any] = { + "bundle_version": "0.1.0", "bundle_type": "mvp_run", "run_id": "h3-" + uuid.uuid4().hex, + "plan_id": frozen["plan_id"], "plan_sha256": plan_sha256, "plan": frozen, + "configuration": configuration, "evidence_kind": "operator_endpoint", + "evidence_caveat": "Operator-managed HTTP endpoint; the client alone cannot establish H3 execution. Model identity is operator-declared, not attested. Mock-server tests are not H3 evidence. Controlled GPU execution requires a separate verified supervisor receipt.", + "started_at": _timestamp(), "finished_at": None, "status": "partial", + "measurement": {"boundary": "submit_to_validated_media", "concurrency": 1, + "warmup_runs": frozen["warmup_runs"], "wall_seconds": 0.0}, + "records": [], "summary": _summary([], scheduled, 0.0), + } + if serving: + run["measurement"].update(boundary="submit_to_downloaded_media", concurrency=serving["concurrency"]) + journal = directory / "events.jsonl" + _write_json(directory / "run.json", run) + measured_start = None + abort_reason = None + abort_code = "not_started_after_uncertain_remote_completion" + interrupted = None + def validate_media(record: dict, deadline: float) -> None: + validation_started = time.monotonic() + try: + media = analyze_media(directory / record["artifact_path"], {**record["expected_media"], "timeout_seconds": _remaining(deadline)}) + finally: + record["media_validation_seconds"] = time.monotonic() - validation_started + if not isinstance(media, dict) or not isinstance(media.get("valid"), bool): + raise _RequestError("media analyzer returned an invalid contract result") + canonical_json_bytes(media) + _remaining(deadline) + record.update(status="succeeded", media=media, outcome="completed" if media["valid"] else "invalid_media") + + def attempt(slot: dict, *, defer_validation: bool = False) -> dict: + nonlocal abort_reason, interrupted + expected = {**frozen["generation"], "requires_motion": slot["requires_motion"], + "requires_sound": slot["requires_sound"], "audio_required": True} + expected["duration_seconds"] = expected["frame_count"] / expected["fps"] + record: dict[str, Any] = { + **{key: slot[key] for key in ("slot_id", "case_id", "prompt", "seed", "repetition", "phase")}, + "status": "failed", "artifact_path": None, "sha256": None, + "latency_seconds": 0.0, "media": None, "error": None, + "submit_to_terminal_seconds": None, "submit_to_media_seconds": None, + "media_validation_seconds": None, + "attempted": False, "expected_media": expected, "outcome": "not_started", + "job_id": None, "provider_status": None, "submit_to_accepted_seconds": None, + "server_timings": None, + } + if abort_reason: + record["error"] = abort_code + _event(journal, "slot_not_started", slot_id=slot["slot_id"], reason=record["error"]) + else: + payload = _payload(frozen, slot, runtime) + _write_json(directory / "requests" / (slot["slot_id"] + ".json"), payload, exclusive=True) + body, content_type = (canonical_json_bytes(payload), "application/json") if runtime == "sglang" else _multipart(payload) + if len(body) > MAX_JSON_BYTES: + raise ValueError("request exceeds byte limit") + # fsync the intent BEFORE the first byte can leave this client. + _event(journal, "attempt_started", slot_id=slot["slot_id"], payload_sha256=_digest(payload)) + record["attempted"] = True + record["outcome"] = "transport_error" + start = time.monotonic() + record["timing_window"] = {"start_monotonic_seconds": start, "start_utc": _timestamp(), + "terminal_monotonic_seconds": None, "end_monotonic_seconds": None} + deadline = start + timeout_seconds + remote_terminal = False + try: + reply = _json_request(parts, "POST", api_path, deadline=deadline, + credential=credential, body=body, content_type=content_type) + record["submit_to_accepted_seconds"] = time.monotonic() - start + identifier = reply.get("id") + if not isinstance(identifier, str) or not re.fullmatch(r"[A-Za-z0-9_-][A-Za-z0-9_.-]{0,199}", identifier): + raise _RequestError("submission returned a missing or unsafe job identifier") + record["job_id"] = identifier + _event(journal, "job_submitted", slot_id=slot["slot_id"], job_id=identifier) + for poll in range(MAX_POLLS + 1): + status = reply.get("status") + record["provider_status"] = status if isinstance(status, str) and status in {"queued", "pending", "in_progress", "processing", "running", "failed", "cancelled", "canceled", "completed", "succeeded", "success"} else None + if status in {"failed", "cancelled", "canceled"}: + record["outcome"] = "provider_failed" if status == "failed" else "provider_cancelled" + remote_terminal = True + record["submit_to_terminal_seconds"] = time.monotonic() - start + record["timing_window"]["terminal_monotonic_seconds"] = start + record["submit_to_terminal_seconds"] + raise _RequestError(f"provider job reported {status}; provider text omitted") + if status in {"completed", "succeeded", "success"}: + remote_terminal = True + record["submit_to_terminal_seconds"] = time.monotonic() - start + record["timing_window"]["terminal_monotonic_seconds"] = start + record["submit_to_terminal_seconds"] + break + if status not in {"queued", "pending", "in_progress", "processing", "running"}: + raise _RequestError("provider job returned an unsupported status") + if poll == MAX_POLLS: + raise _RequestError("job exceeded maximum polling requests") + time.sleep(min(POLL_INTERVAL_SECONDS, _remaining(deadline))) + reply = _json_request(parts, "GET", api_path + "/" + identifier, + deadline=deadline, credential=credential) + artifact = directory / "artifacts" / (slot["slot_id"] + ".mp4") + record["sha256"] = _transfer(parts, "GET", api_path + "/" + identifier + "/content", + deadline=deadline, credential=credential, destination=artifact) + record["artifact_path"] = artifact.relative_to(directory).as_posix() + record["submit_to_media_seconds"] = time.monotonic() - start + record["outcome"] = "downloaded" + if not defer_validation: + validate_media(record, deadline) + except (Exception, KeyboardInterrupt) as exc: + # The deadline watchdog closes sockets, which can surface as EOF instead of socket.timeout. + if not isinstance(exc, KeyboardInterrupt) and record["outcome"] not in {"provider_failed", "provider_cancelled"} and time.monotonic() >= deadline: + exc = TimeoutError("attempt deadline exceeded") + record["error"] = _safe_error(exc) + if isinstance(exc, (TimeoutError, socket.timeout)): + record["outcome"] = "timed_out" + elif isinstance(exc, KeyboardInterrupt): + record["outcome"] = "interrupted" + elif record["outcome"] == "downloaded": + record["outcome"] = "validation_error" + if not remote_terminal: + abort_reason = record["error"] + if isinstance(exc, KeyboardInterrupt): + interrupted = exc + abort_reason = record["error"] + finally: + record["latency_seconds"] = max(0.0, time.monotonic() - start) + record["timing_window"]["end_monotonic_seconds"] = start + record["latency_seconds"] + if defer_validation: + record["timing_window"]["transport_end_monotonic_seconds"] = time.monotonic() + _event(journal, "transport_finished", slot_id=record["slot_id"]) + return record + + order = {slot["slot_id"]: index for index, slot in enumerate(slots)} + + def retain(record: dict) -> None: + if record["attempted"]: + _event(journal, "attempt_finished", record=record) + run["records"].append(record) + run["records"].sort(key=lambda item: order[item["slot_id"]]) + if not serving: + run["measurement"]["wall_seconds"] = max(0.0, time.monotonic() - measured_start) if measured_start is not None else 0.0 + run["summary"] = _summary(run["records"], scheduled, run["measurement"]["wall_seconds"]) + _write_json(directory / "run.json", run) + + serial_slots = slots if not serving else [slot for slot in slots if slot["phase"] == "warmup"] + for slot in serial_slots: + if slot["phase"] == "measurement" and measured_start is None: + measured_start = time.monotonic() + record = attempt(slot) + if slot["phase"] == "warmup" and not (record["status"] == "succeeded" and record["media"]["valid"] is True): + abort_reason = abort_reason or "warmup failed technical media contract" + abort_code = "not_started_after_failed_warmup" + retain(record) + if serving: + measured_start = time.monotonic() + run["measurement"]["start_monotonic_seconds"] = measured_start + # Keep CPU validation off transport workers so it cannot throttle offered concurrency. + with ThreadPoolExecutor(max_workers=serving["concurrency"]) as pool: + futures = [pool.submit(attempt, slot, defer_validation=True) for slot in slots if slot["phase"] == "measurement"] + pending = set(futures) + while pending: + try: + done, _ = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED) + except KeyboardInterrupt as exc: + interrupted, abort_reason = exc, "interrupted by operator" + continue + for future in done: + pending.remove(future) + record = future.result() + if record["outcome"] == "downloaded": + _event(journal, "validation_started", slot_id=record["slot_id"]) + try: + validate_media(record, time.monotonic() + timeout_seconds) + except (Exception, KeyboardInterrupt) as exc: + record["error"] = _safe_error(exc) + record["outcome"] = "timed_out" if isinstance(exc, (TimeoutError, socket.timeout)) else "validation_error" + if isinstance(exc, KeyboardInterrupt): + record["outcome"] = "interrupted" + interrupted, abort_reason = exc, record["error"] + finally: + record["latency_seconds"] = time.monotonic() - record["timing_window"]["start_monotonic_seconds"] + record["timing_window"]["end_monotonic_seconds"] = record["timing_window"]["start_monotonic_seconds"] + record["latency_seconds"] + retain(record) + end = max((r.get("timing_window", {}).get("transport_end_monotonic_seconds", measured_start) + for r in run["records"] if r["phase"] == "measurement"), default=measured_start) + if end == measured_start: + end = time.monotonic() + run["measurement"].update(end_monotonic_seconds=end, wall_seconds=end - measured_start) + run["serving"] = serving_summary(run) + else: + run["measurement"]["wall_seconds"] = max(0.0, time.monotonic() - measured_start) if measured_start is not None else 0.0 + wall = run["measurement"]["wall_seconds"] + warmup_records = [record for record in run["records"] if record["phase"] == "warmup"] + run["measurement"]["warmup_qualified"] = bool(warmup_records) and all( + record["status"] == "succeeded" and record["media"]["valid"] is True for record in warmup_records + ) + run["measurement"]["warmup_status"] = ( + "not_requested" if not warmup_records else ("qualified" if run["measurement"]["warmup_qualified"] else "failed") + ) + run["summary"] = _summary(run["records"], scheduled, wall) + run["finished_at"] = _timestamp() + run["status"] = "complete" if run["summary"]["valid"] == scheduled else ("partial" if run["summary"]["valid"] else "failed") + run["abort_reason"] = abort_reason + _event(journal, "run_finished", status=run["status"], summary=run["summary"]) + _write_json(directory / "run.json", run) + if interrupted: + raise interrupted + return run diff --git a/experimental/video-generation/evaluator/mvp_serving.py b/experimental/video-generation/evaluator/mvp_serving.py new file mode 100644 index 0000000000..71896fdf38 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_serving.py @@ -0,0 +1,106 @@ +"""Closed-loop serving measurements using the existing H3 request records.""" + +from __future__ import annotations + +import math +import statistics + + +def settings(concurrency: int | None, deadline_seconds: float | None = None) -> dict | None: + if concurrency is None: + if deadline_seconds is not None: + raise ValueError("delivery deadline requires --serving-concurrency") + return None + if type(concurrency) is not int or not 1 <= concurrency <= 32: + raise ValueError("serving concurrency must be an integer in [1, 32]") + if deadline_seconds is not None and ( + type(deadline_seconds) not in (int, float) or not math.isfinite(deadline_seconds) + or not 0 < deadline_seconds <= 86400 + ): + raise ValueError("delivery deadline must be finite and in (0, 86400] seconds") + return {"mode": "closed_loop", "concurrency": concurrency, + "delivery_deadline_seconds": deadline_seconds} + + +def _finite(value: object) -> bool: + return type(value) in (int, float) and math.isfinite(value) + + +def validate_window(run: dict) -> int: + """Verify wall time and observed client concurrency from request intervals.""" + load = run["configuration"].get("serving") + if not isinstance(load, dict) or load != settings(load.get("concurrency"), load.get("delivery_deadline_seconds")): + raise ValueError("invalid serving configuration") + window = run["measurement"] + start, end, wall = (window.get(key) for key in ("start_monotonic_seconds", "end_monotonic_seconds", "wall_seconds")) + if (window.get("boundary") != "submit_to_downloaded_media" or window.get("concurrency") != load["concurrency"] + or not all(_finite(value) for value in (start, end, wall)) or end <= start + or abs(end - start - wall) > 1e-6): + raise ValueError("invalid serving measurement window") + events = [] + if not isinstance(run.get("records"), list): + raise ValueError("serving records must be a list") + for record in run["records"]: + if not isinstance(record, dict): + raise ValueError("serving request must be an object") + if record["phase"] != "measurement" or not record["attempted"]: + continue + timing = record.get("timing_window", {}) + if not isinstance(timing, dict): + raise ValueError("serving request lacks a transport window") + begin, finish = timing.get("start_monotonic_seconds"), timing.get("transport_end_monotonic_seconds") + if (not all(_finite(value) for value in (begin, finish)) or not start <= begin < finish <= end + or record.get("submit_to_media_seconds") is not None and ( + not _finite(record["submit_to_media_seconds"]) or record["submit_to_media_seconds"] < 0 + or begin + record["submit_to_media_seconds"] > finish + 1e-6)): + raise ValueError("request transport falls outside serving window") + events.extend(((begin, 1), (finish, -1))) + active = peak = 0 + for _, delta in sorted(events, key=lambda item: (item[0], item[1])): + active += delta + peak = max(peak, active) + if peak > load["concurrency"]: + raise ValueError("observed requests exceed declared serving concurrency") + return peak + + +def summarize(run: dict) -> dict: + load = run["configuration"]["serving"] + records = [r for r in run["records"] if r["phase"] == "measurement"] + valid = [r for r in records if r["status"] == "succeeded" and (r.get("media") or {}).get("valid") is True] + values = [r.get("submit_to_media_seconds") for r in valid] + complete = bool(values) and all(_finite(value) and value >= 0 for value in values) + values = sorted(values) if complete else [] + wall = run["measurement"]["wall_seconds"] + deadline = load["delivery_deadline_seconds"] + on_time = sum(value <= deadline for value in values) if deadline is not None and (complete or not valid) else None + durations = [(r.get("media") or {}).get("video", {}).get("duration_seconds") for r in valid] + seconds = sum(durations) if all(_finite(value) and value > 0 for value in durations) else None + return { + **load, "capacity_qualified": False, + "client_ready_latency_seconds": { + "values": values, "sample_count": len(values), "valid_clip_count": len(valid), + "population": "technically valid measured requests; failures remain in completion counts", + "p50": statistics.median(values) if values else None, + "p90": values[math.ceil(len(values) * .9) - 1] if len(values) >= 10 else None, + "p95": values[math.ceil(len(values) * .95) - 1] if len(values) >= 20 else None, + "quantile_method": "nearest_rank for P90/P95; sample floors 10/20 are not statistical qualification", + }, + "submitted": sum(r["attempted"] for r in records), + "outcomes": {name: sum(r.get("outcome") == name for r in records) for name in ( + "completed", "invalid_media", "provider_failed", "provider_cancelled", "timed_out", + "transport_error", "validation_error", "interrupted", "not_started")}, + "observed_submission_rate_per_second": sum(r["attempted"] for r in records) / wall if wall > 0 else None, + "offered_request_rate_per_second": None, + "peak_client_in_flight": validate_window(run), + "valid_video_seconds_per_second": seconds / wall if seconds is not None and wall > 0 else None, + "deadline_met_valid_clips": on_time, + "deadline_attainment_fraction": on_time / len(records) if on_time is not None and records else None, + "deadline_goodput_clips_per_second": on_time / wall if on_time is not None and wall > 0 else None, + "queue_delay_seconds": None, "server_ready_latency_seconds": None, + "observed_batch_sizes": None, + "limitations": ["Closed-loop delivery load, not a fixed arrival-rate or sustainable-capacity test.", + "Local validation follows delivery and is excluded from the throughput window.", + "Client polling observations are not server-side queue or execution timestamps.", + "Deadline goodput requires technical validity, not calibrated perceptual quality."], + } diff --git a/experimental/video-generation/evaluator/mvp_serving_smoke.py b/experimental/video-generation/evaluator/mvp_serving_smoke.py new file mode 100644 index 0000000000..7a08483b9f --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_serving_smoke.py @@ -0,0 +1,130 @@ +"""Bounded C1/C2/C4 smoke using the existing supervisor and raw run contract.""" + +from __future__ import annotations + +import copy +import html +import json +import time +from pathlib import Path +from urllib.parse import quote + +from . import mvp_gpu_job as gpu +from .mvp_gpu_evidence import verify_measurement_job +from .mvp_report import _CSS, _number +from .mvp_serving import settings +from .mvp_runner import _summary + +CONCURRENCIES = (1, 2, 4) + + +def validate_spec(spec: dict) -> dict: + frozen = gpu.validate_gpu_job(spec) + if not frozen.get("serving") or len(frozen["plan"]["cases"]) * frozen["plan"]["repetitions"] != 4: + raise ValueError("serving smoke requires exactly four measured requests per configuration") + return frozen + + +def _report(root: Path, matrix: dict) -> None: + rows, media = [], [] + for cell in matrix["cells"]: + summary = cell["completion"] + metrics = cell.get("metrics") or {} + rows.append("" + "".join(f"{html.escape(str(value))}" for value in ( + cell["concurrency"], cell["status"], summary["scheduled"], summary["attempted"], + summary["valid"], summary["failed"], summary["not_started"], + _number(metrics.get("client_ready_p50_seconds")), _number(metrics.get("valid_clips_per_second")), + )) + "") + if cell.get("verified"): + run = gpu._read(root / cell["run"]["path"]) + base = Path(cell["run"]["path"]).parent + for record in run["records"]: + if record["phase"] == "measurement" and record.get("artifact_path"): + url = "../" + quote((base / record["artifact_path"]).as_posix(), safe="/") + label = html.escape(f"C{cell['concurrency']} · {record['slot_id']} · {record['outcome']}") + media.append(f'
    {label}
    ') + report = root / "report" + report.mkdir(exist_ok=True) + (report / "index.html").write_text( + '' + f'H3 serving smoke

    H3 serving smoke

    ' + '

    One hardware configuration; four measured requests at each concurrency. Warmups are separate. ' + 'Latency is submit → downloaded media for technically valid clips. Throughput is valid clips / delivery wall seconds. ' + 'Four samples do not establish P90/P95 or sustainable capacity. Failed and unstarted requests remain counted.

    ' + '

    Download summary and raw-evidence links

    ' + '
    ' + '' + '' + ''.join(rows) + '
    ConcurrencyStatusScheduledAttemptedValidFailedNot startedDelivery median (s)Valid clips/s
    ' + '
    Frozen workload and runtime
    '
+        + html.escape(json.dumps({key: matrix[key] for key in ("plan", "runtime", "gpu_uuids")}, indent=2))
+        + '
    ' + ''.join(media) + '
    ', encoding="utf-8") + + +def run_matrix(spec: dict, root: Path) -> dict: + from .mvp_power import analyze_power + + spec = validate_spec(spec) + deadline = time.monotonic() + spec["limits"]["job_seconds"] + matrix = {"schema_version": "1.0.0", "bundle_type": "h3_serving_smoke_matrix", "status": "running", + "started_at": gpu._now(), "plan": spec["plan"], "runtime": spec["baseline"], "gpu_uuids": spec["gpu_uuids"], + "scheduled": 12, "warmup_per_configuration": spec["plan"]["warmup_runs"], + "ci_accepted": False, "release_qualified": False, + "cells": [{"concurrency": concurrency, "status": "not_started", "verified": False, + "completion": {"scheduled": 4, "attempted": 0, "completed": 0, "valid": 0, "failed": 4, "not_started": 4, "unfinished": 0}} + for concurrency in CONCURRENCIES]} + path = root / "serving-smoke.json" + gpu._write(path, matrix) + try: + for cell in matrix["cells"]: + remaining = deadline - time.monotonic() + if remaining <= max(spec["limits"]["startup_seconds"], spec["limits"]["request_seconds"]) + 2 * spec["limits"]["cleanup_seconds"]: + raise TimeoutError("remaining matrix budget cannot bound another configuration") + current = copy.deepcopy(spec) + current["limits"]["job_seconds"] = remaining + current["serving"] = settings(cell["concurrency"], spec["serving"]["delivery_deadline_seconds"]) + current["job_id"] = f"{spec['job_id']}-c{cell['concurrency']}" + directory = root / "gpu" / f"c{cell['concurrency']}" + cell["status"] = "running" + gpu._write(path, matrix) + receipt = gpu.run_gpu_job(current, directory, serving_smoke=True) + cell["status"] = "failed" + cell["receipt"] = {"path": (directory / "gpu-job.json").relative_to(root).as_posix(), "sha256": gpu._hash(directory / "gpu-job.json")} + run_path = directory / "baseline/run.json" + if run_path.is_file(): + raw = gpu._read(run_path) + cell["run"] = {"path": run_path.relative_to(root).as_posix(), "sha256": gpu._hash(run_path)} + summary = _summary(raw["records"], 4, raw["measurement"]["wall_seconds"]) + cell["completion"] = {key: summary[key] for key in ("scheduled", "completed", "valid", "failed")} + finished = {r["slot_id"] for r in raw["records"] if r["phase"] == "measurement" and r["attempted"]} + journal = directory / "baseline/events.jsonl" + events = [json.loads(line) for line in journal.read_text().splitlines()] if journal.exists() else [] + started = finished | {event["slot_id"] for event in events if event["event"] == "attempt_started" and event["slot_id"].startswith("measurement-")} + cell["completion"].update(attempted=len(started), not_started=4-len(started), unfinished=len(started-finished)) + gpu._write(path, matrix) + verified = verify_measurement_job(directory, deadline=deadline, require_success=True, serving_smoke=True) + run, role = verified["runs"]["baseline"], receipt["roles"]["baseline"] + cell.update(status="complete", verified=True, metrics={ + "client_ready_p50_seconds": run["serving"]["client_ready_latency_seconds"]["p50"], + "valid_clips_per_second": _summary(run["records"], 4, run["measurement"]["wall_seconds"])["valid_clips_per_second"], + "serving": run["serving"], "measurement": run["measurement"], + }) + samples = [json.loads(line) for line in (directory / role["telemetry_path"]).read_text().splitlines()] + events = [json.loads(line) for line in (directory / "baseline/events.jsonl").read_text().splitlines()] + power = analyze_power(role, run, samples, events, spec["gpu_uuids"], interval_seconds=spec["limits"]["telemetry_interval_seconds"]) + power_path = directory / "power.json" + gpu._write(power_path, power) + cell["power"] = {"path": power_path.relative_to(root).as_posix(), "sha256": gpu._hash(power_path), "phases": power["phases"]} + gpu._write(path, matrix) + matrix["status"] = "complete" + except (Exception, KeyboardInterrupt) as error: + matrix.update(status="failed", error=str(error) if isinstance(error, (ValueError, RuntimeError, TimeoutError)) else type(error).__name__) + for cell in matrix["cells"]: + if cell["status"] == "running": + cell["status"] = "failed" + finally: + matrix["finished_at"] = gpu._now() + matrix["completion"] = {key: sum(cell["completion"][key] for cell in matrix["cells"]) + for key in ("scheduled", "attempted", "completed", "valid", "failed", "not_started", "unfinished")} + gpu._write(path, matrix) + _report(root, matrix) + return matrix diff --git a/experimental/video-generation/export_ci.py b/experimental/video-generation/export_ci.py new file mode 100644 index 0000000000..9874e82cd4 --- /dev/null +++ b/experimental/video-generation/export_ci.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Publish new result contracts from immutable, GitHub-verified H3 executions.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +from urllib.request import Request, urlopen + +import ci +from evaluator.mvp_result import write_result + + +REPOSITORY = "SemiAnalysisAI/InferenceX" + + +def files_with_nested_seals(root: Path) -> dict[str, str]: + files = ci.inventory(root) + for path in root.rglob("SHA256SUMS"): + if path != root / "SHA256SUMS": + ci.need(path.is_file() and not path.is_symlink(), "Nonregular nested checksum file") + files[path.relative_to(root).as_posix()] = ci.digest(path) + return dict(sorted(files.items())) + + +def source_ids(value: str) -> list[str]: + values = value.split(",") + ci.need(1 <= len(values) <= 2 and len(set(values)) == len(values) + and all(re.fullmatch(r"[1-9][0-9]{0,19}", item) for item in values), + "Supply one or two distinct numeric source run IDs") + return values + + +def api(path: str) -> dict: + request = Request("https://api.github.com/repos/" + REPOSITORY + "/" + path, + headers={"Authorization": "Bearer " + os.environ["GH_TOKEN"], + "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}) + with urlopen(request, timeout=30) as response: + payload = response.read(16 * 1024 * 1024 + 1) + ci.need(len(payload) <= 16 * 1024 * 1024, "GitHub response exceeds metadata limit") + return json.loads(payload) + + +def verified_execution(run_id: str, *, inventory: bool = False) -> tuple[dict, dict]: + run = api("actions/runs/" + run_id) + current_export = (run_id == os.environ.get("GITHUB_RUN_ID") + and str(run["run_attempt"]) == os.environ.get("GITHUB_RUN_ATTEMPT") + and run["head_sha"] == os.environ.get("GITHUB_SHA") and run["status"] == "in_progress") + ci.need(str(run["id"]) == run_id and run["repository"]["full_name"] == REPOSITORY + and run["head_repository"]["full_name"] == REPOSITORY + and run["event"] == "workflow_dispatch" + and ((run["status"] == "completed" and run["conclusion"] == "success") or current_export), + "Source must be a successful manual InferenceX execution or this run's completed H3 job") + jobs = api(f"actions/runs/{run_id}/attempts/{run['run_attempt']}/jobs") + job_name = "H3 H200 hardware inventory" if inventory else "H3 video H200 smoke" + selected = [job for job in jobs["jobs"] if re.fullmatch( + r"(?:h3-video / )?p[0-9]+(?:\.[0-9]+)? \| " + re.escape(job_name), job["name"])] + ci.need(len(selected) == 1 and selected[0]["status"] == "completed" + and selected[0]["conclusion"] == "success", "Source lacks a successful H3 Slurm job") + name = f"h3-{'hardware' if inventory else 'video'}-{run_id}-{run['run_attempt']}" + artifacts = api(f"actions/runs/{run_id}/artifacts") + selected_artifacts = [item for item in artifacts["artifacts"] if item["name"] == name] + ci.need(len(selected_artifacts) == 1, "Expected exactly one original H3 artifact") + artifact = selected_artifacts[0] + ci.need(not artifact["expired"] and 0 < artifact["size_in_bytes"] <= 2 * 1024**3 + and artifact["workflow_run"]["id"] == run["id"] + and artifact["workflow_run"]["head_sha"] == run["head_sha"], "Artifact identity, size or retention invalid") + public_artifact = {key: artifact.get(key) for key in + ("id", "name", "digest", "size_in_bytes", "expired")} + public_artifact["workflow_run"] = {key: artifact["workflow_run"][key] for key in ("id", "head_sha")} + return ({"databaseId": run["id"], "headSha": run["head_sha"], "runAttempt": run["run_attempt"], + "event": run["event"], "status": run["status"], "conclusion": run["conclusion"], + "url": run["html_url"], "jobs": [{key: job[key] for key in ("id", "name", "status", "conclusion")} + for job in jobs["jobs"]]}, public_artifact) + + +def verified_hardware(root: Path, run_id: str, attempt: str, sha: str) -> dict: + """Join the downloaded inventory seal, CI identity, Slurm step and teardown.""" + expected = {} + for line in (root / "SHA256SUMS").read_text().splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) + ci.need(match is not None, "Malformed hardware checksum entry") + digest, name = match.groups() + ci.need(name not in expected, "Duplicate hardware checksum entry") + expected[name] = digest + ci.need(expected and ci.inventory(root) == expected, "Hardware artifact checksum mismatch") + profile, state, manifest, binding, step = (ci.read(root / name) for name in ( + "hardware-profile.json", "ci.json", "manifest.json", "binding.json", "step-result.json")) + for record in (profile, state, manifest): + ci.need(str(record["run_id"]) == run_id and str(record["run_attempt"]) == attempt + and record["source_sha"] == sha, "Hardware observation belongs to a different CI producer") + ci.need(profile["git_commit"] == sha and profile["ci"]["repository"] == REPOSITORY + and str(profile["ci"]["run_id"]) == run_id and str(profile["ci"]["run_attempt"]) == attempt, + "Hardware profile CI identity mismatch") + ci.need(state["phase"] == "complete" and state["exit_code"] == manifest["exit_code"] == step["exit_code"] == 0 + and step["inventory_completed"] is True and state["step_cleanup"]["status"] == "ended" + and state["allocation_cleanup"]["status"] == ("retained" if state["allocation_reused"] else "released"), + "Hardware inventory or cleanup did not complete") + ci.need(profile["slurm"] == binding and binding["job_id"] == manifest["slurm_allocation"]["identity"]["JobId"] + == state["allocation"]["identity"]["JobId"] + and state["step_cleanup"]["step_id"] == binding["job_id"] + "." + binding["step_id"] + and set(profile["gpu_uuids"]) == set(binding["gpu_uuids"]), "Hardware Slurm/GPU identity mismatch") + from inventory_ci import classify_tdp + classified = classify_tdp((root / "nvidia-smi.xml").read_text(), profile["gpu_uuids"]) + ci.need(profile["tdp"] == classified or (profile["tdp"]["status"] == "unknown" and profile["tdp"]["watts_per_gpu"] is None), + "Hardware TDP profile differs from raw PCI identity") + ci.need(all(path in expected for path in profile["raw"].values()), "Hardware raw evidence is missing") + profile["recorded_tdp_classification"] = profile["tdp"] + profile["tdp"] = classified + profile["hardware_variant"] = classified["hardware_variant"] + profile["variant_status"] = classified["status"] + profile["tdp_classifier_git_commit"] = os.environ.get("GITHUB_SHA") + profile["evidence_root"] = "hardware" + profile["tdp"]["evidence"]["raw_path"] = "hardware/nvidia-smi.xml" + profile["raw"] = {name: "hardware/" + path for name, path in profile["raw"].items()} + return profile + + +def publish(run_ids: list[str], output: Path, hardware: Path | None, *, hardware_run_id: str | None = None) -> int: + output.mkdir(parents=True, exist_ok=False) + sha = os.environ.get("GITHUB_SHA", "") + ci.need(re.fullmatch(r"[0-9a-f]{40}", sha) + and ci.command(["git", "rev-parse", "HEAD"]).strip() == sha, "Exact exporter checkout required") + run_id, attempt = os.environ.get("GITHUB_RUN_ID", ""), os.environ.get("GITHUB_RUN_ATTEMPT", "") + ci.need(run_id.isdigit() and attempt.isdigit() and os.environ.get("GITHUB_REPOSITORY") == REPOSITORY, + "GitHub export identity required") + ci.need(hardware is None or hardware_run_id is None, "Choose a local hardware artifact or a verified inventory run") + if hardware_run_id is not None: + ci.need(source_ids(hardware_run_id) == [hardware_run_id], "Expected one inventory run ID") + inventory_ci, inventory_artifact = verified_execution(hardware_run_id, inventory=True) + hardware = output / "hardware" + subprocess.run(["gh", "run", "download", hardware_run_id, "--repo", REPOSITORY, + "--name", inventory_artifact["name"], "--dir", str(hardware)], check=True, timeout=180) + profile = verified_hardware(hardware, hardware_run_id, str(inventory_ci["runAttempt"]), inventory_ci["headSha"]) + profile["verified_ci"] = inventory_ci + profile["source_artifact"] = inventory_artifact + else: + profile = verified_hardware(hardware, run_id, attempt, sha) if hardware is not None else None + producer = {"git_commit": sha, "ci": {"repository": REPOSITORY, "run_id": run_id, + "run_attempt": attempt, "run_url": f"https://github.com/{REPOSITORY}/actions/runs/{run_id}"}, + "mode": "same_run_export" if run_ids == [run_id] else "verified_artifact_reprocessing; no_new_H3_generation"} + catalog = {"schema_version": "1.0.0", "producer": producer, "results": [], "status": "complete"} + exit_code = 0 + for source in run_ids: + target = output / ("source-" + source) + try: + source_ci, artifact = verified_execution(source) + subprocess.run(["gh", "run", "download", source, "--repo", REPOSITORY, + "--name", artifact["name"], "--dir", str(target)], check=True, timeout=180) + original_checksums = (target / "SHA256SUMS").read_bytes() + result = write_result(target, producer={**producer, "source_artifact": artifact}, + source_ci=source_ci, hardware_profile=profile) + (target / "source-SHA256SUMS").write_bytes(original_checksums) + ci.write(target / "source-ci.json", source_ci) + ci.write(target / "source-artifact.json", artifact) + if hardware is not None: + shutil.copytree(hardware, target / "hardware") + result["files"] = [{"path": name, "sha256": digest} for name, digest in files_with_nested_seals(target).items() + if name != "result.json"] + ci.write(target / "result.json", result) + catalog["results"].append({"source_run_id": source, "manifest": f"source-{source}/result.json", + "status": result.get("status", "complete")}) + except (Exception, KeyboardInterrupt) as error: + target.mkdir(parents=True, exist_ok=True) + ci.write(target / "export-error.json", {"error": str(error), "source_run_id": source, "exit_code": 2}) + catalog["results"].append({"source_run_id": source, "status": "failed", "error": str(error)}) + catalog["status"], exit_code = "partial", 2 + finally: + if target.exists(): + if (target / "SHA256SUMS").exists() and not (target / "source-SHA256SUMS").exists(): + (target / "source-SHA256SUMS").write_bytes((target / "SHA256SUMS").read_bytes()) + (target / "SHA256SUMS").unlink(missing_ok=True) + files = files_with_nested_seals(target) + (target / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in sorted(files.items()))) + ci.write(output / "index.json", catalog) + for name in ("result.schema.json", "RESULTS.md", "RESULTS_zh.md"): + shutil.copyfile(Path(__file__).parent / name, output / name) + (output / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in files_with_nested_seals(output).items())) + return exit_code + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-run-ids", required=True) + parser.add_argument("--hardware", type=Path) + parser.add_argument("--hardware-run-id") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + return publish(source_ids(args.source_run_ids), args.output, args.hardware, hardware_run_id=args.hardware_run_id) + except (Exception, KeyboardInterrupt) as error: + args.output.mkdir(parents=True, exist_ok=True) + ci.write(args.output / "export-error.json", {"error": str(error), "exit_code": 2}) + print(str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experimental/video-generation/inventory_ci.py b/experimental/video-generation/inventory_ci.py new file mode 100644 index 0000000000..8567b55b61 --- /dev/null +++ b/experimental/video-generation/inventory_ci.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Bounded H200 hardware inventory using the prepared H3 CI allocation route. + +This records current hardware and power settings. It never loads the model and +cannot establish the power limits of an earlier benchmark. +""" +from __future__ import annotations + +import argparse +import copy +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET + +import ci +from evaluator.mvp_gpu_job import GpuProbe, cuda_devices, validate_gpu_job + +RESOURCES = {"gpus": 4, "cpus": 4, "memory_gb": 8, "minutes": 10} + + + +def classify_tdp(xml_text: str, devices: list[str]) -> dict: + """Identify an advertised maximum TDP from observed PCI IDs, never power draw.""" + sources = { + "specification": "https://www.nvidia.com/en-us/data-center/h200/", + "supported_products": "https://download.nvidia.com/XFree86/Linux-x86_64/580.173.02/README/supportedchips.html", + "pci_variant_mapping": "https://raw.githubusercontent.com/NVIDIA/k8s-launch-kit/db32e4b98170/pkg/networkoperatorplugin/internal/pciids/nvidia.ids", + } + result = {"status": "unknown", "watts_per_gpu": None, "hardware_variant": None, + "source_url": sources["specification"], "semantics": "manufacturer maximum configurable TDP; not an observed power limit", + "evidence": {"sources": sources, "raw_path": "nvidia-smi.xml", "devices": []}} + # Full NVIDIA subsystem IDs narrow the mapping to published supported boards. + supported = { + ("0x233510de", "0x18be10de", "NVIDIA H200"): ("H200 SXM", 700), + ("0x233510de", "0x18bf10de", "NVIDIA H200"): ("H200 SXM", 700), + ("0x233b10de", "0x199610de", "NVIDIA H200 NVL"): ("H200 NVL", 600), + } + try: + root = ET.fromstring(xml_text) + rows = root.findall("gpu") + uuids = [row.findtext("uuid") for row in rows] + ci.need(root.tag == "nvidia_smi_log" and devices and len(set(devices)) == len(devices) + and len(uuids) == len(set(uuids)) and set(devices) <= set(uuids), "XML inventory lacks unique assigned UUIDs") + variants = set() + for row in rows: + uuid = row.findtext("uuid") + if uuid not in devices: + continue + device = {"uuid": uuid, "product_name": row.findtext("product_name"), + "pci_device_id": row.findtext("pci/pci_device_id"), + "pci_sub_system_id": row.findtext("pci/pci_sub_system_id")} + result["evidence"]["devices"].append(device) + pci_ids = [str(device[key]).strip().lower() for key in ("pci_device_id", "pci_sub_system_id")] + ci.need(all(re.fullmatch(r"(?:0x)?[0-9a-f]{8}", value) for value in pci_ids), "Invalid PCI hexadecimal identity") + identity = (*("0x" + value.removeprefix("0x") for value in pci_ids), device["product_name"]) + ci.need(identity in supported, "Unrecognized or inconsistent PCI/product identity") + variants.add(supported[identity]) + ci.need(len(variants) == 1, "Selected GPUs have different hardware variants") + variant, watts = variants.pop() + result.update(status="verified", watts_per_gpu=watts, hardware_variant=variant) + except (ET.ParseError, ValueError) as error: + result["reason"] = str(error) + return result + +def prepare(config: dict) -> tuple[dict, dict]: + config = copy.deepcopy(ci.validate_config(config)) + runtime = config["runtime"] + ci.need(Path(runtime["rootfs"]).is_dir() and Path(runtime["ready_marker"]).is_file(), "Prepared persistent runtime/readiness missing") + ci.need(ci.digest(runtime["entry"]) == runtime["entry_sha256"], "Persistent entry script changed") + interpreter = ci.host_path(config, runtime["python"]) + # Absolute symlinks resolve inside Enroot, not against the submit host root. + ci.need(interpreter.is_file() or interpreter.is_symlink(), "Prepared interpreter missing") + ci.need(ci.digest(config["spec"]["path"]) == config["spec"]["sha256"], "Prepared specification changed") + spec = validate_gpu_job(ci.read(config["spec"]["path"])) + approval = spec["authorization"] + ci.need(approval["compute_approved"] and approval["model_license_reviewed"] and approval["approval_reference"].strip(), "Prepared specification lacks approval") + config["resources"] = dict(RESOURCES) + return config, approval + + + +def source_target(config: dict, run_ids: list[str]) -> dict: + """Join verified GitHub runs to sealed task-owned persistent hardware receipts.""" + from export_ci import REPOSITORY, source_ids, verified_execution + run_ids = source_ids(",".join(run_ids)) + results = Path(config["workspace"]["host"]) / "results" / config["task_id"] + target = {"node": None, "gpu_uuids": None, "sources": []} + for run_id in run_ids: + source_ci, artifact = verified_execution(run_id) + ci.need(source_ci["status"] == "completed" and source_ci["conclusion"] == "success", "Inventory requires completed source execution") + attempt, sha = str(source_ci["runAttempt"]), source_ci["headSha"] + ci.need(attempt.isdigit() and re.fullmatch(r"[0-9a-f]{40}", sha), "Invalid verified source identity") + directory = results / f"github-{run_id}-{attempt}" + ci.need(directory.is_dir() and not directory.is_symlink(), "Original task-owned persistent source missing") + sealed = {} + for line in (directory / "SHA256SUMS").read_text().splitlines(): + digest, separator, name = line.partition(" ") + ci.need(separator and ci.SHA.fullmatch(digest) and name not in sealed, "Invalid source checksum seal") + sealed[name] = digest + names = ("manifest.json", "ci.json", "context.json", "binding.json", "gpu/spec.json") + for name in names: + path = directory / name + ci.need(not path.is_symlink() and path.is_file() and ci.digest(path) == sealed.get(name), "Source receipt differs from its seal: " + name) + manifest, state, context, binding, spec = (ci.read(directory / name) for name in names) + ci.need(all(manifest["evidence"].get(name) == sealed[name] for name in names[1:]), "Source manifest receipt hashes differ") + ci.need(manifest["task_id"] == state["task_id"] == context["config"]["task_id"] == config["task_id"] + and str(manifest["run_id"]) == str(state["run_id"]) == run_id + and str(manifest["run_attempt"]) == str(state["run_attempt"]) == attempt + and context["run_id"] == directory.name + and manifest["git_commit"] == state["source_sha"] == context["source_sha"] == sha, + "Source CI/Git/task identities disagree") + ci.need(manifest["ci"] == state["ci"] and manifest["ci"]["repository"] == REPOSITORY + and manifest["ci"]["run_url"] == source_ci["url"] and manifest["exit_code"] == state["exit_code"] == 0 + and state["phase"] == "complete" and state.get("smoke_completed") is True, "Source execution did not complete cleanly") + allocation = manifest["slurm_allocation"] + ci.need(allocation == state["allocation"] == context["allocation"], "Source allocation receipts disagree") + ci.verify_identity(allocation, state["slurm_job"], config["task_id"]) + node, devices = binding["node"], sorted(binding["gpu_uuids"]) + ci.need(ci.NAME.fullmatch(node) and node == context["node"] == state["slurm_job"]["NodeList"] + and binding["job_id"] == allocation["identity"]["JobId"] + and state["step_cleanup"].get("status") == "ended" + and state["step_cleanup"]["step_id"] == binding["job_id"] + "." + binding["step_id"], "Source Slurm bindings disagree") + ci.need(len(devices) == RESOURCES["gpus"] and len(set(devices)) == len(devices) + and all(re.fullmatch(r"GPU-[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", device) for device in devices) + and sorted(spec["gpu_uuids"]) == devices, "Source GPU assignments disagree") + ci.need(target["node"] in (None, node) and target["gpu_uuids"] in (None, devices), "Source runs used different physical hardware") + target.update(node=node, gpu_uuids=devices) + target["sources"].append({"run_id": run_id, "run_attempt": attempt, "git_commit": sha, + "verified_ci": source_ci, "artifact": artifact, "receipt_hashes": {name: sealed[name] for name in names}}) + return target + +def collect_inventory(config: dict, output: Path, *, source_run_ids: list[str] | None = None) -> int: + config, approval = prepare(config) + run, attempt, sha = (os.environ.get(key, "") for key in ("GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "H3_SOURCE_SHA")) + ci.need(re.fullmatch(r"[0-9]+", run) and re.fullmatch(r"[0-9]+", attempt) and re.fullmatch(r"[0-9a-f]{40}", sha), "Exact CI run/attempt/source required") + source = Path(__file__).resolve().parent + ci.need(ci.command(["git", "-C", str(source), "rev-parse", "HEAD"]).strip() == sha, "Checkout differs from admitted source") + ci.need(not ci.command(["git", "-C", str(source), "status", "--porcelain"]).strip(), "Inventory checkout must be clean and committed") + workspace = Path(config["workspace"]["host"]) + ci.need(workspace.is_dir(), "Persistent workspace missing") + results, control = workspace / "results" / config["task_id"], workspace / "campaigns" / config["task_id"] / "control" + results.mkdir(parents=True, exist_ok=True) + control.mkdir(parents=True, exist_ok=True) + run_dir = results / f"inventory-{run}-{attempt}" + ci_identity = {"run_id": run, "run_attempt": attempt, "repository": os.environ.get("GITHUB_REPOSITORY"), + "run_url": f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{os.environ.get('GITHUB_REPOSITORY', '')}/actions/runs/{run}"} + state = {"schema_version": 1, "operation": "hardware_inventory", "task_id": config["task_id"], + "run_id": run, "run_attempt": attempt, "source_sha": sha, "started_at": ci.now(), "phase": "preparing", + "resources": {"requested": config["resources"], "new_allocation_gpus": 8, "new_allocation_gpu_hours_cap": 8 * 10 / 60}, + "authorization": approval, "ci": {key: os.environ.get(key) for key in ("GITHUB_REPOSITORY", "GITHUB_WORKFLOW_REF", "GITHUB_WORKFLOW_SHA", "GITHUB_ACTOR", "GITHUB_TRIGGERING_ACTOR")}} + with ci.task_lock(control / "ci.lock"): + run_dir.mkdir(exist_ok=False) + ci.write(run_dir / "ci.json", state) + receipt, reused, code = None, False, 2 + try: + shutil.copyfile(config["runtime"]["entry"], run_dir / "runtime-entry.sh") + shutil.copyfile(config["runtime"]["ready_marker"], run_dir / "runtime-readiness.record") + package = workspace / "campaigns" / config["task_id"] / "packages" / (sha + "-inventory") + package_files = ci.stage_package(source, package) + target = source_target(config, source_run_ids if source_run_ids is not None else os.environ.get("H3_SOURCE_RUN_IDS", "").split(",")) + ci.write(run_dir / "target-source.json", target) + decision = ci.recover(config, results, node=target["node"]) + ci.write(run_dir / "recovery.json", decision) + ci.need(decision["action"] != "wait", "Task-owned allocation pending; no duplicate submitted") + if decision["action"] == "reuse": + receipt, reused = decision["receipt"], True + ci.write(run_dir / "allocation.json", receipt) + else: + receipt = ci.allocate(config, run_dir, node=target["node"]) + record = ci.job_record(receipt["identity"]["JobId"]) + ci.verify_identity(receipt, record, config["task_id"]) + ci.need(record["JobState"] == "RUNNING" and record["NodeList"] == target["node"] + and ci.capacity(record, config["resources"]) is None, "Allocation cannot serve pinned inventory step") + state.update(phase="starting", allocation=receipt, allocation_reused=reused, slurm_job=record) + ci.write(run_dir / "ci.json", state) + ci.write(run_dir / "context.json", {"config": config, "allocation": receipt, "node": record["NodeList"], + "source_sha": sha, "package_files": package_files, "run_id": run, "run_attempt": attempt, "ci": ci_identity, "target": target, + "active_steps": ci.command(["squeue", "--steps", "--noheader", "--jobs=" + record["JobId"], "--format=%i|%N"])}) + argv = ci.step_argv(config, receipt, record, run_dir, package) + argv[-3] = str(package / "inventory_ci.py") + ci.write(run_dir / "step-command.json", argv) + code = ci.run_step(argv, run_dir / "srun.log", 360) + result = ci.read(run_dir / "step-result.json") + ci.need(code == result["exit_code"], "Slurm exit and inventory receipt differ") + ci.need(code != 0 or (result.get("inventory_completed") is True and (run_dir / "hardware-profile.json").is_file()), "Inventory success requires a completed hardware profile") + state.update(phase="complete" if code == 0 else "failed", **result) + except (Exception, KeyboardInterrupt) as error: + state.update(phase="failed", error=str(error)) + code = 2 + finally: + if receipt is None and (run_dir / "allocation.json").is_file(): + receipt = ci.read(run_dir / "allocation.json") + try: + if receipt is not None: + state["step_cleanup"] = ci.drain_step(receipt, config["task_id"], run_dir) + except Exception as error: + state.update(phase="failed", step_cleanup_error=str(error)) + code = 2 + try: + if receipt is not None and not reused: + state["allocation_cleanup"] = ci.stop_allocation(receipt, config["task_id"]) + elif reused: + state["allocation_cleanup"] = {"status": "retained", "reason": "attached step does not own parent allocation"} + except Exception as error: + state.update(phase="failed", cleanup_error=str(error)) + code = 2 + state.update(finished_at=ci.now(), exit_code=code) + ci.write(run_dir / "ci.json", state) + ci.write(run_dir / "manifest.json", {"schema_version": 1, "operation": "hardware_inventory", "source_sha": sha, + "run_id": run, "run_attempt": attempt, "task_id": config["task_id"], "ci": state["ci"], + "slurm_allocation": receipt, "runtime": config["runtime"], "prepared_spec": config["spec"], + "evidence": ci.inventory(run_dir), "exit_code": code, "artifact_checksums": "SHA256SUMS"}) + ci.collect(run_dir, output) + return code + + +def enter(run_dir: Path) -> None: + context = ci.read(run_dir / "context.json") + config = ci.validate_config(context["config"]) + job, step = os.environ.get("SLURM_JOB_ID"), os.environ.get("SLURM_STEP_ID", "") + ci.need(job == context["allocation"]["identity"]["JobId"] and re.fullmatch(r"[0-9]+", step) + and os.environ.get("SLURMD_NODENAME") == context["node"], "Wrong Slurm step assignment") + ci.write(run_dir / "binding.json", {"job_id": job, "step_id": step, "node": context["node"], + "cpu_affinity": sorted(os.sched_getaffinity(0)), "observed_at": ci.now(), "phase": "entering_runtime"}) + ci.need(ci.digest(config["runtime"]["entry"]) == config["runtime"]["entry_sha256"], "Entry changed on compute node") + argv = ["/bin/bash", config["runtime"]["entry"], config["runtime"]["python"], + str(ci.mapped(config, Path(__file__).parent) / "inventory_ci.py"), "--inside", str(ci.mapped(config, run_dir))] + os.execv(argv[0], argv) + + +def capture(argv: list[str], path: Path) -> None: + try: + result = subprocess.run(argv, text=True, capture_output=True, timeout=20, env=ci.environment()) + except subprocess.TimeoutExpired as error: + for target, value in ((path, error.stdout), (path.with_suffix(path.suffix + ".stderr.log"), error.stderr)): + target.write_bytes(value.encode() if isinstance(value, str) else (value or b"")) + raise + path.write_text(result.stdout) + path.with_suffix(path.suffix + ".stderr.log").write_text(result.stderr) + ci.need(result.returncode == 0, "Inventory query failed: " + argv[0]) + + +def inside(run_dir: Path) -> int: + result = {"exit_code": 2, "inventory_completed": False} + try: + context = ci.read(run_dir / "context.json") + config = ci.validate_config(context["config"]) + ci.need(os.access(config["runtime"]["python"], os.X_OK) + and Path(config["runtime"]["python"]).samefile(sys.executable), "Inventory is not running the configured container interpreter") + interpreter = {"configured_path": config["runtime"]["python"], "executable": sys.executable, + "version": sys.version.split()[0], "sha256": ci.digest(sys.executable)} + job, step = context["allocation"]["identity"]["JobId"], os.environ.get("SLURM_STEP_ID", "") + ci.need(os.environ.get("SLURM_JOB_ID") == job and re.fullmatch(r"[0-9]+", step) + and os.environ.get("SLURMD_NODENAME") == context["node"] + and os.environ.get("SLURM_PROCID") == "0" and os.environ.get("SLURM_NTASKS") == "1", "Wrong single-node Slurm task") + ci.need(ci.inventory(Path(__file__).parent) == context["package_files"], "Staged inventory bytes changed") + devices, cpus = cuda_devices(), sorted(os.sched_getaffinity(0)) + assigned = os.environ.get("H3_ASSIGNED_GPU_UUIDS", "").split(",") + ci.need(len(devices) == RESOURCES["gpus"] and len(set(devices)) == len(devices) and set(devices) == set(assigned), "CUDA UUIDs differ from assigned GPUs") + ci.need(context["node"] == context["target"]["node"] and sorted(devices) == context["target"]["gpu_uuids"], "Inventory assignment differs from historical source hardware") + binding = ci.read(run_dir / "binding.json") + ci.need(binding["job_id"] == job and binding["step_id"] == step and binding["cpu_affinity"] == cpus + and len(cpus) >= RESOURCES["cpus"], "Container changed assigned CPU/step binding") + binding.update(gpu_uuids=devices, observed_at=ci.now(), phase="inventory", + slurm={key: os.environ.get(key) for key in ("CUDA_VISIBLE_DEVICES", "H3_ORIGINAL_CUDA_VISIBLE_DEVICES", "SLURM_JOB_GPUS", "SLURM_STEP_GPUS", "SLURM_CPU_BIND", "SLURM_CPUS_PER_TASK")}) + ci.write(run_dir / "binding.json", binding) + probe = GpuProbe(devices, timeout=15) + before = probe.snapshot() + ci.write(run_dir / "gpu-before.json", before) + ci.need(not before["compute_apps"], "Assigned inventory GPUs have active compute; no workload started") + capture(["nvidia-smi", "-q", "-x"], run_dir / "nvidia-smi.xml") + capture(["nvidia-smi", "topo", "-m"], run_dir / "topology.txt") + power = probe.power_configuration() + ci.write(run_dir / "power-configuration.json", power) + dmi = {} + for name in ("product_name", "sys_vendor"): + try: + dmi[name] = {"value": (Path("/sys/class/dmi/id") / name).read_text().strip()} + except OSError as error: + dmi[name] = {"value": None, "reason": type(error).__name__} + after = probe.snapshot() + ci.write(run_dir / "gpu-after.json", after) + ci.need(not after["compute_apps"], "Compute appeared on assigned inventory GPUs") + tdp = classify_tdp((run_dir / "nvidia-smi.xml").read_text(), devices) + ci.write(run_dir / "hardware-profile.json", {"schema_version": 1, "observation_kind": "read_only_inventory", + "observed_at": ci.now(), "source_sha": context["source_sha"], "git_commit": context["source_sha"], + "ci": context["ci"], "source_files": context["package_files"], "source_hardware": context["target"], "interpreter": interpreter, + "run_id": context["run_id"], "run_attempt": context["run_attempt"], "slurm": binding, + "gpu_uuids": devices, "gpus": after["gpus"], "dmi": dmi, "power_configuration": power, + "hardware_variant": tdp["hardware_variant"], "variant_status": tdp["status"], "tdp": tdp, + "historical_benchmark_power_limits": "not_observed", "raw": {"inventory": "nvidia-smi.xml", "topology": "topology.txt"}}) + result.update(exit_code=0, inventory_completed=True) + except (Exception, KeyboardInterrupt) as error: + result["error"] = str(error) + finally: + ci.write(run_dir / "step-result.json", result) + return result["exit_code"] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--source-run-ids", help="One or two successful original H3 CI run IDs") + parser.add_argument("--enter", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--inside", type=Path, help=argparse.SUPPRESS) + args = parser.parse_args() + if args.enter: + enter(args.enter) + return 2 + if args.inside: + return inside(args.inside) + ci.need(args.config is not None and args.output is not None, "--config and --output required") + try: + from export_ci import source_ids + return collect_inventory(ci.read(args.config), args.output, source_run_ids=source_ids(args.source_run_ids or os.environ.get("H3_SOURCE_RUN_IDS", ""))) + except (Exception, KeyboardInterrupt) as error: + args.output.mkdir(parents=True, exist_ok=True) + ci.write(args.output / "preflight-error.json", {"operation": "hardware_inventory", "error": str(error), "exit_code": 2, "recorded_at": ci.now()}) + print(str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experimental/video-generation/mvp/example-uncalibrated.policy.json b/experimental/video-generation/mvp/example-uncalibrated.policy.json new file mode 100644 index 0000000000..5a94c57ff0 --- /dev/null +++ b/experimental/video-generation/mvp/example-uncalibrated.policy.json @@ -0,0 +1,8 @@ +{ + "policy_id": "h3-demo-policy-uncalibrated-v1", + "calibration_status": "uncalibrated", + "max_latency_increase_fraction": 0.1, + "min_video_psnr_db": 30.0, + "min_audio_spectral_cosine": 0.95, + "max_audio_rms_ratio_error": 0.1 +} diff --git a/experimental/video-generation/mvp/h3-gpu-job.template.json b/experimental/video-generation/mvp/h3-gpu-job.template.json new file mode 100644 index 0000000000..3655d429c1 --- /dev/null +++ b/experimental/video-generation/mvp/h3-gpu-job.template.json @@ -0,0 +1,113 @@ +{ + "schema_version": "0.1.0", + "job_id": "h3-repeatability-REPLACE_WITH_UNIQUE_ID", + "authorization": { + "compute_approved": false, + "model_license_reviewed": false, + "approval_reference": "" + }, + "allocation": { + "mode": "cooperative_shared", + "label": "REPLACE_WITH_ACTUAL_APPROVED_ALLOCATION" + }, + "gpu_uuids": [ + "REPLACE_WITH_APPROVED_FULL_GPU_UUID_1", + "REPLACE_WITH_APPROVED_FULL_GPU_UUID_2", + "REPLACE_WITH_APPROVED_FULL_GPU_UUID_3", + "REPLACE_WITH_APPROVED_FULL_GPU_UUID_4" + ], + "port": 30283, + "lock_directory": "/REPLACE_WITH_OWNED_PRIVATE_LOCK_DIRECTORY", + "baseline": { + "python": "/REPLACE_WITH_BASELINE_ENV/bin/python", + "source": "/REPLACE_WITH_BASELINE_SOURCE", + "revision": "REPLACE_WITH_MANIFEST_REVISION", + "source_sha256": "REPLACE_WITH_MANIFEST_SOURCE_SHA256" + }, + "candidate": { + "python": "/REPLACE_WITH_CANDIDATE_ENV/bin/python", + "source": "/REPLACE_WITH_CANDIDATE_SOURCE", + "revision": "REPLACE_WITH_MANIFEST_REVISION", + "source_sha256": "REPLACE_WITH_MANIFEST_SOURCE_SHA256" + }, + "model": { + "path": "/REPLACE_WITH_READABLE_MATERIALIZED_H3_SNAPSHOT", + "revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "files": [] + }, + "server": { + "ulysses_degree": 4, + "dit_cpu_offload": false, + "tp_size": 1, + "encoder_parallel": "auto", + "performance_mode": "speed" + }, + "plan": { + "plan_id": "h3-fl2va-t2va-smoke-v1", + "model_id": "MiniMaxAI/MiniMax-H3", + "model_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "generation": { + "duration_seconds": 4, + "aspect_ratio": "16:9", + "width": 1344, + "height": 768, + "frame_count": 107, + "fps": 24, + "audio_sample_rate_hz": 32000, + "audio_channels": 2, + "num_inference_steps": 50, + "flow_shift": 12.0, + "audio_flow_shift": 3.0 + }, + "cases": [ + { + "case_id": "drum-taps", + "prompt": "A single continuous medium shot of a drummer tapping a snare drum three times in a quiet rehearsal room. The drumsticks visibly contact the drumhead on each tap. The three crisp snare hits and the natural room ambience are audible. The camera remains still. No music, speech, subtitles, or scene cuts.", + "seed": 11, + "requires_motion": true, + "requires_sound": true + }, + { + "case_id": "pouring-water", + "prompt": "A single continuous close-up of a hand steadily pouring water from a clear glass jug into an empty drinking glass on a wooden table. The water level visibly rises and the pouring and splashing sounds are clearly audible. Daylight, a still camera, no speech, no music, no subtitles, and no scene cuts.", + "seed": 23, + "requires_motion": true, + "requires_sound": true + }, + { + "case_id": "bicycle-pass", + "prompt": "A single continuous wide shot of a cyclist riding from the left side of the frame to the right along a quiet park path. The moving wheels remain visible and the bicycle bell rings once as the cyclist passes the camera. Soft outdoor ambience is audible. The camera stays still, with no music, speech, subtitles, or scene cuts.", + "seed": 37, + "requires_motion": true, + "requires_sound": true + }, + { + "case_id": "footsteps", + "prompt": "A single continuous low-angle shot of a person in ordinary shoes walking across a wooden floor from left to right. Each step visibly lands on the floor and the corresponding footsteps are audible in the quiet room. The camera stays still. No music, speech, subtitles, or scene cuts.", + "seed": 53, + "requires_motion": true, + "requires_sound": true + } + ], + "repetitions": 2, + "warmup_runs": 1 + }, + "policy": { + "policy_id": "h3-example-uncalibrated-gpu-v1", + "calibration_status": "uncalibrated", + "max_latency_increase_fraction": 0.1, + "min_video_psnr_db": 30, + "min_audio_spectral_cosine": 0.95, + "max_audio_rms_ratio_error": 0.1, + "max_memory_increase_fraction": 0.1 + }, + "limits": { + "job_seconds": 4500, + "startup_seconds": 900, + "request_seconds": 900, + "cleanup_seconds": 60, + "telemetry_interval_seconds": 1, + "command_seconds": 10, + "max_idle_memory_mib": 64 + } +} diff --git a/experimental/video-generation/mvp/h3-smoke.plan.json b/experimental/video-generation/mvp/h3-smoke.plan.json new file mode 100644 index 0000000000..1b93203c7f --- /dev/null +++ b/experimental/video-generation/mvp/h3-smoke.plan.json @@ -0,0 +1,50 @@ +{ + "plan_id": "h3-fl2va-t2va-smoke-v1", + "model_id": "MiniMaxAI/MiniMax-H3", + "model_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "generation": { + "duration_seconds": 4, + "aspect_ratio": "16:9", + "width": 1344, + "height": 768, + "frame_count": 107, + "fps": 24, + "audio_sample_rate_hz": 32000, + "audio_channels": 2, + "num_inference_steps": 50, + "flow_shift": 12.0, + "audio_flow_shift": 3.0 + }, + "cases": [ + { + "case_id": "drum-taps", + "prompt": "A single continuous medium shot of a drummer tapping a snare drum three times in a quiet rehearsal room. The drumsticks visibly contact the drumhead on each tap. The three crisp snare hits and the natural room ambience are audible. The camera remains still. No music, speech, subtitles, or scene cuts.", + "seed": 11, + "requires_motion": true, + "requires_sound": true + }, + { + "case_id": "pouring-water", + "prompt": "A single continuous close-up of a hand steadily pouring water from a clear glass jug into an empty drinking glass on a wooden table. The water level visibly rises and the pouring and splashing sounds are clearly audible. Daylight, a still camera, no speech, no music, no subtitles, and no scene cuts.", + "seed": 23, + "requires_motion": true, + "requires_sound": true + }, + { + "case_id": "bicycle-pass", + "prompt": "A single continuous wide shot of a cyclist riding from the left side of the frame to the right along a quiet park path. The moving wheels remain visible and the bicycle bell rings once as the cyclist passes the camera. Soft outdoor ambience is audible. The camera stays still, with no music, speech, subtitles, or scene cuts.", + "seed": 37, + "requires_motion": true, + "requires_sound": true + }, + { + "case_id": "footsteps", + "prompt": "A single continuous low-angle shot of a person in ordinary shoes walking across a wooden floor from left to right. Each step visibly lands on the floor and the corresponding footsteps are audible in the quiet room. The camera stays still. No music, speech, subtitles, or scene cuts.", + "seed": 53, + "requires_motion": true, + "requires_sound": true + } + ], + "repetitions": 2, + "warmup_runs": 1 +} diff --git a/experimental/video-generation/mvp/h3-startup-pair.plan.json b/experimental/video-generation/mvp/h3-startup-pair.plan.json new file mode 100644 index 0000000000..6eac97031b --- /dev/null +++ b/experimental/video-generation/mvp/h3-startup-pair.plan.json @@ -0,0 +1,29 @@ +{ + "plan_id": "h3-fl2va-t2va-startup-pair-v1", + "model_id": "MiniMaxAI/MiniMax-H3", + "model_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "generation": { + "duration_seconds": 4, + "aspect_ratio": "16:9", + "width": 1344, + "height": 768, + "frame_count": 107, + "fps": 24, + "audio_sample_rate_hz": 32000, + "audio_channels": 2, + "num_inference_steps": 50, + "flow_shift": 12.0, + "audio_flow_shift": 3.0 + }, + "cases": [ + { + "case_id": "drum-taps", + "prompt": "A single continuous medium shot of a drummer tapping a snare drum three times in a quiet rehearsal room. The drumsticks visibly contact the drumhead on each tap. The three crisp snare hits and the natural room ambience are audible. The camera remains still. No music, speech, subtitles, or scene cuts.", + "seed": 11, + "requires_motion": true, + "requires_sound": true + } + ], + "repetitions": 1, + "warmup_runs": 1 +} diff --git a/experimental/video-generation/pyproject.toml b/experimental/video-generation/pyproject.toml new file mode 100644 index 0000000000..cb62b4164b --- /dev/null +++ b/experimental/video-generation/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "inferencex-h3-video-ci" +version = "0.1.0" +description = "Controlled H3 runtime regression measurements for InferenceX CI" +requires-python = ">=3.12" +dependencies = ["av==16.1.0", "numpy==2.3.5"] + +[project.optional-dependencies] +dev = ["pytest==8.4.2"] + +[project.scripts] +vgbench = "evaluator.cli:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["evaluator*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = [".", "tests"] diff --git a/experimental/video-generation/result.schema.json b/experimental/video-generation/result.schema.json new file mode 100644 index 0000000000..4f0e1a110e --- /dev/null +++ b/experimental/video-generation/result.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SemiAnalysisAI/InferenceX/blob/main/experimental/video-generation/result.schema.json", + "title": "H3 benchmark frontend result 1.0.0", + "type": "object", + "required": ["schema_version", "bundle_type", "created_at", "producer", "status", "invalid_reasons", "workload_status", "regression_status", "release_qualified", "definitions", "execution", "hardware", "workload", "policy", "roles", "paired_fidelity", "files", "report", "report_links", "checksums", "limitations"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "1.0.0"}, + "bundle_type": {"const": "h3_benchmark_result"}, + "created_at": {"type": "string", "format": "date-time"}, + "producer": {"type": "object", "required": ["git_commit"], "properties": {"git_commit": {"$ref": "#/$defs/gitCommit"}, "ci": {"type": "object"}}}, + "status": {"enum": ["complete", "failed"]}, + "invalid_reasons": {"$ref": "#/$defs/reasons"}, + "workload_status": {"enum": ["passed", "failed", "unknown"]}, + "regression_status": {"enum": ["pass", "fail", "inconclusive"]}, + "release_qualified": {"const": false}, + "definitions": {"type": "object", "required": ["latency", "valid_clips_per_second", "completion", "gpu_memory", "gpu_power", "gpu_energy", "gpu_energy_per_valid_clip", "media_integrity", "paired_fidelity", "hardware_tdp"], "additionalProperties": {"type": "object", "required": ["unit"], "properties": {"unit": {"type": "string"}}}}, + "execution": {"type": ["object", "null"], "required": ["ci", "slurm", "source_manifest", "model", "runtime", "cleanup_status"], "properties": {"ci": {"type": "object", "required": ["git_commit", "run_id", "run_attempt", "run_url", "external_ci_verification"], "properties": {"git_commit": {"$ref": "#/$defs/gitCommit"}, "run_id": {"type": "string", "pattern": "^[0-9]+$"}, "run_attempt": {"type": "string", "pattern": "^[1-9][0-9]*$"}, "external_ci_verification": {"enum": ["passed", "not_supplied"]}}}, "source_manifest": {"$ref": "#/$defs/file"}, "cleanup_status": {"const": "clean"}}}, + "hardware": {"type": ["object", "null"], "required": ["selected_gpu_count", "reserved_gpu_count", "gpu_uuids", "devices", "tdp", "configured_power_limits"], "properties": {"selected_gpu_count": {"type": "integer", "minimum": 1}, "reserved_gpu_count": {"type": ["integer", "null"], "minimum": 1}, "gpu_uuids": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string"}}, "tdp": {"type": "object", "required": ["status", "watts_per_gpu"], "properties": {"status": {"enum": ["verified", "unavailable"]}, "watts_per_gpu": {"type": ["number", "null"], "exclusiveMinimum": 0}}}}}, + "workload": {"type": ["object", "null"], "required": ["plan", "plan_sha256", "server", "comparison"], "properties": {"plan_sha256": {"$ref": "#/$defs/sha256"}, "comparison": {"enum": ["same_revision_A/A", "baseline_candidate"]}}}, + "policy": {"type": ["object", "null"]}, + "roles": {"type": "object", "additionalProperties": false, "properties": {"baseline": {"$ref": "#/$defs/role"}, "candidate": {"$ref": "#/$defs/role"}}}, + "paired_fidelity": {"type": ["object", "null"]}, + "files": {"type": "array", "items": {"$ref": "#/$defs/file"}}, + "report": {"anyOf": [{"$ref": "#/$defs/file"}, {"type": "null"}]}, + "report_links": {"type": "object", "properties": {"original": {"$ref": "#/$defs/file"}, "power": {"$ref": "#/$defs/file"}}, "additionalProperties": false}, + "checksums": {"type": "object", "required": ["path", "scope"], "properties": {"path": {"const": "SHA256SUMS"}, "scope": {"type": "string"}}}, + "limitations": {"$ref": "#/$defs/reasons"} + }, + "allOf": [{"if": {"properties": {"status": {"const": "complete"}}}, "then": {"properties": {"workload_status": {"const": "passed"}, "invalid_reasons": {"maxItems": 0}, "execution": {"type": "object"}, "hardware": {"type": "object"}, "workload": {"type": "object"}, "roles": {"required": ["baseline", "candidate"]}, "report_links": {"required": ["original", "power"]}}}}], + "$defs": { + "serving": { + "type": "object", "required": ["mode", "concurrency", "capacity_qualified", "client_ready_latency_seconds", "submitted", "outcomes", "peak_client_in_flight", "deadline_met_valid_clips", "deadline_goodput_clips_per_second", "limitations"], + "properties": { + "mode": {"const": "closed_loop"}, "concurrency": {"type": "integer", "minimum": 1, "maximum": 32}, + "capacity_qualified": {"const": false}, "submitted": {"type": "integer", "minimum": 0}, + "peak_client_in_flight": {"type": "integer", "minimum": 0, "maximum": 32}, + "delivery_deadline_seconds": {"type": ["number", "null"], "exclusiveMinimum": 0}, + "deadline_met_valid_clips": {"type": ["integer", "null"], "minimum": 0}, + "deadline_attainment_fraction": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "deadline_goodput_clips_per_second": {"type": ["number", "null"], "minimum": 0}, + "client_ready_latency_seconds": {"type": "object", "required": ["values", "sample_count", "valid_clip_count", "p50", "p90", "p95", "population", "quantile_method"], "properties": {"values": {"type": "array", "items": {"type": "number", "minimum": 0}}, "sample_count": {"type": "integer", "minimum": 0}, "p50": {"type": ["number", "null"], "minimum": 0}, "p90": {"type": ["number", "null"], "minimum": 0}, "p95": {"type": ["number", "null"], "minimum": 0}}} + } + }, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "gitCommit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "reasons": {"type": "array", "items": {"type": "string"}}, + "file": {"type": "object", "required": ["path", "sha256"], "additionalProperties": false, "properties": {"path": {"type": "string", "minLength": 1, "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$))[^\\\\]+$"}, "sha256": {"$ref": "#/$defs/sha256"}}}, + "role": {"type": "object", "required": ["run_id", "metrics", "records", "power", "raw_telemetry", "media_evaluator"], "additionalProperties": false, "properties": { + "run_id": {"type": "string"}, + "metrics": {"type": "object", "required": ["status"], "properties": {"status": {"enum": ["valid", "withheld"]}, "valid_clips_per_second": {"type": "number", "minimum": 0}, "serving": {"anyOf": [{"$ref": "#/$defs/serving"}, {"type": "null"}]}}}, + "records": {"type": "array", "items": {"type": "object", "required": ["slot_id", "phase", "status", "attempted", "media", "media_file"], "properties": {"job_id": {"type": ["string", "null"]}, "submit_to_accepted_seconds": {"type": ["number", "null"], "minimum": 0}, "outcome": {"enum": [null, "completed", "invalid_media", "provider_failed", "provider_cancelled", "timed_out", "transport_error", "validation_error", "interrupted", "not_started"]}, "phase": {"enum": ["warmup", "measurement"]}, "status": {"enum": ["succeeded", "failed"]}, "media_file": {"anyOf": [{"$ref": "#/$defs/file"}, {"type": "null"}]}}}}, + "power": {"type": "object", "required": ["path", "sha256", "status", "phases", "windows"], "properties": {"path": {"type": "string", "pattern": "^power/(baseline|candidate)\\.json$"}, "sha256": {"$ref": "#/$defs/sha256"}, "status": {"enum": ["valid", "partial", "invalid"]}, "phases": {"type": "object", "required": ["startup", "warmup", "measurement"], "additionalProperties": {"$ref": "#/$defs/phase"}}, "windows": {"type": "array"}}}, + "raw_telemetry": {"$ref": "#/$defs/file"}, "media_evaluator": {"type": ["object", "null"]} + }}, + "phase": {"type": "object", "required": ["status", "valid", "invalid_reasons", "window_count", "valid_window_count", "aggregate", "per_gpu"], "properties": {"status": {"enum": ["valid", "invalid", "not_requested"]}, "valid": {"type": "boolean"}, "invalid_reasons": {"$ref": "#/$defs/reasons"}, "window_count": {"type": "integer", "minimum": 0}, "valid_window_count": {"type": "integer", "minimum": 0}, "aggregate": {"type": ["object", "null"], "required": ["energy_j", "avg_power_w", "observed_peak_power_w", "joules_per_valid_clip"], "additionalProperties": {"type": ["number", "null"], "minimum": 0}}, "per_gpu": {"type": ["object", "null"], "additionalProperties": {"type": "object", "required": ["energy_j", "avg_power_w", "observed_peak_power_w"], "additionalProperties": false, "properties": {"energy_j": {"type": "number", "minimum": 0}, "avg_power_w": {"type": "number", "minimum": 0}, "observed_peak_power_w": {"type": "number", "minimum": 0}}}}}, "allOf": [{"if": {"properties": {"valid": {"const": false}}}, "then": {"properties": {"aggregate": {"type": "null"}, "per_gpu": {"type": "null"}}}}, {"if": {"properties": {"valid": {"const": true}}}, "then": {"properties": {"status": {"const": "valid"}, "aggregate": {"type": "object"}, "per_gpu": {"type": "object", "minProperties": 1}, "invalid_reasons": {"maxItems": 0}}}}]} + } +} diff --git a/experimental/video-generation/runtime-entry.example.sh b/experimental/video-generation/runtime-entry.example.sh new file mode 100644 index 0000000000..e8421a13a7 --- /dev/null +++ b/experimental/video-generation/runtime-entry.example.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copy beside the PREPARED runtime, fill the three paths, then pin its SHA256 +# in the site config. This entry never allocates, imports, installs, or downloads. +set -euo pipefail +runtime_root=/REPLACE_WITH_EXISTING_RUNTIME_PARENT +workspace=/REPLACE_WITH_PERSISTENT_WORKSPACE +container_name=REPLACE_WITH_EXISTING_ENROOT_NAME + +: "${SLURM_JOB_ID:?must enter through srun}" +: "${SLURM_STEP_ID:?must enter through an allocated step}" +: "${SLURM_STEP_GPUS:?Slurm must assign GPUs}" +[[ -d "$runtime_root/enroot-data/$container_name" && -f "$runtime_root/.rootfs-ready" ]] +[[ -d "$workspace" && $# -gt 0 ]] +exec 9>"$runtime_root/.session.lock" +flock -n 9 || { echo "Prepared runtime has an active session" >&2; exit 2; } +export ENROOT_DATA_PATH="$runtime_root/enroot-data" +export ENROOT_RUNTIME_PATH="/tmp/inferencex-h3-${SLURM_JOB_ID}-${SLURM_STEP_ID}/runtime" +export ENROOT_TEMP_PATH="/tmp/inferencex-h3-${SLURM_JOB_ID}-${SLURM_STEP_ID}/tmp" +export ENROOT_CACHE_PATH="$runtime_root/cache" + +# This site uses AutoDetect=nvidia and /dev/nvidia minor-number ordering. +# NVML query indices use PCI ordering and differ under partial allocations. +h3_gpu_uuids=$(python3 - <<'PY' +import os, re +from pathlib import Path +value = os.environ['SLURM_STEP_GPUS'] +if not re.fullmatch(r'[0-9]+(?:-[0-9]+)?(?:,[0-9]+(?:-[0-9]+)?)*', value): + raise SystemExit('Unsupported Slurm GPU assignment; no index fallback') +ids = [] +for part in value.split(','): + bounds = list(map(int, part.split('-'))) + if len(bounds) == 1: + ids.append(bounds[0]) + elif 0 <= bounds[0] <= bounds[1] < 8: + ids.extend(range(bounds[0], bounds[1] + 1)) + else: + raise SystemExit('GPU range outside the single H200 node') +if len(ids) != len(set(ids)) or not ids or any(i >= 8 for i in ids): + raise SystemExit('Invalid global GPU assignment') +devices = {} +for path in Path('/proc/driver/nvidia/gpus').glob('*/information'): + fields = dict(line.split(':', 1) for line in path.read_text().splitlines() if ':' in line) + minor, identity = fields.get('Device Minor', '').strip(), fields.get('GPU UUID', '').strip() + if minor.isdigit() and re.fullmatch(r'GPU-[0-9a-fA-F-]{36}', identity): + if int(minor) in devices: + raise SystemExit('Duplicate NVIDIA device minor') + devices[int(minor)] = identity +if any(index not in devices for index in ids): + raise SystemExit('Assigned Slurm device files lack NVIDIA UUIDs') +print(','.join(devices[index] for index in ids)) +PY +) +h3_gpu_rows=$(nvidia-smi --id="$h3_gpu_uuids" --query-gpu=uuid,name --format=csv,noheader) +H3_ASSIGNED_GPU_UUIDS=$(python3 - "$h3_gpu_rows" "$h3_gpu_uuids" <<'PY' +import csv, re, sys +rows = list(csv.reader(sys.argv[1].splitlines())) +if not rows or any(len(row) != 2 or 'H200' not in row[1] or not re.fullmatch(r'GPU-[0-9a-fA-F-]{36}', row[0].strip()) for row in rows): + raise SystemExit('Assigned hardware is not a physical H200 GPU set') +observed = [row[0].strip() for row in rows] +if sorted(observed) != sorted(sys.argv[2].split(',')): + raise SystemExit('NVIDIA query differs from assigned physical UUIDs') +print(','.join(observed)) +PY +) +export H3_ASSIGNED_GPU_UUIDS +export H3_ORIGINAL_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-} +export NVIDIA_VISIBLE_DEVICES=$H3_ASSIGNED_GPU_UUIDS +export CUDA_VISIBLE_DEVICES=$H3_ASSIGNED_GPU_UUIDS +export NVIDIA_DRIVER_CAPABILITIES=compute,utility +h3_env=(NVIDIA_VISIBLE_DEVICES NVIDIA_DRIVER_CAPABILITIES CUDA_VISIBLE_DEVICES + H3_ASSIGNED_GPU_UUIDS H3_ORIGINAL_CUDA_VISIBLE_DEVICES PYTHONDONTWRITEBYTECODE + SLURM_JOB_ID SLURM_STEP_ID SLURMD_NODENAME SLURM_PROCID SLURM_NTASKS + SLURM_JOB_GPUS SLURM_STEP_GPUS SLURM_CPU_BIND SLURM_CPUS_PER_TASK) +h3_args=() +for name in "${h3_env[@]}"; do + if [[ -v "$name" ]]; then h3_args+=(--env "$name"); fi +done +# Keep the existing cache/mount layout; add other already-prepared mounts here. +# This saved image has /etc/rc -> exec bash "$@", so it expects -c. Adapt +# the tail to the verified entrypoint when reusing a different prepared image. +exec enroot start --root --rw --mount "$workspace:/work" \ + "${h3_args[@]}" --env HF_HOME=/work/.cache/huggingface \ + -- "$container_name" -c 'cd /work && exec "$@"' bash "$@" diff --git a/experimental/video-generation/site.example.json b/experimental/video-generation/site.example.json new file mode 100644 index 0000000000..f701cbff10 --- /dev/null +++ b/experimental/video-generation/site.example.json @@ -0,0 +1,22 @@ +{ + "schema_version": 1, + "task_id": "h3-video-ci", + "workspace": { + "host": "/REPLACE_WITH_PERSISTENT_WORKSPACE", + "container": "/work" + }, + "runtime": { + "entry": "/REPLACE_WITH_REVIEWED_ENTRY_ONLY_SCRIPT.sh", + "entry_sha256": "REPLACE_WITH_SHA256", + "rootfs": "/REPLACE_WITH_EXISTING_ROOTFS", + "ready_marker": "/REPLACE_WITH_PREPARATION_RECEIPT", + "python": "/REPLACE_WITH_PREPARED_PYTHON" + }, + "spec": { + "path": "/REPLACE_WITH_FROZEN_GPU_SPEC.json", + "sha256": "REPLACE_WITH_SHA256" + }, + "resources": {"gpus": 4, "cpus": 32, "memory_gb": 1024, "minutes": 90}, + "allocation_receipts": [], + "mode": "smoke" +} diff --git a/experimental/video-generation/tests/test_ci.py b/experimental/video-generation/tests/test_ci.py new file mode 100644 index 0000000000..c8420d13b3 --- /dev/null +++ b/experimental/video-generation/tests/test_ci.py @@ -0,0 +1,340 @@ +"""Control-path tests use a fake scheduler. They do not claim GPU execution.""" +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +import ci + + +def config(tmp_path): + return {"schema_version": 1, "task_id": "h3-test", "workspace": {"host": str(tmp_path), "container": "/work"}, + "runtime": {"entry": str(tmp_path / "entry.sh"), "entry_sha256": "a" * 64, + "rootfs": str(tmp_path / "rootfs"), "ready_marker": str(tmp_path / "ready"), "python": "/opt/harness/bin/python"}, + "spec": {"path": str(tmp_path / "spec.json"), "sha256": "b" * 64}, + "resources": {"gpus": 4, "cpus": 32, "memory_gb": 256, "minutes": 90}, + "allocation_receipts": [], "mode": "smoke"} + + +def allocation(tmp_path, job="123"): + identity = {"JobId": job, "JobName": "owned-holder", "Comment": "h3:owner-nonce", "WorkDir": str(tmp_path), + "Account": "sa-shared", "Partition": "main", "UserId": f"tester({os.getuid()})"} + receipt = {"task_id": "h3-test", "identity": identity} + record = {**identity, "JobState": "RUNNING", "NumNodes": "1", "NodeList": "h200-node", + "AllocTRES": "cpu=64,mem=512G,node=1,gres/gpu=8", "NumCPUs": "64", "OverSubscribe": "NO", + "EndTime": (datetime.now(timezone.utc) + timedelta(hours=2)).strftime("%Y-%m-%dT%H:%M:%S")} + return receipt, record + + +def save_receipt(root, receipt): + path = root / "old" / "allocation.json" + path.parent.mkdir(parents=True) + ci.write(path, receipt) + return path + + +@pytest.mark.parametrize("mode", ["smoke", "serving-smoke"]) +def test_allocation_submits_from_receipted_work_directory(tmp_path, monkeypatch, mode): + run_dir = tmp_path / "results" + run_dir.mkdir() + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + scheduler = bin_dir / "salloc" + scheduler.write_text('#!/bin/sh\npwd > "$H3_TEST_SCHEDULER_CWD"\necho "salloc: Granted job allocation 123"\n') + scheduler.chmod(0o755) + observed = tmp_path / "scheduler-cwd" + monkeypatch.setenv("RUNNER_NAME", "h3-test-runner") + monkeypatch.setenv("PATH", str(bin_dir) + os.pathsep + os.environ["PATH"]) + monkeypatch.setenv("H3_TEST_SCHEDULER_CWD", str(observed)) + cfg = config(tmp_path) + cfg["mode"] = mode + cfg["resources"]["gpus"] = 2 + receipt = ci.allocate(cfg, run_dir) + assert observed.read_text().strip() == receipt["identity"]["WorkDir"] == str(run_dir) + argv = json.loads((run_dir / "allocation-command.json").read_text()) + assert ("--exclusive" in argv) is (mode == "smoke") + assert ("--gres=gpu:2" if mode == "serving-smoke" else "--gres=gpu:8") in argv + + +def test_reuse_checks_identity_and_retains_active_step_evidence(tmp_path, monkeypatch): + receipt, record = allocation(tmp_path) + save_receipt(tmp_path, receipt) + calls = [] + def command(argv, **kwargs): + calls.append(argv) + if "--steps" in argv: + assert "--format=%i|%N" in argv + return "123.0|h200-node\n" + return "123\n" + monkeypatch.setattr(ci, "command", command) + monkeypatch.setattr(ci, "job_record", lambda job: record) + found = ci.recover(config(tmp_path), tmp_path) + assert found["action"] == "reuse" + assert "123.0" in found["active_steps"] + assert not any(argv[0] in {"salloc", "scancel"} for argv in calls) + record["Comment"] = "another-task" + with pytest.raises(ValueError, match="identity differs"): + ci.recover(config(tmp_path), tmp_path) + + +def test_expired_capacity_records_reason_before_new_allocation(tmp_path, monkeypatch): + receipt, record = allocation(tmp_path) + save_receipt(tmp_path, receipt) + record["EndTime"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + monkeypatch.setattr(ci, "command", lambda argv, **kw: "123\n") + monkeypatch.setattr(ci, "job_record", lambda job: record) + found = ci.recover(config(tmp_path), tmp_path) + assert found["action"] == "allocate" + assert "remaining" in found["reasons"][0]["reason"] + + +def test_pending_or_unknown_submission_never_allocates(tmp_path, monkeypatch): + receipt, record = allocation(tmp_path) + path = save_receipt(tmp_path, receipt) + record["JobState"] = "PENDING" + monkeypatch.setattr(ci, "command", lambda argv, **kw: "123\n") + monkeypatch.setattr(ci, "job_record", lambda job: record) + assert ci.recover(config(tmp_path), tmp_path)["action"] == "wait" + path.unlink() + ci.write(path.with_name("allocation-intent.json"), {}) + with pytest.raises(ValueError, match="Unresolved allocation intent"): + ci.recover(config(tmp_path), tmp_path) + + +def test_scheduler_failure_does_not_mean_empty_queue(tmp_path, monkeypatch): + def failed(argv): + raise subprocess.CalledProcessError(1, argv) + monkeypatch.setattr(ci, "command", failed) + with pytest.raises(subprocess.CalledProcessError): + ci.recover(config(tmp_path), tmp_path) + + +def test_step_uses_bound_suballocation_and_persistent_entry(tmp_path, monkeypatch): + cfg = config(tmp_path) + receipt, record = allocation(tmp_path) + monkeypatch.setenv("SLURM_GPUS_PER_NODE", "8") + monkeypatch.setenv("SLURM_TRES_PER_TASK", "gres/gpu:8") + monkeypatch.setenv("SALLOC_PARTITION", "another-provider") + env = ci.environment() + assert not any(key.startswith(("SLURM_", "SALLOC_")) for key in env) + argv = ci.step_argv(cfg, receipt, record, tmp_path / "results", tmp_path / "package") + assert "--gpus-per-task=4" in argv and "--exclusive" in argv and "--exact" in argv + assert "--cpus-per-task=32" in argv and "--cpu-bind=verbose,cores" in argv + assert "--time=85" in argv + assert argv[-4:] == ["python3", str(tmp_path / "package" / "ci.py"), "--enter", str(tmp_path / "results")] + assert not any("container-image" in arg or "overlap" in arg for arg in argv) + + +def test_cleanup_only_cancels_bound_owned_step(tmp_path, monkeypatch): + receipt, record = allocation(tmp_path) + ci.write(tmp_path / "binding.json", {"job_id": "123", "step_id": "7"}) + queues = iter(["123.7\n123.4\n", "123.4\n"]) + calls = [] + def command(argv, **kw): + calls.append(argv) + return next(queues) if argv[0] == "squeue" else "" + monkeypatch.setattr(ci, "command", command) + monkeypatch.setattr(ci, "job_record", lambda job: record) + assert ci.drain_step(receipt, "h3-test", tmp_path)["status"] == "ended" + assert [call for call in calls if call[0] == "scancel"] == [["scancel", "123.7"]] + + +@pytest.mark.parametrize("finishes", [True, False]) +def test_allocation_cleanup_waits_for_slurm_epilog_with_finite_deadline(tmp_path, monkeypatch, finishes): + receipt, record = allocation(tmp_path) + elapsed = [0] + monkeypatch.setattr(ci, "time", SimpleNamespace(monotonic=lambda: elapsed[0], sleep=lambda seconds: elapsed.__setitem__(0, elapsed[0] + seconds))) + monkeypatch.setattr(ci, "job_record", lambda job: record) + def command(argv): + if argv[0] == "scancel": + return "" + return "" if finishes and elapsed[0] >= 20 else "COMPLETING\n" + monkeypatch.setattr(ci, "command", command) + if finishes: + assert ci.stop_allocation(receipt, "h3-test")["status"] == "released" + else: + with pytest.raises(RuntimeError, match="terminal state"): + ci.stop_allocation(receipt, "h3-test") + assert elapsed[0] <= 120 + + +@pytest.mark.parametrize("comparison_status", ["pass", "fail", "inconclusive"]) +def test_smoke_completion_is_separate_from_regression(comparison_status): + receipt = {"regression_status": "inconclusive", "ci_accepted": False} + verified = {"comparison": {"overall_status": comparison_status, "checks": [], "slots": [{ + role: {"status": "succeeded", "media": {"valid": True}, "analysis_error": None} + for role in ("baseline", "candidate")}]}, "runs": { + role: {"summary": {"scheduled": 1, "valid": 1}} for role in ("baseline", "candidate")}} + assert ci.smoke_exit(verified, receipt, "smoke") == 0 + assert receipt["ci_accepted"] is False + assert ci.smoke_exit(verified, receipt, "regression") == 2 + receipt["regression_status"] = "fail" + assert ci.smoke_exit(verified, receipt, "smoke") == 0 + assert ci.smoke_exit(verified, receipt, "regression") == 1 + verified["runs"]["candidate"]["summary"]["valid"] = 0 + assert ci.smoke_exit(verified, receipt, "smoke") == 1 + + +@pytest.mark.parametrize("failure", ["decode", "analysis", "warmup"]) +def test_smoke_rejects_fresh_media_failure_despite_recorded_success(failure): + observation = {"status": "succeeded", "media": {"valid": True}, "analysis_error": None} + comparison = {"slots": [{role: dict(observation) for role in ("baseline", "candidate")}], "checks": []} + verified = {"comparison": comparison, "runs": { + role: {"summary": {"scheduled": 1, "valid": 1}} for role in ("baseline", "candidate")}} + if failure == "decode": + comparison["slots"][0]["candidate"]["media"] = {"valid": False} + elif failure == "analysis": + comparison["slots"][0]["candidate"]["analysis_error"] = "decoder failed" + else: + comparison["checks"] = [{"name": "candidate.warmup", "status": "inconclusive"}] + assert ci.smoke_exit(verified, {"regression_status": "inconclusive"}, "smoke") == 1 + + +def test_changed_runtime_blocks_before_scheduler(tmp_path): + cfg = config(tmp_path) + Path(cfg["runtime"]["rootfs"]).mkdir() + Path(cfg["runtime"]["ready_marker"]).touch() + Path(cfg["runtime"]["entry"]).write_text("changed") + with pytest.raises(ValueError, match="entry script changed"): + ci.prepared_spec(cfg) + + +def test_staging_reuses_identical_source_and_refuses_drift(tmp_path): + source = tmp_path / "source" + (source / "evaluator").mkdir(parents=True) + (source / "ci.py").write_text("entry") + (source / "evaluator" / "__init__.py").write_text("") + dest = tmp_path / "package" + original = ci.stage_package(source, dest) + assert ci.stage_package(source, dest) == original + (dest / "ci.py").write_text("tampered") + with pytest.raises(ValueError, match="source differs"): + ci.stage_package(source, dest) + + +@pytest.mark.parametrize("reused", [False, True]) +def test_failure_collects_original_outputs_and_preserves_holder(tmp_path, monkeypatch, reused): + cfg = config(tmp_path) + entry = Path(cfg["runtime"]["entry"]) + entry.write_text("entry") + Path(cfg["runtime"]["ready_marker"]).write_text("pinned image and preparation identity") + cfg["runtime"]["entry_sha256"] = ci.digest(entry) + receipt, record = allocation(tmp_path) + monkeypatch.setenv("GITHUB_RUN_ID", "456") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + monkeypatch.setenv("H3_SOURCE_SHA", "a" * 40) + monkeypatch.setattr(ci, "prepared_spec", lambda cfg: {"test_only": True}) + monkeypatch.setattr(ci, "command", lambda argv, **kw: "a" * 40 if "rev-parse" in argv else "") + monkeypatch.setattr(ci, "stage_package", lambda source, destination: {}) + decision = {"action": "reuse", "receipt": receipt} if reused else {"action": "allocate"} + monkeypatch.setattr(ci, "recover", lambda cfg, results: decision) + monkeypatch.setattr(ci, "allocate", lambda cfg, path: receipt) + monkeypatch.setattr(ci, "job_record", lambda job: record) + monkeypatch.setattr(ci, "drain_step", lambda *args: {"status": "ended"}) + canceled = [] + monkeypatch.setattr(ci, "stop_allocation", lambda receipt, task: canceled.append(receipt["identity"]["JobId"]) or {"status": "released"}) + def fail(argv, log, seconds): + log.write_text("test-only infrastructure failure") + (log.parent / "partial.mp4").write_bytes(b"retained failed-output bytes; not video evidence") + raise RuntimeError("server exited before readiness") + monkeypatch.setattr(ci, "run_step", fail) + output = tmp_path / "download" + assert ci.launch(cfg, output) == 2 + assert (output / "partial.mp4").read_bytes().startswith(b"retained") + state = ci.read(output / "ci.json") + assert "server exited" in state["error"] and state["ci_accepted"] is False + assert canceled == ([] if reused else ["123"]) + assert "partial.mp4" in (output / "SHA256SUMS").read_text() + manifest = ci.read(output / "manifest.json") + assert manifest["git_commit"] == "a" * 40 + assert manifest["slurm_allocation"]["identity"]["JobId"] == "123" + assert manifest["evidence"]["ci.json"] == ci.digest(output / "ci.json") + assert (output / "runtime-readiness.record").read_text() == "pinned image and preparation identity" + assert manifest["evidence"]["runtime-entry.sh"] == ci.digest(entry) + + +def test_timeout_stops_only_its_local_process_group(tmp_path): + with pytest.raises(subprocess.TimeoutExpired): + ci.run_step([sys.executable, "-c", "import time; time.sleep(30)"], tmp_path / "log", 0.1) + + +def test_missing_weights_fail_preparation_before_any_slurm_call(tmp_path, monkeypatch): + from evaluator import mvp_gpu_job + cfg = config(tmp_path) + Path(cfg["runtime"]["rootfs"]).mkdir() + Path(cfg["runtime"]["ready_marker"]).touch() + Path(cfg["runtime"]["entry"]).write_text("entry") + cfg["runtime"]["entry_sha256"] = ci.digest(cfg["runtime"]["entry"]) + (tmp_path / "source").mkdir() + spec = {"baseline": {"source": "/work/source"}, "candidate": {"source": "/work/source"}, + "authorization": {"compute_approved": True, "model_license_reviewed": True, "approval_reference": "test-only"}, + "limits": {"job_seconds": 4500}, "model": {"path": "/work/models", "files": [{"path": "model.safetensors", "size_bytes": 4}]}} + ci.write(cfg["spec"]["path"], spec) + cfg["spec"]["sha256"] = ci.digest(cfg["spec"]["path"]) + monkeypatch.setattr(mvp_gpu_job, "validate_gpu_job", lambda spec: spec) + monkeypatch.setattr(ci, "command", lambda *args, **kwargs: pytest.fail("Preparation must not call Slurm")) + with pytest.raises(ValueError, match="stage weights before allocating"): + ci.prepared_spec(cfg) + (tmp_path / "models").mkdir() + (tmp_path / "models" / "model.safetensors").write_bytes(b"test") + assert ci.prepared_spec(cfg)["model"] == spec["model"] + + +def test_entry_failure_retains_step_binding_before_runtime(tmp_path, monkeypatch): + cfg = config(tmp_path) + receipt, record = allocation(tmp_path) + Path(cfg["runtime"]["entry"]).write_text("changed runtime entry") + ci.write(tmp_path / "context.json", {"config": cfg, "allocation": receipt, "node": record["NodeList"]}) + for key, value in {"SLURM_JOB_ID": "123", "SLURM_STEP_ID": "9", "SLURMD_NODENAME": "h200-node"}.items(): + monkeypatch.setenv(key, value) + monkeypatch.setattr(ci.os, "sched_getaffinity", lambda pid: {2, 3, 4, 5}, raising=False) + with pytest.raises(ValueError, match="Entry changed"): + ci.enter(tmp_path) + binding = ci.read(tmp_path / "binding.json") + assert binding["job_id"] == "123" and binding["step_id"] == "9" + assert binding["cpu_affinity"] == [2, 3, 4, 5] + + +def test_export_excludes_only_known_caches_and_preserves_media(tmp_path): + source, target = tmp_path / "source", tmp_path / "export" + cache = source / "gpu" / "supervisor" / "baseline" / "cache" + cache.mkdir(parents=True) + (cache / "kernel-link").symlink_to("/not-a-readable-cache-target") + media = source / "gpu" / "baseline" / "outputs" / "sample.mp4" + media.parent.mkdir(parents=True) + media.write_bytes(b"test-only media bytes") + report = source / "report" / "index.html" + report.parent.mkdir() + report.write_text("test-only report") + ci.collect(source, target) + assert (target / "gpu/baseline/outputs/sample.mp4").read_bytes() == b"test-only media bytes" + assert (target / "report/index.html").read_text() == "test-only report" + assert not (target / "gpu/supervisor/baseline/cache").exists() + assert (cache / "kernel-link").is_symlink() + assert "kernel-link" not in (target / "SHA256SUMS").read_text() + + +@pytest.mark.parametrize('assignment,success', [('4,5', True), ('4-5', True), ('0,1', False)]) +def test_entry_resolves_device_minors_instead_of_nvml_indices(tmp_path, assignment, success): + driver = tmp_path / 'driver' + ids = ['GPU-12441e19-6453-d8c4-69a8-9fe1cd8b770c', 'GPU-994fd357-abc0-57a9-12d5-f7d40e741530'] + for minor, identity in zip((4, 5), ids): + info = driver / f'pci-{minor}' / 'information' + info.parent.mkdir(parents=True) + info.write_text(f'Model: NVIDIA H200\nDevice Minor: {minor}\nGPU UUID: {identity}\n') + entry = (Path(ci.__file__).parent / 'runtime-entry.example.sh').read_text() + program = entry.split("h3_gpu_uuids=$(python3 - <<'PY'\n", 1)[1].split('\nPY\n)', 1)[0] + program = program.replace('/proc/driver/nvidia/gpus', str(driver)) + result = subprocess.run([sys.executable, '-c', program], env={**os.environ, 'SLURM_STEP_GPUS': assignment}, + capture_output=True, text=True) + if success: + assert result.returncode == 0 + assert result.stdout.strip() == ','.join(ids) + else: + assert result.returncode != 0 + assert 'lack NVIDIA UUIDs' in result.stderr diff --git a/experimental/video-generation/tests/test_export_ci.py b/experimental/video-generation/tests/test_export_ci.py new file mode 100644 index 0000000000..54922b6808 --- /dev/null +++ b/experimental/video-generation/tests/test_export_ci.py @@ -0,0 +1,321 @@ +"""Exporter trust and failure paths, with GitHub/download collaborators faked.""" + +from copy import deepcopy +import hashlib +from io import BytesIO +import json +from pathlib import Path +import sys + +import pytest + +import export_ci + + +REPO = "SemiAnalysisAI/InferenceX" +SHA = "a" * 40 +REAL_API = export_ci.api + + +@pytest.fixture +def github(monkeypatch): + run = {"id": 123, "repository": {"full_name": REPO}, "head_repository": {"full_name": REPO}, + "head_sha": SHA, "run_attempt": 1, "event": "workflow_dispatch", "status": "completed", + "conclusion": "success", "html_url": f"https://github.com/{REPO}/actions/runs/123"} + jobs = {"jobs": [{"id": 456, "name": "h3-video / p1 | H3 video H200 smoke", + "status": "completed", "conclusion": "success"}]} + artifact = {"id": 789, "name": "h3-video-123-1", "expired": False, "size_in_bytes": 1000, + "workflow_run": {"id": 123, "head_sha": SHA}} + responses = {"actions/runs/123": run, "actions/runs/123/attempts/1/jobs": jobs, + "actions/runs/123/artifacts": {"artifacts": [artifact]}} + monkeypatch.setattr(export_ci, "api", lambda path: deepcopy(responses[path])) + monkeypatch.setenv("GITHUB_SHA", SHA) + monkeypatch.setenv("GITHUB_RUN_ID", "999") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + monkeypatch.setenv("GITHUB_REPOSITORY", REPO) + return run, jobs, artifact + + +@pytest.mark.parametrize("value", ["", "0", "123,123", "1,2,3", "../123", "1;touch x", "1, 2", "123\n", "1" * 21]) +def test_source_ids_reject_ambiguous_or_non_numeric_dispatch_input(value): + with pytest.raises(ValueError): + export_ci.source_ids(value) + + +def test_source_ids_preserve_order_without_shell_interpretation(): + assert export_ci.source_ids("34293342829,34291306687") == ["34293342829", "34291306687"] + + +def test_inventory_reuse_requires_the_hardware_job_and_artifact(github): + with pytest.raises(ValueError): + export_ci.verified_execution("123", inventory=True) + github[1]["jobs"][0]["name"] = "h3-video / p1.500 | H3 H200 hardware inventory" + github[2]["name"] = "h3-hardware-123-1" + source_ci, artifact = export_ci.verified_execution("123", inventory=True) + assert source_ci["databaseId"] == 123 and artifact["name"] == "h3-hardware-123-1" + + +def test_authentication_stays_out_of_saved_public_ci_receipts(github, monkeypatch, tmp_path): + run, jobs, artifact = github + sentinel = "CPU-test-authentication-sentinel" + for value in (run, jobs["jobs"][0], artifact): + value["unexpected_auth_field"] = sentinel + responses = iter((run, jobs, {"artifacts": [artifact]})) + def response(request, timeout): + assert request.get_header("Authorization") == "Bearer " + sentinel + assert request.full_url.startswith("https://api.github.com/repos/" + REPO + "/") + return BytesIO(json.dumps(next(responses)).encode()) + monkeypatch.setenv("GH_TOKEN", sentinel) + monkeypatch.setattr(export_ci, "api", REAL_API) + monkeypatch.setattr(export_ci, "urlopen", response) + source_ci, source_artifact = export_ci.verified_execution("123") + path = tmp_path / "public-ci.json" + export_ci.ci.write(path, {"source_ci": source_ci, "artifact": source_artifact}) + assert sentinel not in path.read_text() + assert source_ci["databaseId"] == 123 and source_artifact["id"] == 789 + + +@pytest.mark.parametrize("job_name", ["h3-video / p1.500 | H3 video H200 smoke", "p1 | H3 video H200 smoke"]) +def test_verified_execution_returns_exact_attempt_and_artifact(github, job_name): + github[1]["jobs"][0]["name"] = job_name + source_ci, artifact = export_ci.verified_execution("123") + assert (source_ci["databaseId"], source_ci["headSha"], source_ci["runAttempt"]) == (123, SHA, 1) + assert source_ci["jobs"][0]["id"] == 456 + assert artifact["id"] == 789 + + +@pytest.mark.parametrize("defect", ["fork", "repository", "event", "failed", "other_in_progress", + "run_id", "job_failed", "job_not_completed", "job_name", "duplicate_job", + "expired", "artifact_commit", "artifact_run", "empty_artifact", "oversized_artifact"]) +def test_verified_execution_rejects_wrong_execution_or_artifact_identity(github, defect): + run, jobs, artifact = github + if defect == "fork": + run["head_repository"]["full_name"] = "someone/InferenceX" + elif defect == "repository": + run["repository"]["full_name"] = "someone/InferenceX" + elif defect == "event": + run["event"] = "pull_request" + elif defect == "failed": + run["conclusion"] = "failure" + elif defect == "other_in_progress": + run.update(status="in_progress", conclusion=None) + elif defect == "run_id": + run["id"] = 999 + elif defect == "job_failed": + jobs["jobs"][0]["conclusion"] = "failure" + elif defect == "job_not_completed": + jobs["jobs"][0]["status"] = "in_progress" + elif defect == "job_name": + jobs["jobs"][0]["name"] = "unrelated H3 video H200 smoke fixture" + elif defect == "duplicate_job": + jobs["jobs"].append(deepcopy(jobs["jobs"][0])) + elif defect == "expired": + artifact["expired"] = True + elif defect == "artifact_commit": + artifact["workflow_run"]["head_sha"] = "b" * 40 + elif defect == "artifact_run": + artifact["workflow_run"]["id"] = 124 + elif defect == "empty_artifact": + artifact["size_in_bytes"] = 0 + else: + artifact["size_in_bytes"] = 2 * 1024**3 + 1 + with pytest.raises(ValueError): + export_ci.verified_execution("123") + + +@pytest.mark.parametrize("mismatch", [None, "commit", "attempt"]) +def test_current_run_exception_requires_exact_producer_and_finished_h3_job(github, monkeypatch, mismatch): + run, _, _ = github + run.update(status="in_progress", conclusion=None) + monkeypatch.setenv("GITHUB_RUN_ID", "123") + if mismatch == "commit": + monkeypatch.setenv("GITHUB_SHA", "b" * 40) + elif mismatch == "attempt": + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") + if mismatch: + with pytest.raises(ValueError): + export_ci.verified_execution("123") + else: + source_ci, _ = export_ci.verified_execution("123") + assert source_ci["status"] == "in_progress" + assert source_ci["jobs"][0]["conclusion"] == "success" + + +def fake_download(monkeypatch): + original = b"raw runtime log\n" + seal = f"{hashlib.sha256(original).hexdigest()} runtime.log\n".encode() + def download(argv, **kwargs): + assert kwargs == {"check": True, "timeout": 180} + assert argv[:5] == ["gh", "run", "download", "123", "--repo"] + destination = Path(argv[argv.index("--dir") + 1]) + destination.mkdir(parents=True) + (destination / "runtime.log").write_bytes(original) + (destination / "SHA256SUMS").write_bytes(seal) + monkeypatch.setattr(export_ci.subprocess, "run", download) + monkeypatch.setattr(export_ci.ci, "command", lambda argv, **kwargs: SHA) + return original, seal + + +def test_current_run_export_keeps_independent_github_metadata_for_manifest_join(github, monkeypatch, tmp_path): + run, _, _ = github + run.update(status="in_progress", conclusion=None) + monkeypatch.setenv("GITHUB_RUN_ID", "123") + fake_download(monkeypatch) + observed = [] + def write_result(target, **kwargs): + observed.append(kwargs) + (target / "result.json").write_text('{"status":"complete"}') + return {"status": "complete"} + monkeypatch.setattr(export_ci, "write_result", write_result) + assert export_ci.publish(["123"], tmp_path / "output", None) == 0 + assert observed[0]["producer"]["mode"] == "same_run_export" + assert observed[0]["source_ci"]["databaseId"] == 123 + assert observed[0]["source_ci"]["headSha"] == SHA + assert observed[0]["source_ci"]["runAttempt"] == 1 + + +def test_failed_result_export_preserves_raw_files_old_seal_and_error_seal(github, monkeypatch, tmp_path): + original, seal = fake_download(monkeypatch) + def write_result(target, **kwargs): + (target / "result.json").write_text('{"status":"failed"}') + raise ValueError("Media reference is missing") + monkeypatch.setattr(export_ci, "write_result", write_result) + output = tmp_path / "output" + assert export_ci.publish(["123"], output, None) == 2 + target = output / "source-123" + assert (target / "runtime.log").read_bytes() == original + assert (target / "source-SHA256SUMS").read_bytes() == seal + assert json.loads((target / "export-error.json").read_text())["exit_code"] == 2 + assert json.loads((output / "index.json").read_text())["status"] == "partial" + checksums = dict(line.split(" ", 1)[::-1] for line in (target / "SHA256SUMS").read_text().splitlines()) + assert {"runtime.log", "source-SHA256SUMS", "result.json", "export-error.json"} <= checksums.keys() + for name, expected in checksums.items(): + assert hashlib.sha256((target / name).read_bytes()).hexdigest() == expected + + +def seal(root): + files = [(path.name, hashlib.sha256(path.read_bytes()).hexdigest()) for path in root.iterdir() if path.name != "SHA256SUMS"] + (root / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in sorted(files))) + + +def hardware_artifact(root): + from inventory_ci import classify_tdp + + root.mkdir() + xml = "GPU-fixtureCPU TEST" + identity = {"run_id": "999", "run_attempt": "1", "source_sha": SHA} + binding = {"job_id": "7", "step_id": "0", "gpu_uuids": ["GPU-fixture"], "node": "fixture-node"} + profile = {**identity, "git_commit": SHA, "ci": {"repository": REPO, "run_id": "999", "run_attempt": "1"}, + "slurm": deepcopy(binding), "gpu_uuids": ["GPU-fixture"], "tdp": classify_tdp(xml, ["GPU-fixture"]), + "raw": {"inventory": "nvidia-smi.xml", "topology": "topology.txt"}} + files = {"hardware-profile.json": profile, "binding.json": binding, + "ci.json": {**identity, "phase": "complete", "exit_code": 0, + "step_cleanup": {"status": "ended", "step_id": "7.0"}, + "allocation_cleanup": {"status": "released"}, "allocation_reused": False, + "allocation": {"identity": {"JobId": "7"}}}, + "manifest.json": {**identity, "exit_code": 0, "slurm_allocation": {"identity": {"JobId": "7"}}}, + "step-result.json": {"exit_code": 0, "inventory_completed": True}} + for name, value in files.items(): + (root / name).write_text(json.dumps(value)) + (root / "nvidia-smi.xml").write_text(xml) + (root / "topology.txt").write_text("CPU TEST topology placeholder") + seal(root) + return profile + + +def test_verified_hardware_preserves_source_and_joins_slurm_identity(tmp_path): + root = tmp_path / "hardware" + hardware_artifact(root) + original = (root / "hardware-profile.json").read_bytes() + profile = export_ci.verified_hardware(root, "999", "1", SHA) + assert profile["slurm"]["job_id"] == "7" + assert profile["raw"]["inventory"] == "hardware/nvidia-smi.xml" + assert (root / "hardware-profile.json").read_bytes() == original + + +def test_reclassify_retained_raw_pci_without_rewriting_the_inventory(tmp_path): + root = tmp_path / "hardware" + hardware_artifact(root) + (root / "nvidia-smi.xml").write_text('GPU-fixtureNVIDIA H200' + '233510DE18BE10DE') + seal(root) + original = (root / "hardware-profile.json").read_bytes() + profile = export_ci.verified_hardware(root, "999", "1", SHA) + assert profile["tdp"]["status"] == "verified" and profile["tdp"]["watts_per_gpu"] == 700 + assert profile["recorded_tdp_classification"]["status"] == "unknown" + assert (root / "hardware-profile.json").read_bytes() == original + + +def test_cpu_export_reuses_independently_verified_inventory_commit(github, monkeypatch, tmp_path): + fake_download(monkeypatch) + original_download = export_ci.subprocess.run + execution = export_ci.verified_execution + def admission(run_id, *, inventory=False): + if inventory: + assert run_id == "456" + return {"runAttempt": 2, "headSha": "b" * 40}, {"name": "h3-hardware-456-2"} + return execution(run_id) + def download(argv, **kwargs): + if argv[3] == "456": + hardware_artifact(Path(argv[argv.index("--dir") + 1])) + else: + original_download(argv, **kwargs) + observed = [] + def verify(root, run_id, attempt, sha): + observed.append((run_id, attempt, sha)) + return {"gpu_uuids": ["GPU-fixture"]} + monkeypatch.setattr(export_ci, "verified_execution", admission) + monkeypatch.setattr(export_ci.subprocess, "run", download) + monkeypatch.setattr(export_ci, "verified_hardware", verify) + monkeypatch.setattr(export_ci, "write_result", lambda *args, **kwargs: {"status": "complete"}) + monkeypatch.setattr(export_ci.ci, "allocate", lambda *args, **kwargs: pytest.fail("CPU export allocated GPUs")) + output = tmp_path / "output" + assert export_ci.publish(["123"], output, None, hardware_run_id="456") == 0 + assert observed == [("456", "2", "b" * 40)] + assert (output / "source-123/hardware/nvidia-smi.xml").is_file() + + +@pytest.mark.parametrize("defect", ["checksum", "cleanup", "step", "gpu", "missing_raw", "invented_tdp"]) +def test_verified_hardware_rejects_bad_seal_or_unproven_teardown(tmp_path, defect): + root = tmp_path / "hardware" + hardware_artifact(root) + if defect == "checksum": + (root / "nvidia-smi.xml").write_text("changed after seal") + elif defect == "missing_raw": + (root / "topology.txt").unlink() + seal(root) + else: + name = "hardware-profile.json" if defect in ("gpu", "invented_tdp") else "ci.json" + value = json.loads((root / name).read_text()) + if defect == "cleanup": + value["allocation_cleanup"]["status"] = "failed" + elif defect == "step": + value["step_cleanup"]["step_id"] = "8.0" + elif defect == "gpu": + value["gpu_uuids"] = ["GPU-other"] + else: + value["tdp"]["watts_per_gpu"] = 700 + (root / name).write_text(json.dumps(value)) + seal(root) + with pytest.raises(ValueError): + export_ci.verified_hardware(root, "999", "1", SHA) + + +@pytest.mark.parametrize("field", ["git_commit", "run_id", "run_attempt"]) +def test_hardware_profile_mismatch_fails_before_download_and_retains_error(github, monkeypatch, tmp_path, field): + monkeypatch.setattr(export_ci.ci, "command", lambda argv, **kwargs: SHA) + monkeypatch.setattr(export_ci.subprocess, "run", lambda *a, **k: pytest.fail("Mismatched hardware must not start download")) + hardware = tmp_path / "hardware" + profile = hardware_artifact(hardware) + if field == "git_commit": + profile[field] = "b" * 40 + else: + profile["ci"][field] = "2" + (hardware / "hardware-profile.json").write_text(json.dumps(profile)) + seal(hardware) + output = tmp_path / "output" + monkeypatch.setattr(sys, "argv", ["export_ci.py", "--source-run-ids", "123", "--hardware", str(hardware), "--output", str(output)]) + assert export_ci.main() == 2 + error = json.loads((output / "export-error.json").read_text()) + assert error["exit_code"] == 2 + assert "Hardware" in error["error"] diff --git a/experimental/video-generation/tests/test_inventory_ci.py b/experimental/video-generation/tests/test_inventory_ci.py new file mode 100644 index 0000000000..5a0583de03 --- /dev/null +++ b/experimental/video-generation/tests/test_inventory_ci.py @@ -0,0 +1,335 @@ +"""Fake-scheduler inventory tests; no GPU or model execution.""" +import copy +import sys +from pathlib import Path + +import pytest + +import ci +import inventory_ci as inv +from test_ci import allocation, config +from test_mvp_gpu_job import spec + + +def prepared(tmp_path, spec): + cfg = config(tmp_path) + Path(cfg["runtime"]["rootfs"]).mkdir() + Path(cfg["runtime"]["ready_marker"]).write_text("prepared test runtime") + Path(cfg["runtime"]["entry"]).write_text("test-only entry") + cfg["runtime"]["entry_sha256"] = ci.digest(cfg["runtime"]["entry"]) + python = ci.host_path(cfg, cfg["runtime"]["python"]) + python.parent.mkdir(parents=True) + python.touch() + ci.write(cfg["spec"]["path"], spec) + cfg["spec"]["sha256"] = ci.digest(cfg["spec"]["path"]) + return cfg + + +def test_inventory_keeps_pins_without_loading_or_requiring_model_files(tmp_path, spec, monkeypatch): + cfg = prepared(tmp_path, spec) + original = copy.deepcopy(cfg) + for item in spec["model"]["files"]: + (Path(spec["model"]["path"]) / item["path"]).unlink() + monkeypatch.setattr(ci, "prepared_spec", lambda cfg: pytest.fail("Model preparation must not run")) + bounded, approval = inv.prepare(cfg) + assert bounded["resources"] == {"gpus": 4, "cpus": 4, "memory_gb": 8, "minutes": 10} + assert cfg == original and approval["compute_approved"] is True + Path(cfg["runtime"]["entry"]).write_text("changed") + with pytest.raises(ValueError, match="entry script changed"): + inv.prepare(cfg) + + +def test_inventory_rejects_missing_approval_before_scheduler(tmp_path, spec, monkeypatch): + spec["authorization"]["compute_approved"] = False + cfg = prepared(tmp_path, spec) + monkeypatch.setattr(ci, "command", lambda *args: pytest.fail("Must reject before scheduler")) + with pytest.raises(ValueError, match="lacks approval"): + inv.collect_inventory(cfg, tmp_path / "out") + + +@pytest.mark.parametrize("reused", [False, True]) +@pytest.mark.parametrize("step_fails", [False, True]) +def test_inventory_lifecycle_retains_logs_and_only_releases_owned_holder(tmp_path, spec, monkeypatch, reused, step_fails): + cfg = prepared(tmp_path, spec) + receipt, record = allocation(tmp_path) + monkeypatch.setenv("GITHUB_RUN_ID", "456") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") + monkeypatch.setenv("H3_SOURCE_SHA", "a" * 40) + monkeypatch.setattr(ci, "command", lambda argv, **kw: "a" * 40 if "rev-parse" in argv else "") + monkeypatch.setattr(ci, "stage_package", lambda *args: {}) + monkeypatch.setattr(inv, "source_target", lambda *args: {"node": "h200-node", "gpu_uuids": [], "sources": []}) + monkeypatch.setattr(ci, "recover", lambda *args, **kwargs: {"action": "reuse", "receipt": receipt} if reused else {"action": "allocate"}) + monkeypatch.setattr(ci, "allocate", lambda *args, **kwargs: receipt) + monkeypatch.setattr(ci, "job_record", lambda job: record) + drained, released = [], [] + monkeypatch.setattr(ci, "drain_step", lambda *args: drained.append("step") or {"status": "ended"}) + monkeypatch.setattr(ci, "stop_allocation", lambda *args: released.append("holder") or {"status": "released"}) + def run_step(argv, log, seconds): + assert "--gpus-per-task=4" in argv and "--cpus-per-task=4" in argv + assert "--mem=8G" in argv and "--time=5" in argv + assert Path(argv[-3]).name == "inventory_ci.py" and argv[-2] == "--enter" + assert seconds == 360 + log.write_text("retained inventory query output") + if step_fails: + raise RuntimeError("test inventory failure") + ci.write(log.parent / "step-result.json", {"exit_code": 0, "inventory_completed": True}) + ci.write(log.parent / "hardware-profile.json", {"observation_kind": "test-only"}) + return 0 + monkeypatch.setattr(ci, "run_step", run_step) + output = tmp_path / "out" + assert inv.collect_inventory(cfg, output) == (2 if step_fails else 0) + state = ci.read(output / "ci.json") + assert state["phase"] == ("failed" if step_fails else "complete") + assert drained == ["step"] and released == ([] if reused else ["holder"]) + assert (output / "srun.log").read_text() == "retained inventory query output" + assert ci.read(output / "manifest.json")["evidence"]["ci.json"] == ci.digest(output / "ci.json") + assert "srun.log" in (output / "SHA256SUMS").read_text() + + +def test_entry_retains_step_before_rejecting_changed_runtime(tmp_path, monkeypatch): + cfg = config(tmp_path) + Path(cfg["runtime"]["entry"]).write_text("drift") + receipt, record = allocation(tmp_path) + ci.write(tmp_path / "context.json", {"config": cfg, "allocation": receipt, "node": record["NodeList"]}) + monkeypatch.setenv("SLURM_JOB_ID", "123") + monkeypatch.setenv("SLURM_STEP_ID", "7") + monkeypatch.setenv("SLURMD_NODENAME", record["NodeList"]) + monkeypatch.setattr(inv.os, "sched_getaffinity", lambda pid: {1, 2, 3, 4}, raising=False) + monkeypatch.setattr(inv.os, "execv", lambda *args: pytest.fail("Drifted runtime must not start")) + with pytest.raises(ValueError, match="Entry changed"): + inv.enter(tmp_path) + assert ci.read(tmp_path / "binding.json")["step_id"] == "7" + + +def inside_context(tmp_path, monkeypatch): + cfg = config(tmp_path) + cfg["runtime"]["python"] = sys.executable + cfg["resources"] = dict(inv.RESOURCES) + receipt, record = allocation(tmp_path) + ci.write(tmp_path / "context.json", {"config": cfg, "allocation": receipt, "node": record["NodeList"], + "package_files": {}, "source_sha": "a" * 40, "run_id": "456", "run_attempt": "1", + "target": {"node": record["NodeList"], "gpu_uuids": ["GPU-a", "GPU-b", "GPU-c", "GPU-d"], "sources": []}, + "ci": {"run_id": "456", "run_attempt": "1", "repository": "SemiAnalysisAI/InferenceX", "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/456"}}) + ci.write(tmp_path / "binding.json", {"job_id": "123", "step_id": "7", "node": record["NodeList"], "cpu_affinity": [1, 2, 3, 4]}) + for key, value in {"SLURM_JOB_ID": "123", "SLURM_STEP_ID": "7", "SLURMD_NODENAME": record["NodeList"], "SLURM_PROCID": "0", "SLURM_NTASKS": "1", "H3_ASSIGNED_GPU_UUIDS": "GPU-a,GPU-b,GPU-c,GPU-d"}.items(): + monkeypatch.setenv(key, value) + monkeypatch.setattr(ci, "inventory", lambda *args: {}) + monkeypatch.setattr(inv.os, "sched_getaffinity", lambda pid: {1, 2, 3, 4}, raising=False) + monkeypatch.setattr(inv, "cuda_devices", lambda: ["GPU-a", "GPU-b", "GPU-c", "GPU-d"]) + + +def test_inventory_rejects_uuid_mismatch_before_any_query(tmp_path, monkeypatch): + inside_context(tmp_path, monkeypatch) + monkeypatch.setenv("H3_ASSIGNED_GPU_UUIDS", "GPU-foreign,GPU-b,GPU-c,GPU-d") + monkeypatch.setattr(inv, "GpuProbe", lambda *args, **kwargs: pytest.fail("Must reject before probe")) + assert inv.inside(tmp_path) == 2 + assert "UUIDs differ" in ci.read(tmp_path / "step-result.json")["error"] + + +def test_inventory_collects_current_limits_without_inventing_variant_or_history(tmp_path, monkeypatch): + inside_context(tmp_path, monkeypatch) + calls = [] + class Probe: + def __init__(self, devices, timeout): + self.devices = devices + def snapshot(self): + calls.append("snapshot") + return {"compute_apps": [], "gpus": [{"uuid": uuid, "name": "NVIDIA H200"} for uuid in self.devices]} + def power_configuration(self): + calls.append("power_configuration") + return {"status": "recorded", "gpus": [{"uuid": uuid, "configured_limit_w": 700} for uuid in self.devices]} + monkeypatch.setattr(inv, "GpuProbe", Probe) + def capture(argv, path): + calls.append(argv) + path.write_text("test-only raw inventory") + monkeypatch.setattr(inv, "capture", capture) + assert inv.inside(tmp_path) == 0 + profile = ci.read(tmp_path / "hardware-profile.json") + assert calls.count("snapshot") == 2 and "power_configuration" in calls + assert ["nvidia-smi", "-q", "-x"] in calls and ["nvidia-smi", "topo", "-m"] in calls + assert profile["hardware_variant"] is None + assert profile["historical_benchmark_power_limits"] == "not_observed" + assert profile["power_configuration"]["gpus"][0]["configured_limit_w"] == 700 + assert profile["run_id"] == "456" and profile["slurm"]["step_id"] == "7" + + +def test_capture_preserves_partial_query_output_on_timeout(tmp_path, monkeypatch): + import subprocess + def timeout(*args, **kwargs): + raise subprocess.TimeoutExpired(["nvidia-smi"], 20, output=b"partial XML", stderr=b"query stalled") + monkeypatch.setattr(inv.subprocess, "run", timeout) + with pytest.raises(subprocess.TimeoutExpired): + inv.capture(["nvidia-smi", "-q", "-x"], tmp_path / "inventory.xml") + assert (tmp_path / "inventory.xml").read_text() == "partial XML" + assert (tmp_path / "inventory.xml.stderr.log").read_text() == "query stalled" + + +def test_inventory_preserves_foreign_compute_evidence_and_stops_queries(tmp_path, monkeypatch): + inside_context(tmp_path, monkeypatch) + class BusyProbe: + def __init__(self, *args, **kwargs): + pass + def snapshot(self): + return {"compute_apps": [{"gpu_uuid": "GPU-a", "pid": 999}], "gpus": []} + monkeypatch.setattr(inv, "GpuProbe", BusyProbe) + monkeypatch.setattr(inv, "capture", lambda *args: pytest.fail("Busy GPU must stop inventory")) + assert inv.inside(tmp_path) == 2 + assert ci.read(tmp_path / "gpu-before.json")["compute_apps"][0]["pid"] == 999 + assert "active compute" in ci.read(tmp_path / "step-result.json")["error"] + + +def xml_gpu(uuid="GPU-a", device="0x233510DE", subsystem="0x18BE10DE", name="NVIDIA H200"): + return f"{uuid}{name}{device}{subsystem}" + + +def test_tdp_uses_vendor_pci_identity_and_distinguishes_nvl(): + sxm = inv.classify_tdp("" + xml_gpu() + "", ["GPU-a"]) + assert sxm["status"] == "verified" and sxm["watts_per_gpu"] == 700 + assert sxm["hardware_variant"] == "H200 SXM" + nvl = inv.classify_tdp("" + xml_gpu(device="0x233B10DE", subsystem="0x199610DE", name="NVIDIA H200 NVL") + "", ["GPU-a"]) + assert nvl["status"] == "verified" and nvl["watts_per_gpu"] == 600 + assert nvl["hardware_variant"] == "H200 NVL" + + +@pytest.mark.parametrize("rows,devices", [ + (xml_gpu(device="0x233B10DE", subsystem="0x199610DE"), ["GPU-a"]), + (xml_gpu(subsystem="0xFFFFFFFF"), ["GPU-a"]), + (xml_gpu(), ["GPU-foreign"]), + (xml_gpu() + xml_gpu(), ["GPU-a"]), + (xml_gpu() + xml_gpu(uuid="GPU-b", device="0x233B10DE", subsystem="0x199610DE", name="NVIDIA H200 NVL"), ["GPU-a", "GPU-b"]), +]) +def test_tdp_never_guesses_from_generic_name_or_incomplete_identity(rows, devices): + result = inv.classify_tdp("" + rows + "", devices) + assert result["status"] == "unknown" and result["watts_per_gpu"] is None + + +def saved_source(tmp_path, run_id, *, node="h200-node", devices=None): + import export_ci + cfg = config(tmp_path) + directory = tmp_path / "results" / cfg["task_id"] / f"github-{run_id}-1" + (directory / "gpu").mkdir(parents=True) + receipt, record = allocation(tmp_path) + record["NodeList"] = node + devices = devices or [f"GPU-00000000-0000-0000-0000-{index:012d}" for index in range(4)] + url = f"https://github.com/{export_ci.REPOSITORY}/actions/runs/{run_id}" + metadata = {"repository": export_ci.REPOSITORY, "run_url": url} + state = {"task_id": cfg["task_id"], "run_id": run_id, "run_attempt": "1", "source_sha": "b" * 40, + "ci": metadata, "exit_code": 0, "phase": "complete", "smoke_completed": True, + "allocation": receipt, "slurm_job": record, "step_cleanup": {"status": "ended", "step_id": "123.0"}} + context = {"config": cfg, "run_id": directory.name, "source_sha": "b" * 40, "allocation": receipt, "node": node} + binding = {"node": node, "job_id": "123", "step_id": "0", "gpu_uuids": devices} + for name, value in {"ci.json": state, "context.json": context, "binding.json": binding, "gpu/spec.json": {"gpu_uuids": devices}}.items(): + ci.write(directory / name, value) + manifest = {"task_id": cfg["task_id"], "run_id": run_id, "run_attempt": "1", "git_commit": "b" * 40, + "ci": metadata, "exit_code": 0, "slurm_allocation": receipt, "evidence": ci.inventory(directory)} + ci.write(directory / "manifest.json", manifest) + (directory / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in ci.inventory(directory).items())) + source_ci = {"databaseId": int(run_id), "runAttempt": 1, "headSha": "b" * 40, "status": "completed", "conclusion": "success", "url": url} + return directory, source_ci + + +def test_source_pin_joins_both_successful_runs_to_same_hardware(tmp_path, monkeypatch): + import export_ci + _, first = saved_source(tmp_path, "101") + _, second = saved_source(tmp_path, "102") + monkeypatch.setattr(export_ci, "verified_execution", lambda run: ({"101": first, "102": second}[run], {"name": "source-artifact"})) + target = inv.source_target(config(tmp_path), ["101", "102"]) + assert target["node"] == "h200-node" and len(target["gpu_uuids"]) == 4 + assert [source["run_id"] for source in target["sources"]] == ["101", "102"] + assert target["sources"][0]["receipt_hashes"]["binding.json"] + + +def test_source_pin_rejects_tampered_receipt_before_allocation(tmp_path, monkeypatch): + import export_ci + directory, source_ci = saved_source(tmp_path, "101") + (directory / "binding.json").write_text("{}") + monkeypatch.setattr(export_ci, "verified_execution", lambda run: (source_ci, {})) + with pytest.raises(ValueError, match="differs from its seal"): + inv.source_target(config(tmp_path), ["101"]) + + +def test_source_pin_rejects_different_historical_nodes(tmp_path, monkeypatch): + import export_ci + _, first = saved_source(tmp_path, "101") + _, second = saved_source(tmp_path, "102", node="other-h200") + monkeypatch.setattr(export_ci, "verified_execution", lambda run: ({"101": first, "102": second}[run], {})) + with pytest.raises(ValueError, match="different physical hardware"): + inv.source_target(config(tmp_path), ["101", "102"]) + + +def test_inventory_rejects_valid_slurm_assignment_of_different_historical_gpu(tmp_path, monkeypatch): + inside_context(tmp_path, monkeypatch) + context = ci.read(tmp_path / "context.json") + context["target"]["gpu_uuids"][0] = "GPU-other-historical" + ci.write(tmp_path / "context.json", context) + monkeypatch.setattr(inv, "GpuProbe", lambda *args, **kwargs: pytest.fail("Historical mismatch must stop before probe")) + assert inv.inside(tmp_path) == 2 + assert "historical source hardware" in ci.read(tmp_path / "step-result.json")["error"] + + +def test_source_pin_rejects_local_receipts_for_another_trusted_commit(tmp_path, monkeypatch): + import export_ci + _, source_ci = saved_source(tmp_path, "101") + source_ci["headSha"] = "c" * 40 + monkeypatch.setattr(export_ci, "verified_execution", lambda run: (source_ci, {})) + with pytest.raises(ValueError, match="CI/Git/task identities"): + inv.source_target(config(tmp_path), ["101"]) + + +def test_prepare_accepts_interpreter_symlink_resolved_only_inside_container(tmp_path, spec): + cfg = prepared(tmp_path, spec) + cfg["runtime"]["python"] = "/usr/bin/python3" + rootfs = Path(cfg["runtime"]["rootfs"]) + interpreter = rootfs / "usr/bin/python3" + interpreter.parent.mkdir(parents=True) + interpreter.symlink_to("/etc/alternatives/h3-inventory-test-python") + target = rootfs / "etc/alternatives/h3-inventory-test-python" + target.parent.mkdir(parents=True) + target.write_text("container-only interpreter target") + assert interpreter.is_symlink() and not interpreter.is_file() + assert inv.prepare(cfg)[0]["runtime"]["python"] == "/usr/bin/python3" + interpreter.unlink() + with pytest.raises(ValueError, match="Prepared interpreter missing"): + inv.prepare(cfg) + + +@pytest.mark.parametrize("fault", ["source_ids", "config"]) +def test_cli_retains_preflight_errors_without_allocation(tmp_path, monkeypatch, capsys, fault): + import sys + config_path = tmp_path / "invalid.json" + ci.write(config_path, {}) + output = tmp_path / "diagnostics" + monkeypatch.setattr(sys, "argv", ["inventory_ci.py", "--config", str(config_path), "--output", str(output), + "--source-run-ids", "invalid" if fault == "source_ids" else "101"]) + monkeypatch.setattr(ci, "allocate", lambda *args, **kwargs: pytest.fail("Preflight must not allocate")) + assert inv.main() == 2 + error = ci.read(output / "preflight-error.json") + assert error["exit_code"] == 2 and error["error"] + assert error["error"] in capsys.readouterr().err + + +def test_inside_rejects_different_interpreter_before_gpu_inventory(tmp_path, monkeypatch): + inside_context(tmp_path, monkeypatch) + other = tmp_path / "different-python" + other.write_text("test-only executable") + other.chmod(0o755) + context = ci.read(tmp_path / "context.json") + context["config"]["runtime"]["python"] = str(other) + ci.write(tmp_path / "context.json", context) + monkeypatch.setattr(inv, "cuda_devices", lambda: pytest.fail("Reject wrong interpreter before any GPU query")) + assert inv.inside(tmp_path) == 2 + assert "configured container interpreter" in ci.read(tmp_path / "step-result.json")["error"] + + +def test_tdp_accepts_unprefixed_hex_from_live_nvidia_smi_xml(): + xml = "" + xml_gpu(device="233510DE", subsystem="18BE10DE") + "" + result = inv.classify_tdp(xml, ["GPU-a"]) + assert result["status"] == "verified" + assert result["hardware_variant"] == "H200 SXM" and result["watts_per_gpu"] == 700 + assert result["evidence"]["devices"][0]["pci_device_id"] == "233510DE" + + +def test_tdp_rejects_invalid_hex_even_with_matching_product_name(): + xml = "" + xml_gpu(device="233510DG", subsystem="18BE10DE") + "" + result = inv.classify_tdp(xml, ["GPU-a"]) + assert result["status"] == "unknown" and result["watts_per_gpu"] is None diff --git a/experimental/video-generation/tests/test_mvp_compare.py b/experimental/video-generation/tests/test_mvp_compare.py new file mode 100644 index 0000000000..92e0ae123d --- /dev/null +++ b/experimental/video-generation/tests/test_mvp_compare.py @@ -0,0 +1,585 @@ +import copy +import hashlib +import json +from pathlib import Path + +import pytest + +from evaluator import mvp_compare +from evaluator.mvp_compare import compare_runs +from evaluator.mvp_report import write_report + + +POLICY = { + "policy_id": "explicit-test-policy", + "calibration_status": "fixture_control", + "max_latency_increase_fraction": 0.1, + "min_video_psnr_db": 40.0, + "min_audio_spectral_cosine": 0.99, + "max_audio_rms_ratio_error": 0.05, +} + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode()).hexdigest() + + +def write_bundle(directory, document): + (directory / "run.json").write_text(json.dumps(document), encoding="utf-8") + + +def bundle(tmp_path, name, *, warmups=1): + directory = tmp_path / name + directory.mkdir() + (directory / "media").mkdir() + plan = { + "plan_id": "test-plan", + "model_id": "fixture/no-model", + "model_revision": "fixed-model-revision", + "generation": { + "width": 320, "height": 180, "fps": 24, "frame_count": 48, + "duration_seconds": 2, "audio_sample_rate_hz": 32000, "audio_channels": 2, + }, + "repetitions": 1, + "warmup_runs": warmups, + "cases": [ + {"case_id": f"case-{index}", "prompt": f"test prompt {index}", "seed": index, + "requires_motion": True, "requires_sound": True} + for index in (1, 2) + ], + } + configuration = { + "runtime": "test-runtime", "runtime_revision": name, + "model_id": plan["model_id"], "model_revision": plan["model_revision"], + "hardware_label": "same-test-device", "endpoint": "http://localhost:1234", + "identity_verification": "operator_declared", + } + records = [] + slots = [(f"warmup-{index:03d}", plan["cases"][(index - 1) % 2], "warmup", 0) for index in range(1, warmups + 1)] + slots += [(f"measurement-r001-c{index:03d}", case, "measurement", 1) for index, case in enumerate(plan["cases"], 1)] + for slot_id, case, phase, repetition in slots: + relative = f"media/{slot_id}.mp4" + data = f"mock-{name}-{slot_id}".encode() + (directory / relative).write_bytes(data) + records.append({ + "slot_id": slot_id, "case_id": case["case_id"], "prompt": case["prompt"], + "seed": case["seed"], "repetition": repetition, "phase": phase, + "status": "succeeded", "attempted": True, + "artifact_path": relative, "sha256": hashlib.sha256(data).hexdigest(), + "latency_seconds": 1000.0 if phase == "warmup" else 10.0, + "media": {"valid": True}, "error": None, + }) + document = { + "bundle_version": "0.1.0", "bundle_type": "mvp_run", "run_id": name, + "plan_id": plan["plan_id"], "plan_sha256": digest(plan), "plan": plan, + "configuration": configuration, "configuration_sha256": digest(configuration), + "evidence_kind": "fixture", + "status": "complete", "started_at": "2026-09-01T00:00:00+00:00", "finished_at": "2026-09-01T00:30:00+00:00", + "measurement": {"boundary": "submit_to_validated_media", "concurrency": 1, + "wall_seconds": 20.5, "timing_evidence": "synthetic_fixture_control"}, + "records": records, + "summary": {"valid": 9999, "latency_median_seconds": 0.00001}, + } + write_bundle(directory, document) + return directory, document + + +@pytest.fixture +def media_stub(monkeypatch): + analysis = {"valid": True, "video": {"present": True, "width": 320, "height": 180, "frame_count": 48}, + "audio": {"present": True, "channels": 2, "sample_rate_hz": 32000}, "checks": []} + comparison = { + "compatible": True, + "metrics": { + "video_mae": 0.0, "video_psnr_db": None, "video_identical": True, + "video_compared_frames": 48, "video_total_frames": 48, "video_sample_coverage_fraction": 1.0, + "audio_spectral_cosine": 1.0, "audio_spectral_cosine_channels": [1.0, 1.0], + "audio_rms_ratio": 1.0, "audio_rms_ratio_channels": [1.0, 1.0], + }, + "checks": [], "notes": [], + } + monkeypatch.setattr(mvp_compare, "analyze_media", lambda path, expected=None: copy.deepcopy(analysis)) + monkeypatch.setattr(mvp_compare, "compare_media", lambda left, right: copy.deepcopy(comparison)) + return analysis, comparison + + +def test_recomputed_summary_pairs_only_measurements_and_retains_explicit_limits(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, _ = bundle(tmp_path, "candidate") + result = compare_runs(baseline, candidate, policy=POLICY) + assert result["overall_status"] == "pass" + assert result["release_qualified"] is False + assert result["summary"]["measurement_slots"] == 2 + assert result["candidate"]["summary"]["valid"] == 2 + assert result["candidate"]["summary"]["latency_median_seconds"] == 10.0 + assert result["candidate"]["summary"]["valid_clips_per_second"] == pytest.approx(2 / 20.5) + assert result["measurement"]["timing_evidence"]["candidate"] == "synthetic_fixture_control" + json.dumps(result, allow_nan=False) + + +def test_latency_regression_uses_declared_threshold(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + for record in document["records"]: + if record["phase"] == "measurement": + record["latency_seconds"] = 12.0 + document["measurement"]["wall_seconds"] = 24.5 + write_bundle(candidate, document) + result = compare_runs(baseline, candidate, policy=POLICY) + assert result["overall_status"] == "fail" + assert result["measurement"]["latency_increase_fraction"] == pytest.approx(0.2) + assert next(check for check in result["checks"] if check["name"] == "performance.median_latency")["status"] == "fail" + + +def test_optional_client_timing_boundaries_are_nested_and_legacy_records_still_work(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + for record in document["records"]: + record.update(submit_to_terminal_seconds=7.0, submit_to_media_seconds=8.0, + media_validation_seconds=1.5) + write_bundle(candidate, document) + assert compare_runs(baseline, candidate, policy=POLICY)["overall_status"] == "pass" + + +@pytest.mark.parametrize("updates,match", [ + ({"submit_to_terminal_seconds": -1}, "finite nonnegative"), + ({"submit_to_terminal_seconds": True}, "finite nonnegative"), + ({"submit_to_terminal_seconds": 9}, "follows completed media"), + ({"submit_to_media_seconds": 11}, "exceeds total latency"), + ({"media_validation_seconds": 3}, "download plus validation"), + ({"submit_to_terminal_seconds": None}, "missing client timing"), + ({"media_validation_seconds": None}, "missing client timing"), +]) +def test_impossible_client_timing_boundaries_are_rejected(tmp_path, media_stub, updates, match): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + document["records"][-1].update(submit_to_terminal_seconds=7.0, + submit_to_media_seconds=8.0, media_validation_seconds=1.5) + document["records"][-1].update(updates) + write_bundle(candidate, document) + with pytest.raises(ValueError, match=match): + compare_runs(baseline, candidate, policy=POLICY) + + +def test_failed_attempt_may_have_only_terminal_timing(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + document["records"][-1].update(status="failed", artifact_path=None, sha256=None, + media=None, error="provider failed", submit_to_terminal_seconds=7.0, + submit_to_media_seconds=None, media_validation_seconds=None) + write_bundle(candidate, document) + assert compare_runs(baseline, candidate, policy=POLICY)["overall_status"] == "fail" + + +@pytest.mark.parametrize("field", ["hardware_label", "runtime"]) +def test_cross_configuration_latency_is_descriptive_not_a_regression_gate(tmp_path, media_stub, field): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + document["configuration"][field] = "different-class" + document["configuration_sha256"] = digest(document["configuration"]) + for record in document["records"]: + record["latency_seconds"] = 100.0 + document["measurement"]["wall_seconds"] = 200.5 + write_bundle(candidate, document) + result = compare_runs(baseline, candidate, policy=POLICY) + assert result["overall_status"] == "pass" + assert result["measurement"]["performance_mode"] == "descriptive_only" + assert next(check for check in result["checks"] if check["name"] == "performance.median_latency")["status"] == "descriptive" + + +def test_worst_audio_channel_cannot_be_hidden_by_aggregate(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, _ = bundle(tmp_path, "candidate") + media_stub[1]["metrics"]["audio_rms_ratio_channels"] = [1.0, 0.0] + media_stub[1]["metrics"]["audio_spectral_cosine_channels"] = [1.0, 0.5] + result = compare_runs(baseline, candidate, policy=POLICY) + assert result["overall_status"] == "fail" + names = {check["name"] for check in result["slots"][0]["checks"] if check["status"] == "fail"} + assert names == {"fidelity.audio_rms_ratio_error", "fidelity.audio_spectral_cosine"} + + +@pytest.mark.parametrize("value", [None, float("nan"), float("inf")]) +def test_missing_or_nonfinite_quality_metric_is_not_a_pass(tmp_path, media_stub, value): + baseline, _ = bundle(tmp_path, "baseline") + candidate, _ = bundle(tmp_path, "candidate") + media_stub[1]["metrics"].update(video_identical=False, video_psnr_db=value) + result = compare_runs(baseline, candidate, policy=POLICY) + assert result["overall_status"] == "inconclusive" + assert result["slots"][0]["metrics"]["video_psnr_db"] is None + json.dumps(result, allow_nan=False) + + +def test_audio_undefined_from_silence_is_inconclusive(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, _ = bundle(tmp_path, "candidate") + media_stub[1]["metrics"]["audio_spectral_cosine_channels"] = [1.0, None] + media_stub[1]["metrics"]["audio_rms_ratio_channels"] = [1.0, None] + result = compare_runs(baseline, candidate, policy=POLICY) + assert result["overall_status"] == "inconclusive" + + +def test_not_started_failure_is_retained_without_zero_latency_imputation(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + document["records"][-1].update(status="failed", attempted=False, latency_seconds=0, + artifact_path=None, sha256=None, media=None, error="Not started after uncertain remote completion") + write_bundle(candidate, document) + result = compare_runs(baseline, candidate, policy=POLICY) + summary = result["candidate"]["summary"] + assert result["overall_status"] == "fail" + assert summary["scheduled"] == 2 + assert summary["valid"] == summary["failed"] == summary["not_started"] == 1 + assert summary["technical_success_rate"] == 0.5 + assert summary["latency_median_seconds"] == summary["attempt_latency_median_seconds"] == 10.0 + assert result["slots"][-1]["metrics"].get("video_psnr_db") is None + assert result["slots"][-1]["metrics"]["latency_increase_fraction"] is None + assert result["slots"][-1]["candidate"]["latency_seconds"] is None + + +def test_baseline_failure_is_inconclusive_not_evidence_of_candidate_quality(tmp_path, media_stub): + baseline, document = bundle(tmp_path, "baseline") + candidate, _ = bundle(tmp_path, "candidate") + document["records"][-1].update(status="failed", artifact_path=None, sha256=None, media=None, error="baseline failed") + write_bundle(baseline, document) + result = compare_runs(baseline, candidate, policy=POLICY) + assert result["overall_status"] == "inconclusive" + + +def test_media_validation_is_recomputed_not_trusted(tmp_path, media_stub, monkeypatch): + baseline, _ = bundle(tmp_path, "baseline") + candidate, _ = bundle(tmp_path, "candidate") + analysis = media_stub[0] + monkeypatch.setattr(mvp_compare, "analyze_media", lambda path, expected=None: { + **analysis, "valid": "candidate" not in path.parts, + "checks": [{"name": "video.decode", "status": "failed", "detail": "corrupted frame"}], + }) + result = compare_runs(baseline, candidate, policy=POLICY) + assert result["overall_status"] == "fail" + assert result["candidate"]["summary"]["valid"] == 0 + + +def test_native_duration_is_resolved_from_frames_not_rounded_request(tmp_path, media_stub, monkeypatch): + baseline, left = bundle(tmp_path, "baseline") + candidate, right = bundle(tmp_path, "candidate") + for directory, document in ((baseline, left), (candidate, right)): + document["plan"]["generation"].update(duration_seconds=4, frame_count=107) + document["plan_sha256"] = digest(document["plan"]) + write_bundle(directory, document) + expected_values = [] + monkeypatch.setattr(mvp_compare, "analyze_media", lambda path, expected=None: expected_values.append(expected) or copy.deepcopy(media_stub[0])) + compare_runs(baseline, candidate, policy=POLICY) + assert all(expected["duration_seconds"] == 107 / 24 for expected in expected_values) + + +@pytest.mark.parametrize("mutation,match", [ + (lambda d: d["records"].pop(), "missing measurement"), + (lambda d: d["records"].append(copy.deepcopy(d["records"][-1])), "duplicate slot"), + (lambda d: d["records"][-1].update(prompt="changed prompt"), "prompt differs"), + (lambda d: d["records"][-1].update(repetition=True), "repetition differs"), + (lambda d: d["measurement"].update(concurrency=True), "concurrency=1"), + (lambda d: d.update(plan_sha256="0" * 64), "plan SHA256"), + (lambda d: d["configuration"].update(model_revision="changed"), "does not match the frozen plan"), + (lambda d: d["configuration"].update(runtime_revision="changed"), "configuration SHA256"), + (lambda d: d["records"][-1].update(sha256="0" * 64), "SHA256 mismatch"), + (lambda d: d["records"][-1].update(artifact_path="../outside.mp4"), "escapes"), + (lambda d: d["records"][-1].update(artifact_path="media/missing.mp4"), "missing artifact"), + (lambda d: d["records"].pop(0), "missing warmup"), + (lambda d: d["records"][0].update(slot_id="warmup-999"), "unexpected warmup"), + (lambda d: d["records"].append(d["records"].pop(0)), "warmup slots must precede"), + (lambda d: d["measurement"].update(wall_seconds=0.000001), "shorter than summed"), + (lambda d: d["records"][-1].update(attempted=False), "not-started slot"), + (lambda d: d.update(finished_at=None), "not finalized"), + (lambda d: d.pop("configuration_sha256"), "declare its configuration SHA256"), + (lambda d: d["records"].insert(1, d["records"].pop()), "frozen execution order"), +]) +def test_malformed_or_tampered_bundles_are_refused(tmp_path, media_stub, mutation, match): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + mutation(document) + write_bundle(candidate, document) + with pytest.raises(ValueError, match=match): + compare_runs(baseline, candidate, policy=POLICY) + + +def test_artifact_symlink_may_not_escape_run_directory(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + outside = tmp_path / "outside.mp4" + outside.write_bytes(b"outside") + link = candidate / "media" / "link.mp4" + link.symlink_to(outside) + document["records"][-1].update(artifact_path="media/link.mp4", sha256=hashlib.sha256(b"outside").hexdigest()) + write_bundle(candidate, document) + with pytest.raises(ValueError, match="inside its run directory"): + compare_runs(baseline, candidate, policy=POLICY) + + +def test_duplicate_json_keys_are_rejected(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + (candidate / "run.json").write_text('{"run_id":"hidden",' + json.dumps(document)[1:], encoding="utf-8") + with pytest.raises(ValueError, match="duplicate JSON key: run_id"): + compare_runs(baseline, candidate, policy=POLICY) + + +@pytest.mark.parametrize("field,value,match", [ + ("repetitions", 10001, "10000 total slots"), + ("generation", {}, "generation.width"), + ("warmup_runs", -1, "nonnegative"), +]) +def test_invalid_plan_cannot_omit_expected_checks(tmp_path, media_stub, field, value, match): + baseline, _ = bundle(tmp_path, "baseline") + candidate, document = bundle(tmp_path, "candidate") + document["plan"][field] = value + document["plan_sha256"] = digest(document["plan"]) + write_bundle(candidate, document) + with pytest.raises(ValueError, match=match): + compare_runs(baseline, candidate, policy=POLICY) + + +@pytest.mark.parametrize("value", [None, float("nan"), float("inf"), True, -0.1]) +def test_threshold_policy_has_no_implicit_or_nonfinite_defaults(tmp_path, media_stub, value): + policy = {**POLICY, "max_latency_increase_fraction": value} + with pytest.raises(ValueError, match="explicit, finite"): + compare_runs(tmp_path / "unused", tmp_path / "unused", policy=policy) + + +def test_report_is_portable_script_free_escaped_and_hash_checked(tmp_path, media_stub): + baseline, _ = bundle(tmp_path, "baseline") + candidate, _ = bundle(tmp_path, "candidate") + result = compare_runs(baseline, candidate, policy=POLICY) + result["slots"][0]["prompt"] = ' & injection' + destination = tmp_path / "presentation" / "report.html" + write_report(result, destination) + content = destination.read_text(encoding="utf-8") + assert "Synthetic fixture evidence" in content + assert "not an H3 result" in content + assert '' + spec["job_id"] = attack + write_json(job / "spec.json", spec) + receipt["spec_sha256"] = digest(spec) + receipt["failures"] = [attack] + receipt["roles"]["candidate"]["source_identity"]["revision"] = attack + write_json(job / "gpu-job.json", receipt) + result, output, page = render(tmp_path, job) + assert "", "status": "fail", "observed": 0.2, "threshold": 0.1, "unit": "fraction", "reason": "synthetic fixture"}]} + receipt.update(comparison_path="comparison.json", comparison_sha256=write_json(job / "comparison.json", comparison)) + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["comparison_status"] == "fail" + assert "<unsafe>" in page and "" not in page + assert "fraction" in page + assert result["ci_status"] == "inconclusive" + + +def test_telemetry_tamper_is_explicit(tmp_path): + job, _, receipt = fixture_job(tmp_path) + (job / receipt["roles"]["candidate"]["telemetry_path"]).write_text("tampered") + result, _, page = render(tmp_path, job) + assert not result["roles"]["candidate"]["telemetry"]["file_sha256_verified"] + assert "Telemetry SHA256 does not match" in page + + +def test_unverified_different_gpu_uuid_is_not_same_device_regression(tmp_path): + job, _, receipt = fixture_job(tmp_path) + receipt["roles"]["candidate"]["telemetry_summary"]["gpu_identity"][0]["uuid"] = "GPU-OTHER-SYNTHETIC" + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["same_gpu_uuid_set"] is None + assert "GPU identity missing" in page + assert "Cross-GPU results are descriptive" in page + + +def test_display_limit_preserves_full_denominators_and_export(tmp_path, monkeypatch): + job, _, _ = fixture_job(tmp_path) + monkeypatch.setattr(viewer, "MAX_DISPLAY_SLOTS", 1) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["summary"]["scheduled"] == 3 + assert len(result["roles"]["candidate"]["observations"]) == 3 + assert "first 1 of 3 scheduled slots" in page + + +def test_not_bound_run_withholds_gpu_metrics(tmp_path): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + del receipt["roles"]["candidate"]["run_sha256"] + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert not result["roles"]["candidate"]["gpu_timing_presented"] + assert "GPU timings withheld" in page + + +def test_inconsistent_configuration_hash_withholds_gpu_metrics(tmp_path): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + change_run(job, receipt, "candidate", lambda run: run["configuration"].update(configuration_sha256="0" * 64)) + result, _, page = render(tmp_path, job) + assert not result["roles"]["candidate"]["gpu_timing_presented"] + assert "configuration hash is absent or mismatched" in page + + +def test_unknown_slots_cannot_qualify_timing_population(tmp_path): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + change_run(job, receipt, "candidate", lambda run: run["records"].append(copy.deepcopy(run["records"][1]))) + result, _, page = render(tmp_path, job) + assert not result["roles"]["candidate"]["gpu_timing_presented"] + assert "duplicate" in page + + +def test_impossible_serial_wall_time_is_not_a_throughput_result(tmp_path): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + change_run(job, receipt, "candidate", lambda run: run["measurement"].update(wall_seconds=1.0)) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["summary"]["valid_clips_per_second"] is None + assert "throughput withheld" in page + + +def test_real_cpu_encoded_fixture_has_playable_media_but_no_gpu_timing(tmp_path): + """Also produces an explicitly CPU-fixture-labeled artifact for visual QA.""" + pytest.importorskip("av") + pytest.importorskip("numpy") + from test_mvp_fixtures import FIXTURE_POLICY, _fixture_plan, _fixture_run + from evaluator.mvp_compare import compare_runs + + job = tmp_path / "cpu-fixture-job" + job.mkdir() + plan = _fixture_plan() + _fixture_run(job / "baseline", "baseline", plan) + _fixture_run(job / "candidate", "candidate", plan, baseline_dir=job / "baseline", defect="muted") + spec = {"schema_version": "0.1.0", "job_id": "CPU FIXTURE VISUAL QA — NOT AN H3 RUN", "plan": plan, + "allocation": {"mode": "cooperative_shared", "label": "CPU-only encoded moving shapes and tones"}, "policy": FIXTURE_POLICY} + write_json(job / "spec.json", spec) + receipt = {"schema_version": "0.1.0", "bundle_type": "controlled_gpu_job", "status": "complete", "measurement_status": "complete", + "evidence_kind": "no_gpu_measurement", "spec_sha256": digest(spec), "roles": {}, "regression_status": "inconclusive", "ci_accepted": False, + "acceptance_reasons": ["CPU fixture only. No GPU exists in this test and no model inference ran."]} + for label in ("baseline", "candidate"): + receipt["roles"][label] = {"status": "complete", "run_path": f"{label}/run.json", "run_sha256": hashlib.sha256((job / label / "run.json").read_bytes()).hexdigest(), "cleanup": {"status": "not_applicable", "reason": "No GPU resources used by CPU fixture"}} + comparison = compare_runs(job / "baseline", job / "candidate", policy=FIXTURE_POLICY) + receipt.update(comparison_path="comparison.json", comparison_sha256=write_json(job / "comparison.json", comparison)) + write_json(job / "gpu-job.json", receipt) + result, output, page = render(tmp_path, job) + assert result["comparison_status"] == "fail" + assert result["roles"]["baseline"]["summary"]["valid"] == 2 + assert result["roles"]["candidate"]["summary"]["valid"] == 1 + assert result["roles"]["candidate"]["summary"]["latency_median_seconds"] is None + assert result["slot_comparisons"] + assert "CPU FIXTURE VISUAL QA" in page and "Harness-only / imported evidence" in page + assert "Recorded paired fidelity metrics" in page + assert "Recorded supervisor acceptance" in page + assert all(path.stat().st_size > 1000 for path in (output / "assets").iterdir()) + + +@pytest.mark.parametrize("field,value", [ + ("revision", "0" * 40), ("source_sha256", "0" * 64), + ("python", "/not/the/pinned/python"), ("sglang_module", "/wrong/sglang/__init__.py"), +]) +def test_gpu_label_requires_matching_observed_runtime_pins(tmp_path, field, value): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + receipt["roles"]["candidate"]["source_identity"][field] = value + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["gpu_timing_presented"] is False + assert "Observed source/Python identity does not match" in page + + +@pytest.mark.parametrize("field,value", [("pid", 1), ("pgid", 12), ("session_id", 12), ("start_ticks", 0), ("launch_nonce", "not-a-launch-nonce")]) +def test_gpu_label_requires_plausible_owned_process_receipt(tmp_path, field, value): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + receipt["roles"]["candidate"]["process_identity"][field] = value + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["gpu_timing_presented"] is False + assert "plausible owned process/session" in page + + +def test_telemetry_identity_annotations_preserve_timing_without_clearing_cleanup_failure(tmp_path): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + for role in receipt["roles"].values(): + telemetry = job / role["telemetry_path"] + samples = [json.loads(line) for line in telemetry.read_text().splitlines()] + for sample in samples: + for app in sample["owned_compute_apps"]: + app["process_identity"] = {"pid": app["pid"], "pgid": 123, "session_id": 123, "start_ticks": 999} + telemetry.write_text("\n".join(json.dumps(sample) for sample in samples) + "\n") + role["telemetry_sha256"] = hashlib.sha256(telemetry.read_bytes()).hexdigest() + receipt.update(status="failed", measurement_status="incomplete", cleanup_status="failed", + failures=["owned runtime cleanup did not establish idle GPUs"]) + receipt["roles"]["candidate"]["cleanup"].update(status="failed", idle_after=False) + write_json(job / "gpu-job.json", receipt) + result, _, _ = render(tmp_path, job) + assert result["issues"] == [] + assert all(role["telemetry"]["samples_consistent"] and role["gpu_timing_presented"] for role in result["roles"].values()) + assert result["status"] == "failed" and result["ci_accepted"] is False + assert result["failures"] == receipt["failures"] + assert result["roles"]["candidate"]["cleanup"]["status"] == "failed" + + +@pytest.mark.parametrize("defect", ["missing", "overlap", "duplicate", "wrong_pid", "memory"]) +def test_telemetry_ownership_partition_rejects_mismatched_observations(tmp_path, defect): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + role = receipt["roles"]["candidate"] + telemetry = job / role["telemetry_path"] + samples = [json.loads(line) for line in telemetry.read_text().splitlines()] + sample = samples[0] + app = sample["owned_compute_apps"][0] + if defect == "missing": + sample["owned_compute_apps"] = [] + elif defect == "overlap": + sample["unowned_compute_apps"] = [dict(app, ownership_observation="not_owned")] + elif defect == "duplicate": + sample["compute_apps"].append(dict(app, pid=124)) + sample["owned_compute_apps"].append(dict(app)) + elif defect == "wrong_pid": + app["pid"] = 124 + else: + app["memory_used_mib"] += 1 + telemetry.write_text("\n".join(json.dumps(sample) for sample in samples) + "\n") + role["telemetry_sha256"] = hashlib.sha256(telemetry.read_bytes()).hexdigest() + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["telemetry"]["samples_consistent"] is False + assert result["roles"]["candidate"]["gpu_timing_presented"] is False + assert "Telemetry compute ownership partition is inconsistent" in page + + +def test_gpu_label_requires_owned_compute_during_measurement(tmp_path): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + role = receipt["roles"]["candidate"] + telemetry = job / role["telemetry_path"] + samples = [json.loads(line) for line in telemetry.read_text().splitlines()] + for sample in samples: + sample["owned_compute_apps"] = [] + sample["compute_apps"] = [] + telemetry.write_text("\n".join(json.dumps(sample) for sample in samples) + "\n") + role["telemetry_sha256"] = hashlib.sha256(telemetry.read_bytes()).hexdigest() + role["telemetry_summary"]["observed_owned_compute_by_gpu"] = {"GPU-SYNTHETIC-TEST-ONLY": 0} + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["telemetry"]["samples_consistent"] is True + assert result["roles"]["candidate"]["gpu_timing_presented"] is False + assert "do not establish owned compute" in page + + +def test_gpu_label_requires_summary_to_match_raw_telemetry(tmp_path): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + receipt["roles"]["candidate"]["telemetry_summary"]["observed_memory_peak_mib_by_gpu"] = {"GPU-SYNTHETIC-TEST-ONLY": 1} + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["gpu_timing_presented"] is False + assert "does not match its hash-verified samples" in page + + +@pytest.mark.parametrize("edit", [ + lambda row: row.update(submit_to_terminal_seconds=20.0), + lambda row: row.update(submit_to_media_seconds=-1.0), + lambda row: row.update(media_validation_seconds=5.0), + lambda row: row.update(submit_to_terminal_seconds=None), +]) +def test_invalid_nested_timings_are_never_presented(tmp_path, edit): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + change_run(job, receipt, "candidate", lambda run: edit(run["records"][1])) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["gpu_timing_presented"] is False + assert result["roles"]["candidate"]["summary"]["latency_median_seconds"] is None + assert "Invalid nested client timing boundaries" in page + + +def test_recorded_failed_ci_verdict_is_not_overwritten_as_inconclusive(tmp_path): + job, _, receipt = fixture_job(tmp_path) + receipt["regression_status"] = "fail" + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["ci_status"] == "fail" + assert 'Recorded CI verdict: fail' in page + assert not result["release_qualified"] + + +def test_recorded_ci_pass_cannot_be_green_with_unqualified_evidence(tmp_path): + job, _, receipt = fixture_job(tmp_path) + receipt.update(regression_status="pass", ci_accepted=True) + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["ci_status"] == "inconclusive" + assert result["ci_accepted"] is False + assert 'Recorded CI verdict: pass' not in page + assert "recorded pass is not shown as passing" in page + + +def test_nonfinalized_run_never_presents_gpu_timing(tmp_path): + job, _, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + change_run(job, receipt, "candidate", lambda run: run.update(finished_at=None)) + result, _, page = render(tmp_path, job) + assert result["roles"]["candidate"]["gpu_timing_presented"] is False + assert "not finalized" in page + + +def qualified_fixture_job(tmp_path): + """Structurally qualified synthetic receipt; never actual GPU/CI evidence.""" + job, spec, receipt = fixture_job(tmp_path, evidence="operator_endpoint", receipt_kind="controlled_h3_gpu") + spec["policy"]["calibration_status"] = "operator_calibrated" + spec["allocation"]["mode"] = "dedicated_ci" + write_json(job / "spec.json", spec) + receipt["spec_sha256"] = digest(spec) + receipt.update(regression_status="pass", ci_accepted=True) + comparison = {"bundle_type": "mvp_comparison", "overall_status": "pass", "policy": spec["policy"], + "baseline": {"run_bundle_sha256": receipt["roles"]["baseline"]["run_sha256"]}, + "candidate": {"run_bundle_sha256": receipt["roles"]["candidate"]["run_sha256"]}} + receipt.update(comparison_path="comparison.json", comparison_sha256=write_json(job / "comparison.json", comparison)) + write_json(job / "gpu-job.json", receipt) + return job, receipt + + +@pytest.mark.parametrize("changes", [ + {"status": "failed"}, {"status": "running"}, {"status": "aborted"}, + {"failures": ["Synthetic infrastructure failure"]}, {"failures": None}, + {"cleanup_status": "failed"}, {"cleanup_status": None}, + {"finished_at": None}, {"started_at": None}, + {"finished_at": "not a timestamp"}, + {"finished_at": "2026-09-01T01:05:00"}, + {"finished_at": "2026-09-01T00:59:59Z"}, +]) +def test_otherwise_qualified_ci_pass_requires_finalized_clean_success(tmp_path, changes): + job, receipt = qualified_fixture_job(tmp_path) + receipt.update(changes) + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["ci_status"] == "inconclusive" + assert result["ci_accepted"] is False + assert result["release_qualified"] is False + assert 'Recorded CI verdict: pass' not in page + assert "recorded pass is not shown as passing" in page + + +def test_recorded_ci_pass_requires_finalization_fields_to_be_present(tmp_path): + job, receipt = qualified_fixture_job(tmp_path) + del receipt["finished_at"] + write_json(job / "gpu-job.json", receipt) + result, _, _ = render(tmp_path, job) + assert result["ci_status"] == "inconclusive" and result["ci_accepted"] is False + + +@pytest.mark.parametrize("condition", ["cooperative_shared", "uncalibrated"]) +def test_otherwise_qualified_receipt_cannot_promote_shared_or_uncalibrated_job(tmp_path, condition): + job, receipt = qualified_fixture_job(tmp_path) + spec = json.loads((job / "spec.json").read_text()) + if condition == "cooperative_shared": + spec["allocation"]["mode"] = condition + else: + spec["policy"]["calibration_status"] = condition + comparison = json.loads((job / "comparison.json").read_text()) + comparison["policy"] = spec["policy"] + receipt["comparison_sha256"] = write_json(job / "comparison.json", comparison) + write_json(job / "spec.json", spec) + receipt["spec_sha256"] = digest(spec) + write_json(job / "gpu-job.json", receipt) + result, _, page = render(tmp_path, job) + assert result["ci_status"] == "inconclusive" and result["ci_accepted"] is False + assert 'Recorded CI verdict: pass' not in page + + +def test_recorded_accepted_ci_is_labeled_reported_not_independently_qualified(tmp_path): + job, _ = qualified_fixture_job(tmp_path) + result, _, page = render(tmp_path, job) + assert result["ci_status"] == "pass" and result["ci_accepted"] is True + assert result["release_qualified"] is False + assert 'Recorded CI verdict: pass' in page + assert "does not independently qualify CI" in page diff --git a/experimental/video-generation/tests/test_mvp_media.py b/experimental/video-generation/tests/test_mvp_media.py new file mode 100644 index 0000000000..0b9053cc15 --- /dev/null +++ b/experimental/video-generation/tests/test_mvp_media.py @@ -0,0 +1,291 @@ +"""Real encoded fixtures: no decoder, filesystem, or signal-analysis mocks.""" + +from fractions import Fraction +import hashlib +import json +from pathlib import Path + +import pytest + +av = pytest.importorskip("av") +np = pytest.importorskip("numpy") + +from evaluator.mvp_media import analyze_media, compare_media + + +def encode_media( + path: Path, + *, + width: int = 64, + height: int = 48, + fps: int = 10, + frame_count: int = 8, + audio_mode: str = "stereo", + video_change: bool = False, + frozen: bool = False, + blank: bool = False, + timestamp_offset_seconds: float = 0.0, + audio_offset_seconds: float = 0.0, + audio_duration_scale: float = 1.0, + freeze_after_frame: int | None = None, +) -> Path: + """Encode exact RGB+PCM in MKV, or browser-playable H.264+AAC in MP4. + + All media is artificial diagnostic material, never model output. MP4 is + intentionally lossy; assertions requiring sample identity should use MKV. + """ + is_mp4 = path.suffix.lower() == ".mp4" + sample_rate = 24000 + samples = round(frame_count / fps * sample_rate * audio_duration_scale) + phase = np.arange(samples, dtype=np.float64) / sample_rate + left = 0.35 * np.sin(2 * np.pi * 440 * phase) + right = 0.25 * np.sin(2 * np.pi * 730 * phase) + if audio_mode == "silent": + left[:], right[:] = 0, 0 + elif audio_mode == "silent_right": + right[:] = 0 + elif audio_mode == "collapsed": + right = left.copy() + elif audio_mode == "loud": + left[:], right[:] = 1, -1 + signal = np.stack([left, right]) + with av.open(str(path), "w") as container: + video = container.add_stream("libx264" if is_mp4 else "ffv1", rate=fps) + video.width, video.height = width, height + video.pix_fmt = "yuv420p" if is_mp4 else "bgr0" + if is_mp4: + video.options = {"preset": "ultrafast", "crf": "18"} + audio = None + if audio_mode != "absent": + audio = container.add_stream("aac" if is_mp4 else "pcm_s16le", rate=sample_rate) + audio.layout = "stereo" + for index in range(frame_count): + pixels = np.zeros((height, width, 3), dtype=np.uint8) + if not blank: + pixels[:, :, 0] = 40 + pixels[:, :, 1] = np.arange(width, dtype=np.uint8)[None, :] * 3 + moving_index = min(index, freeze_after_frame) if freeze_after_frame is not None else index + x = 4 if frozen else (4 + moving_index * 3) % (width - 12) + pixels[8:24, x : x + 12] = [220, 90, 20] + if video_change: + pixels[:, :, 2] = 160 + frame = av.VideoFrame.from_ndarray(pixels, format="rgb24") + frame.pts = index + round(timestamp_offset_seconds * fps) + frame.time_base = Fraction(1, fps) + for packet in video.encode(frame): + container.mux(packet) + for packet in video.encode(): + container.mux(packet) + if audio is not None: + for start in range(0, samples, 1024): + chunk = signal[:, start : start + 1024] + if is_mp4: + values, fmt = chunk.astype(np.float32), "fltp" + else: + values = np.round(np.clip(chunk, -1, 32767 / 32768) * 32768).astype(np.int16).T.reshape(1, -1) + fmt = "s16" + frame = av.AudioFrame.from_ndarray(values, format=fmt, layout="stereo") + frame.sample_rate = sample_rate + frame.pts = start + round((timestamp_offset_seconds + audio_offset_seconds) * sample_rate) + frame.time_base = Fraction(1, sample_rate) + for packet in audio.encode(frame): + container.mux(packet) + for packet in audio.encode(): + container.mux(packet) + return path + + +def expectation() -> dict: + return { + "width": 64, + "height": 48, + "frame_count": 8, + "fps": 10, + "duration_seconds": 0.8, + "duration_tolerance_seconds": 0.005, + "audio_required": True, + "audio_sample_rate_hz": 24000, + "audio_channels": 2, + "requires_motion": True, + "requires_sound": True, + } + + +def test_full_decode_contract_and_channel_statistics(tmp_path): + path = encode_media(tmp_path / "real.mkv") + result = analyze_media(path, expectation()) + assert result["valid"], result + assert result["decode_ok"] + assert result["sha256"] == hashlib.sha256(path.read_bytes()).hexdigest() + assert result["byte_size"] == path.stat().st_size + assert result["video"]["frame_count"] == 8 + assert result["video"]["fps"] == pytest.approx(10) + assert result["video"]["duration_seconds"] == pytest.approx(0.8) + assert result["audio"]["sample_count"] == 19200 + assert result["audio"]["channels"] == 2 + assert result["audio"]["rms_channels"] == pytest.approx([0.35 / 2**0.5, 0.25 / 2**0.5], abs=5e-5) + assert result["audio"]["silent_channels"] == [] + assert result["metrics"]["av_start_skew_seconds"] == pytest.approx(0.0) + assert result["metrics"]["av_end_skew_seconds"] == pytest.approx(0.0, abs=0.001) + json.dumps(result, allow_nan=False) + + +def test_corrupt_file_fails_without_pretending_it_is_a_clip(tmp_path): + path = tmp_path / "broken.mp4" + path.write_bytes(b"not a media container\x00" * 20) + result = analyze_media(path, expectation()) + assert not result["valid"] + assert not result["decode_ok"] + assert result["errors"] + assert result["sha256"] + assert result["video"]["present"] is False + json.dumps(result, allow_nan=False) + + +def test_wrong_media_contract_fails(tmp_path): + path = encode_media(tmp_path / "mismatch.mkv") + expected = expectation() | {"width": 128, "frame_count": 9, "fps": 24, "duration_seconds": 5, "audio_channels": 1, "audio_sample_rate_hz": 48000} + result = analyze_media(path, expected) + assert result["decode_ok"] + assert not result["valid"] + failed = {item["name"] for item in result["checks"] if item["status"] == "failed"} + assert {"video.width", "video.frame_count", "video.fps", "video.duration_seconds", "audio.channels", "audio.sample_rate_hz"} <= failed + + +def test_silence_freeze_and_blank_are_detected_without_quality_claims(tmp_path): + path = encode_media(tmp_path / "silent-frozen.mkv", audio_mode="silent", blank=True) + measured = analyze_media(path) + assert measured["valid"] # Silence/static scenes are not universal failures. + assert measured["audio"]["rms_channels"] == [0.0, 0.0] + assert measured["audio"]["rms_dbfs_channels"] == [None, None] + assert measured["audio"]["silent_channels"] == [0, 1] + assert measured["video"]["blank_fraction"] == 1.0 + assert measured["video"]["duplicate_fraction"] == 1.0 + assert measured["video"]["frozen_fraction"] == 1.0 + result = analyze_media(path, expectation()) + failed = {item["name"] for item in result["checks"] if item["status"] == "failed"} + assert {"video.motion_presence", "audio.sound_presence"} <= failed + json.dumps(result, allow_nan=False) + + +def test_channel_collapse_and_full_scale_are_reported_per_channel(tmp_path): + collapsed = analyze_media(encode_media(tmp_path / "collapsed.mkv", audio_mode="collapsed")) + assert collapsed["audio"]["identical_channel_pairs"] == [[0, 1]] + partial_silence = analyze_media(encode_media(tmp_path / "right-silent.mkv", audio_mode="silent_right")) + assert partial_silence["audio"]["silent_channels"] == [1] + assert partial_silence["audio"]["rms_channels"][0] > 0.2 + loud = analyze_media(encode_media(tmp_path / "loud.mkv", audio_mode="loud")) + assert loud["audio"]["clipping_fraction_channels"] == [1.0, 1.0] + + +def test_identical_pair_has_complete_coverage_and_no_infinity(tmp_path): + baseline = encode_media(tmp_path / "baseline.mkv") + candidate = tmp_path / "candidate.mkv" + candidate.write_bytes(baseline.read_bytes()) + result = compare_media(baseline, candidate) + assert result["compatible"], result + metrics = result["metrics"] + assert metrics["video_mae"] == 0.0 + assert metrics["video_identical"] is True + assert metrics["video_psnr_db"] is None + assert metrics["video_compared_frames"] == 8 + assert metrics["video_sample_coverage_fraction"] == 1.0 + assert metrics["audio_identical"] is True + assert metrics["audio_waveform_mae"] == 0.0 + assert metrics["audio_rms_ratio_channels"] == [1.0, 1.0] + assert metrics["audio_spectral_cosine_channels"] == pytest.approx([1.0, 1.0]) + assert metrics["audio_sample_coverage_fraction"] == 1.0 + json.dumps(result, allow_nan=False) + + +def test_changed_visual_content_and_silent_channel_are_measured(tmp_path): + baseline = encode_media(tmp_path / "baseline.mkv") + candidate = encode_media(tmp_path / "changed.mkv", video_change=True, audio_mode="silent_right") + result = compare_media(baseline, candidate) + assert result["compatible"], result + metrics = result["metrics"] + assert metrics["video_mae"] > 0.1 + assert metrics["video_psnr_db"] < 15 + assert metrics["video_identical"] is False + assert metrics["audio_rms_ratio_channels"] == [1.0, 0.0] + assert metrics["audio_spectral_cosine_channels"][0] == pytest.approx(1.0) + assert metrics["audio_spectral_cosine_channels"][1] is None + assert metrics["audio_spectral_cosine"] is None + assert metrics["audio_newly_silent_channels"] == [1] + assert metrics["audio_waveform_mae_channels"][1] > 0.1 + json.dumps(result, allow_nan=False) + + +@pytest.mark.parametrize("change", [{"width": 80}, {"frame_count": 7}, {"fps": 20}, {"timestamp_offset_seconds": 0.2}, {"audio_mode": "absent"}]) +def test_incompatible_media_is_not_resized_trimmed_or_time_shifted(tmp_path, change): + baseline = encode_media(tmp_path / "baseline.mkv") + candidate = encode_media(tmp_path / "incompatible.mkv", **change) + result = compare_media(baseline, candidate) + assert not result["compatible"], result + assert result["metrics"]["video_mae"] is None + assert any(check["status"] == "failed" for check in result["checks"]) + + +def test_audio_optional_and_identically_silent_pairs_are_explicit(tmp_path): + absent = encode_media(tmp_path / "absent.mkv", audio_mode="absent") + result = analyze_media(absent) + assert result["valid"] + assert result["audio"]["present"] is False + assert result["metrics"]["av_end_skew_seconds"] is None + assert not analyze_media(absent, {"audio_required": True})["valid"] + no_audio_pair = compare_media(absent, absent) + assert no_audio_pair["compatible"] + assert no_audio_pair["metrics"]["audio_spectral_cosine"] is None + silent = encode_media(tmp_path / "silent.mkv", audio_mode="silent") + silent_pair = compare_media(silent, silent) + assert silent_pair["compatible"] + assert silent_pair["metrics"]["audio_identical"] is True + assert silent_pair["metrics"]["audio_rms_ratio"] is None + assert silent_pair["metrics"]["audio_spectral_cosine"] is None + json.dumps(silent_pair, allow_nan=False) + + +def test_browser_playable_mp4_is_decoded(tmp_path): + path = encode_media(tmp_path / "browser.mp4") + result = analyze_media(path, expectation()) + assert result["valid"], result + assert result["video"]["codec"] == "h264" + assert result["audio"]["codec"] == "aac" + + +def test_tiny_analysis_deadline_fails_without_partial_success(tmp_path): + path = encode_media(tmp_path / "deadline.mkv") + result = analyze_media(path, {"timeout_seconds": 1e-12}) + assert not result["valid"] + assert not result["decode_ok"] + assert "TimeoutError" in result["errors"][0] + + +def test_optional_av_boundary_and_partial_freeze_gates_are_explicit(tmp_path): + offset = encode_media(tmp_path / "offset-audio.mkv", audio_offset_seconds=0.2) + unconstrained = analyze_media(offset) + assert unconstrained["valid"] + assert unconstrained["metrics"]["av_start_skew_seconds"] == pytest.approx(0.2) + constrained = analyze_media(offset, {"max_av_start_skew_seconds": 0.05}) + assert not constrained["valid"] + assert any(item["name"] == "av.start_skew_seconds" and item["status"] == "failed" for item in constrained["checks"]) + short_audio = encode_media(tmp_path / "short-audio.mkv", audio_duration_scale=0.5) + assert analyze_media(short_audio)["valid"] + constrained = analyze_media(short_audio, {"max_av_end_skew_seconds": 0.05}) + assert not constrained["valid"] + assert constrained["metrics"]["av_end_skew_seconds"] == pytest.approx(-0.4, abs=0.001) + partial_freeze = encode_media(tmp_path / "partial-freeze.mkv", freeze_after_frame=2) + unconstrained = analyze_media(partial_freeze, {"requires_motion": True}) + assert unconstrained["valid"] + assert unconstrained["video"]["frozen_fraction"] == pytest.approx(5 / 7) + assert not analyze_media(partial_freeze, {"max_frozen_fraction": 0.2})["valid"] + + +@pytest.mark.parametrize("expected", [{"fps": float("nan")}, {"timeout_seconds": "oops"}, {"duration_tolerance_seconds": -1}, {"max_av_end_skew_seconds": float("inf")}, {"max_frozen_fraction": 1.1}, {"width": 4.5}, {"audio_required": "false"}]) +def test_invalid_expectations_never_produce_nonfinite_or_accepted_results(tmp_path, expected): + path = encode_media(tmp_path / "valid.mkv") + report = analyze_media(path, expected) + assert not report["valid"] + assert not report["decode_ok"] + assert report["errors"] + json.dumps(report, allow_nan=False) diff --git a/experimental/video-generation/tests/test_mvp_power.py b/experimental/video-generation/tests/test_mvp_power.py new file mode 100644 index 0000000000..2642807570 --- /dev/null +++ b/experimental/video-generation/tests/test_mvp_power.py @@ -0,0 +1,211 @@ +"""Hand-worked phase integration and fail-closed telemetry boundaries.""" + +import json +from copy import deepcopy +from datetime import datetime, timezone + +import pytest + +from evaluator.mvp_power import analyze_power + + +def utc(seconds): + return datetime.fromtimestamp(1_000_000 + seconds, timezone.utc).isoformat() + + +def record(slot, phase, start, terminal, *, valid=True): + return {"slot_id": slot, "case_id": "case", "phase": phase, "attempted": True, + "status": "succeeded", "media": {"valid": valid}, + "submit_to_terminal_seconds": terminal - start, + "latency_seconds": terminal - start + 0.25, + "timing_window": {"start_monotonic_seconds": start, + "terminal_monotonic_seconds": terminal, + "end_monotonic_seconds": terminal + 0.25, "start_utc": utc(start)}} + + +def data(): + role = {"process_identity": {"pgid": 10, "session_id": 10}, "startup_seconds": 2, + "startup_timing_window": {"start_monotonic_seconds": 0, "end_monotonic_seconds": 2, + "start_utc": utc(0)}} + run = {"records": [record("warmup", "warmup", 3, 5), record("measured", "measurement", 6.5, 9.5)]} + samples = [] + for time in range(-1, 14): + apps = [{"gpu_uuid": gpu, "pid": pid, "memory_used_mib": 20} for gpu, pid in (("a", 11), ("b", 12))] + samples.append({"at": utc(time), "monotonic_seconds": time, + "gpus": [{"uuid": "a", "power_watts": 10 + 5 * time}, {"uuid": "b", "power_watts": 20 + 10 * time}], + "compute_apps": apps, "unowned_compute_apps": [], + "owned_compute_apps": [{**app, "process_identity": {"pid": app["pid"], "pgid": 10, "session_id": 10, "start_ticks": 1}} for app in apps]}) + return role, run, samples + + +def analyze(role, run, samples, events=None): + return analyze_power(role, run, samples, events or [], ["a", "b"], interval_seconds=1) + + +def test_ramp_clipping_separates_phases_and_excludes_client_decode(): + result = analyze(*data()) + assert result["valid"] is True + measured = result["phases"]["measurement"] + assert measured["duration_seconds"] == 3 + assert measured["per_gpu"]["a"] == {"energy_j": 150, "avg_power_w": 50, "observed_peak_power_w": 55} + assert measured["aggregate"] == {"energy_j": 450, "avg_power_w": 150, "observed_peak_power_w": 165, "joules_per_valid_clip": 450} + assert result["phases"]["startup"]["aggregate"]["energy_j"] == 90 + assert result["phases"]["warmup"]["aggregate"]["energy_j"] == 180 + assert result["windows"][2]["coverage"]["coverage_fraction"] == 1 + assert result["sample_series"][8]["aggregate_watts"] == 135 + + +@pytest.mark.parametrize("defect", ["missing_power", "nan", "negative", "missing_gpu", "duplicate_gpu", "foreign", "identity", "partition", "nonmonotonic", "clock_jump"]) +def test_invalid_telemetry_withholds_measurement_metrics(defect): + role, run, samples = data() + sample = samples[9] + if defect in ("missing_power", "nan", "negative"): + sample["gpus"][0]["power_watts"] = {"missing_power": None, "nan": float("nan"), "negative": -1}[defect] + elif defect == "missing_gpu": + sample["gpus"].pop() + elif defect == "duplicate_gpu": + sample["gpus"][1]["uuid"] = "a" + elif defect == "foreign": + sample["unowned_compute_apps"] = [sample["compute_apps"][0]] + elif defect == "identity": + sample["owned_compute_apps"][0]["process_identity"]["pgid"] = 99 + elif defect == "partition": + sample["owned_compute_apps"].pop() + elif defect == "nonmonotonic": + sample["monotonic_seconds"] = 6 + else: + sample["at"] = utc(88) + result = analyze(role, run, samples) + assert result["phases"]["measurement"]["valid"] is False + assert result["phases"]["measurement"]["aggregate"] is None + assert result["phases"]["measurement"]["per_gpu"] is None + assert result["phases"]["measurement"]["invalid_reasons"] + + +@pytest.mark.parametrize("defect", ["gap", "end_missing", "empty", "no_owned"]) +def test_missing_coverage_and_ownership_do_not_extrapolate(defect): + role, run, samples = data() + if defect == "gap": + samples = [sample for sample in samples if sample["monotonic_seconds"] not in (7, 8, 9)] + elif defect == "end_missing": + samples = samples[:11] # Last sample t=9, request terminal t=9.5. + elif defect == "empty": + samples = [] + else: + for sample in samples: + sample["compute_apps"] = sample["owned_compute_apps"] = [] + measured = analyze(role, run, samples)["phases"]["measurement"] + assert measured["valid"] is False + assert measured["aggregate"] is None + + +def test_missing_startup_bracket_does_not_discard_valid_generation(): + role, run, samples = data() + result = analyze(role, run, samples[2:]) + assert result["status"] == "partial" + assert result["phases"]["startup"]["valid"] is False + assert result["phases"]["measurement"]["aggregate"]["energy_j"] == 450 + + +def test_invalid_media_energy_counts_but_not_invalid_clip_denominator(): + role, run, samples = data() + run["records"].append(record("bad-media", "measurement", 10, 11, valid=False)) + measured = analyze(role, run, samples)["phases"]["measurement"] + assert measured["valid"] is True + assert (measured["attempted"], measured["completed"], measured["valid_clips"]) == (2, 2, 1) + assert measured["aggregate"]["energy_j"] == 637.5 + assert measured["aggregate"]["joules_per_valid_clip"] == 637.5 + run["records"][1]["media"]["valid"] = False + assert analyze(role, run, samples)["phases"]["measurement"]["aggregate"]["joules_per_valid_clip"] is None + + +def test_legacy_journal_mapping_is_bounded_and_must_agree_with_record(): + role, run, samples = data() + item = run["records"][1] + item.pop("timing_window") + events = [{"event": "attempt_started", "at": utc(6.39), "slot_id": "measured"}, + {"event": "attempt_finished", "at": utc(9.75), "record": deepcopy(item)}] + window = analyze(role, run, samples, events)["windows"][2] + assert window["valid"] is True + assert window["start_monotonic_seconds"] == pytest.approx(6.5) + assert window["timing_uncertainty_seconds"] == pytest.approx(0.11) + assert window["aggregate"]["energy_j"] == pytest.approx(450) + events[0]["at"] = utc(6.0) + assert analyze(role, run, samples, events)["phases"]["measurement"]["aggregate"] is None + events[0]["at"] = utc(6.39) + events[1]["record"]["latency_seconds"] += 1 + assert analyze(role, run, samples, events)["phases"]["measurement"]["aggregate"] is None + + +@pytest.mark.parametrize("defect", ["reversed", "duration", "overlap", "missing_terminal"]) +def test_malformed_generation_windows_withhold_power(defect): + role, run, samples = data() + item = run["records"][1] + if defect == "reversed": + item["timing_window"]["end_monotonic_seconds"] = 6 + elif defect == "duration": + item["submit_to_terminal_seconds"] = 2 + elif defect == "overlap": + run["records"][1] = record("measured", "measurement", 4, 7) + else: + item["submit_to_terminal_seconds"] = None + assert analyze(role, run, samples)["phases"]["measurement"]["aggregate"] is None + + +@pytest.mark.parametrize("watts", [1e308, 6e307]) +def test_finite_inputs_cannot_publish_overflowed_power_or_energy(watts): + role, run, samples = data() + for sample in samples: + for device in sample["gpus"]: + device["power_watts"] = watts + result = analyze(role, run, samples) + assert result["phases"]["measurement"]["aggregate"] is None + assert result["phases"]["measurement"]["valid"] is False + json.dumps(result, allow_nan=False) + + +def test_phase_sum_overflow_is_withheld_even_if_each_request_integrates(): + role, run, samples = data() + run["records"] = [record("one", "measurement", 6, 7), record("two", "measurement", 8, 9)] + for sample in samples: + for device in sample["gpus"]: + device["power_watts"] = 6e307 + result = analyze(role, run, samples) + assert all(window["valid"] for window in result["windows"] if window["phase"] == "measurement") + assert result["phases"]["measurement"]["aggregate"] is None + assert "nonfinite_phase_power_integration" in result["phases"]["measurement"]["invalid_reasons"] + json.dumps(result, allow_nan=False) + + +@pytest.mark.parametrize("query", [None, {"start_monotonic_seconds": 8, "end_monotonic_seconds": 7, "start_utc": utc(8)}, + {"start_monotonic_seconds": 7, "end_monotonic_seconds": 9, "start_utc": utc(7)}]) +def test_malformed_timing_evidence_cannot_publish_power(query): + role, run, samples = data() + if query is None: + run["records"][1].pop("timing_window") + events = [{"event": "attempt_finished", "record": None}] + else: + samples[9]["power_query"] = query + events = [] + result = analyze(role, run, samples, events) + assert result["phases"]["measurement"]["aggregate"] is None + + +def test_serving_integrates_overlapping_gpu_work_once(): + from evaluator.mvp_serving import settings + role, run, samples = data() + run['records'].append(record('overlap', 'measurement', 7, 9.5)) + for row in run['records']: + row['timing_window']['transport_end_monotonic_seconds'] = row['timing_window']['end_monotonic_seconds'] + run['configuration'] = {'serving': settings(2)} + run['measurement'] = {'concurrency': 2, 'boundary': 'submit_to_downloaded_media', + 'start_monotonic_seconds': 6.5, 'end_monotonic_seconds': 9.75, 'wall_seconds': 3.25} + measured = analyze(role, run, samples)['phases']['measurement'] + assert measured['valid'] is True + assert measured['window_count'] == 1 + assert measured['valid_clips'] == 2 + assert measured['duration_seconds'] == 3 + assert measured['aggregate']['energy_j'] == 450 + assert measured['aggregate']['joules_per_valid_clip'] == 225 + run['records'][-1]['submit_to_terminal_seconds'] = None + assert analyze(role, run, samples)['phases']['measurement']['aggregate'] is None diff --git a/experimental/video-generation/tests/test_mvp_result.py b/experimental/video-generation/tests/test_mvp_result.py new file mode 100644 index 0000000000..99b677f283 --- /dev/null +++ b/experimental/video-generation/tests/test_mvp_result.py @@ -0,0 +1,220 @@ +"""CPU-only exporter tests reuse the supervisor's synthetic evidence fixture.""" +import json + +import pytest + +from evaluator import mvp_gpu_job as gpu +from evaluator.mvp_result import write_result +from test_mvp_gpu_job import GPU, saved_job, spec # noqa: F401 + +COMMIT = "a" * 40 +PRODUCER = {"git_commit": "b" * 40, "ci": {"run_id": "999"}} +SOURCE_CI = {"databaseId": 123, "runAttempt": 1, "headSha": COMMIT, "url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/123", "status": "completed", "conclusion": "success", "jobs": [{"name": "h3-video / H3 video H200 smoke", "status": "completed", "conclusion": "success"}]} + + +def _seal(root): + (root / "SHA256SUMS").write_text("".join(f"{gpu._hash(path)} {path.relative_to(root).as_posix()}\n" for path in sorted(root.rglob("*")) if path.is_file() and path.name != "SHA256SUMS")) + + +@pytest.fixture +def bundle(spec, tmp_path, request): + root = tmp_path / "artifact" + root.mkdir() + spec["plan"]["cases"] = spec["plan"]["cases"][:1] + spec["plan"]["repetitions"] = 1 + spec["allocation"] = {"mode": "dedicated_ci", "label": "Slurm 456.0 on test-node"} + if getattr(request, "param", None): + spec["serving"] = request.param + saved_job(spec, root / "gpu") + receipt = gpu._read(root / "gpu/gpu-job.json") + receipt.update(regression_status="inconclusive", ci_accepted=False, release_qualified=False) + gpu._write(root / "gpu/gpu-job.json", receipt) + for role in ("baseline", "candidate"): + (root / f"gpu/{role}/events.jsonl").write_text("") + for name in ("runtime.stdout.log", "runtime.stderr.log", "client.stderr.log", "client.stdout.json"): + (root / f"gpu/supervisor/{role}/{name}").write_text("synthetic test log\n") + (root / "report").mkdir() + (root / "report/index.html").write_text("Synthetic CPU test report") + allocation = {"identity": {"JobId": "456"}} + ci_identity = {"repository": "SemiAnalysisAI/InferenceX", "workflow_sha": COMMIT, "run_url": SOURCE_CI["url"]} + ci = {"schema_version": 1, "source_sha": COMMIT, "run_id": "123", "run_attempt": "1", "ci": ci_identity, + "allocation": allocation, "slurm_job": {"JobId": "456", "NodeList": "test-node", "AllocTRES": "cpu=8,gres/gpu=8"}, + "step_cleanup": {"status": "ended", "step_id": "456.0"}, "allocation_cleanup": {"status": "released"}, "exit_code": 0} + gpu._write(root / "ci.json", ci) + gpu._write(root / "allocation.json", allocation) + gpu._write(root / "binding.json", {"job_id": "456", "step_id": "0", "node": "test-node", "gpu_uuids": [GPU]}) + gpu._write(root / "step-result.json", {"exit_code": 0}) + gpu._write(root / "manifest.json", {"schema_version": 1, "git_commit": COMMIT, "run_id": "123", "run_attempt": "1", "ci": ci_identity, + "slurm_allocation": allocation, "workload_plan": spec["plan"], "exit_code": 0, + "evidence": {"ci.json": gpu._hash(root / "ci.json")}}) + _seal(root) + return root + + +def test_export_preserves_execution_provenance_and_withholds_missing_power(bundle): + original = {path.relative_to(bundle): path.read_bytes() for path in bundle.rglob("*") if path.is_file()} + result = write_result(bundle, producer=PRODUCER, source_ci=SOURCE_CI) + assert result["schema_version"] == "1.0.0" + assert result["status"] == "complete" and result["workload_status"] == "passed" + assert result["regression_status"] == "inconclusive" and result["release_qualified"] is False + assert result["execution"]["ci"]["git_commit"] == COMMIT != result["producer"]["git_commit"] + assert result["execution"]["ci"]["run_id"] == "123" != result["producer"]["ci"]["run_id"] + assert result["hardware"]["selected_gpu_count"] == 1 and result["hardware"]["reserved_gpu_count"] == 8 + assert result["hardware"]["tdp"]["watts_per_gpu"] is None + baseline = result["roles"]["baseline"] + assert baseline["metrics"]["latency_seconds"]["mean"] == 0.001 + assert baseline["metrics"]["valid_clips_per_second"] == 100.0 + assert baseline["power"]["phases"]["measurement"]["valid"] is False + assert baseline["power"]["phases"]["measurement"]["aggregate"] is None + assert all((bundle / path).read_bytes() == content for path, content in original.items()) + assert all((bundle / item["path"]).is_file() and gpu._hash(bundle / item["path"]) == item["sha256"] for item in result["files"]) + + +@pytest.mark.parametrize("mutation", ["tamper", "extra_file", "symlink", "missing_telemetry", "unknown_version", "nonzero_exit", "slurm_mismatch", "trusted_ci_mismatch", "missing_report_asset"]) +def test_invalid_bundle_writes_failed_result_and_propagates_error(bundle, mutation): + trusted = dict(SOURCE_CI) + if mutation == "tamper": + (bundle / "report/index.html").write_text("changed") + elif mutation == "extra_file": + (bundle / "unlisted.txt").write_text("unsealed") + elif mutation == "symlink": + (bundle / "media-link").symlink_to(bundle / "report/index.html") + elif mutation == "trusted_ci_mismatch": + trusted["headSha"] = "c" * 40 + else: + if mutation == "missing_report_asset": + (bundle / "report/index.html").write_text("") + elif mutation == "missing_telemetry": + (bundle / "gpu/supervisor/baseline/telemetry.jsonl").unlink() + else: + path = bundle / ("step-result.json" if mutation == "nonzero_exit" else "manifest.json" if mutation == "unknown_version" else "binding.json") + value = json.loads(path.read_text()) + value.update({"exit_code": 2} if mutation == "nonzero_exit" else {"schema_version": 99} if mutation == "unknown_version" else {"job_id": "987"}) + gpu._write(path, value) + _seal(bundle) + with pytest.raises(ValueError): + write_result(bundle, producer=PRODUCER, source_ci=trusted) + failed = json.loads((bundle / "result.json").read_text()) + assert failed["status"] == "failed" and failed["invalid_reasons"] + assert failed["release_qualified"] is False + assert all(role["metrics"]["status"] == "withheld" for role in failed["roles"].values()) + + +def test_frontend_schema_rejects_unknown_version_and_invalid_power_values(bundle): + from copy import deepcopy + from pathlib import Path + import jsonschema + + schema = json.loads((Path(__file__).parents[1] / "result.schema.json").read_text()) + result = write_result(bundle, producer=PRODUCER, source_ci=SOURCE_CI) + jsonschema.Draft202012Validator(schema).validate(result) + invalid = deepcopy(result) + invalid["schema_version"] = "2.0.0" + with pytest.raises(jsonschema.ValidationError): + jsonschema.Draft202012Validator(schema).validate(invalid) + invalid = deepcopy(result) + invalid["roles"]["baseline"]["power"]["phases"]["measurement"]["aggregate"] = {"energy_j": 99, "avg_power_w": 99, "observed_peak_power_w": 99, "joules_per_valid_clip": 99} + with pytest.raises(jsonschema.ValidationError): + jsonschema.Draft202012Validator(schema).validate(invalid) + + +def test_later_hardware_profile_never_backfills_historical_limits_and_html_escapes(bundle): + profile = {"schema_version": 1, "observation_kind": "read_only_inventory", "observed_at": "2026-09-10T00:00:00Z", + "gpu_uuids": [GPU], "slurm": {"gpu_uuids": [GPU]}, + "power_configuration": {"gpus": [{"uuid": GPU, "configured_limit_w": 700}]}, + "tdp": {"status": "verified", "watts_per_gpu": 700, "hardware_variant": "synthetic SXM test", + "source_url": "https://example.org/test-hardware", "evidence": ""}} + result = write_result(bundle, producer=PRODUCER, source_ci=SOURCE_CI, hardware_profile=profile) + assert result["hardware"]["tdp"]["watts_per_gpu"] == 700 + assert result["hardware"]["configured_power_limits"]["watts_by_gpu"] is None + assert result["hardware"]["later_hardware_observation"]["observed_at"] == "2026-09-10T00:00:00Z" + power_report = (bundle / "power-report.html").read_text() + assert "" not in power_report and "<script>bad()</script>" in power_report + assert "report/index.html" in power_report and "Baseline samples and coverage" in power_report + + +def test_same_run_export_keeps_workflow_pending_and_verifies_completed_gpu_job(bundle): + from copy import deepcopy + + trusted = deepcopy(SOURCE_CI) | {"status": "in_progress", "conclusion": None} + trusted["jobs"][0]["name"] = "p1.500 | H3 video H200 smoke" + producer = {"git_commit": COMMIT, "mode": "same_run_export", "ci": { + "run_id": "123", "run_attempt": "1", "repository": "SemiAnalysisAI/InferenceX"}} + result = write_result(bundle, producer=producer, source_ci=trusted) + assert result["workload_status"] == "passed" + assert result["execution"]["ci"]["external_ci_verification"] == "passed" + assert result["execution"]["ci"]["workflow_status_at_export"] == "in_progress" + assert result["execution"]["ci"]["workflow_conclusion_at_export"] is None + + +@pytest.mark.parametrize("mismatch", ["run", "attempt", "repository", "commit", "mode", "source", "pending_job", "failed_job"]) +def test_same_run_exception_does_not_skip_execution_identity_or_job_status(bundle, mismatch): + from copy import deepcopy + + trusted = deepcopy(SOURCE_CI) | {"status": "in_progress", "conclusion": None} + producer = {"git_commit": COMMIT, "mode": "same_run_export", "ci": { + "run_id": "123", "run_attempt": "1", "repository": "SemiAnalysisAI/InferenceX"}} + if mismatch in {"run", "attempt", "repository"}: + producer["ci"][{"run": "run_id", "attempt": "run_attempt", "repository": "repository"}[mismatch]] = "wrong" + elif mismatch == "commit": + producer["git_commit"] = "e" * 40 + elif mismatch == "mode": + producer["mode"] = "historical_replay" + elif mismatch == "source": + trusted["headSha"] = "f" * 40 + elif mismatch == "pending_job": + trusted["jobs"][0]["status"] = "in_progress" + else: + trusted["jobs"][0]["conclusion"] = "failure" + with pytest.raises(ValueError): + write_result(bundle, producer=producer, source_ci=trusted) + assert json.loads((bundle / "result.json").read_text())["status"] == "failed" + + +@pytest.mark.parametrize("change", ["different_setting", "wrong_device"]) +def test_contemporaneous_limits_preserve_each_snapshot_without_claiming_stability(bundle, change): + receipt_path = bundle / "gpu/gpu-job.json" + receipt = gpu._read(receipt_path) + for role in receipt["roles"].values(): + role.update(started_at="2026-08-03T00:00:01Z", finished_at="2026-08-03T00:59:59Z") + for when, timestamp in (("before", "2026-08-03T00:00:00Z"), ("after", "2026-08-03T01:00:00Z")): + role[f"power_configuration_{when}"] = {"status": "recorded", "observed_at": timestamp, "gpus": [{ + "uuid": GPU, "configured_limit_w": 600, "enforced_limit_w": 600, "default_limit_w": 700, "maximum_limit_w": 700}]} + after = receipt["roles"]["baseline"]["power_configuration_after"]["gpus"][0] + after["configured_limit_w" if change == "different_setting" else "uuid"] = 700 if change == "different_setting" else "GPU-WRONG" + gpu._write(receipt_path, receipt) + _seal(bundle) + result = write_result(bundle, producer=PRODUCER, source_ci=SOURCE_CI) + limits = result["hardware"]["configured_power_limits"] + assert limits["watts_by_gpu"] is None + assert limits["by_role"]["baseline"]["before"]["gpus"][0]["configured_limit_w"] == 600 + if change == "different_setting": + assert limits["status"] == "recorded" + assert limits["by_role"]["baseline"]["after"]["gpus"][0]["configured_limit_w"] == 700 + assert limits["by_role"]["baseline"]["same_observed_values"] is False + else: + assert limits["status"] == "partial" + assert limits["by_role"]["baseline"]["after"]["gpus"] is None + assert result["workload_status"] == "passed" + + +@pytest.mark.parametrize('bundle', [{'concurrency': 2, 'delivery_deadline_seconds': 1}], indirect=True) +def test_serving_export_keeps_contract_media_and_deployment_identity(bundle): + from pathlib import Path + import jsonschema + result = write_result(bundle, producer=PRODUCER, source_ci=SOURCE_CI) + jsonschema.Draft202012Validator(json.loads((Path(__file__).parents[1] / 'result.schema.json').read_text())).validate(result) + assert result['schema_version'] == '1.0.0' + assert result['execution']['deployment']['replica_count'] == 1 + assert result['execution']['deployment']['gpus_per_replica'] == 1 + assert result['execution']['deployment']['configured_batch_size'] is None + assert result['hardware']['reserved_gpu_count'] == 8 + stats = result['roles']['baseline']['metrics']['serving'] + assert stats['concurrency'] == 2 + assert stats['deadline_met_valid_clips'] == 1 + assert stats['client_ready_latency_seconds']['p50'] == .0005 + assert stats['client_ready_latency_seconds']['p90'] is None + assert stats['capacity_qualified'] is False + assert result['roles']['baseline']['records'][1]['job_id'] == 'fixture-1' + media = result['roles']['baseline']['records'][1]['media_file'] + assert gpu._hash(bundle / media['path']) == media['sha256'] diff --git a/experimental/video-generation/tests/test_mvp_runner.py b/experimental/video-generation/tests/test_mvp_runner.py new file mode 100644 index 0000000000..4b6fef612d --- /dev/null +++ b/experimental/video-generation/tests/test_mvp_runner.py @@ -0,0 +1,593 @@ +"""Offline transport tests using a local HTTP fixture, never an H3 model. + +The fixture returns deliberately non-video bytes and a mocked media analyzer. +Actual decoding/corruption tests belong to the media engine's test suite. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import math +import sys +import threading +import time +from email import policy +from email.parser import BytesParser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from evaluator import mvp_runner + + +MODEL_REVISION = "42ed227ee7df40d41602854ae760620d6eb651fe" +FIXTURE_BYTES = b"LOCAL_HTTP_FIXTURE_NOT_H3_VIDEO" + + +@pytest.fixture +def plan(): + return { + "plan_id": "offline-http-fixture-plan", + "model_id": "MiniMaxAI/MiniMax-H3", + "model_revision": MODEL_REVISION, + "generation": { + "duration_seconds": 4, + "aspect_ratio": "16:9", + "width": 1344, + "height": 768, + "frame_count": 107, + "fps": 24, + "audio_sample_rate_hz": 32000, + "audio_channels": 2, + "num_inference_steps": 50, + "flow_shift": 12.0, + "audio_flow_shift": 3.0, + }, + "cases": [ + {"case_id": "fixture-a", "prompt": "LOCAL TEST: motion and sound.", "seed": 42, + "requires_motion": True, "requires_sound": True}, + {"case_id": "fixture-b", "prompt": "LOCAL TEST: a quiet still scene.", "seed": 91, + "requires_motion": False, "requires_sound": False}, + ], + "repetitions": 2, + "warmup_runs": 1, + } + + +@pytest.fixture +def mocked_media(monkeypatch): + calls = [] + state = {"valid": True} + + def analyze(path, expected): + assert path.read_bytes() == FIXTURE_BYTES + calls.append((path, expected)) + return { + "decode_ok": True, + "video": {"width": expected["width"], "height": expected["height"], + "frame_count": expected["frame_count"], "fps": expected["fps"]}, + "audio": {"sample_rate_hz": expected["audio_sample_rate_hz"], "channels": expected["audio_channels"]}, + "checks": {"fixture_analysis_only": state["valid"]}, + "valid": state["valid"], + "metrics": {}, + } + + monkeypatch.setitem(sys.modules, "evaluator.mvp_media", SimpleNamespace( + analyze_media=analyze, IMPLEMENTATION_VERSION="local-test-fixture", __file__=__file__, + )) + monkeypatch.setitem(sys.modules, "av", SimpleNamespace( + __version__="local-test-fixture", library_versions={"libavcodec": (62, 1, 2)}, + )) + monkeypatch.setitem(sys.modules, "numpy", SimpleNamespace(__version__="local-test-fixture")) + monkeypatch.setattr(mvp_runner, "POLL_INTERVAL_SECONDS", 0.001) + return calls, state + + +@pytest.fixture +def fixture_server(): + servers = [] + + def start(*, mode="success", before_submit=None): + state = {"requests": [], "posts": [], "polls": {}, "auth": [], "mode": mode} + submission_lock = threading.Lock() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_): + pass + + def send(self, status, body, *, headers=None, announced_length=None): + self.send_response(status) + self.send_header("Content-Length", str(len(body) if announced_length is None else announced_length)) + self.send_header("Connection", "close") + for key, value in (headers or {}).items(): + self.send_header(key, value) + self.end_headers() + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass + self.close_connection = True + + def send_json(self, status, body): + self.send(status, json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + + def do_POST(self): + state["requests"].append(("POST", self.path)) + state["auth"].append(self.headers.get("Authorization")) + if before_submit: + before_submit() + body = self.rfile.read(int(self.headers["Content-Length"])) + content_type = self.headers.get("Content-Type", "") + if content_type.startswith("multipart/form-data"): + message = BytesParser(policy=policy.default).parsebytes( + b"Content-Type: " + content_type.encode() + b"\r\nMIME-Version: 1.0\r\n\r\n" + body + ) + payload = {part.get_param("name", header="content-disposition"): part.get_payload(decode=True).decode() + for part in message.iter_parts()} + else: + payload = json.loads(body) + with submission_lock: + state["posts"].append(payload) + ordinal = len(state["posts"]) + if state["mode"] == "http_error": + self.send_json(500, {"error": "do-not-log-this-server-secret"}) + elif state["mode"] == "redirect": + self.send(307, b"", headers={"Location": "/stolen-credentials"}) + elif state["mode"] == "unsafe_id": + self.send_json(200, {"id": "../../external?token=secret", "status": "queued"}) + else: + self.send_json(200, {"id": f"fixture-{ordinal}", "status": "queued"}) + + def do_GET(self): + state["requests"].append(("GET", self.path)) + state["auth"].append(self.headers.get("Authorization")) + if self.path.endswith("/content"): + if state["mode"] == "oversized": + self.send(200, FIXTURE_BYTES, announced_length=mvp_runner.MAX_MEDIA_BYTES + 1) + elif state["mode"] == "truncated": + self.send(200, FIXTURE_BYTES, announced_length=len(FIXTURE_BYTES) + 10) + else: + self.send(200, FIXTURE_BYTES, headers={"Content-Type": "video/mp4"}) + else: + state["polls"][self.path] = state["polls"].get(self.path, 0) + 1 + status = "queued" if state["mode"] == "timeout" else ( + "failed" if state["mode"] == "job_failed" else "completed" + ) + self.send_json(200, {"id": self.path.rsplit("/", 1)[-1], "status": status, + "error": "do-not-log-this-server-secret"}) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.005}, daemon=True) + thread.start() + servers.append((server, thread)) + return f"http://127.0.0.1:{server.server_port}", state + + yield start + for server, thread in servers: + server.shutdown() + server.server_close() + thread.join(timeout=1) + + +def execute(plan, output_dir, endpoint, **kwargs): + return mvp_runner.run_plan( + plan, output_dir, endpoint=endpoint, + runtime_revision="253020450290328e9deb307eece1e402fa17f35e", + hardware_label="LOCAL HTTP FIXTURE; no GPU and no H3 execution", + model_revision=MODEL_REVISION, + **kwargs, + ) + + +def test_preview_is_network_free_and_reproduces_frozen_slots(plan, monkeypatch): + monkeypatch.setattr(mvp_runner, "_open_response", lambda *args, **kwargs: pytest.fail("preview made a network request")) + preview = mvp_runner.preview_plan(plan) + assert preview["evidence_kind"] == "request_preview_no_generation" + assert preview["total_requests"] == 5 + assert preview["measurement_count"] == 4 + assert [slot["slot_id"] for slot in preview["slots"]] == [ + "warmup-001", "measurement-r001-c001", "measurement-r001-c002", + "measurement-r002-c001", "measurement-r002-c002", + ] + assert preview["slots"][1]["request"]["seed"] == 42 + assert preview["slots"][3]["request"]["seed"] == 42 + assert "fps" not in preview["slots"][0]["request"] + assert "num_frames" not in preview["slots"][0]["request"] + assert preview["plan_sha256"] == hashlib.sha256(mvp_runner.canonical_json_bytes(plan)).hexdigest() + + +@pytest.mark.parametrize("runtime", ["sglang", "vllm-omni"]) +@pytest.mark.parametrize(("duration", "frames"), [(4, 107), (8, 192)]) +def test_local_http_fixture_lifecycle_and_identity(plan, tmp_path, mocked_media, fixture_server, runtime, duration, frames): + plan["generation"].update(duration_seconds=duration, frame_count=frames) + output = tmp_path / runtime + intents_seen = [] + + def before_submit(): + events = [json.loads(line) for line in (output / "events.jsonl").read_text().splitlines()] + intents_seen.append(events[-1]["event"]) + assert events[-1]["event"] == "attempt_started" + + endpoint, server = fixture_server(before_submit=before_submit) + run = execute(plan, output, endpoint, runtime=runtime) + assert run["status"] == "complete" + assert run["evidence_kind"] == "operator_endpoint" + assert len(run["records"]) == 5 + assert len(server["posts"]) == 5 + assert len(intents_seen) == 5 + assert run["summary"]["scheduled"] == run["summary"]["completed"] == run["summary"]["valid"] == 4 + assert run["summary"]["failed"] == 0 + assert run["summary"]["technical_success_rate"] == 1 + assert math.isclose(run["summary"]["valid_clips_per_second"], 4 / run["measurement"]["wall_seconds"]) + assert all(record["latency_seconds"] > 0 for record in run["records"]) + for record in run["records"]: + assert 0 < record["submit_to_terminal_seconds"] <= record["submit_to_media_seconds"] <= record["latency_seconds"] + assert 0 <= record["media_validation_seconds"] <= record["latency_seconds"] + assert run["summary"]["latency_samples"] == 4 + assert run["summary"]["latency_sample_stddev_seconds"] is not None + assert run["summary"]["submit_to_terminal_median_seconds"] > 0 + assert run["configuration"]["identity_verification"] == "operator_declared" + assert "not remotely verified" in run["configuration"]["server_identity_caveat"] + assert "Mock-server tests are not H3 evidence" in run["evidence_caveat"] + assert run["configuration"]["hardware_label"].startswith("LOCAL HTTP FIXTURE") + assert run["measurement"]["warmup_qualified"] is True + config = copy.deepcopy(run["configuration"]) + digest = config.pop("configuration_sha256") + assert hashlib.sha256(mvp_runner.canonical_json_bytes(config)).hexdigest() == digest + assert (output / "plan.json").read_bytes() == mvp_runner.canonical_json_bytes(plan) + assert json.loads((output / "run.json").read_text()) == run + for record in run["records"]: + assert (output / record["artifact_path"]).read_bytes() == FIXTURE_BYTES + assert record["sha256"] == hashlib.sha256(FIXTURE_BYTES).hexdigest() + assert math.isclose(record["expected_media"]["duration_seconds"], frames / 24) + assert record["expected_media"]["audio_required"] is True + calls, _ = mocked_media + assert len(calls) == 5 + assert all(expected["timeout_seconds"] > 0 for _, expected in calls) + first = server["posts"][0] + if runtime == "sglang": + assert first["target"] == {"duration_seconds": duration, "aspect_ratio": "16:9", "short_edge": 768} + assert "fps" not in first and "num_frames" not in first + assert first["audio_flow_shift"] == 3 + else: + assert first["width"] == "1344" + assert first["num_frames"] == str(frames) + assert json.loads(first["extra_params"])["audio_flow_shift"] == 3 + + +def test_http_failure_is_durable_never_retried_and_aborts_unknown_jobs(plan, tmp_path, mocked_media, fixture_server): + plan["warmup_runs"] = 0 + endpoint, state = fixture_server(mode="http_error") + output = tmp_path / "failed-http-fixture" + run = execute(plan, output, endpoint) + assert run["status"] == "failed" + assert len(state["posts"]) == 1 + assert run["summary"]["failed"] == 4 + assert run["summary"]["failed_attempts"] == 1 + assert run["summary"]["not_started"] == 3 + assert run["summary"]["technical_success_rate"] == 0 + assert run["summary"]["latency_median_seconds"] is None + assert run["summary"]["submit_to_terminal_median_seconds"] is None + assert run["summary"]["latency_samples"] == 0 + assert run["records"][0]["submit_to_terminal_seconds"] is None + assert run["records"][1]["submit_to_media_seconds"] is None + assert "HTTP 500" in run["records"][0]["error"] + assert all(record["error"] == "not_started_after_uncertain_remote_completion" for record in run["records"][1:]) + assert "do-not-log-this-server-secret" not in (output / "events.jsonl").read_text() + assert "do-not-log-this-server-secret" not in (output / "run.json").read_text() + + +def test_media_evaluator_fingerprint_is_covered_by_configuration_hash(plan, tmp_path, mocked_media, fixture_server): + plan["warmup_runs"] = 0 + plan["repetitions"] = 1 + plan["cases"] = plan["cases"][:1] + endpoint, _ = fixture_server() + run = execute(plan, tmp_path / "evaluator-fingerprint-fixture", endpoint) + configuration = copy.deepcopy(run["configuration"]) + assert configuration["media_evaluator"] == { + "implementation_version": "local-test-fixture", + "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "pyav_version": "local-test-fixture", + "numpy_version": "local-test-fixture", + "ffmpeg_libraries": {"libavcodec": "62.1.2"}, + } + original_digest = configuration.pop("configuration_sha256") + assert hashlib.sha256(mvp_runner.canonical_json_bytes(configuration)).hexdigest() == original_digest + configuration["media_evaluator"]["ffmpeg_libraries"]["libavcodec"] = "62.2.0" + assert hashlib.sha256(mvp_runner.canonical_json_bytes(configuration)).hexdigest() != original_digest + + +def test_known_failed_job_does_not_remove_other_scheduled_attempts(plan, tmp_path, mocked_media, fixture_server): + plan["warmup_runs"] = 0 + endpoint, state = fixture_server(mode="job_failed") + run = execute(plan, tmp_path / "failed-jobs-fixture", endpoint) + assert len(state["posts"]) == 4 + assert run["summary"]["failed_attempts"] == 4 + assert run["summary"]["not_started"] == 0 + assert run["abort_reason"] is None + + +@pytest.mark.parametrize("mode", ["success", "job_failed", "http_error"]) +def test_request_timing_windows_preserve_observed_terminal_and_end(plan, tmp_path, mocked_media, fixture_server, mode): + plan.update(warmup_runs=0, repetitions=1, cases=plan["cases"][:1]) + endpoint, _ = fixture_server(mode=mode) + before = time.monotonic() + run = execute(plan, tmp_path / mode, endpoint) + after = time.monotonic() + record = run["records"][0] + timing = record["timing_window"] + start, end = timing["start_monotonic_seconds"], timing["end_monotonic_seconds"] + assert before <= start < end <= after + assert math.isclose(end - start, record["latency_seconds"], rel_tol=1e-6, abs_tol=1e-9) + if mode == "http_error": + assert timing["terminal_monotonic_seconds"] is None + assert record["submit_to_terminal_seconds"] is None + else: + terminal = timing["terminal_monotonic_seconds"] + assert start < terminal <= end + assert math.isclose(terminal - start, record["submit_to_terminal_seconds"], rel_tol=1e-6, abs_tol=1e-9) + events = [json.loads(line) for line in (tmp_path / mode / "events.jsonl").read_text().splitlines()] + assert next(event["record"] for event in events if event["event"] == "attempt_finished") == record + + +def test_timeout_is_bounded_and_retains_all_denominators(plan, tmp_path, mocked_media, fixture_server): + plan["warmup_runs"] = 0 + endpoint, state = fixture_server(mode="timeout") + start = time.monotonic() + run = execute(plan, tmp_path / "timeout-fixture", endpoint, timeout_seconds=0.05) + assert time.monotonic() - start < 3 + assert len(state["posts"]) == 1 + assert run["summary"]["scheduled"] == run["summary"]["failed"] == 4 + assert run["records"][0]["status"] == "failed" + assert run["abort_reason"] + assert run["records"][1]["attempted"] is False + + +def test_completed_but_invalid_media_is_not_technical_success(plan, tmp_path, mocked_media, fixture_server): + plan["warmup_runs"] = 0 + _, media = mocked_media + media["valid"] = False + endpoint, state = fixture_server() + run = execute(plan, tmp_path / "invalid-media-fixture", endpoint) + assert len(state["posts"]) == 4 + assert run["summary"]["completed"] == 4 + assert run["summary"]["valid"] == 0 + assert run["summary"]["invalid_completed"] == 4 + assert run["summary"]["failed_attempts"] == 0 + assert run["summary"]["failed"] == 4 + + +def test_failed_warmup_cannot_qualify_a_measured_run(plan, tmp_path, mocked_media, fixture_server): + _, media = mocked_media + media["valid"] = False + endpoint, state = fixture_server() + run = execute(plan, tmp_path / "failed-warmup-fixture", endpoint) + assert len(state["posts"]) == 1 + assert run["measurement"]["warmup_qualified"] is False + assert run["measurement"]["warmup_status"] == "failed" + assert run["summary"]["scheduled"] == run["summary"]["not_started"] == 4 + assert all(record["error"] == "not_started_after_failed_warmup" for record in run["records"][1:]) + + +@pytest.mark.parametrize("mode", ["oversized", "truncated"]) +def test_download_size_and_integrity_limits(plan, tmp_path, mocked_media, fixture_server, mode): + plan["warmup_runs"] = 0 + plan["repetitions"] = 1 + plan["cases"] = plan["cases"][:1] + endpoint, state = fixture_server(mode=mode) + run = execute(plan, tmp_path / mode, endpoint) + assert run["summary"]["valid"] == 0 + assert run["records"][0]["artifact_path"] is None + assert run["records"][0]["sha256"] is None + assert len(state["posts"]) == 1 + assert not mocked_media[0] + + +def test_refuses_overwrite_without_network_or_file_changes(plan, tmp_path, mocked_media, fixture_server): + output = tmp_path / "existing" + output.mkdir() + marker = output / "user-owned.txt" + marker.write_text("preserve this file") + endpoint, state = fixture_server() + with pytest.raises(FileExistsError): + execute(plan, output, endpoint) + assert marker.read_text() == "preserve this file" + assert list(output.iterdir()) == [marker] + assert not state["requests"] + + +def test_credential_is_not_persisted_or_forwarded_on_redirect(plan, tmp_path, mocked_media, fixture_server, monkeypatch): + monkeypatch.setenv("VGBENCH_TEST_ONLY_TOKEN", "fixture-credential-never-a-real-key") + endpoint, state = fixture_server(mode="redirect") + output = tmp_path / "redirect-fixture" + run = execute(plan, output, endpoint, api_key_env="VGBENCH_TEST_ONLY_TOKEN") + assert len(state["requests"]) == 1 + assert state["auth"] == ["Bearer fixture-credential-never-a-real-key"] + assert run["records"][0]["status"] == "failed" + for path in output.rglob("*"): + if path.is_file(): + assert b"fixture-credential-never-a-real-key" not in path.read_bytes() + + +@pytest.mark.parametrize("endpoint", [ + "https://user:secret@example.com", "https://example.com?token=secret", + "https://example.com#secret", "https://api.minimax.io", "file:///tmp/video", +]) +def test_refuses_unsafe_or_hosted_endpoint_before_creating_output(plan, tmp_path, mocked_media, endpoint): + output = tmp_path / "must-not-exist" + with pytest.raises(ValueError): + execute(plan, output, endpoint) + assert not output.exists() + + +def test_untrusted_job_id_cannot_change_download_destination(plan, tmp_path, mocked_media, fixture_server): + endpoint, state = fixture_server(mode="unsafe_id") + run = execute(plan, tmp_path / "unsafe-id-fixture", endpoint) + assert len(state["requests"]) == 1 + assert run["records"][0]["error"] == "submission returned a missing or unsafe job identifier" + assert all(record["artifact_path"] is None for record in run["records"]) + + +def test_missing_controls_and_revision_drift_fail_before_output(plan, tmp_path, mocked_media): + invalid = copy.deepcopy(plan) + del invalid["generation"]["num_inference_steps"] + with pytest.raises(ValueError, match="num_inference_steps"): + mvp_runner.preview_plan(invalid) + invalid = copy.deepcopy(plan) + invalid["model_revision"] = "1" * 40 + output = tmp_path / "not-created" + with pytest.raises(ValueError, match="does not match"): + execute(invalid, output, "http://127.0.0.1:1") + assert not output.exists() + + +@pytest.mark.parametrize("change", [ + {"num_frames": 107}, {"frame_count": 96}, {"width": 1366}, {"scheduler": "secretly-changed"}, + {"duration_seconds": 8}, {"duration_seconds": 8, "frame_count": 193}, + {"frame_count": 192}, {"duration_seconds": 6, "frame_count": 158}, +]) +def test_unsupported_or_inconsistent_generation_contracts_are_rejected(plan, change): + plan["generation"].update(change) + with pytest.raises(ValueError): + mvp_runner.preview_plan(plan) + + +def test_missing_media_dependency_prevents_live_submission(plan, tmp_path, mocked_media, fixture_server, monkeypatch): + monkeypatch.setitem(sys.modules, "av", None) + endpoint, server = fixture_server() + output = tmp_path / "dependency-missing" + with pytest.raises(ImportError): + execute(plan, output, endpoint) + assert not output.exists() + assert not server["requests"] + + +def test_dns_timeout_cannot_submit_late_request(plan, tmp_path, mocked_media, fixture_server, monkeypatch): + plan["warmup_runs"] = 0 + endpoint, server = fixture_server() + release_resolver = threading.Event() + original = mvp_runner.socket.getaddrinfo + + def slow_resolution(*args, **kwargs): + release_resolver.wait(timeout=2) + return original(*args, **kwargs) + + monkeypatch.setattr(mvp_runner.socket, "getaddrinfo", slow_resolution) + try: + run = execute(plan, tmp_path / "slow-dns-fixture", endpoint, timeout_seconds=0.03) + assert run["summary"]["failed"] == 4 + assert "timed out" in run["records"][0]["error"] + assert not server["requests"] + finally: + release_resolver.set() + + +def test_keyboard_interrupt_retains_inflight_outcome_and_schedule(plan, tmp_path, mocked_media, fixture_server, monkeypatch): + plan["warmup_runs"] = 0 + endpoint, server = fixture_server() + + def interrupt_analysis(*_): + raise KeyboardInterrupt + + monkeypatch.setattr(sys.modules["evaluator.mvp_media"], "analyze_media", interrupt_analysis) + output = tmp_path / "interrupted-fixture" + with pytest.raises(KeyboardInterrupt): + execute(plan, output, endpoint) + run = json.loads((output / "run.json").read_text()) + assert len(server["posts"]) == 1 + assert run["status"] == "failed" + assert run["summary"]["failed"] == 4 + assert run["summary"]["not_started"] == 3 + assert run["records"][0]["error"].startswith("interrupted by operator") + + +def test_serving_overlaps_requests_without_waiting_for_local_validation(plan, tmp_path, mocked_media, fixture_server, monkeypatch): + plan['warmup_runs'] = 0 + pair = threading.Barrier(2) + submitted = threading.Event() + count = [0] + lock = threading.Lock() + + def before_submit(): + with lock: + count[0] += 1 + if count[0] == 4: + submitted.set() + pair.wait(timeout=2) + + endpoint, state = fixture_server(before_submit=before_submit) + analyzer = sys.modules['evaluator.mvp_media'].analyze_media + + def validate_after_submissions(path, expected): + assert submitted.wait(2), 'local validation blocked later submissions' + return analyzer(path, expected) + + monkeypatch.setattr(sys.modules['evaluator.mvp_media'], 'analyze_media', validate_after_submissions) + run = execute(plan, tmp_path / 'serving', endpoint, serving_concurrency=2, delivery_deadline_seconds=10) + assert run['status'] == 'complete' + assert run['serving']['peak_client_in_flight'] == 2 + assert run['summary']['valid'] == 4 + assert run['serving']['deadline_met_valid_clips'] == 4 + assert run['serving']['client_ready_latency_seconds']['p90'] is None + assert run['serving']['observed_batch_sizes'] is None + assert len({r['job_id'] for r in run['records']}) == 4 + assert [r['slot_id'] for r in run['records']] == [s['slot_id'] for s in mvp_runner._slots(plan)] + window = run['measurement'] + assert window['boundary'] == 'submit_to_downloaded_media' + assert window['end_monotonic_seconds'] == max(r['timing_window']['transport_end_monotonic_seconds'] for r in run['records']) + assert run['summary']['valid_clips_per_second'] == 4 / window['wall_seconds'] + for record in run['records']: + assert 0 <= record['submit_to_accepted_seconds'] <= record['submit_to_terminal_seconds'] <= record['submit_to_media_seconds'] <= record['latency_seconds'] + assert record['server_timings'] is None + assert len(state['posts']) == 4 + + +def test_serving_unknown_remote_completion_stops_queued_requests(plan, tmp_path, mocked_media, fixture_server): + plan['warmup_runs'] = 0 + pair = threading.Barrier(2) + endpoint, state = fixture_server(mode='timeout', before_submit=lambda: pair.wait(timeout=2)) + run = execute(plan, tmp_path / 'timeout', endpoint, serving_concurrency=2, timeout_seconds=.1) + assert len(state['posts']) == 2 + assert run['summary']['scheduled'] == run['summary']['failed'] == 4 + assert run['summary']['not_started'] == 2 + assert run['serving']['outcomes']['timed_out'] == 2, run['records'] + assert run['serving']['outcomes']['not_started'] == 2 + assert run['serving']['peak_client_in_flight'] == 2 + assert run['serving']['client_ready_latency_seconds']['sample_count'] == 0 + + +def test_serving_failed_warmup_never_submits_measured_requests(plan, tmp_path, mocked_media, fixture_server): + mocked_media[1]['valid'] = False + endpoint, state = fixture_server() + run = execute(plan, tmp_path / 'warmup-failure', endpoint, serving_concurrency=2) + assert len(state['posts']) == 1 + assert run['summary']['not_started'] == 4 + assert run['measurement']['warmup_status'] == 'failed' + assert run['serving']['peak_client_in_flight'] == 0 + + +@pytest.mark.parametrize('concurrency,deadline', [(0, None), (33, None), (True, None), (1, float('nan')), (None, 10)]) +def test_invalid_serving_settings_fail_before_output(plan, tmp_path, mocked_media, concurrency, deadline): + destination = tmp_path / 'invalid-serving' + with pytest.raises(ValueError): + execute(plan, destination, 'http://localhost:9', serving_concurrency=concurrency, delivery_deadline_seconds=deadline) + assert not destination.exists() + + +@pytest.mark.parametrize("exception", [ConnectionError, AttributeError, ValueError]) +def test_watchdog_deadline_wins_over_socket_teardown_errors(plan, tmp_path, mocked_media, monkeypatch, exception): + plan['warmup_runs'] = 0 + def stopped_transfer(*args, deadline, **kwargs): + time.sleep(max(0, deadline-time.monotonic()) + .001) + raise exception('CPU fixture: watchdog closed transport') + monkeypatch.setattr(mvp_runner, '_json_request', stopped_transfer) + run = execute(plan, tmp_path / 'deadline', 'http://127.0.0.1:1', serving_concurrency=1, timeout_seconds=.01) + assert run['serving']['outcomes']['timed_out'] == 1 + assert run['summary']['not_started'] == 3 diff --git a/experimental/video-generation/tests/test_mvp_serving.py b/experimental/video-generation/tests/test_mvp_serving.py new file mode 100644 index 0000000000..0730942130 --- /dev/null +++ b/experimental/video-generation/tests/test_mvp_serving.py @@ -0,0 +1,69 @@ +"""Hand-worked serving metrics; synthetic request timings, never GPU evidence.""" + +from copy import deepcopy + +import pytest + +from evaluator.mvp_serving import settings, summarize, validate_window + + +def sample_run(): + records = [] + for index in range(11): + latency = index + 1 + start = index * 12 + records.append({ + 'slot_id': str(index), 'phase': 'measurement', 'attempted': True, + 'status': 'succeeded' if index < 10 else 'failed', + 'outcome': 'completed' if index < 10 else 'provider_failed', + 'submit_to_media_seconds': latency if index < 10 else None, + 'media': {'valid': True, 'video': {'duration_seconds': 8}} if index < 10 else None, + 'timing_window': {'start_monotonic_seconds': start, 'transport_end_monotonic_seconds': start + latency}, + }) + return {'configuration': {'serving': settings(1, 5)}, 'records': records, + 'measurement': {'boundary': 'submit_to_downloaded_media', 'concurrency': 1, + 'start_monotonic_seconds': 0, 'end_monotonic_seconds': 132, 'wall_seconds': 132}} + + +def test_percentiles_goodput_and_failure_denominators(): + metrics = summarize(sample_run()) + latency = metrics['client_ready_latency_seconds'] + assert latency['sample_count'] == 10 + assert latency['p50'] == 5.5 + assert latency['p90'] == 9 + assert latency['p95'] is None + assert metrics['deadline_met_valid_clips'] == 5 + assert metrics['deadline_attainment_fraction'] == 5 / 11 + assert metrics['deadline_goodput_clips_per_second'] == 5 / 132 + assert metrics['valid_video_seconds_per_second'] == 80 / 132 + assert metrics['outcomes']['provider_failed'] == 1 + assert metrics['capacity_qualified'] is False + assert metrics['offered_request_rate_per_second'] is None + + +def test_missing_sample_withholds_percentiles_and_goodput(): + run = sample_run() + run['records'][0]['submit_to_media_seconds'] = None + metrics = summarize(run) + assert metrics['client_ready_latency_seconds']['sample_count'] == 0 + assert metrics['client_ready_latency_seconds']['valid_clip_count'] == 10 + assert metrics['client_ready_latency_seconds']['p90'] is None + assert metrics['deadline_goodput_clips_per_second'] is None + + +@pytest.mark.parametrize('defect', ['wall', 'boundary', 'outside', 'concurrency', 'empty_interval']) +def test_tampered_serving_windows_fail_closed(defect): + run = deepcopy(sample_run()) + if defect == 'wall': + run['measurement']['wall_seconds'] = 130 + elif defect == 'boundary': + run['measurement']['boundary'] = 'submit_to_validated_media' + elif defect == 'outside': + run['records'][0]['timing_window']['transport_end_monotonic_seconds'] = 133 + elif defect == 'empty_interval': + run['records'][0]['timing_window']['transport_end_monotonic_seconds'] = 0 + else: + run['records'][1]['timing_window']['start_monotonic_seconds'] = .5 + run['records'][1]['timing_window']['transport_end_monotonic_seconds'] = 2.5 + with pytest.raises(ValueError): + validate_window(run) diff --git a/experimental/video-generation/tests/test_mvp_serving_smoke.py b/experimental/video-generation/tests/test_mvp_serving_smoke.py new file mode 100644 index 0000000000..2cefef538f --- /dev/null +++ b/experimental/video-generation/tests/test_mvp_serving_smoke.py @@ -0,0 +1,110 @@ +"""CPU-only matrix and receipt tests; fixture bytes are not generated media.""" + +import copy +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from evaluator import mvp_gpu_job as gpu, mvp_gpu_evidence as evidence, mvp_serving_smoke as smoke +from test_mvp_gpu_job import saved_job, spec # noqa: F401 + + +def saved_single(spec, directory): + directory.parent.mkdir(parents=True, exist_ok=True) + saved_job(spec, directory) + receipt = gpu._read(directory / "gpu-job.json") + receipt.update(bundle_type="controlled_serving_smoke", comparison_path=None, ci_accepted=False, release_qualified=False) + del receipt["roles"]["candidate"] + gpu._write(directory / "gpu-job.json", receipt) + (directory / "baseline/events.jsonl").write_text("") + return receipt + + +def test_single_runtime_smoke_cannot_be_accepted_as_paired_evidence(spec, tmp_path): + spec["serving"] = {"concurrency": 2} + directory = tmp_path / "single" + saved_single(spec, directory) + verified = evidence.verify_measurement_job(directory, deadline=time.monotonic() + 5, serving_smoke=True) + assert set(verified["runs"]) == {"baseline"} + assert verified["comparison"] is None + with pytest.raises(ValueError, match="identity is not verified"): + evidence.verify_measurement_job(directory, deadline=time.monotonic() + 5) + (directory / "baseline/artifacts/1.mp4").write_bytes(b"tampered") + with pytest.raises(ValueError, match="media bytes"): + evidence.verify_measurement_job(directory, deadline=time.monotonic() + 5, serving_smoke=True) + + +@pytest.mark.parametrize("fail_second", [False, True]) +def test_matrix_preserves_twelve_requests_without_doubling_roles(spec, tmp_path, monkeypatch, fail_second): + spec["plan"]["cases"] = spec["plan"]["cases"][:1] + spec["plan"]["repetitions"] = 4 + spec["serving"] = {"concurrency": 1} + submitted = [] + def execute(current, directory, *, serving_smoke): + assert serving_smoke is True + submitted.append(copy.deepcopy(current)) + if fail_second and len(submitted) == 2: + raise RuntimeError("CPU fake runtime startup failure") + return saved_single(current, directory) + monkeypatch.setattr(gpu, "run_gpu_job", execute) + result = smoke.run_matrix(spec, tmp_path) + assert result["completion"]["scheduled"] == 12 + assert [s["serving"]["concurrency"] for s in submitted] == ([1, 2] if fail_second else [1, 2, 4]) + assert all(s["plan"] == spec["plan"] for s in submitted) + assert result["completion"]["valid"] == (4 if fail_second else 12) + assert result["completion"]["not_started"] == (8 if fail_second else 0) + assert result["status"] == ("failed" if fail_second else "complete") + assert not result["ci_accepted"] + report = (tmp_path / "report/index.html").read_text() + assert report.count("