From 81242b5f70d0286bb5f8cb46449f396b135665fd Mon Sep 17 00:00:00 2001 From: IIIllllIlIlllII <306726845+IIIllllIlIlllII@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:53:12 +0000 Subject: [PATCH 1/2] Publish reproducible Vision v8 ONNX release artifacts Add an ONNX release workflow, triggered manually or by an onnx-v8-* tag, that downloads a pinned checkpoint revision, exports both prediction branches, runs the parity gates, writes a manifest and verifies every checksum before it uploads anything. Any failure in that chain aborts before publication. All logic lives in scripts/build_onnx_release.py so it can run and be tested outside CI; the workflow only installs the pinned toolchain and calls it. docs/onnx/release.json pins the checkpoint repository and revision, the opset, and the export toolchain, and the workflow reads its version pins from that same file so the manifest cannot drift from what CI installs. Pinning the toolchain is what makes a repository commit sufficient to reproduce a digest: re-exporting the same checkpoint under torch 2.13.0 instead of 2.6.0 produces binaries one byte larger on both branches, because torch.onnx.export stamps its own version into producer_version. The graph is unchanged and the parity gates pass identically, but the SHA-256 does not match. A moving checkpoint ref is rejected: the revision must be a full 40-character sha, otherwise reproducibility would silently depend on when the build ran. Also ignore *.onnx, *.engine and /dist/ so exported graphs cannot reach a source commit, and link the release path from the validation report. Closes #11 Co-Authored-By: Claude Opus 5 --- .github/workflows/detector-export.yml | 9 + .github/workflows/onnx-release.yml | 87 +++++ .gitignore | 5 + docs/onnx/release.json | 25 ++ docs/onnx/tr_hash_v8_validation_report.md | 46 ++- scripts/build_onnx_release.py | 446 ++++++++++++++++++++++ tests/test_onnx_release.py | 205 ++++++++++ 7 files changed, 815 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/onnx-release.yml create mode 100644 docs/onnx/release.json create mode 100644 scripts/build_onnx_release.py create mode 100644 tests/test_onnx_release.py diff --git a/.github/workflows/detector-export.yml b/.github/workflows/detector-export.yml index 350a706d..6e72fbe9 100644 --- a/.github/workflows/detector-export.yml +++ b/.github/workflows/detector-export.yml @@ -6,11 +6,14 @@ on: - ".github/workflows/detector-export.yml" - "complexity/deploy/onnx_detector/**" - "complexity/generative/detection/**" + - "scripts/build_onnx_release.py" - "scripts/check_onnx_parity.py" - "scripts/onnx_detect.py" - "scripts/export_onnx.py" - "scripts/export_tensorrt.py" + - "docs/onnx/release.json" - "tests/test_detector_export.py" + - "tests/test_onnx_release.py" - "tests/test_onnx_detect_cli.py" - "tests/test_onnx_detector_*.py" - "pyproject.toml" @@ -20,11 +23,14 @@ on: - ".github/workflows/detector-export.yml" - "complexity/deploy/onnx_detector/**" - "complexity/generative/detection/**" + - "scripts/build_onnx_release.py" - "scripts/check_onnx_parity.py" - "scripts/onnx_detect.py" - "scripts/export_onnx.py" - "scripts/export_tensorrt.py" + - "docs/onnx/release.json" - "tests/test_detector_export.py" + - "tests/test_onnx_release.py" - "tests/test_onnx_detect_cli.py" - "tests/test_onnx_detector_*.py" - "pyproject.toml" @@ -52,6 +58,7 @@ jobs: ruff check complexity/deploy/onnx_detector complexity/generative/detection/exporting.py + scripts/build_onnx_release.py scripts/check_onnx_parity.py scripts/onnx_detect.py scripts/export_onnx.py @@ -62,6 +69,7 @@ jobs: tests/test_onnx_detector_metadata.py tests/test_onnx_detector_pipeline.py tests/test_onnx_detector_skeleton.py + tests/test_onnx_release.py - name: Test ONNX branches and dynamic batch parity run: >- pytest -q @@ -71,3 +79,4 @@ jobs: tests/test_onnx_detector_metadata.py tests/test_onnx_detector_pipeline.py tests/test_onnx_detector_skeleton.py + tests/test_onnx_release.py diff --git a/.github/workflows/onnx-release.yml b/.github/workflows/onnx-release.yml new file mode 100644 index 00000000..c89db553 --- /dev/null +++ b/.github/workflows/onnx-release.yml @@ -0,0 +1,87 @@ +name: ONNX release + +on: + workflow_dispatch: + inputs: + tag: + description: "Release tag to create or update (e.g. onnx-v8-2026.08)" + required: true + type: string + draft: + description: "Publish as a draft release" + required: false + default: true + type: boolean + push: + tags: + - "onnx-v8-*" + +permissions: + contents: write + +concurrency: + group: onnx-release-${{ github.event.inputs.tag || github.ref_name }} + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Read the pinned toolchain + id: pins + run: | + python - <<'PY' >> "$GITHUB_OUTPUT" + import json + toolchain = json.load(open("docs/onnx/release.json"))["toolchain"] + for package, version in toolchain.items(): + print(f"{package}={version}") + PY + + - name: Install the pinned export toolchain + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev,export]" huggingface_hub + # Installed last so the pins win over the extras' version ranges. + python -m pip install "torch==${{ steps.pins.outputs.torch }}" \ + --index-url https://download.pytorch.org/whl/cpu + python -m pip install \ + "onnx==${{ steps.pins.outputs.onnx }}" \ + "onnxruntime==${{ steps.pins.outputs.onnxruntime }}" + + - name: Test the release tooling + run: pytest -q tests/test_onnx_release.py + + - name: Build, gate and verify the release + env: + PYTHONPATH: . + run: python scripts/build_onnx_release.py --output-dir dist/onnx + + - name: Re-verify the published files against the manifest + run: python scripts/build_onnx_release.py --verify dist/onnx/manifest.json + + - name: Upload release assets + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.inputs.tag || github.ref_name }} + DRAFT: ${{ github.event.inputs.draft == 'true' && '--draft' || '' }} + run: | + gh release view "$TAG" >/dev/null 2>&1 \ + || gh release create "$TAG" \ + --title "TR-HASH Vision v8 ONNX artifacts" \ + --notes-file dist/onnx/RELEASE_NOTES.md \ + $DRAFT + gh release upload "$TAG" \ + dist/onnx/manifest.json \ + dist/onnx/tr_hash_v8_o2m.onnx \ + dist/onnx/tr_hash_v8_o2m.json \ + dist/onnx/tr_hash_v8_nms_free.onnx \ + dist/onnx/tr_hash_v8_nms_free.json \ + --clobber diff --git a/.gitignore b/.gitignore index 9909ab6b..bd59827d 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,11 @@ weights/ *.gguf *.ggml +# Exported inference graphs: published as release assets, never committed +*.onnx +*.engine +/dist/ + # Training outputs artifacts/ runs/ diff --git a/docs/onnx/release.json b/docs/onnx/release.json new file mode 100644 index 00000000..9656633e --- /dev/null +++ b/docs/onnx/release.json @@ -0,0 +1,25 @@ +{ + "checkpoint_repo": "AETHORIA-AI/TR-HASH-Vision-v8-2M-COCO-SFT", + "checkpoint_revision": "f3b3e659612e543ca9ff91892c0662d38dc1a1d6", + "opset": 17, + "parity_num_tests": 5, + "toolchain": { + "torch": "2.13.0", + "onnx": "1.21.0", + "onnxruntime": "1.24.4" + }, + "branches": [ + { + "branch": "o2m", + "checkpoint_subdir": null, + "stem": "tr_hash_v8_o2m", + "post_processing": "decode plus NMS" + }, + { + "branch": "nms-free", + "checkpoint_subdir": "best_nms_free", + "stem": "tr_hash_v8_nms_free", + "post_processing": "decode plus confidence filtering" + } + ] +} diff --git a/docs/onnx/tr_hash_v8_validation_report.md b/docs/onnx/tr_hash_v8_validation_report.md index 95141679..68503e7c 100644 --- a/docs/onnx/tr_hash_v8_validation_report.md +++ b/docs/onnx/tr_hash_v8_validation_report.md @@ -5,16 +5,46 @@ Validation was run from commit `0fcf05c146f84a857d392b7da7d2947a41eb6d62` `AETHORIA-AI/TR-HASH-Vision-v8-2M-COCO-SFT`. The generated ONNX binaries are intentionally not committed to the source -repository. Upload them as GitHub Release assets and link the release asset URLs -from the release notes or from a follow-up update to this report. The release -assets should include both ONNX binaries and their export metadata sidecars: +repository — `*.onnx` is ignored — and are published as GitHub Release assets +instead. + +## Releases + +The `ONNX release` workflow (`.github/workflows/onnx-release.yml`) builds and +publishes them. It is triggered manually, or by pushing a tag matching +`onnx-v8-*`, and it aborts before uploading anything if the export fails, a +parity gate fails, or a checksum does not match the manifest. + +Every input is pinned by [`release.json`](release.json): checkpoint repository +and revision, opset, and the export toolchain. The toolchain pin is what makes +the artifacts reproducible from a repository commit — a commit and a checkpoint +revision alone do not determine the digest. + +The cheapest demonstration: re-exporting this same checkpoint under PyTorch +`2.13.0` instead of `2.6.0` yields binaries exactly **one byte larger** on both +branches (`11,104,477` and `11,108,684` against `11,104,476` and `11,108,683` +above), and therefore entirely different digests. `torch.onnx.export` stamps its +own version into the model's `producer_version` field, and `"2.13.0"` is one +character longer than `"2.6.0"`. The graph is otherwise unchanged — the parity +gates pass identically — but the bytes are not. + +Each release publishes five assets: both `.onnx` binaries, both metadata +sidecars, and `manifest.json`. The manifest records the checkpoint revision, the +framework commit, the opset, the input/output contract, and the size and +SHA-256 of every asset. Verify a download against it: -- `tr_hash_v8_o2m.onnx` -- `tr_hash_v8_o2m.json` -- `tr_hash_v8_nms_free.onnx` -- `tr_hash_v8_nms_free.json` +```bash +python scripts/build_onnx_release.py --verify manifest.json +``` + +Rebuild the same artifacts locally with: + +```bash +PYTHONPATH=. python scripts/build_onnx_release.py --output-dir dist/onnx +``` -Use the following hashes and sizes to verify the uploaded release artifacts: +The historical artifacts below predate this workflow. Use the following hashes +and sizes to verify them: | Branch | Artifact | Size | SHA-256 | |---|---:|---:|---| diff --git a/scripts/build_onnx_release.py b/scripts/build_onnx_release.py new file mode 100644 index 00000000..4b09767f --- /dev/null +++ b/scripts/build_onnx_release.py @@ -0,0 +1,446 @@ +"""Build and verify reproducible Vision v8 ONNX release artifacts. + +The release is pinned by ``docs/onnx/release.json``: checkpoint repository and +revision, opset, and the exact export toolchain. Pinning the toolchain is what +makes the artifacts reproducible from a repository commit — the ONNX graph, and +therefore the SHA-256 of the binary, depends on the PyTorch version that traced +it. + +Publication is fail-closed: a toolchain mismatch, a failed export, a failed +parity gate, or a checksum mismatch aborts before anything is uploaded. + +Usage: + python scripts/build_onnx_release.py --output-dir dist/onnx + python scripts/build_onnx_release.py --verify dist/onnx/manifest.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +os.environ.setdefault("COMPLEXITY_DISABLE_KERNELS", "1") + +MANIFEST_NAME = "manifest.json" +RELEASE_NOTES_NAME = "RELEASE_NOTES.md" +MANIFEST_VERSION = 1 +DEFAULT_CONFIG_PATH = Path("docs/onnx/release.json") + + +@dataclass(frozen=True) +class BranchSpec: + """One exported prediction branch.""" + + branch: str + checkpoint_subdir: str | None + stem: str + post_processing: str + + @property + def model_name(self) -> str: + return f"{self.stem}.onnx" + + @property + def sidecar_name(self) -> str: + return f"{self.stem}.json" + + +@dataclass(frozen=True) +class ReleaseConfig: + """Pinned inputs for a reproducible release build.""" + + checkpoint_repo: str + checkpoint_revision: str + opset: int + parity_num_tests: int + toolchain: Mapping[str, str] + branches: tuple[BranchSpec, ...] + + +class ReleaseError(RuntimeError): + """Any condition that must prevent publication.""" + + +def load_config(path: Path = DEFAULT_CONFIG_PATH) -> ReleaseConfig: + """Load and validate the pinned release configuration.""" + + return config_from_mapping(json.loads(Path(path).read_text())) + + +def config_from_mapping(values: Mapping[str, Any]) -> ReleaseConfig: + branches = tuple( + BranchSpec( + branch=str(entry["branch"]), + checkpoint_subdir=( + None if entry.get("checkpoint_subdir") is None + else str(entry["checkpoint_subdir"]) + ), + stem=str(entry["stem"]), + post_processing=str(entry["post_processing"]), + ) + for entry in values["branches"] + ) + if not branches: + raise ReleaseError("release config must declare at least one branch") + if len({spec.branch for spec in branches}) != len(branches): + raise ReleaseError("release config declares a branch twice") + + revision = str(values["checkpoint_revision"]) + if len(revision) != 40 or not all(c in "0123456789abcdef" for c in revision): + raise ReleaseError( + "checkpoint_revision must be a full 40-character commit sha, " + f"got {revision!r}; a moving ref would break reproducibility" + ) + + toolchain = values["toolchain"] + if not isinstance(toolchain, Mapping) or not toolchain: + raise ReleaseError("release config must pin a toolchain") + + return ReleaseConfig( + checkpoint_repo=str(values["checkpoint_repo"]), + checkpoint_revision=revision, + opset=int(values["opset"]), + parity_num_tests=int(values.get("parity_num_tests", 5)), + toolchain={str(k): str(v) for k, v in toolchain.items()}, + branches=branches, + ) + + +def installed_toolchain() -> dict[str, str]: + """Return the installed versions of the packages that shape the export.""" + + import onnx + import onnxruntime + import torch + + # Local version tags (``+cu130``) identify the wheel build, not the graph. + return { + "torch": torch.__version__.split("+")[0], + "onnx": onnx.__version__, + "onnxruntime": onnxruntime.__version__, + } + + +def toolchain_mismatches( + pinned: Mapping[str, str], + installed: Mapping[str, str], +) -> list[str]: + """Return one message per package whose version departs from the pin.""" + + return [ + f"{package}: pinned {version}, installed {installed.get(package, 'missing')}" + for package, version in pinned.items() + if installed.get(package) != version + ] + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def framework_commit() -> str: + """Return the commit the release is built from.""" + + from_ci = os.environ.get("GITHUB_SHA") + if from_ci: + return from_ci + result = subprocess.run( + ("git", "rev-parse", "HEAD"), + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def artifact_entry(path: Path, **fields: Any) -> dict[str, Any]: + """Describe one published file by name, size and digest.""" + + resolved = Path(path) + return { + "name": resolved.name, + "size_bytes": resolved.stat().st_size, + "sha256": sha256_file(resolved), + **fields, + } + + +def output_contract(sidecar: Mapping[str, Any]) -> dict[str, Any]: + """Describe the ONNX input/output contract from an export sidecar.""" + + image_size = int(sidecar["image_size"]) + prediction_width = int(sidecar["regression_width"]) + int(sidecar["num_classes"]) + return { + "input_name": "pixel_values", + "input_shape": [1, 3, image_size, image_size], + "output_name": "predictions", + "output_shape": [1, int(sidecar["num_cells"]), prediction_width], + "dtype": "float32", + "output_semantics": sidecar.get("output_semantics", ""), + "regression_width": int(sidecar["regression_width"]), + "num_classes": int(sidecar["num_classes"]), + "grid_sizes": list(sidecar["grid_sizes"]), + } + + +def build_manifest( + config: ReleaseConfig, + artifacts: Sequence[Mapping[str, Any]], + *, + commit: str, + toolchain: Mapping[str, str], +) -> dict[str, Any]: + """Assemble the machine-readable release manifest.""" + + return { + "manifest_version": MANIFEST_VERSION, + "framework_commit": commit, + "checkpoint_repo": config.checkpoint_repo, + "checkpoint_revision": config.checkpoint_revision, + "opset": config.opset, + "parity_num_tests": config.parity_num_tests, + "toolchain": dict(toolchain), + "artifacts": [dict(artifact) for artifact in artifacts], + } + + +def verify_manifest(manifest: Mapping[str, Any], directory: Path) -> list[str]: + """Return one message per artifact that is missing, resized or altered.""" + + problems: list[str] = [] + for artifact in manifest["artifacts"]: + path = Path(directory) / str(artifact["name"]) + if not path.is_file(): + problems.append(f"{artifact['name']}: missing") + continue + actual_size = path.stat().st_size + if actual_size != int(artifact["size_bytes"]): + problems.append( + f"{artifact['name']}: size {actual_size}, " + f"manifest {artifact['size_bytes']}" + ) + actual_digest = sha256_file(path) + if actual_digest != str(artifact["sha256"]): + problems.append( + f"{artifact['name']}: sha256 {actual_digest}, " + f"manifest {artifact['sha256']}" + ) + return problems + + +def render_release_notes(manifest: Mapping[str, Any]) -> str: + """Render release notes that keep the two branches distinguishable.""" + + lines = [ + "# TR-HASH Vision v8 ONNX artifacts", + "", + f"Built from framework commit `{manifest['framework_commit']}` and " + f"checkpoint `{manifest['checkpoint_repo']}` at revision " + f"`{manifest['checkpoint_revision']}`, opset `{manifest['opset']}`.", + "", + "Both models expose raw detector logits only; decode and post-processing " + "run outside the graph.", + "", + "| Branch | Model | Post-processing | Size | SHA-256 |", + "|---|---|---|---:|---|", + ] + for artifact in manifest["artifacts"]: + if artifact.get("kind") != "model": + continue + lines.append( + f"| {artifact['branch']} | `{artifact['name']}` | " + f"{artifact['post_processing']} | {artifact['size_bytes']:,} bytes | " + f"`{artifact['sha256']}` |" + ) + + lines += [ + "", + "## Verifying a download", + "", + "```bash", + "sha256sum -c <(python - <<'PY'", + "import json", + f"manifest = json.load(open('{MANIFEST_NAME}'))", + "for a in manifest['artifacts']:", + " print(f\"{a['sha256']} {a['name']}\")", + "PY", + ")", + "```", + "", + "## Toolchain", + "", + "The exported graph depends on the tracing toolchain, so reproducing " + "these digests requires the pinned versions:", + "", + "| Package | Version |", + "|---|---|", + ] + for package, version in manifest["toolchain"].items(): + lines.append(f"| `{package}` | `{version}` |") + lines.append("") + return "\n".join(lines) + + +def build_release( + config: ReleaseConfig, + output_dir: Path, + *, + allow_toolchain_drift: bool = False, +) -> dict[str, Any]: + """Download, export, gate and describe the release. Raises on any failure.""" + + from huggingface_hub import snapshot_download + + from scripts.check_onnx_parity import check_parity + from scripts.export_onnx import export_onnx + + installed = installed_toolchain() + mismatches = toolchain_mismatches(config.toolchain, installed) + if mismatches: + message = "toolchain does not match the pinned release toolchain:\n " + "\n ".join( + mismatches + ) + if not allow_toolchain_drift: + raise ReleaseError( + f"{message}\nDigests would not be reproducible. " + "Pass --allow-toolchain-drift for a local dry run." + ) + print(f"WARNING: {message}") + + destination = Path(output_dir) + destination.mkdir(parents=True, exist_ok=True) + + print(f"Downloading {config.checkpoint_repo}@{config.checkpoint_revision}") + checkpoint_root = Path( + snapshot_download( + repo_id=config.checkpoint_repo, + revision=config.checkpoint_revision, + ) + ) + + artifacts: list[dict[str, Any]] = [] + for spec in config.branches: + checkpoint = ( + checkpoint_root + if spec.checkpoint_subdir is None + else checkpoint_root / spec.checkpoint_subdir + ) + model_path = destination / spec.model_name + + print(f"\n=== {spec.branch} ===") + export_onnx( + checkpoint, + model_path, + opset_version=config.opset, + check=True, + branch=spec.branch, + ) + + if not check_parity( + checkpoint, + model_path, + branch=spec.branch, + num_tests=config.parity_num_tests, + ): + raise ReleaseError(f"parity gates failed for branch {spec.branch}") + + sidecar_path = model_path.with_suffix(".json") + sidecar = json.loads(sidecar_path.read_text()) + artifacts.append( + artifact_entry( + model_path, + kind="model", + branch=spec.branch, + requires_nms=bool(sidecar["requires_nms"]), + post_processing=spec.post_processing, + contract=output_contract(sidecar), + ) + ) + artifacts.append( + artifact_entry(sidecar_path, kind="metadata", branch=spec.branch) + ) + + manifest = build_manifest( + config, + artifacts, + commit=framework_commit(), + toolchain=installed, + ) + manifest_path = destination / MANIFEST_NAME + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + problems = verify_manifest(manifest, destination) + if problems: + raise ReleaseError("checksum verification failed:\n " + "\n ".join(problems)) + + (destination / RELEASE_NOTES_NAME).write_text(render_release_notes(manifest)) + print(f"\nManifest: {manifest_path}") + print(f"Verified {len(manifest['artifacts'])} artifacts") + return manifest + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", + type=Path, + default=DEFAULT_CONFIG_PATH, + help="Pinned release configuration (default: %(default)s)", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("dist/onnx"), + help="Directory receiving the release artifacts (default: %(default)s)", + ) + parser.add_argument( + "--verify", + type=Path, + default=None, + help="Verify an existing manifest against the files beside it, then exit", + ) + parser.add_argument( + "--allow-toolchain-drift", + action="store_true", + help="Warn instead of failing when versions differ from the pin", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + if args.verify is not None: + manifest = json.loads(args.verify.read_text()) + problems = verify_manifest(manifest, args.verify.parent) + if problems: + print("Verification FAILED:") + for problem in problems: + print(f" {problem}") + raise SystemExit(1) + print(f"Verification PASSED: {len(manifest['artifacts'])} artifacts") + return + + try: + build_release( + load_config(args.config), + args.output_dir, + allow_toolchain_drift=args.allow_toolchain_drift, + ) + except ReleaseError as error: + print(f"Release ABORTED: {error}") + raise SystemExit(1) from error + + +if __name__ == "__main__": + main() diff --git a/tests/test_onnx_release.py b/tests/test_onnx_release.py new file mode 100644 index 00000000..d91056fb --- /dev/null +++ b/tests/test_onnx_release.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from scripts.build_onnx_release import ( + MANIFEST_NAME, + ReleaseConfig, + ReleaseError, + artifact_entry, + build_manifest, + config_from_mapping, + load_config, + output_contract, + render_release_notes, + sha256_file, + toolchain_mismatches, + verify_manifest, +) + +CONFIG_PATH = Path("docs/onnx/release.json") + +SIDECAR = { + "architecture_version": 8, + "image_size": 640, + "num_classes": 80, + "num_cells": 34000, + "regression_width": 68, + "reg_max": 16, + "grid_sizes": [160, 80, 40, 20], + "branch": "o2m", + "requires_nms": True, + "output_semantics": "raw_ltrb_dfl_and_quality_class_logits", +} + + +def config_mapping(**overrides: Any) -> dict[str, Any]: + values: dict[str, Any] = { + "checkpoint_repo": "AETHORIA-AI/TR-HASH-Vision-v8-2M-COCO-SFT", + "checkpoint_revision": "f3b3e659612e543ca9ff91892c0662d38dc1a1d6", + "opset": 17, + "parity_num_tests": 5, + "toolchain": {"torch": "2.13.0", "onnx": "1.21.0", "onnxruntime": "1.24.4"}, + "branches": [ + { + "branch": "o2m", + "checkpoint_subdir": None, + "stem": "tr_hash_v8_o2m", + "post_processing": "decode plus NMS", + }, + { + "branch": "nms-free", + "checkpoint_subdir": "best_nms_free", + "stem": "tr_hash_v8_nms_free", + "post_processing": "decode plus confidence filtering", + }, + ], + } + values.update(overrides) + return values + + +def written_release(directory: Path) -> dict[str, Any]: + """Write two fake artifacts and return their verified manifest.""" + + config = config_from_mapping(config_mapping()) + artifacts = [] + for spec in config.branches: + model = directory / spec.model_name + model.write_bytes(spec.branch.encode() * 64) + sidecar = directory / spec.sidecar_name + sidecar.write_text(json.dumps({**SIDECAR, "branch": spec.branch})) + artifacts.append( + artifact_entry( + model, + kind="model", + branch=spec.branch, + requires_nms=spec.branch == "o2m", + post_processing=spec.post_processing, + contract=output_contract(SIDECAR), + ) + ) + artifacts.append(artifact_entry(sidecar, kind="metadata", branch=spec.branch)) + + return build_manifest( + config, + artifacts, + commit="0" * 40, + toolchain=config.toolchain, + ) + + +def test_committed_release_config_is_valid_and_pins_both_branches() -> None: + config = load_config(CONFIG_PATH) + + assert isinstance(config, ReleaseConfig) + assert {spec.branch for spec in config.branches} == {"o2m", "nms-free"} + assert config.opset == 17 + assert set(config.toolchain) == {"torch", "onnx", "onnxruntime"} + + +def test_a_moving_checkpoint_ref_is_rejected() -> None: + with pytest.raises(ReleaseError, match="40-character commit sha"): + config_from_mapping(config_mapping(checkpoint_revision="main")) + + +def test_a_duplicated_branch_is_rejected() -> None: + branches = config_mapping()["branches"] + with pytest.raises(ReleaseError, match="branch twice"): + config_from_mapping(config_mapping(branches=[branches[0], branches[0]])) + + +def test_an_unpinned_toolchain_is_rejected() -> None: + with pytest.raises(ReleaseError, match="pin a toolchain"): + config_from_mapping(config_mapping(toolchain={})) + + +def test_toolchain_drift_is_reported_per_package() -> None: + pinned = {"torch": "2.13.0", "onnx": "1.21.0"} + + assert toolchain_mismatches(pinned, {"torch": "2.13.0", "onnx": "1.21.0"}) == [] + assert toolchain_mismatches(pinned, {"torch": "2.6.0", "onnx": "1.21.0"}) == [ + "torch: pinned 2.13.0, installed 2.6.0" + ] + assert toolchain_mismatches(pinned, {"torch": "2.13.0"}) == [ + "onnx: pinned 1.21.0, installed missing" + ] + + +def test_output_contract_derives_the_prediction_width_from_the_sidecar() -> None: + contract = output_contract(SIDECAR) + + assert contract["input_shape"] == [1, 3, 640, 640] + assert contract["output_shape"] == [1, 34000, 148] + assert contract["dtype"] == "float32" + + +def test_manifest_carries_every_field_the_release_must_document(tmp_path: Path) -> None: + manifest = written_release(tmp_path) + + assert manifest["checkpoint_revision"] == "f3b3e659612e543ca9ff91892c0662d38dc1a1d6" + assert manifest["framework_commit"] == "0" * 40 + assert manifest["opset"] == 17 + assert manifest["toolchain"]["torch"] == "2.13.0" + assert len(manifest["artifacts"]) == 4 + for artifact in manifest["artifacts"]: + assert artifact["size_bytes"] > 0 + assert len(artifact["sha256"]) == 64 + + +def test_verification_passes_on_an_untouched_release(tmp_path: Path) -> None: + manifest = written_release(tmp_path) + + assert verify_manifest(manifest, tmp_path) == [] + + +def test_verification_catches_an_altered_binary(tmp_path: Path) -> None: + manifest = written_release(tmp_path) + target = tmp_path / "tr_hash_v8_o2m.onnx" + original = target.read_bytes() + # Same length, different content: only the digest can catch this. + target.write_bytes(b"x" + original[1:]) + + problems = verify_manifest(manifest, tmp_path) + + assert len(problems) == 1 + assert "tr_hash_v8_o2m.onnx" in problems[0] + assert "sha256" in problems[0] + + +def test_verification_catches_a_truncated_or_missing_artifact(tmp_path: Path) -> None: + manifest = written_release(tmp_path) + (tmp_path / "tr_hash_v8_o2m.onnx").write_bytes(b"short") + (tmp_path / "tr_hash_v8_nms_free.json").unlink() + + problems = "\n".join(verify_manifest(manifest, tmp_path)) + + assert "size" in problems + assert "missing" in problems + + +def test_release_notes_distinguish_the_two_branches(tmp_path: Path) -> None: + manifest = written_release(tmp_path) + + notes = render_release_notes(manifest) + + assert "decode plus NMS" in notes + assert "decode plus confidence filtering" in notes + assert "tr_hash_v8_o2m.onnx" in notes + assert "tr_hash_v8_nms_free.onnx" in notes + # Sidecars are published but are not models: they must not add table rows. + assert notes.count("| o2m |") == 1 + assert MANIFEST_NAME in notes + + +def test_sha256_matches_a_known_digest(tmp_path: Path) -> None: + target = tmp_path / "empty.bin" + target.write_bytes(b"") + + assert sha256_file(target) == ( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ) From c42102da3f340ed9862782d3c8f851ae679760bd Mon Sep 17 00:00:00 2001 From: IIIllllIlIlllII <306726845+IIIllllIlIlllII@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:40:33 +0000 Subject: [PATCH 2/2] Bind the release tag to the built commit and deepen the parity gate Nothing tied the published tag to the commit the manifest records. gh release create without --target tags the default branch head, so a dispatch run could publish a manifest whose framework_commit cannot be checked out from the release itself, which defeats the point of pinning anything. The release is now created with --target "$GITHUB_SHA"; an existing tag is resolved through the API and publication aborts when it points elsewhere; and --verify gained --expect-commit so the manifest is bound to the publishing commit by tested code rather than by shell in the workflow. Raise parity_num_tests from 5 to 50. Five seeds underestimate the observed maximum by about 60%, so a release gate has to be deeper than the development default; a test keeps the committed value at 20 or above. At 50 seeds the maxima are 2.79e-03 (o2m) and 4.79e-03 (nms-free) against thresholds of 6.0e-03 and 1.0e-02, so the deeper gate keeps roughly a 2x margin. Job timeout raised to 60 minutes to cover two exports plus 100 parity runs. Co-Authored-By: Claude Opus 5 --- .github/workflows/onnx-release.yml | 25 ++++++++++++++++-- docs/onnx/release.json | 2 +- docs/onnx/tr_hash_v8_validation_report.md | 14 ++++++++-- scripts/build_onnx_release.py | 31 ++++++++++++++++++++--- tests/test_onnx_release.py | 14 ++++++++++ 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/.github/workflows/onnx-release.yml b/.github/workflows/onnx-release.yml index c89db553..ee13cc7c 100644 --- a/.github/workflows/onnx-release.yml +++ b/.github/workflows/onnx-release.yml @@ -26,7 +26,8 @@ concurrency: jobs: publish: runs-on: ubuntu-latest - timeout-minutes: 30 + # Two 640px exports plus 50-seed parity gates on both branches. + timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -65,7 +66,26 @@ jobs: run: python scripts/build_onnx_release.py --output-dir dist/onnx - name: Re-verify the published files against the manifest - run: python scripts/build_onnx_release.py --verify dist/onnx/manifest.json + run: >- + python scripts/build_onnx_release.py + --verify dist/onnx/manifest.json + --expect-commit "$GITHUB_SHA" + + - name: Require the release tag to resolve to the built commit + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.inputs.tag || github.ref_name }} + run: | + # An existing tag pointing elsewhere would publish a manifest whose + # framework_commit cannot be checked out from the release itself. + TAG_SHA=$(gh api "repos/$GITHUB_REPOSITORY/commits/$TAG" --jq .sha 2>/dev/null || true) + if [ -z "$TAG_SHA" ]; then + echo "Tag $TAG does not exist yet; it will be created at $GITHUB_SHA." + elif [ "$TAG_SHA" != "$GITHUB_SHA" ]; then + echo "Tag $TAG resolves to $TAG_SHA but the manifest records $GITHUB_SHA." + echo "Refusing to publish artifacts the tagged tree cannot reproduce." + exit 1 + fi - name: Upload release assets env: @@ -75,6 +95,7 @@ jobs: run: | gh release view "$TAG" >/dev/null 2>&1 \ || gh release create "$TAG" \ + --target "$GITHUB_SHA" \ --title "TR-HASH Vision v8 ONNX artifacts" \ --notes-file dist/onnx/RELEASE_NOTES.md \ $DRAFT diff --git a/docs/onnx/release.json b/docs/onnx/release.json index 9656633e..77f6c2db 100644 --- a/docs/onnx/release.json +++ b/docs/onnx/release.json @@ -2,7 +2,7 @@ "checkpoint_repo": "AETHORIA-AI/TR-HASH-Vision-v8-2M-COCO-SFT", "checkpoint_revision": "f3b3e659612e543ca9ff91892c0662d38dc1a1d6", "opset": 17, - "parity_num_tests": 5, + "parity_num_tests": 50, "toolchain": { "torch": "2.13.0", "onnx": "1.21.0", diff --git a/docs/onnx/tr_hash_v8_validation_report.md b/docs/onnx/tr_hash_v8_validation_report.md index 68503e7c..4f22dfc3 100644 --- a/docs/onnx/tr_hash_v8_validation_report.md +++ b/docs/onnx/tr_hash_v8_validation_report.md @@ -28,10 +28,20 @@ own version into the model's `producer_version` field, and `"2.13.0"` is one character longer than `"2.6.0"`. The graph is otherwise unchanged — the parity gates pass identically — but the bytes are not. +The release gate runs the parity checks at 50 seeds rather than the development +default of 5, for the reason given under Sample-size sensitivity: five seeds +underestimate the observed maximum by about 60%. A release is published rarely +enough that the extra cost is irrelevant. + +The published tag is bound to the commit the manifest records: the workflow +creates it at that commit, and refuses to publish if an existing tag resolves +anywhere else. Otherwise a release could document digests that its own tagged +tree cannot reproduce. + Each release publishes five assets: both `.onnx` binaries, both metadata sidecars, and `manifest.json`. The manifest records the checkpoint revision, the -framework commit, the opset, the input/output contract, and the size and -SHA-256 of every asset. Verify a download against it: +framework commit, the opset, the parity depth, the input/output contract, and +the size and SHA-256 of every asset. Verify a download against it: ```bash python scripts/build_onnx_release.py --verify manifest.json diff --git a/scripts/build_onnx_release.py b/scripts/build_onnx_release.py index 4b09767f..7c311010 100644 --- a/scripts/build_onnx_release.py +++ b/scripts/build_onnx_release.py @@ -214,10 +214,26 @@ def build_manifest( } -def verify_manifest(manifest: Mapping[str, Any], directory: Path) -> list[str]: - """Return one message per artifact that is missing, resized or altered.""" +def verify_manifest( + manifest: Mapping[str, Any], + directory: Path, + *, + expect_commit: str | None = None, +) -> list[str]: + """Return one message per artifact that is missing, resized or altered. + + ``expect_commit`` additionally binds the manifest to the commit publication + happens from: a release whose tag resolves elsewhere would document digests + that its own source tree cannot reproduce. + """ problems: list[str] = [] + if expect_commit is not None: + recorded = str(manifest.get("framework_commit", "")) + if recorded != expect_commit: + problems.append( + f"framework_commit {recorded or 'missing'}, expected {expect_commit}" + ) for artifact in manifest["artifacts"]: path = Path(directory) / str(artifact["name"]) if not path.is_file(): @@ -409,6 +425,11 @@ def parse_args() -> argparse.Namespace: default=None, help="Verify an existing manifest against the files beside it, then exit", ) + parser.add_argument( + "--expect-commit", + default=None, + help="With --verify, require the manifest to record this framework commit", + ) parser.add_argument( "--allow-toolchain-drift", action="store_true", @@ -422,7 +443,11 @@ def main() -> None: if args.verify is not None: manifest = json.loads(args.verify.read_text()) - problems = verify_manifest(manifest, args.verify.parent) + problems = verify_manifest( + manifest, + args.verify.parent, + expect_commit=args.expect_commit, + ) if problems: print("Verification FAILED:") for problem in problems: diff --git a/tests/test_onnx_release.py b/tests/test_onnx_release.py index d91056fb..ca60084e 100644 --- a/tests/test_onnx_release.py +++ b/tests/test_onnx_release.py @@ -100,6 +100,9 @@ def test_committed_release_config_is_valid_and_pins_both_branches() -> None: assert {spec.branch for spec in config.branches} == {"o2m", "nms-free"} assert config.opset == 17 assert set(config.toolchain) == {"torch", "onnx", "onnxruntime"} + # Five seeds underestimate the observed maxima (see the validation report), + # so a release gate has to be deeper than the development default. + assert config.parity_num_tests >= 20 def test_a_moving_checkpoint_ref_is_rejected() -> None: @@ -182,6 +185,17 @@ def test_verification_catches_a_truncated_or_missing_artifact(tmp_path: Path) -> assert "missing" in problems +def test_verification_binds_the_manifest_to_the_publishing_commit(tmp_path: Path) -> None: + manifest = written_release(tmp_path) + + assert verify_manifest(manifest, tmp_path, expect_commit="0" * 40) == [] + + problems = verify_manifest(manifest, tmp_path, expect_commit="1" * 40) + + assert len(problems) == 1 + assert "framework_commit" in problems[0] + + def test_release_notes_distinguish_the_two_branches(tmp_path: Path) -> None: manifest = written_release(tmp_path)