diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index b695725..643c86b 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -28,6 +28,9 @@ description: >- Use file_cfg.filename for MXQ and derive the same-stem ONNX artifact unless onnx_filename is required. - Every post_cfg declares dataset; resolve output taxonomy from the dataset/task pair. +- For Hub linkage changes, run `tests/test_hub_validation.py` as described in `tests/TEST.md`. + The metadata checker follows runtime board paths without downloading weights; live commits + are diagnostics, not substitutes for missing historical approval provenance. - For opt-in YAML family inheritance, follow `mblt_vision/README.md`'s configuration inheritance rules. Keep legacy `update` shallow and run `tests/test_config_inheritance.py` to prove every existing variant's values and processing order remain identical. diff --git a/AGENTS.md b/AGENTS.md index 06a243b..a6f72b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,6 +60,9 @@ The current ownership boundary is deliberate: onnx_path as compatibility aliases. - Update task-package exports and lazy top-level exports together. Confirm that list_models() discovers every public model class. +- Verify Hub configuration changes with the metadata-only checks in + `tests/test_hub_validation.py` (see `tests/TEST.md`). Use exact runtime board paths; + treat live Hub commits as diagnostics, never as invented approval provenance. - Keep each model YAML's file_cfg, pre_cfg, and post_cfg shape stable. file_cfg.filename is the canonical MXQ Hub artifact; derive the same-stem ONNX filename unless the Hub artifact requires an explicit onnx_filename. diff --git a/mblt_vision/_hub_validation.py b/mblt_vision/_hub_validation.py new file mode 100644 index 0000000..c387ce1 --- /dev/null +++ b/mblt_vision/_hub_validation.py @@ -0,0 +1,100 @@ +"""Read-only Hub metadata checks for resolved model artifact configurations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from huggingface_hub import HfApi +from mblt_npu import normalize_target_device + + +@dataclass(frozen=True) +class HubArtifactCheck: + """A metadata-only resolution result; a live commit is not approval provenance.""" + + repo_id: str + revision: str + commit: str | None + framework: str + path: str + status: str + + +def check_hub_artifacts( + config: dict[str, Any], + *, + target_device: str | None = None, + frameworks: tuple[str, ...] = ("mxq", "onnx"), + api: HfApi | None = None, +) -> tuple[HubArtifactCheck, ...]: + """Check exact runtime Hub paths without downloading model bytes. + + Accept a resolved configuration (``resolve_model_config`` derives the ONNX + filename). MXQ is checked only under the normalized target board; ONNX is + checked at the repository root, matching the engine. Never fall back to a + different board, revision or filename. Local-only configs are reported as + skipped, without contacting the Hub. Hub/API exceptions propagate so missing + access or network failure cannot be mistaken for a missing file. + + The returned live commit is diagnostic, not evidence that a commit was + approved. This function does not modify YAMLs or infer approved revisions. + """ + + if not frameworks or any( + framework not in {"mxq", "onnx"} for framework in frameworks + ): + raise ValueError("frameworks must contain only 'mxq' and/or 'onnx'.") + files = config.get("file_cfg") + if not isinstance(files, dict): + raise TypeError("Resolved model configuration requires a file_cfg mapping.") + repo_id, revision = files.get("repo_id"), files.get("revision") + if ( + not isinstance(repo_id, str) + or not repo_id + or not isinstance(revision, str) + or not revision + ): + raise ValueError( + "Hub verification requires nonempty repo_id and revision strings." + ) + board = normalize_target_device( + files.get("target_device", "aries-rb") if target_device is None else target_device + ) + paths = [] + for framework in dict.fromkeys(frameworks): + filename = files.get("filename" if framework == "mxq" else "onnx_filename") + if not isinstance(filename, str) or not filename: + raise ValueError( + f"Resolved configuration requires the {framework} artifact filename." + ) + paths.append( + (framework, f"{board}/{filename}" if framework == "mxq" else filename) + ) + if files.get("local_artifact_only"): + return tuple( + HubArtifactCheck( + repo_id, revision, None, framework, path, "local_artifact_only" + ) + for framework, path in paths + ) + client = HfApi() if api is None else api + info = client.model_info(repo_id=repo_id, revision=revision) + if not isinstance(info.sha, str) or not info.sha: + raise ValueError("Hub model metadata did not return a commit SHA.") + # Pin the file listing to the just-resolved commit: symbolic branches can move + # between API calls, and results must describe a single repository snapshot. + available = set( + client.list_repo_files(repo_id=repo_id, revision=info.sha, repo_type="model") + ) + return tuple( + HubArtifactCheck( + repo_id, + revision, + info.sha, + framework, + path, + "resolved" if path in available else "missing", + ) + for framework, path in paths + ) diff --git a/tests/TEST.md b/tests/TEST.md index ffd156d..bea1fcf 100644 --- a/tests/TEST.md +++ b/tests/TEST.md @@ -35,6 +35,22 @@ After authenticating with Hugging Face Hub, exercise representative ONNX models: pytest tests/test_onnx_classification.py -m requires_network ``` +To check representative Hub configurations without downloading model bytes or +using an inference backend: + +```bash +pytest tests/test_hub_validation.py -m requires_network +``` + +These read-only checks use the exact configured `repo_id` and `revision`, verify +MXQ under the normalized target board and ONNX at the repository root, and pin +metadata listing to the resolved commit to avoid races with moving branches. +Local-only artifacts are skipped without a network request. An observed current +commit is diagnostic metadata, not evidence of a historically approved revision; +missing approval provenance must never be replaced with today's `main` tip. +Offline mocked tests cover missing files, wrong boards, API failures and local-only +restrictions. No YAML revision or model artifact is changed by these checks. + To run MXQ inference, add a configured NPU and the shared runtime options: ```bash diff --git a/tests/test_hub_validation.py b/tests/test_hub_validation.py new file mode 100644 index 0000000..06bdb2b --- /dev/null +++ b/tests/test_hub_validation.py @@ -0,0 +1,106 @@ +"""Offline unit tests and explicit metadata-only Hub integration checks.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from mblt_vision._hub_validation import check_hub_artifacts +from mblt_vision.wrapper import resolve_model_config + + +def config(**overrides): + return { + "file_cfg": { + "repo_id": "mobilint/example", + "revision": "approved-commit", + "filename": "model.mxq", + "onnx_filename": "model.onnx", + **overrides, + } + } + + +def api_with_files(files): + api = Mock() + api.model_info.return_value = SimpleNamespace(sha="exact-live-commit") + api.list_repo_files.return_value = files + return api + + +def test_exact_paths_and_commit_snapshot() -> None: + api = api_with_files(["aries-rb/model.mxq", "model.onnx"]) + checks = check_hub_artifacts(config(), api=api) + assert [item.status for item in checks] == ["resolved", "resolved"] + assert [item.path for item in checks] == ["aries-rb/model.mxq", "model.onnx"] + assert all(item.commit == "exact-live-commit" for item in checks) + api.model_info.assert_called_once_with( + repo_id="mobilint/example", revision="approved-commit" + ) + api.list_repo_files.assert_called_once_with( + repo_id="mobilint/example", revision="exact-live-commit", repo_type="model" + ) + + +def test_wrong_board_never_falls_back() -> None: + api = api_with_files(["aries-rb/model.mxq", "model.mxq", "model.onnx"]) + checks = check_hub_artifacts(config(), target_device="regulus-ra", api=api) + assert checks[0].path == "regulus-ra/model.mxq" + assert checks[0].status == "missing" + assert checks[1].status == "resolved" + + +def test_configured_board_and_explicit_override_precedence() -> None: + api = api_with_files(["regulus-ra/model.mxq", "model.onnx"]) + configured = config(target_device="regulus-ra") + assert check_hub_artifacts(configured, api=api)[0].path == "regulus-ra/model.mxq" + override = check_hub_artifacts(configured, target_device="aries-rb", api=api) + assert override[0].path == "aries-rb/model.mxq" + assert override[0].status == "missing" + + +def test_local_artifact_only_does_not_contact_hub() -> None: + api = Mock() + checks = check_hub_artifacts(config(local_artifact_only=True), api=api) + assert all( + item.status == "local_artifact_only" and item.commit is None for item in checks + ) + assert not api.mock_calls + + +def test_api_errors_remain_errors() -> None: + api = Mock() + api.model_info.side_effect = ConnectionError("offline") + with pytest.raises(ConnectionError, match="offline"): + check_hub_artifacts(config(), api=api) + + +@pytest.mark.parametrize("frameworks", [(), ("other",)]) +def test_invalid_frameworks(frameworks) -> None: + with pytest.raises(ValueError, match="frameworks"): + check_hub_artifacts(config(), frameworks=frameworks) + + +@pytest.mark.parametrize( + "override", + [{"repo_id": ""}, {"revision": None}, {"filename": None}, {"onnx_filename": ""}], +) +def test_missing_required_fields(override) -> None: + with pytest.raises(ValueError): + check_hub_artifacts(config(**override), api=Mock()) + + +def test_framework_selection_does_not_require_other_filename() -> None: + api = api_with_files(["model.onnx"]) + checks = check_hub_artifacts(config(filename=None), frameworks=("onnx",), api=api) + assert len(checks) == 1 + assert checks[0].status == "resolved" + + +@pytest.mark.requires_network +@pytest.mark.parametrize("model", ["AlexNet", "ResNet50", "YOLOv8n"]) +def test_representative_hub_metadata_resolves(model: str) -> None: + checks = check_hub_artifacts(resolve_model_config(model)) + assert all(item.status == "resolved" for item in checks), checks