From 281b8683f42342e6fdd57764b76ba1f39ded2356 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:22:52 +0900 Subject: [PATCH 01/13] Add opt-in model YAML family inheritance with identity proofs Preserve existing alias and update semantics across 334 configs and 595 variants. Add ordered resolution and CPU preprocessing regression tests without changing shipped model definitions. Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 3 + AGENTS.md | 4 + mblt_vision/README.md | 34 ++++++ mblt_vision/_config.py | 120 ++++++++++++++++++++ mblt_vision/wrapper.py | 62 +---------- pyproject.toml | 1 + tests/test_config_inheritance.py | 164 ++++++++++++++++++++++++++++ 7 files changed, 329 insertions(+), 59 deletions(-) create mode 100644 mblt_vision/_config.py create mode 100644 tests/test_config_inheritance.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 1b9b9c5..b695725 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -28,6 +28,9 @@ description: >- Use file_cfg.filename for MXQ and derive the same-stem ONNX artifact unless onnx_filename is required. - Every post_cfg declares dataset; resolve output taxonomy from the dataset/task pair. +- For opt-in YAML family inheritance, follow `mblt_vision/README.md`'s configuration + inheritance rules. Keep legacy `update` shallow and run `tests/test_config_inheritance.py` + to prove every existing variant's values and processing order remain identical. - Reuse wrapper.download_hub_artifact for any model that needs more than one Hub artifact rather than duplicating Hub-resolution logic. - Registering a new dataset requires readiness (`_*_ready` + `dataset_ready` diff --git a/AGENTS.md b/AGENTS.md index 5fd0aa6..06a243b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,6 +69,10 @@ The current ownership boundary is deliberate: floors, and rounding differs by a pixel at 224/0.9 and 224/0.95. Several names (ConvNext_Base) resolve in timm yet carry torchvision's transform here, so read the expected value from that model's source.yaml provenance rather than from the name. +- To share processing defaults without changing legacy models, opt new family YAMLs into + `extends: bases/family.yaml#DEFAULT`. See the configuration-inheritance section of + `mblt_vision/README.md` for ordering and merge rules. Keep legacy `update` shallow and + prove every existing variant order-identical with `tests/test_config_inheritance.py`. - Require post_cfg.dataset in every model YAML and resolve output class counts using the dataset/task pair. Do not assume one output taxonomy for every model in a task. - Preserve automatic .mxq/.onnx framework detection and the fail-fast error when a local diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 30bbddc..0187f74 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -60,6 +60,40 @@ and `list_models()` are the preferred discovery and loading APIs for new code. Top-level model imports remain available for compatibility. `obb` is the sole oriented-bounding-box task name. +## Model configuration inheritance + +Model variants may opt into local family YAMLs with `extends`. This keeps the +artifact, preprocessing and dataset connection together without duplicating a +family's processing defaults. Existing `update: VARIANT` and alias behavior is +unchanged; no shipped model YAML has been migrated by this loader refactor. + +```yaml +DEFAULT: + extends: bases/classifier.yaml#DEFAULT + file_cfg: + repo_id: mobilint/YourPublishedModel + filename: model.mxq + revision: approved-hf-commit + pre_cfg: + Resize: + size: 256 +``` + +The example is a schema illustration, not a downloadable model registration. +The base path is relative to the declaring YAML; omit `#DEFAULT` to select the +same default variant. Family bases belong in `models/bases/`, outside top-level +model discovery. `extends` recursively merges mappings and replaces lists and +scalars. Inherited mapping order is preserved: existing preprocessing operations +stay in place and new operations append. Use a standalone pipeline when operation +order must change. In contrast, `update` retains its historical one-section merge, +so replacing `pre_cfg.Resize` replaces that whole operation's parameters. A variant +cannot combine `update` and `extends`; an `update` may reference another variant +that uses `extends`. References are local only and cycles fail before inference. + +`tests/test_config_inheritance.py` compares every existing YAML variant with the +legacy resolver, including ordered processing mappings and derived ONNX filenames, +and checks exact CPU preprocessing equality for representative model families. + ## Pre-Trained Vision Models This section lists the publicly pre-trained models supported by the vision framework. diff --git a/mblt_vision/_config.py b/mblt_vision/_config.py new file mode 100644 index 0000000..ab2fee2 --- /dev/null +++ b/mblt_vision/_config.py @@ -0,0 +1,120 @@ +"""Local YAML model inheritance, independent of inference backends.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any + +import yaml + + +def _merge( + base: dict[str, Any], child: dict[str, Any], *, recursive: bool +) -> dict[str, Any]: + """Merge independent mappings while retaining preprocessing operation order.""" + + result = copy.deepcopy(base) + for key, value in child.items(): + if isinstance(result.get(key), dict) and isinstance(value, dict): + if recursive: + result[key] = _merge(result[key], value, recursive=True) + else: + # Historical `update` merges exactly one section level. + result[key].update(copy.deepcopy(value)) + else: + result[key] = copy.deepcopy(value) + return result + + +def load_model_variant( + config_path: str | Path, variant: str = "DEFAULT" +) -> dict[str, Any]: + """Resolve a local YAML variant, its aliases, and optional family inheritance. + + ``update`` retains the legacy same-file, one-section merge semantics. + ``extends: bases/family.yaml#DEFAULT`` opts into recursive mapping inheritance; + the fragment defaults to DEFAULT and paths are relative to the declaring file. + Lists and scalars replace their base values. Mapping key order is retained, so + inherited preprocessing operations retain their order. New operations append; + replace a complete operation sequence with a standalone config when reordering. + Only local files are read; each result is independently deep-copied. + """ + + documents: dict[Path, dict[str, Any]] = {} + resolving: set[tuple[Path, str]] = set() + + def document(path: Path) -> dict[str, Any]: + if path not in documents: + with path.open(encoding="utf-8") as stream: + parsed = yaml.safe_load(stream) + if not isinstance(parsed, dict): + raise TypeError( + f"Model configuration '{path}' should define a dictionary." + ) + documents[path] = parsed + return documents[path] + + def resolve(path: Path, key: str) -> dict[str, Any]: + identity = (path, key) + if identity in resolving: + raise ValueError( + f"Circular model configuration reference detected for '{path}#{key}'." + ) + resolving.add(identity) + try: + full = document(path) + part = full.get(key) + if part is None: + raise ValueError(f"Model type '{key}' not found in configuration.") + if isinstance(part, str): + if part not in full: + raise ValueError( + f"Model alias '{part}' not found in configuration." + ) + return resolve(path, part) + if not isinstance(part, dict): + raise TypeError( + f"Resolved model configuration for '{key}' is not a dictionary." + ) + child = copy.deepcopy(part) + base_key = child.pop("update", None) + has_extends = "extends" in child + reference = child.pop("extends", None) + if base_key is not None and has_extends: + raise ValueError( + "A model variant cannot combine 'update' and 'extends'." + ) + if base_key is not None: + if not isinstance(base_key, str) or base_key not in full: + raise ValueError( + f"Base configuration '{base_key}' not found for update." + ) + return _merge(resolve(path, base_key), child, recursive=False) + if has_extends: + if not isinstance(reference, str) or not reference: + raise ValueError( + "Model 'extends' must be a nonempty local YAML reference." + ) + filename, separator, base_variant = reference.partition("#") + if ( + not filename + or "://" in filename + or (separator and not base_variant) + ): + raise ValueError( + f"Invalid local model inheritance reference: {reference!r}." + ) + if "#" in base_variant: + raise ValueError( + f"Invalid local model inheritance reference: {reference!r}." + ) + base_path = (path.parent / filename).resolve() + return _merge( + resolve(base_path, base_variant or "DEFAULT"), child, recursive=True + ) + return child + finally: + resolving.remove(identity) + + return resolve(Path(config_path).resolve(), variant) diff --git a/mblt_vision/wrapper.py b/mblt_vision/wrapper.py index a15296f..d451284 100644 --- a/mblt_vision/wrapper.py +++ b/mblt_vision/wrapper.py @@ -16,11 +16,11 @@ import numpy as np import torch -import yaml from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError - from mblt_npu import MobilintNPUBackend, ONNXBackend, normalize_target_device + +from ._config import load_model_variant from ._model_paths import resolve_framework as _resolve_framework from ._model_paths import split_model_paths as _split_model_paths from ._model_paths import ( @@ -365,63 +365,7 @@ def resolve_model_config( if not config_path.is_file(): config_path = MODEL_CONFIG_DIR / _model_name_aliasing(model_cls) - with config_path.open(encoding="utf-8") as config_file: - full_config = yaml.safe_load(config_file) - if not isinstance(full_config, dict): - raise TypeError( - f"Model configuration '{config_path}' should define a dictionary." - ) - - resolving: set[str] = set() - - def _resolve_variant(variant: str) -> dict[str, Any]: - if variant in resolving: - raise ValueError( - f"Circular model configuration reference detected for '{variant}'." - ) - resolving.add(variant) - try: - model_config_part = full_config.get(variant) - if model_config_part is None: - raise ValueError(f"Model type '{variant}' not found in configuration.") - if isinstance(model_config_part, str): - if model_config_part not in full_config: - raise ValueError( - f"Model alias '{model_config_part}' not found in configuration." - ) - return _resolve_variant(model_config_part) - if not isinstance(model_config_part, dict): - raise TypeError( - f"Resolved model configuration for '{variant}' is not a dictionary." - ) - - resolved = copy.deepcopy(model_config_part) - base_config_key = resolved.pop("update", None) - if base_config_key is None: - return resolved - if ( - not isinstance(base_config_key, str) - or base_config_key not in full_config - ): - raise ValueError( - f"Base configuration '{base_config_key}' not found for update." - ) - - merged_config = _resolve_variant(base_config_key) - for key, value in resolved.items(): - if ( - key in merged_config - and isinstance(merged_config[key], dict) - and isinstance(value, dict) - ): - merged_config[key].update(value) - else: - merged_config[key] = value - return merged_config - finally: - resolving.remove(variant) - - return _normalize_model_artifacts(_resolve_variant(model_type)) + return _normalize_model_artifacts(load_model_variant(config_path, model_type)) class MBLT_Engine: diff --git a/pyproject.toml b/pyproject.toml index 5514221..9850c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ version = { attr = "mblt_vision.__version__" } "mblt_vision" = [ "py.typed", "models/*.yaml", + "models/bases/*.yaml", "datasets/*.yaml", "datasets/*.txt", ] diff --git a/tests/test_config_inheritance.py b/tests/test_config_inheritance.py new file mode 100644 index 0000000..a36907b --- /dev/null +++ b/tests/test_config_inheritance.py @@ -0,0 +1,164 @@ +"""CPU proofs for legacy resolution and opt-in local YAML inheritance.""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import numpy as np +import pytest +import yaml + +from mblt_vision._config import load_model_variant +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.wrapper import MODEL_CONFIG_DIR, resolve_model_config + + +def legacy_resolve(document: dict, variant: str) -> dict: + """Independent pre-refactor resolution, including ONNX artifact derivation.""" + + part = document[variant] + if isinstance(part, str): + return legacy_resolve(document, part) + child = copy.deepcopy(part) + base = child.pop("update", None) + if base is not None: + resolved = legacy_resolve(document, base) + for key, value in child.items(): + if isinstance(resolved.get(key), dict) and isinstance(value, dict): + resolved[key].update(value) + else: + resolved[key] = value + return resolved + return child + + +def ordered(value): + """Deep equality alone misses insertion order, which defines pre-op order.""" + + if isinstance(value, dict): + return [(key, ordered(item)) for key, item in value.items()] + if isinstance(value, list): + return [ordered(item) for item in value] + return value + + +def legacy_config(document: dict, variant: str) -> dict: + config = legacy_resolve(document, variant) + files = config.get("file_cfg", {}) + if files.get("filename") and "onnx_filename" not in files: + files["onnx_filename"] = str(Path(files["filename"]).with_suffix(".onnx")) + return config + + +CONFIG_VARIANTS = [ + (path, variant) + for path in sorted(MODEL_CONFIG_DIR.glob("*.yaml")) + for variant in yaml.safe_load(path.read_text(encoding="utf-8")) +] + + +@pytest.mark.parametrize("path,variant", CONFIG_VARIANTS, ids=lambda value: str(value)) +def test_every_legacy_variant_is_order_identical(path: Path, variant: str) -> None: + expected = legacy_config(yaml.safe_load(path.read_text(encoding="utf-8")), variant) + actual = resolve_model_config(str(path), variant) + assert actual == expected + assert ordered(actual) == ordered(expected) + + +def write_config(path: Path, config: dict) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + return path + + +def test_recursive_extends_keeps_order_and_independent_results(tmp_path: Path) -> None: + write_config( + tmp_path / "bases/base.yaml", + { + "DEFAULT": { + "pre_cfg": { + "Reader": {"style": "pil", "color_mode": "RGB"}, + "Resize": {"size": [32, 32], "interpolation": "bilinear"}, + }, + "post_cfg": {"nested": {"labels": [1, 2], "other": True}}, + } + }, + ) + child = write_config( + tmp_path / "child.yaml", + { + "DEFAULT": { + "extends": "bases/base.yaml", + "pre_cfg": {"Resize": {"size": [16, 16]}, "SetOrder": {"shape": "HWC"}}, + "post_cfg": {"nested": {"labels": [3]}}, + } + }, + ) + result = load_model_variant(child) + assert list(result["pre_cfg"]) == ["Reader", "Resize", "SetOrder"] + assert result["pre_cfg"]["Resize"] == { + "size": [16, 16], + "interpolation": "bilinear", + } + assert result["post_cfg"]["nested"] == {"labels": [3], "other": True} + result["post_cfg"]["nested"]["labels"].append(4) + assert load_model_variant(child)["post_cfg"]["nested"]["labels"] == [3] + + +def test_update_stays_shallow_with_extends_base(tmp_path: Path) -> None: + write_config( + tmp_path / "base.yaml", + {"DEFAULT": {"pre_cfg": {"Resize": {"size": 32, "interpolation": "bilinear"}}}}, + ) + path = write_config( + tmp_path / "child.yaml", + { + "BASE": {"extends": "base.yaml"}, + "DEFAULT": {"update": "BASE", "pre_cfg": {"Resize": {"size": 16}}}, + }, + ) + assert load_model_variant(path)["pre_cfg"]["Resize"] == {"size": 16} + + +@pytest.mark.parametrize( + "reference", + [None, "", 123, "#DEFAULT", "base.yaml#", "https://host/base.yaml", "a#b#c"], +) +def test_invalid_extends(tmp_path: Path, reference) -> None: + path = write_config(tmp_path / "child.yaml", {"DEFAULT": {"extends": reference}}) + with pytest.raises(ValueError): + load_model_variant(path) + + +def test_cross_file_cycle(tmp_path: Path) -> None: + a = write_config(tmp_path / "a.yaml", {"DEFAULT": {"extends": "b.yaml"}}) + write_config(tmp_path / "b.yaml", {"DEFAULT": {"extends": "./a.yaml"}}) + with pytest.raises(ValueError, match="Circular"): + load_model_variant(a) + + +def test_missing_base_and_conflicting_modes(tmp_path: Path) -> None: + path = write_config( + tmp_path / "child.yaml", {"DEFAULT": {"extends": "absent.yaml"}} + ) + with pytest.raises(FileNotFoundError): + load_model_variant(path) + write_config( + path, {"BASE": {}, "DEFAULT": {"extends": "absent.yaml", "update": "BASE"}} + ) + with pytest.raises(ValueError, match="cannot combine"): + load_model_variant(path) + + +@pytest.mark.parametrize("model", ["ResNet50", "YOLOv8n", "YOLOX-s", "DAMO-YOLO-T"]) +def test_cpu_preprocessing_numeric_identity(model: str) -> None: + path = MODEL_CONFIG_DIR / f"{model}.yaml" + expected = legacy_config( + yaml.safe_load(path.read_text(encoding="utf-8")), "DEFAULT" + ) + actual = resolve_model_config(str(path)) + image = np.random.default_rng(0).integers(0, 256, (71, 103, 3), dtype=np.uint8) + before = build_preprocess(expected["pre_cfg"])(image.copy()) + after = build_preprocess(actual["pre_cfg"])(image.copy()) + np.testing.assert_array_equal(np.asarray(after), np.asarray(before)) From e8fe25c01c7ec9513485dfa164957cc3ab138a8e Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:30:30 +0900 Subject: [PATCH 02/13] Add read-only Hub artifact resolution checks Verify exact runtime board paths using metadata only, preserve local-only restrictions and distinguish live commits from approval provenance. Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 3 + AGENTS.md | 3 + mblt_vision/_hub_validation.py | 98 +++++++++++++++++++++++++++++ tests/TEST.md | 16 +++++ tests/test_hub_validation.py | 97 ++++++++++++++++++++++++++++ 5 files changed, 217 insertions(+) create mode 100644 mblt_vision/_hub_validation.py create mode 100644 tests/test_hub_validation.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index b695725..643c86b 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -28,6 +28,9 @@ description: >- Use file_cfg.filename for MXQ and derive the same-stem ONNX artifact unless onnx_filename is required. - Every post_cfg declares dataset; resolve output taxonomy from the dataset/task pair. +- For Hub linkage changes, run `tests/test_hub_validation.py` as described in `tests/TEST.md`. + The metadata checker follows runtime board paths without downloading weights; live commits + are diagnostics, not substitutes for missing historical approval provenance. - For opt-in YAML family inheritance, follow `mblt_vision/README.md`'s configuration inheritance rules. Keep legacy `update` shallow and run `tests/test_config_inheritance.py` to prove every existing variant's values and processing order remain identical. diff --git a/AGENTS.md b/AGENTS.md index 06a243b..a6f72b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,6 +60,9 @@ The current ownership boundary is deliberate: onnx_path as compatibility aliases. - Update task-package exports and lazy top-level exports together. Confirm that list_models() discovers every public model class. +- Verify Hub configuration changes with the metadata-only checks in + `tests/test_hub_validation.py` (see `tests/TEST.md`). Use exact runtime board paths; + treat live Hub commits as diagnostics, never as invented approval provenance. - Keep each model YAML's file_cfg, pre_cfg, and post_cfg shape stable. file_cfg.filename is the canonical MXQ Hub artifact; derive the same-stem ONNX filename unless the Hub artifact requires an explicit onnx_filename. diff --git a/mblt_vision/_hub_validation.py b/mblt_vision/_hub_validation.py new file mode 100644 index 0000000..d1f136d --- /dev/null +++ b/mblt_vision/_hub_validation.py @@ -0,0 +1,98 @@ +"""Read-only Hub metadata checks for resolved model artifact configurations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from huggingface_hub import HfApi +from mblt_npu import normalize_target_device + + +@dataclass(frozen=True) +class HubArtifactCheck: + """A metadata-only resolution result; a live commit is not approval provenance.""" + + repo_id: str + revision: str + commit: str | None + framework: str + path: str + status: str + + +def check_hub_artifacts( + config: dict[str, Any], + *, + target_device: str = "aries-rb", + frameworks: tuple[str, ...] = ("mxq", "onnx"), + api: HfApi | None = None, +) -> tuple[HubArtifactCheck, ...]: + """Check exact runtime Hub paths without downloading model bytes. + + Accept a resolved configuration (``resolve_model_config`` derives the ONNX + filename). MXQ is checked only under the normalized target board; ONNX is + checked at the repository root, matching the engine. Never fall back to a + different board, revision or filename. Local-only configs are reported as + skipped, without contacting the Hub. Hub/API exceptions propagate so missing + access or network failure cannot be mistaken for a missing file. + + The returned live commit is diagnostic, not evidence that a commit was + approved. This function does not modify YAMLs or infer approved revisions. + """ + + if not frameworks or any( + framework not in {"mxq", "onnx"} for framework in frameworks + ): + raise ValueError("frameworks must contain only 'mxq' and/or 'onnx'.") + files = config.get("file_cfg") + if not isinstance(files, dict): + raise TypeError("Resolved model configuration requires a file_cfg mapping.") + repo_id, revision = files.get("repo_id"), files.get("revision") + if ( + not isinstance(repo_id, str) + or not repo_id + or not isinstance(revision, str) + or not revision + ): + raise ValueError( + "Hub verification requires nonempty repo_id and revision strings." + ) + board = normalize_target_device(target_device) + paths = [] + for framework in dict.fromkeys(frameworks): + filename = files.get("filename" if framework == "mxq" else "onnx_filename") + if not isinstance(filename, str) or not filename: + raise ValueError( + f"Resolved configuration requires the {framework} artifact filename." + ) + paths.append( + (framework, f"{board}/{filename}" if framework == "mxq" else filename) + ) + if files.get("local_artifact_only"): + return tuple( + HubArtifactCheck( + repo_id, revision, None, framework, path, "local_artifact_only" + ) + for framework, path in paths + ) + client = HfApi() if api is None else api + info = client.model_info(repo_id=repo_id, revision=revision) + if not isinstance(info.sha, str) or not info.sha: + raise ValueError("Hub model metadata did not return a commit SHA.") + # Pin the file listing to the just-resolved commit: symbolic branches can move + # between API calls, and results must describe a single repository snapshot. + available = set( + client.list_repo_files(repo_id=repo_id, revision=info.sha, repo_type="model") + ) + return tuple( + HubArtifactCheck( + repo_id, + revision, + info.sha, + framework, + path, + "resolved" if path in available else "missing", + ) + for framework, path in paths + ) diff --git a/tests/TEST.md b/tests/TEST.md index ffd156d..bea1fcf 100644 --- a/tests/TEST.md +++ b/tests/TEST.md @@ -35,6 +35,22 @@ After authenticating with Hugging Face Hub, exercise representative ONNX models: pytest tests/test_onnx_classification.py -m requires_network ``` +To check representative Hub configurations without downloading model bytes or +using an inference backend: + +```bash +pytest tests/test_hub_validation.py -m requires_network +``` + +These read-only checks use the exact configured `repo_id` and `revision`, verify +MXQ under the normalized target board and ONNX at the repository root, and pin +metadata listing to the resolved commit to avoid races with moving branches. +Local-only artifacts are skipped without a network request. An observed current +commit is diagnostic metadata, not evidence of a historically approved revision; +missing approval provenance must never be replaced with today's `main` tip. +Offline mocked tests cover missing files, wrong boards, API failures and local-only +restrictions. No YAML revision or model artifact is changed by these checks. + To run MXQ inference, add a configured NPU and the shared runtime options: ```bash diff --git a/tests/test_hub_validation.py b/tests/test_hub_validation.py new file mode 100644 index 0000000..f702c88 --- /dev/null +++ b/tests/test_hub_validation.py @@ -0,0 +1,97 @@ +"""Offline unit tests and explicit metadata-only Hub integration checks.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from mblt_vision._hub_validation import check_hub_artifacts +from mblt_vision.wrapper import resolve_model_config + + +def config(**overrides): + return { + "file_cfg": { + "repo_id": "mobilint/example", + "revision": "approved-commit", + "filename": "model.mxq", + "onnx_filename": "model.onnx", + **overrides, + } + } + + +def api_with_files(files): + api = Mock() + api.model_info.return_value = SimpleNamespace(sha="exact-live-commit") + api.list_repo_files.return_value = files + return api + + +def test_exact_paths_and_commit_snapshot() -> None: + api = api_with_files(["aries-rb/model.mxq", "model.onnx"]) + checks = check_hub_artifacts(config(), api=api) + assert [item.status for item in checks] == ["resolved", "resolved"] + assert [item.path for item in checks] == ["aries-rb/model.mxq", "model.onnx"] + assert all(item.commit == "exact-live-commit" for item in checks) + api.model_info.assert_called_once_with( + repo_id="mobilint/example", revision="approved-commit" + ) + api.list_repo_files.assert_called_once_with( + repo_id="mobilint/example", revision="exact-live-commit", repo_type="model" + ) + + +def test_wrong_board_never_falls_back() -> None: + api = api_with_files(["aries-rb/model.mxq", "model.mxq", "model.onnx"]) + checks = check_hub_artifacts(config(), target_device="regulus-ra", api=api) + assert checks[0].path == "regulus-ra/model.mxq" + assert checks[0].status == "missing" + assert checks[1].status == "resolved" + + +def test_local_artifact_only_does_not_contact_hub() -> None: + api = Mock() + checks = check_hub_artifacts(config(local_artifact_only=True), api=api) + assert all( + item.status == "local_artifact_only" and item.commit is None for item in checks + ) + assert not api.mock_calls + + +def test_api_errors_remain_errors() -> None: + api = Mock() + api.model_info.side_effect = ConnectionError("offline") + with pytest.raises(ConnectionError, match="offline"): + check_hub_artifacts(config(), api=api) + + +@pytest.mark.parametrize("frameworks", [(), ("other",)]) +def test_invalid_frameworks(frameworks) -> None: + with pytest.raises(ValueError, match="frameworks"): + check_hub_artifacts(config(), frameworks=frameworks) + + +@pytest.mark.parametrize( + "override", + [{"repo_id": ""}, {"revision": None}, {"filename": None}, {"onnx_filename": ""}], +) +def test_missing_required_fields(override) -> None: + with pytest.raises(ValueError): + check_hub_artifacts(config(**override), api=Mock()) + + +def test_framework_selection_does_not_require_other_filename() -> None: + api = api_with_files(["model.onnx"]) + checks = check_hub_artifacts(config(filename=None), frameworks=("onnx",), api=api) + assert len(checks) == 1 + assert checks[0].status == "resolved" + + +@pytest.mark.requires_network +@pytest.mark.parametrize("model", ["AlexNet", "ResNet50", "YOLOv8n"]) +def test_representative_hub_metadata_resolves(model: str) -> None: + checks = check_hub_artifacts(resolve_model_config(model)) + assert all(item.status == "resolved" for item in checks), checks From c1289eb095bdb4852ca62eaaeee2ee28e8e05f83 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:37:33 +0900 Subject: [PATCH 03/13] Add person retrieval datasets, embedding postprocessing and exact protocol scoring Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 5 + AGENTS.md | 6 + benchmark/benchmark_vision_models.py | 3 +- mblt_vision/README.md | 49 ++++ mblt_vision/__init__.py | 2 + mblt_vision/_tasks.py | 1 + mblt_vision/datasets/dukemtmc-reid.yaml | 35 +++ mblt_vision/datasets/market1501.yaml | 35 +++ mblt_vision/datasets/occluded-duke.yaml | 35 +++ mblt_vision/models/bases/person_reid.yaml | 26 +++ mblt_vision/person_reid/__init__.py | 33 +++ mblt_vision/utils/datasets/reid.py | 96 ++++++++ mblt_vision/utils/evaluation/retrieval.py | 208 +++++++++++++++++ mblt_vision/utils/postprocess/__init__.py | 3 +- mblt_vision/utils/postprocess/build_post.py | 3 + .../utils/postprocess/embedding_post.py | 85 +++++++ mblt_vision/utils/preprocess/normalize.py | 30 ++- tests/reference/ops_reid_metrics.py | 53 +++++ tests/reference/ops_reid_preprocess.py | 92 ++++++++ tests/test_api.py | 1 + tests/test_reid.py | 216 ++++++++++++++++++ tests/test_reid_parity.py | 69 ++++++ 22 files changed, 1082 insertions(+), 4 deletions(-) create mode 100644 mblt_vision/datasets/dukemtmc-reid.yaml create mode 100644 mblt_vision/datasets/market1501.yaml create mode 100644 mblt_vision/datasets/occluded-duke.yaml create mode 100644 mblt_vision/models/bases/person_reid.yaml create mode 100644 mblt_vision/person_reid/__init__.py create mode 100644 mblt_vision/utils/datasets/reid.py create mode 100644 mblt_vision/utils/evaluation/retrieval.py create mode 100644 mblt_vision/utils/postprocess/embedding_post.py create mode 100644 tests/reference/ops_reid_metrics.py create mode 100644 tests/reference/ops_reid_preprocess.py create mode 100644 tests/test_reid.py create mode 100644 tests/test_reid_parity.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index b695725..7d563ab 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -109,6 +109,11 @@ description: >- metric). Load difficult objects as ordinary targets, as Ultralytics' validation does. - Run YOLOv7 models at `iou_thres: 0.65`, WongKinYiu/yolov7's own `test.py` default. +- ReID is utility-only until published wrappers and routing are added. Reuse + `EmbeddingPost` for dense vectors, explicit float32 mean/std for source preprocessing, + and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions + and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. + ## Runtime and Packaging - Route NPU runtime access through mblt-npu-python; do not copy backend classes into Vision. diff --git a/AGENTS.md b/AGENTS.md index 06a243b..5c3c614 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,6 +180,12 @@ The current ownership boundary is deliberate: any future model needing more than one Hub artifact, rather than duplicating Hub-resolution logic. +- ReID utilities live in `mblt_vision.person_reid`; concrete published wrappers are separate. + Use `EmbeddingPost` for validated dense vectors and `PersonReIDPost` for dataset binding. + Retrieval protocols and prepared loaders are pinned to ops source in dataset YAMLs; keep + stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. + Named Normalize styles retain legacy arithmetic; explicit mean/std uses source float32 arithmetic. + ## Python-First Architecture - Keep the public layer independent from a particular backend. Define small internal interfaces diff --git a/benchmark/benchmark_vision_models.py b/benchmark/benchmark_vision_models.py index a5c086d..06f86e3 100644 --- a/benchmark/benchmark_vision_models.py +++ b/benchmark/benchmark_vision_models.py @@ -35,7 +35,8 @@ CORE_MODES: tuple[CoreMode, ...] = cast( tuple[CoreMode, ...], core_modes_for_target_device("aries-rb") ) -TASK_CHOICES = VISION_TASKS +# Utility-only embedding tasks have no benchmark engine/CLI route yet. +TASK_CHOICES = tuple(task for task in VISION_TASKS if task != "person_reid") SUPPORTED_TARGET_DEVICES = frozenset({"aries-rb", "regulus-ra", "regulus-rb"}) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 0187f74..6249bc9 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -489,3 +489,52 @@ validation split, following Ultralytics' depth-validation convention. - Difficult objects count as ordinary targets, as in Ultralytics' validation. + +## Person re-identification utilities + +`mblt_vision.person_reid` provides prepared query/gallery loaders, embedding +postprocessing and retrieval scoring. Published model wrappers/HF artifacts are +not added in this stage; no weights or datasets are downloaded by these utilities. +The CLI/benchmark router is unchanged. + +```python +from mblt_vision.person_reid import Market1501Dataset, Market1501Evaluator + +# root contains query/ and bounding_box_test/; embeddings have one row per sample. +dataset = Market1501Dataset("/path/to/market1501/eval") +# result = Market1501Evaluator().evaluate(embeddings, dataset.samples) +# print(result.as_dict()) +``` + +Also available: `DukeMTMCReIDDataset`, `OccludedDukeDataset` and matching +`DukeMTMCReIDEvaluator`, `OccludedDukeEvaluator`. Loaders emit immutable +`ReIDSample(image, pid, camid, split)` records, sorted query-first then gallery; +cameras are zero-based, identities are not relabeled. Counts are source-checked +by default; caller-owned subsets must explicitly set `validate_counts=False`. +Only Market-1501 drops pid -1 junk; pid 0 background distractors remain. +Duke loaders reject pid -1. Root defaults use `~/.mblt_model_zoo/datasets//eval`. + +`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`. It validates one +floating-point `(N,D)` output or singleton-spatial NCHW/NHWC tensor, optionally +checks `embedding_dim`, and uses `normalize: true` for row-wise L2 normalization +(zero vectors remain zero). Return type is a float32 torch matrix, never class +probabilities. An explicit `post_cfg.dataset` is required. Processing-only base +profiles in `models/bases/person_reid.yaml` can be selected using `extends`; +concrete models must supply their own HF `file_cfg`, dataset and embedding dimension. +`Normalize` accepts either a historical named style or explicit three-element +`mean` and `std` lists; named styles retain unchanged legacy arithmetic. + +`extract_embeddings(batches, infer, postprocess)` runs prepared batches through +the shared bounded inference pipeline, preserving order. Evaluators consume +these embeddings without adding normalization, camera inputs, re-ranking or flip +fusion. Same-ID/same-camera gallery rows are excluded; unmatched queries are +skipped. Squared float32 Euclidean distances use stable ties in gallery order. +Ranks beyond the gallery saturate; mAP is primary, rank-1/5/10 secondary, all +ratios. `as_dict()` rounds to five decimals like ops; raw `mean_ap`/`cmc` remain +unrounded. Dataset YAMLs record exact ops config, loader, evaluator and revision. + +Offline tests vendor the inspected ops functions at +`ec537764e36ca10da963fcb7afab157ea60ca022`. Synthetic embeddings require exact +distance/CMC/mAP equality; synthetic RGB preprocessing requires byte equality +(`atol=0`) against the ops adapter for explicit source mean/std profiles. These +are protocol tests, not measured model accuracy claims. diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index e56c12a..9cf69d2 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -13,6 +13,7 @@ from . import instance_segmentation as instance_segmentation from . import obb as obb from . import object_detection as object_detection +from . import person_reid as person_reid from . import pose_estimation as pose_estimation from . import semantic_segmentation as semantic_segmentation from ._api import list_models as list_models @@ -52,6 +53,7 @@ "object_detection", "obb", "pose_estimation", + "person_reid", "semantic_segmentation", ] + sorted(_LEGACY_MODEL_EXPORTS) # Keep legacy compatibility exports synchronized with their task packages. diff --git a/mblt_vision/_tasks.py b/mblt_vision/_tasks.py index df9a462..c8a4b6d 100644 --- a/mblt_vision/_tasks.py +++ b/mblt_vision/_tasks.py @@ -13,6 +13,7 @@ "obb", "pose_estimation", "face_detection", + "person_reid", ) diff --git a/mblt_vision/datasets/dukemtmc-reid.yaml b/mblt_vision/datasets/dukemtmc-reid.yaml new file mode 100644 index 0000000..c1fb241 --- /dev/null +++ b/mblt_vision/datasets/dukemtmc-reid.yaml @@ -0,0 +1,35 @@ +name: dukemtmc-reid +path: ~/.mblt_model_zoo/datasets/dukemtmc-reid +tasks: +- person_reid +val: eval +cameras: 8 +skip_junk: false +retrieval_splits: + query: + directory: query + count: 2228 + gallery: + directory: bounding_box_test + count: 17661 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/dukemtmc_reid/dukemtmc_reid.yaml + name: dukemtmc-reid + dataloader_path: datasets/dukemtmc_reid/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: dukemtmc-reid-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: dukemtmc-reid-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/dukemtmc_reid/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/market1501.yaml b/mblt_vision/datasets/market1501.yaml new file mode 100644 index 0000000..b638134 --- /dev/null +++ b/mblt_vision/datasets/market1501.yaml @@ -0,0 +1,35 @@ +name: market1501 +path: ~/.mblt_model_zoo/datasets/market1501 +tasks: +- person_reid +val: eval +cameras: 6 +skip_junk: true +retrieval_splits: + query: + directory: query + count: 3368 + gallery: + directory: bounding_box_test + count: 15913 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/market1501/market1501.yaml + name: market1501 + dataloader_path: datasets/market1501/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: market1501-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: market1501-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/market1501/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/occluded-duke.yaml b/mblt_vision/datasets/occluded-duke.yaml new file mode 100644 index 0000000..dc039b2 --- /dev/null +++ b/mblt_vision/datasets/occluded-duke.yaml @@ -0,0 +1,35 @@ +name: occluded-duke +path: ~/.mblt_model_zoo/datasets/occluded-duke +tasks: +- person_reid +val: eval +cameras: 8 +skip_junk: false +retrieval_splits: + query: + directory: query + count: 2210 + gallery: + directory: bounding_box_test + count: 17661 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/occluded_duke/occluded_duke.yaml + name: occluded-duke + dataloader_path: datasets/occluded_duke/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: occluded-duke-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: occluded-duke-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/occluded_duke/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/models/bases/person_reid.yaml b/mblt_vision/models/bases/person_reid.yaml new file mode 100644 index 0000000..cafeaf3 --- /dev/null +++ b/mblt_vision/models/bases/person_reid.yaml @@ -0,0 +1,26 @@ +# Processing only: published HF artifact identities belong to concrete stage-d2 YAMLs. +# Source: mblt-model-ops ec537764e36ca10da963fcb7afab157ea60ca022 +# models/person_reid/CLIP_ReID_RN50_Baseline_DukeMTMC/distribution.yaml +DEFAULT: + pre_cfg: + Reader: + style: pil + Resize: + size: [256, 128] + interpolation: bilinear + Normalize: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + post_cfg: + task: person_reid + dataset: dukemtmc-reid + normalize: true +# Inherit this processing profile and override dataset/dimension/artifacts per model. +MARKET1501: + extends: person_reid.yaml#DEFAULT + post_cfg: + dataset: market1501 +OCCLUDED_DUKE: + extends: person_reid.yaml#DEFAULT + post_cfg: + dataset: occluded-duke diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py new file mode 100644 index 0000000..07fecad --- /dev/null +++ b/mblt_vision/person_reid/__init__.py @@ -0,0 +1,33 @@ +"""Person retrieval utilities; published model wrappers are added separately.""" + +from ..utils.datasets.reid import ( + DukeMTMCReIDDataset, + Market1501Dataset, + OccludedDukeDataset, + ReIDDataset, + ReIDSample, +) +from ..utils.evaluation.retrieval import ( + DukeMTMCReIDEvaluator, + Market1501Evaluator, + OccludedDukeEvaluator, + RetrievalEvaluator, + RetrievalResult, + extract_embeddings, +) +from ..utils.postprocess.embedding_post import PersonReIDPost + +__all__ = [ + "DukeMTMCReIDDataset", + "DukeMTMCReIDEvaluator", + "Market1501Dataset", + "Market1501Evaluator", + "OccludedDukeDataset", + "OccludedDukeEvaluator", + "PersonReIDPost", + "ReIDDataset", + "ReIDSample", + "RetrievalEvaluator", + "RetrievalResult", + "extract_embeddings", +] diff --git a/mblt_vision/utils/datasets/reid.py b/mblt_vision/utils/datasets/reid.py new file mode 100644 index 0000000..af3e68d --- /dev/null +++ b/mblt_vision/utils/datasets/reid.py @@ -0,0 +1,96 @@ +"""Prepared query/gallery person crops; no download or identity relabeling.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from ...datasets import get_dataset_config_for_task + +_FILENAME = re.compile(r"^(-?\d+)_c(\d+)(?:s\d+)?(?:_|\.)") + + +@dataclass(frozen=True) +class ReIDSample: + """One crop with source identity, zero-based camera and query/gallery split.""" + + image: Path + pid: int + camid: int + split: Literal["query", "gallery"] + + +class ReIDDataset: + """Read a prepared benchmark root in source-defined deterministic order. + + ``root`` is the directory containing ``query`` and ``bounding_box_test`` + (the ops prepared ``eval`` directory). Counts are checked by default; + ``validate_counts=False`` explicitly supports caller-owned small subsets. + """ + + name: str + + def __init__( + self, root: str | Path | None = None, *, validate_counts: bool = True + ) -> None: + self.config = get_dataset_config_for_task("person_reid", self.name) + self.root = ( + Path(root).expanduser() + if root is not None + else Path(self.config["path"]) / self.config["val"] + ) + samples = [] + for split in ("query", "gallery"): + options = self.config["retrieval_splits"][split] + directory = self.root / options["directory"] + if not directory.is_dir(): + raise FileNotFoundError(f"ReID split directory is missing: {directory}") + rows = [] + for path in sorted(directory.glob("*.jpg")): + match = _FILENAME.match(path.name) + if match is None: + raise ValueError( + f"ReID file name carries no _c token: {path}" + ) + pid, camera = map(int, match.groups()) + if pid == -1: + if self.config["skip_junk"]: + continue + raise ValueError(f"Unexpected junk identity: {path}") + if not 1 <= camera <= self.config["cameras"]: + raise ValueError(f"Camera {camera} outside benchmark range: {path}") + rows.append(ReIDSample(path, pid, camera - 1, split)) + if not rows: + raise ValueError(f"ReID split directory holds no images: {directory}") + if validate_counts and len(rows) != options["count"]: + raise ValueError( + f"{split} has {len(rows)} crops, expected {options['count']}" + ) + samples.extend(rows) + self.samples = tuple(samples) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> ReIDSample: + return self.samples[index] + + +class Market1501Dataset(ReIDDataset): + """Market-1501: discard pid -1; retain pid 0 background distractors.""" + + name = "market1501" + + +class DukeMTMCReIDDataset(ReIDDataset): + """DukeMTMC-reID prepared held-out query/gallery crops.""" + + name = "dukemtmc-reid" + + +class OccludedDukeDataset(ReIDDataset): + """Occluded-Duke prepared query split and Duke gallery.""" + + name = "occluded-duke" diff --git a/mblt_vision/utils/evaluation/retrieval.py b/mblt_vision/utils/evaluation/retrieval.py new file mode 100644 index 0000000..e3d8ffa --- /dev/null +++ b/mblt_vision/utils/evaluation/retrieval.py @@ -0,0 +1,208 @@ +"""Source-exact Market protocol retrieval metrics (ops ec537764).""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch + +from ..datasets.reid import ReIDSample +from ._pipeline import map_batched_inference + + +def extract_embeddings( + batches: Iterable[Any], infer: Callable, postprocess: Callable +) -> np.ndarray: + """Decode prepared batches in input order through the shared bounded pipeline. + + ``infer`` receives a prepared batch; ``postprocess`` receives raw outputs and + returns a validated ``(N,D)`` tensor. No implicit normalization or flip fusion. + """ + + def decode(batch: Any, output: Any) -> np.ndarray: + value = postprocess(output) + if isinstance(value, torch.Tensor): + value = value.detach().cpu().numpy() + value = np.asarray(value, dtype=np.float32) + if value.ndim != 2 or min(value.shape) == 0 or not np.isfinite(value).all(): + raise ValueError( + "Decoded embeddings require a nonempty finite (N,D) matrix" + ) + return value + + rows = list(map_batched_inference(batches, infer, decode)) + if not rows: + raise ValueError("No embedding batches supplied") + return np.concatenate(rows, axis=0) + + +def euclidean_distance(query: np.ndarray, gallery: np.ndarray) -> np.ndarray: + """Source float32 squared Euclidean distance (not cosine or square root).""" + query = np.asarray(query, dtype=np.float32) + gallery = np.asarray(gallery, dtype=np.float32) + return ( + np.sum(query**2, axis=1, keepdims=True) + + np.sum(gallery**2, axis=1, keepdims=True).T + - 2.0 * (query @ gallery.T) + ) + + +@dataclass(frozen=True) +class RetrievalResult: + """Unrounded mAP and CMC curve; ``as_dict`` matches ops score rounding.""" + + cmc: np.ndarray + mean_ap: float + sample_count: int + + @property + def primary_score(self) -> float: + return self.mean_ap + + @property + def secondary_score(self) -> float: + return self.rank(1) + + def rank(self, k: int) -> float: + """Saturate ranks beyond the gallery length, as in the source scorer.""" + if not isinstance(k, int) or isinstance(k, bool) or k <= 0: + raise ValueError("Rank must be a positive integer") + return float(self.cmc[min(k, len(self.cmc)) - 1]) + + def as_dict(self) -> dict: + """Return source-compatible primary/secondary metrics in ratio units.""" + return { + "primary": {"map": round(self.mean_ap, 5)}, + "secondary": {f"rank{k}": round(self.rank(k), 5) for k in (1, 5, 10)}, + "sample_count": self.sample_count, + } + + +class RetrievalEvaluator: + """Market retrieval protocol shared by three dataset-specific evaluators. + + Stable distance ties retain gallery manifest order. Exclude only same-ID, + same-camera rows; pid 0 distractors remain. Dataset loaders own pid -1 junk + filtering. Queries without cross-camera matches do not enter either mean. + """ + + dataset: str | None = None + + def evaluate( + self, + embeddings: np.ndarray, + samples: Sequence[ReIDSample], + *, + max_rank: int = 50, + ) -> RetrievalResult: + """Score one embedding row per sample in query-then-gallery order.""" + values = np.asarray(embeddings, dtype=np.float32) + if ( + values.ndim != 2 + or min(values.shape) == 0 + or values.shape[0] != len(samples) + or not np.isfinite(values).all() + ): + raise ValueError("Embeddings require one finite nonempty vector per sample") + if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: + raise ValueError("max_rank must be a positive integer") + if any(s.split not in {"query", "gallery"} or s.pid == -1 for s in samples): + raise ValueError( + "Samples require query/gallery splits with loader-filtered junk" + ) + query = [i for i, s in enumerate(samples) if s.split == "query"] + gallery = [i for i, s in enumerate(samples) if s.split == "gallery"] + if not query or not gallery: + raise ValueError("Benchmark needs both a query and a gallery split") + cmc, mean_ap = self.evaluate_distances( + euclidean_distance(values[query], values[gallery]), + np.asarray([samples[i].pid for i in query]), + np.asarray([samples[i].pid for i in gallery]), + np.asarray([samples[i].camid for i in query]), + np.asarray([samples[i].camid for i in gallery]), + max_rank, + ) + return RetrievalResult(cmc, mean_ap, len(samples)) + + @staticmethod + def evaluate_distances( + distmat: np.ndarray, + q_pids: np.ndarray, + g_pids: np.ndarray, + q_camids: np.ndarray, + g_camids: np.ndarray, + max_rank: int = 50, + ) -> tuple[np.ndarray, float]: + """CMC and mAP with the exact source accumulation and padding order.""" + distmat = np.asarray(distmat) + q_pids, g_pids = np.asarray(q_pids), np.asarray(g_pids) + q_camids, g_camids = np.asarray(q_camids), np.asarray(g_camids) + if ( + distmat.ndim != 2 + or min(distmat.shape) == 0 + or not np.isfinite(distmat).all() + ): + raise ValueError("Distances require a nonempty finite matrix") + num_q, num_g = distmat.shape + if any( + v.shape != (n,) + for v, n in ( + (q_pids, num_q), + (g_pids, num_g), + (q_camids, num_q), + (g_camids, num_g), + ) + ): + raise ValueError( + "Identity/camera metadata does not match distance dimensions" + ) + if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: + raise ValueError("max_rank must be a positive integer") + max_rank = min(max_rank, num_g) + indices = np.argsort(distmat, axis=1, kind="stable") + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + all_cmc = [] + all_ap = [] + for q_idx in range(num_q): + order = indices[q_idx] + remove = (g_pids[order] == q_pids[q_idx]) & ( + g_camids[order] == q_camids[q_idx] + ) + orig_cmc = matches[q_idx][~remove] + if not np.any(orig_cmc): + continue + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + all_cmc.append( + np.pad( + cmc[:max_rank], (0, max_rank - min(len(cmc), max_rank)), mode="edge" + ) + ) + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() / np.arange(1, orig_cmc.shape[0] + 1) + all_ap.append(float((tmp_cmc * orig_cmc).sum() / num_rel)) + if not all_cmc: + raise ValueError("No query identity appears in the gallery") + cmc_curve = np.asarray(all_cmc, dtype=np.float64).sum(0) / len(all_cmc) + return cmc_curve, float(np.mean(all_ap)) + + +class Market1501Evaluator(RetrievalEvaluator): + """Market-1501 held-out retrieval protocol v1.""" + + dataset = "market1501" + + +class DukeMTMCReIDEvaluator(RetrievalEvaluator): + """DukeMTMC-reID held-out retrieval protocol v1.""" + + dataset = "dukemtmc-reid" + + +class OccludedDukeEvaluator(RetrievalEvaluator): + """Occluded-Duke held-out retrieval protocol v1.""" + + dataset = "occluded-duke" diff --git a/mblt_vision/utils/postprocess/__init__.py b/mblt_vision/utils/postprocess/__init__.py index e5090d0..879a06f 100644 --- a/mblt_vision/utils/postprocess/__init__.py +++ b/mblt_vision/utils/postprocess/__init__.py @@ -4,6 +4,7 @@ from .build_post import build_postprocess from .depth_post import DepthPost +from .embedding_post import EmbeddingPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost -__all__ = ["DepthPost", "SemanticSegPost", "build_postprocess"] +__all__ = ["DepthPost", "EmbeddingPost", "PersonReIDPost", "SemanticSegPost", "build_postprocess"] diff --git a/mblt_vision/utils/postprocess/build_post.py b/mblt_vision/utils/postprocess/build_post.py index dc89192..eb569ac 100644 --- a/mblt_vision/utils/postprocess/build_post.py +++ b/mblt_vision/utils/postprocess/build_post.py @@ -9,6 +9,7 @@ from .cls_post import ClsPost from .damoyolo_post import DAMOYOLODetectionPost from .depth_post import DepthPost +from .embedding_post import PersonReIDPost from .semantic_seg_post import SemanticSegPost from .yolo_anchor_post import ( YOLOAnchorDetectionPost, @@ -70,6 +71,8 @@ def build_postprocess( f"Unsupported post_cfg.head {head!r}; expected one of {sorted(DETECTION_HEADS)}." ) return DETECTION_HEADS[head](pre_cfg, post_cfg, **kwargs) + if task == "person_reid": + return PersonReIDPost(pre_cfg, post_cfg) if task == "image_classification": return ClsPost(pre_cfg, post_cfg) if task == "depth_estimation": diff --git a/mblt_vision/utils/postprocess/embedding_post.py b/mblt_vision/utils/postprocess/embedding_post.py new file mode 100644 index 0000000..8d55353 --- /dev/null +++ b/mblt_vision/utils/postprocess/embedding_post.py @@ -0,0 +1,85 @@ +"""Validated appearance embeddings, shared by retrieval and verification tasks.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np +import torch +import torch.nn.functional as F + +from ...datasets import get_dataset_config_for_task +from ..types import ListTensorLike, TensorLike +from .base import PostBase + + +class EmbeddingPost(PostBase): + """Return floating-point ``(batch, dimension)`` embeddings without argmax. + + Accept a single matrix or singleton-spatial NCHW/NHWC output (also wrapped + in a one-element runtime output list). L2 normalization is opt-in; zero + vectors stay zero, matching the source embedding decoder. + """ + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__() + self.normalize = post_cfg.get("normalize", False) + if not isinstance(self.normalize, bool): + raise TypeError("post_cfg.normalize must be a boolean") + self.embedding_dim = post_cfg.get("embedding_dim") + if self.embedding_dim is not None and ( + not isinstance(self.embedding_dim, int) + or isinstance(self.embedding_dim, bool) + or self.embedding_dim <= 0 + ): + raise ValueError("post_cfg.embedding_dim must be a positive integer") + + def __call__(self, x: TensorLike | ListTensorLike) -> torch.Tensor: + """Validate one dense output and optionally L2-normalize each row.""" + if isinstance(x, Sequence): + if len(x) != 1: + raise ValueError( + "Embedding postprocessing expects exactly one output tensor" + ) + x = x[0] + if isinstance(x, np.ndarray): + x = torch.from_numpy(x) + if not isinstance(x, torch.Tensor): + raise TypeError("Embedding output must be a NumPy array or torch tensor") + if not x.is_floating_point(): + raise TypeError("Embedding output must have a floating-point dtype") + x = x.to(self.device) + if x.ndim == 4 and (x.shape[2:] == (1, 1) or x.shape[1:3] == (1, 1)): + x = x.reshape(x.shape[0], -1) + if x.ndim != 2 or min(x.shape) <= 0: + raise ValueError( + f"Embedding output requires nonempty (N, D), got {tuple(x.shape)}" + ) + if self.embedding_dim is not None and x.shape[1] != self.embedding_dim: + raise ValueError( + f"Embedding dimension {x.shape[1]} does not match {self.embedding_dim}" + ) + if not bool(torch.isfinite(x).all()): + raise ValueError("Embedding output must contain only finite values") + # Float32 agrees with the ops decoder, including its 1e-12 norm floor. + x = x.float() + if not bool(torch.isfinite(x).all()): + raise ValueError("Embedding values overflow float32") + if self.normalize and not bool( + torch.isfinite(torch.linalg.vector_norm(x, dim=1)).all() + ): + raise ValueError("Embedding norms overflow float32") + if self.normalize: + x = F.normalize(x, p=2, dim=1, eps=1e-12) + return x + + +class PersonReIDPost(EmbeddingPost): + """Appearance embeddings tied to an explicitly configured ReID dataset.""" + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__(pre_cfg, post_cfg) + dataset = post_cfg.get("dataset") + if not isinstance(dataset, str) or not dataset: + raise ValueError("Person ReID postprocessing requires post_cfg.dataset") + self.dataset = get_dataset_config_for_task("person_reid", dataset)["name"] diff --git a/mblt_vision/utils/preprocess/normalize.py b/mblt_vision/utils/preprocess/normalize.py index be7dd1b..ad19c3d 100644 --- a/mblt_vision/utils/preprocess/normalize.py +++ b/mblt_vision/utils/preprocess/normalize.py @@ -30,14 +30,40 @@ class Normalize(PreOps): std: Array of standard deviation values for normalization. """ - def __init__(self, style: str) -> None: + def __init__( + self, + style: str | None = None, + *, + mean: list[float] | None = None, + std: list[float] | None = None, + ) -> None: """Initializes the Normalize layer with a specific style. Args: - style: The preprocessing style to use. Must be one of STYLE_LIST. + style: Historical named style; mutually exclusive with mean/std. + mean: Optional explicit three-element RGB mean, using float32 arithmetic. + std: Optional explicit positive RGB standard deviations. """ super().__init__() + self.explicit_stats = mean is not None or std is not None + if self.explicit_stats: + if style is not None or mean is None or std is None: + raise ValueError("Provide either style or both mean and std") + self.mean = np.asarray(mean, dtype=np.float32) + self.std = np.asarray(std, dtype=np.float32) + if ( + self.mean.shape != (3,) + or self.std.shape != (3,) + or not np.isfinite(self.mean).all() + or not np.isfinite(self.std).all() + or not (self.std > 0).all() + ): + raise ValueError( + "mean/std require three finite RGB values and positive std" + ) + self.style = "explicit" + return if not isinstance(style, str): raise TypeError( f"Normalize style must be a string, got {type(style).__name__}." diff --git a/tests/reference/ops_reid_metrics.py b/tests/reference/ops_reid_metrics.py new file mode 100644 index 0000000..78f10fa --- /dev/null +++ b/tests/reference/ops_reid_metrics.py @@ -0,0 +1,53 @@ +"""Verbatim functions from datasets/_shared/reid.py at +ec537764e36ca10da963fcb7afab157ea60ca022 (mblt-model-ops). +Snapshot permits ordinary offline differential tests without sibling repos. +""" +import numpy as np +MAX_RANK = 50 + +def euclidean_distance(query: np.ndarray, gallery: np.ndarray) -> np.ndarray: + """Squared euclidean distance, the reference ``euclidean_distance`` without the sqrt. + + Ranking is monotone in the square, so CMC and mAP are unchanged. + """ + query = np.asarray(query, dtype=np.float32) + gallery = np.asarray(gallery, dtype=np.float32) + q2 = np.sum(query**2, axis=1, keepdims=True) + g2 = np.sum(gallery**2, axis=1, keepdims=True).T + return q2 + g2 - 2.0 * (query @ gallery.T) + +def evaluate( + distmat: np.ndarray, + q_pids: np.ndarray, + g_pids: np.ndarray, + q_camids: np.ndarray, + g_camids: np.ndarray, + max_rank: int = MAX_RANK, +) -> tuple[np.ndarray, float]: + """Market-1501 protocol CMC curve and mAP (reference ``eval_func``).""" + num_q, num_g = distmat.shape + max_rank = min(max_rank, num_g) + indices = np.argsort(distmat, axis=1, kind="stable") + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + all_cmc = [] + all_ap = [] + for q_idx in range(num_q): + order = indices[q_idx] + remove = (g_pids[order] == q_pids[q_idx]) & (g_camids[order] == q_camids[q_idx]) + orig_cmc = matches[q_idx][~remove] + if not np.any(orig_cmc): + continue # the query identity never appears in the gallery + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + # Same-identity/same-camera removal leaves different gallery lengths. + # Saturate missing ranks with the final CMC value for this query. + all_cmc.append( + np.pad(cmc[:max_rank], (0, max_rank - min(len(cmc), max_rank)), mode="edge") + ) + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() / np.arange(1, orig_cmc.shape[0] + 1) + all_ap.append(float((tmp_cmc * orig_cmc).sum() / num_rel)) + if not all_cmc: + raise ValueError("No query identity appears in the gallery") + cmc_curve = np.asarray(all_cmc, dtype=np.float64).sum(0) / len(all_cmc) + return cmc_curve, float(np.mean(all_ap)) diff --git a/tests/reference/ops_reid_preprocess.py b/tests/reference/ops_reid_preprocess.py new file mode 100644 index 0000000..b2b4ce0 --- /dev/null +++ b/tests/reference/ops_reid_preprocess.py @@ -0,0 +1,92 @@ +# Verbatim source: models/person_reid/_shared/preprocess.py +# mblt-model-ops ec537764e36ca10da963fcb7afab157ea60ca022 +"""Default person_reid preprocess: the CLIP-ReID / TransReID test transform. + +Upstream evaluates with ``Resize((H, W)) -> ToTensor -> Normalize(mean, std)``; there is no +crop, and the two backbone families differ only in the normalization statistics, so the +statistics are explicit ``mean``/``std`` lists in ``pre_cfg`` rather than a named style. + +Operations are applied in a fixed canonical order, not in ``config`` iteration order: +``predict.input.json`` is written with sorted keys, so relying on dict order would run +``Normalize`` before ``Resize``. + +A camera-aware Model (SIE) declares ``CameraOneHot``. The camera index is the ``_c`` +token of the crop's file name, which is how every ReID dataset and the upstream loader +encode it; it is a Model input, not ground truth. The network then takes two inputs, and +the returned tensor is a tuple ``(image, camera_one_hot)`` in that order. + +The return value is ``(source, network_input, camera_index)`` so that the shared +``inference.tensor_of`` finds the network input in the second slot. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +PIPELINE_ORDER = ("Reader", "Resize", "SetOrder", "Normalize", "CameraOneHot") +REQUIRED = ("Resize", "Normalize") +_RESAMPLING = { + "bilinear": Image.Resampling.BILINEAR, + "bicubic": Image.Resampling.BICUBIC, +} +DEFAULT_CAMERA_PATTERN = r"_c(\d+)" + +NetworkInput = np.ndarray | tuple[np.ndarray, np.ndarray] + + +def camera_index(source: Path, config: dict[str, Any]) -> int: + """The 0-based camera index of a crop, from its ``_c`` file-name token.""" + pattern = str(config.get("pattern", DEFAULT_CAMERA_PATTERN)) + match = re.search(pattern, Path(source).name) + if match is None: + raise ValueError(f"{source}: file name carries no camera token {pattern!r}") + cameras = int(config["num_cameras"]) + camera = int(match.group(1)) - 1 + if not 0 <= camera < cameras: + raise ValueError(f"{source}: camera {camera + 1} is outside 1..{cameras}") + return camera + + +def preprocess( + source: Path, config: dict[str, Any] +) -> tuple[Path, NetworkInput, int | None]: + """Return ``(source, network input, camera index or None)``.""" + unknown = set(config) - set(PIPELINE_ORDER) + if unknown: + raise ValueError(f"unsupported preprocess operations: {sorted(unknown)}") + missing = [name for name in REQUIRED if name not in config] + if missing: + raise ValueError(f"pre_cfg is missing required operations: {missing}") + source = Path(source) + value: Any = Image.open(source).convert("RGB") + camera: int | None = None + for operation in PIPELINE_ORDER: + if operation not in config: + continue + options = config[operation] + if operation == "Reader": + continue + if operation == "Resize": + height, width = (int(v) for v in options["size"]) + mode = _RESAMPLING[str(options.get("interpolation", "bilinear"))] + value = value.resize((width, height), mode) + elif operation == "SetOrder": + if options.get("shape", "HWC") != "HWC": + raise ValueError("person_reid preprocess produces HWC tensors only") + elif operation == "Normalize": + mean = np.asarray(options["mean"], dtype=np.float32) + std = np.asarray(options["std"], dtype=np.float32) + value = (np.asarray(value, dtype=np.float32) / 255.0 - mean) / std + elif operation == "CameraOneHot": + camera = camera_index(source, options) + tensor = np.ascontiguousarray(value, dtype=np.float32) + if camera is None: + return source, tensor, None + one_hot = np.zeros(int(config["CameraOneHot"]["num_cameras"]), dtype=np.float32) + one_hot[camera] = 1.0 + return source, (tensor, one_hot), camera diff --git a/tests/test_api.py b/tests/test_api.py index 40b4182..ce184aa 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -21,6 +21,7 @@ def test_public_discovery_exposes_all_supported_tasks() -> None: "obb", "pose_estimation", "face_detection", + "person_reid", ] assert list_models("obb")["obb"] diff --git a/tests/test_reid.py b/tests/test_reid.py new file mode 100644 index 0000000..a6c0fcc --- /dev/null +++ b/tests/test_reid.py @@ -0,0 +1,216 @@ +"""CPU-only embedding, prepared loader and retrieval protocol contracts.""" + +from pathlib import Path + +import numpy as np +import pytest +import torch +from mblt_vision._config import load_model_variant +from mblt_vision.datasets import get_dataset_config +from mblt_vision.utils.datasets.reid import ( + DukeMTMCReIDDataset, + Market1501Dataset, + OccludedDukeDataset, + ReIDSample, +) +from mblt_vision.utils.evaluation.retrieval import ( + DukeMTMCReIDEvaluator, + Market1501Evaluator, + OccludedDukeEvaluator, + RetrievalEvaluator, + extract_embeddings, +) +from mblt_vision.utils.postprocess import build_postprocess +from mblt_vision.utils.postprocess.embedding_post import EmbeddingPost, PersonReIDPost +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.utils.preprocess.normalize import Normalize + + +@pytest.mark.parametrize("shape", [(2, 3), (2, 3, 1, 1), (2, 1, 1, 3)]) +def test_embedding_shapes(shape): + raw = np.array([[3.0, 4.0, 0.0], [0.0, 0.0, 0.0]], dtype=np.float32).reshape(shape) + post = build_postprocess( + {}, + { + "task": "person_reid", + "dataset": "market1501", + "normalize": True, + "embedding_dim": 3, + }, + ) + assert isinstance(post, (EmbeddingPost, PersonReIDPost)) + np.testing.assert_array_equal( + post([raw]), np.array([[0.6, 0.8, 0.0], [0.0, 0.0, 0.0]], dtype=np.float32) + ) + np.testing.assert_array_equal(raw.reshape(2, 3)[0], [3.0, 4.0, 0.0]) + + +@pytest.mark.parametrize( + "raw", + [ + np.empty((0, 3)), + np.ones(3), + np.ones((2, 3, 2, 2)), + np.ones((2, 3, 1)), + np.array([[np.nan]]), + np.array([[np.inf]]), + ], +) +def test_embedding_rejects_invalid(raw): + with pytest.raises(ValueError): + EmbeddingPost({}, {})(raw) + + +def test_embedding_validation_and_optional_normalization(): + post = EmbeddingPost({}, {}) + raw = torch.tensor([[3.0, 4.0]]) + assert torch.equal(post(raw), raw) + with pytest.raises(TypeError): + post(np.ones((1, 2), dtype=np.int32)) + with pytest.raises(ValueError): + post([raw, raw]) + with pytest.raises(ValueError): + EmbeddingPost({}, {"embedding_dim": 3})(raw) + with pytest.raises(ValueError): + PersonReIDPost({}, {"dataset": "coco"}) + with pytest.raises(ValueError): + PersonReIDPost({}, {}) + with pytest.raises(TypeError): + EmbeddingPost({}, {"normalize": "yes"}) + + +@pytest.mark.parametrize( + "dataset,loader", + [ + ("market1501", Market1501Dataset), + ("dukemtmc-reid", DukeMTMCReIDDataset), + ("occluded-duke", OccludedDukeDataset), + ], +) +def test_loader_metadata_and_source(tmp_path, dataset, loader): + for folder in ("query", "bounding_box_test"): + (tmp_path / folder).mkdir() + for filename in ("0002_c2_b.jpg", "0001_c1s1_a.jpg"): + (tmp_path / "query" / filename).touch() + (tmp_path / "bounding_box_test" / "0001_c2_a.jpg").touch() + if dataset == "market1501": + (tmp_path / "query" / "-1_c1_junk.jpg").touch() + (tmp_path / "bounding_box_test" / "0000_c1_background.jpg").touch() + data = loader(tmp_path, validate_counts=False) + assert [(s.pid, s.camid, s.split) for s in data][:2] == [ + (1, 0, "query"), + (2, 1, "query"), + ] + assert data[-1].split == "gallery" + assert all(s.pid != -1 for s in data) + if dataset == "market1501": + assert data[2].pid == 0 + with pytest.raises(ValueError, match="expected"): + loader(tmp_path) + cfg = get_dataset_config(dataset) + assert cfg["source"]["revision"] == "ec537764e36ca10da963fcb7afab157ea60ca022" + assert cfg["eval_protocol"]["id"] == cfg["evaluation"]["protocol"] + + +@pytest.mark.parametrize("filename", ["bad.jpg", "1_c9_bad.jpg", "-1_c1_junk.jpg"]) +def test_duke_invalid_names(tmp_path, filename): + (tmp_path / "query").mkdir() + (tmp_path / "bounding_box_test").mkdir() + (tmp_path / "query" / filename).touch() + with pytest.raises(ValueError): + DukeMTMCReIDDataset(tmp_path, validate_counts=False) + + +def test_missing_and_empty_splits(tmp_path): + with pytest.raises(FileNotFoundError): + Market1501Dataset(tmp_path) + (tmp_path / "query").mkdir() + with pytest.raises(ValueError, match="no images"): + Market1501Dataset(tmp_path) + + +def test_handbuilt_exclusions_ties_and_missing_identity(): + # same-camera positive is removed; tied background stays before the positive. + distances = np.array([[0.0, 1.0, 1.0, 2.0], [4.0, 3.0, 2.0, 1.0]]) + cmc, mean_ap = RetrievalEvaluator.evaluate_distances( + distances, + np.array([1, 99]), + np.array([1, 0, 1, 1]), + np.array([0, 0]), + np.array([0, 1, 1, 2]), + ) + np.testing.assert_array_equal(cmc, [0.0, 1.0, 1.0, 1.0]) + assert mean_ap == (1 / 2 + 2 / 3) / 2 + # Stable ties are manifest order, not identity order. + cmc, mean_ap = RetrievalEvaluator.evaluate_distances( + np.zeros((1, 2)), + np.array([1]), + np.array([1, 0]), + np.array([0]), + np.array([1, 1]), + ) + assert mean_ap == 1.0 + np.testing.assert_array_equal(cmc, [1.0, 1.0]) + + +@pytest.mark.parametrize( + "evaluator", [Market1501Evaluator, DukeMTMCReIDEvaluator, OccludedDukeEvaluator] +) +def test_score_and_rank_saturation(evaluator): + samples = [ + ReIDSample(Path("q"), 1, 0, "query"), + ReIDSample(Path("g"), 1, 1, "gallery"), + ] + result = evaluator().evaluate(np.array([[1.0, 0.0], [1.0, 0.0]]), samples) + assert result.as_dict() == { + "primary": {"map": 1.0}, + "secondary": {"rank1": 1.0, "rank5": 1.0, "rank10": 1.0}, + "sample_count": 2, + } + assert result.primary_score == result.secondary_score == result.rank(50) == 1.0 + with pytest.raises(ValueError): + result.rank(0) + + +def test_no_valid_queries(): + with pytest.raises(ValueError, match="No query"): + RetrievalEvaluator.evaluate_distances( + np.zeros((1, 1)), np.array([1]), np.array([1]), np.array([0]), np.array([0]) + ) + + +def test_extraction_order(): + batches = [np.full((2, 3), i, dtype=np.float32) for i in range(5)] + actual = extract_embeddings(batches, lambda x: [x], EmbeddingPost({}, {})) + np.testing.assert_array_equal(actual, np.concatenate(batches)) + with pytest.raises(ValueError): + extract_embeddings([], lambda x: x, lambda x: x) + + +@pytest.mark.parametrize( + "options", + [ + {"mean": [0, 0, 0]}, + {"mean": [0, 0, 0], "std": [1, 0, 1]}, + {"mean": [0, 0], "std": [1, 1]}, + {"style": "torch", "mean": [0, 0, 0], "std": [1, 1, 1]}, + ], +) +def test_explicit_normalization_validation(options): + with pytest.raises(ValueError): + Normalize(**options) + + +def test_family_base_inheritance(tmp_path): + base = Path(__file__).parents[1] / "mblt_vision/models/bases/person_reid.yaml" + for variant, name in [ + ("DEFAULT", "dukemtmc-reid"), + ("MARKET1501", "market1501"), + ("OCCLUDED_DUKE", "occluded-duke"), + ]: + cfg = load_model_variant(base, variant) + assert cfg["post_cfg"]["dataset"] == name + assert isinstance( + build_postprocess(cfg["pre_cfg"], cfg["post_cfg"]), PersonReIDPost + ) + assert build_preprocess(cfg["pre_cfg"]) diff --git a/tests/test_reid_parity.py b/tests/test_reid_parity.py new file mode 100644 index 0000000..84bdfa6 --- /dev/null +++ b/tests/test_reid_parity.py @@ -0,0 +1,69 @@ +"""Offline numeric differential tests against pinned, verbatim ops functions.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from mblt_vision._config import load_model_variant +from mblt_vision.utils.evaluation.retrieval import ( + RetrievalEvaluator, + euclidean_distance, +) +from mblt_vision.utils.preprocess import build_preprocess +from PIL import Image + + +def reference(name): + path = Path(__file__).parent / "reference" / f"ops_reid_{name}.py" + spec = importlib.util.spec_from_file_location(f"ops_reid_{name}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("seed", range(10)) +def test_ops_evaluator_exact_numeric_parity(seed): + ops = reference("metrics") + rng = np.random.default_rng(seed) + # Quantized integer vectors produce numerous exact distance ties. + query = rng.integers(-2, 3, size=(17, 8)).astype(np.float32) + gallery = rng.integers(-2, 3, size=(61, 8)).astype(np.float32) + q_ids = rng.integers(1, 6, 17) + g_ids = rng.integers(0, 6, 61) # zero-valued distractors remain ordinary rows + q_cam = rng.integers(0, 3, 17) + g_cam = rng.integers(0, 3, 61) + expected_distances = ops.euclidean_distance(query, gallery) + actual_distances = euclidean_distance(query, gallery) + np.testing.assert_array_equal(actual_distances, expected_distances) + for max_rank in (1, 5, 50, 100): + expected_cmc, expected_map = ops.evaluate( + expected_distances, q_ids, g_ids, q_cam, g_cam, max_rank + ) + actual_cmc, actual_map = RetrievalEvaluator.evaluate_distances( + actual_distances, q_ids, g_ids, q_cam, g_cam, max_rank + ) + np.testing.assert_array_equal(actual_cmc, expected_cmc) + assert actual_map == expected_map + + +@pytest.mark.parametrize("shape", [(19, 11), (128, 256), (383, 191)]) +@pytest.mark.parametrize("style", ["torch", "tf"]) +def test_ops_preprocessing_parity(tmp_path, shape, style): + ops = reference("preprocess") + cfg = load_model_variant( + Path(__file__).parents[1] / "mblt_vision/models/bases/person_reid.yaml", + "DEFAULT", + ) + config = cfg["pre_cfg"] + if style == "tf": + # CLIP_ReID_ViT_B_16_Baseline_DukeMTMC/distribution.yaml at ec537764. + config["Normalize"] = {"mean": [0.5, 0.5, 0.5], "std": [0.5, 0.5, 0.5]} + rng = np.random.default_rng(0) + path = tmp_path / "0001_c2_synthetic.png" + Image.fromarray(rng.integers(0, 256, size=(*shape, 3), dtype=np.uint8)).save(path) + expected = ops.preprocess(path, config)[1] + actual = build_preprocess(config)(path) + # Both use PIL bilinear + explicit float32 stats; no tolerance needed. + np.testing.assert_array_equal(actual, expected) + assert np.max(np.abs(actual - expected)) == 0.0 From 511b8a36e7dd1116eed5fd40128cc378d4c995b0 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:38:17 +0900 Subject: [PATCH 04/13] Add published ImageNet classifier family configurations Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 3 + AGENTS.md | 4 + mblt_vision/README.md | 19 +++ mblt_vision/image_classification/__init__.py | 82 ++++++++++--- mblt_vision/models/EfficientViT_B3.yaml | 12 ++ mblt_vision/models/EfficientViT_L1.yaml | 8 ++ mblt_vision/models/EfficientViT_L2.yaml | 8 ++ mblt_vision/models/EfficientViT_L3.yaml | 8 ++ mblt_vision/models/LeViT_128.yaml | 12 ++ mblt_vision/models/LeViT_128S.yaml | 12 ++ mblt_vision/models/LeViT_192.yaml | 12 ++ mblt_vision/models/LeViT_256.yaml | 12 ++ mblt_vision/models/LeViT_384.yaml | 12 ++ .../models/ViT_Base_Patch16_CLIP_224.yaml | 14 +++ .../models/ViT_Base_Patch16_RPN_224.yaml | 14 +++ .../models/ViT_Base_Patch32_CLIP_224.yaml | 14 +++ .../models/ViT_Base_Patch32_CLIP_384.yaml | 18 +++ .../models/ViT_Base_Patch32_CLIP_448.yaml | 18 +++ .../models/ViT_Huge_Patch14_CLIP_224.yaml | 11 ++ .../models/ViT_Large_Patch14_CLIP_224.yaml | 11 ++ .../models/ViT_Medium_Patch16_GAP_256.yaml | 18 +++ .../models/ViT_RelPos_Base_Patch16_224.yaml | 14 +++ .../ViT_RelPos_Base_Patch32_Plus_RPN_256.yaml | 18 +++ .../models/ViT_RelPos_Medium_Patch16_224.yaml | 14 +++ .../ViT_RelPos_Medium_Patch16_RPN_224.yaml | 14 +++ .../models/ViT_RelPos_Small_Patch16_224.yaml | 14 +++ .../ViT_SRelPos_Medium_Patch16_224.yaml | 14 +++ .../models/ViT_SRelPos_Small_Patch16_224.yaml | 14 +++ .../models/bases/imagenet_classifier.yaml | 19 +++ tests/test_config_inheritance.py | 4 +- tests/test_published_classifiers.py | 112 ++++++++++++++++++ 31 files changed, 541 insertions(+), 18 deletions(-) create mode 100644 mblt_vision/models/EfficientViT_B3.yaml create mode 100644 mblt_vision/models/EfficientViT_L1.yaml create mode 100644 mblt_vision/models/EfficientViT_L2.yaml create mode 100644 mblt_vision/models/EfficientViT_L3.yaml create mode 100644 mblt_vision/models/LeViT_128.yaml create mode 100644 mblt_vision/models/LeViT_128S.yaml create mode 100644 mblt_vision/models/LeViT_192.yaml create mode 100644 mblt_vision/models/LeViT_256.yaml create mode 100644 mblt_vision/models/LeViT_384.yaml create mode 100644 mblt_vision/models/ViT_Base_Patch16_CLIP_224.yaml create mode 100644 mblt_vision/models/ViT_Base_Patch16_RPN_224.yaml create mode 100644 mblt_vision/models/ViT_Base_Patch32_CLIP_224.yaml create mode 100644 mblt_vision/models/ViT_Base_Patch32_CLIP_384.yaml create mode 100644 mblt_vision/models/ViT_Base_Patch32_CLIP_448.yaml create mode 100644 mblt_vision/models/ViT_Huge_Patch14_CLIP_224.yaml create mode 100644 mblt_vision/models/ViT_Large_Patch14_CLIP_224.yaml create mode 100644 mblt_vision/models/ViT_Medium_Patch16_GAP_256.yaml create mode 100644 mblt_vision/models/ViT_RelPos_Base_Patch16_224.yaml create mode 100644 mblt_vision/models/ViT_RelPos_Base_Patch32_Plus_RPN_256.yaml create mode 100644 mblt_vision/models/ViT_RelPos_Medium_Patch16_224.yaml create mode 100644 mblt_vision/models/ViT_RelPos_Medium_Patch16_RPN_224.yaml create mode 100644 mblt_vision/models/ViT_RelPos_Small_Patch16_224.yaml create mode 100644 mblt_vision/models/ViT_SRelPos_Medium_Patch16_224.yaml create mode 100644 mblt_vision/models/ViT_SRelPos_Small_Patch16_224.yaml create mode 100644 mblt_vision/models/bases/imagenet_classifier.yaml create mode 100644 tests/test_published_classifiers.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index b695725..2093143 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -28,6 +28,9 @@ description: >- Use file_cfg.filename for MXQ and derive the same-stem ONNX artifact unless onnx_filename is required. - Every post_cfg declares dataset; resolve output taxonomy from the dataset/task pair. +- Additional published EfficientViT/LeViT/ViT classifiers use `bases/imagenet_classifier.yaml`. + Preserve the recorded ops transforms checked by `tests/test_published_classifiers.py`, + including the TF-normalized `ViT_Large_Patch14_CLIP_224` exception. - For opt-in YAML family inheritance, follow `mblt_vision/README.md`'s configuration inheritance rules. Keep legacy `update` shallow and run `tests/test_config_inheritance.py` to prove every existing variant's values and processing order remain identical. diff --git a/AGENTS.md b/AGENTS.md index 06a243b..ac74d0e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,6 +69,10 @@ The current ownership boundary is deliberate: floors, and rounding differs by a pixel at 224/0.9 and 224/0.95. Several names (ConvNext_Base) resolve in timm yet carry torchvision's transform here, so read the expected value from that model's source.yaml provenance rather than from the name. +- Published EfficientViT, non-convolutional LeViT and additional ViT classifiers inherit + `bases/imagenet_classifier.yaml`. Their exact recorded preprocessing and dataset connection + are pinned by `tests/test_published_classifiers.py`; preserve TF normalization for + `ViT_Large_Patch14_CLIP_224` rather than inferring normalization from its name. - To share processing defaults without changing legacy models, opt new family YAMLs into `extends: bases/family.yaml#DEFAULT`. See the configuration-inheritance section of `mblt_vision/README.md` for ordering and merge rules. Keep legacy `update` shallow and diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 0187f74..f3dfecd 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -100,6 +100,25 @@ This section lists the publicly pre-trained models supported by the vision frame ### Image Classification +Additional published ImageNet classifiers are available through the same task +subpackage, top-level lazy exports and `MBLT_Engine`: + +- EfficientViT: `EfficientViT_B3`, `EfficientViT_L1`, `EfficientViT_L2`, `EfficientViT_L3`. +- Non-convolutional LeViT: `LeViT_128`, `LeViT_128S`, `LeViT_192`, `LeViT_256`, `LeViT_384`. +- ViT RPN, relative-position, SRelPos, GAP and CLIP families: discover exact names with + `list_models("image_classification")`. + +These configurations extend `models/bases/imagenet_classifier.yaml` and override +only recorded processing differences. The source is each model's +`models/image_classification//_source/predict_float.input.json` and matching +`distribution.yaml` in mblt-model-ops at `6fdbf471a3bd6ec22035a42c8730fe88a68881e4`. +Input crop/resize and normalization are model-specific: in particular, +`ViT_Large_Patch14_CLIP_224` uses TF normalization despite its CLIP name. +All 24 additions retain the existing `revision: main` convention because their +historical approved artifact HF commits are not recorded; current Hub tips and +upstream source-model commits are not substituted as approval pins. This is a +configuration/CPU processing parity addition, not a new inference accuracy claim. + | Model | Input Size
(H,W,C) | AccTop1
(NPU) | AccTop1
(GPU) | FLOPs (B) | params (M) | Source | Note | | --- | --- | --- | --- | --- | --- | --- | --- | | AlexNet | (224,224,3) | 56.084 | 56.556 | 1.43 | 61.10 | [Link](https://docs.pytorch.org/vision/stable/models/generated/torchvision.models.alexnet.html) | | diff --git a/mblt_vision/image_classification/__init__.py b/mblt_vision/image_classification/__init__.py index 3223c59..085defb 100644 --- a/mblt_vision/image_classification/__init__.py +++ b/mblt_vision/image_classification/__init__.py @@ -5,6 +5,14 @@ from .._compat import create_model_class __all__: list[str] = [ + "VGG11", + "VGG11_BN", + "VGG13", + "VGG13_BN", + "VGG16", + "VGG16_BN", + "VGG19", + "VGG19_BN", "AlexNet", "CAFormer_B36", "CAFormer_M36", @@ -21,10 +29,6 @@ "ConvNeXt_Large", "ConvNeXt_Small", "ConvNeXt_Tiny", - "DeiT_Base_Patch16_224", - "DeiT_Base_Patch16_384", - "DeiT_Small_Patch16_224", - "DeiT_Tiny_Patch16_224", "DeiT3_Base_Patch16_224", "DeiT3_Base_Patch16_384", "DeiT3_Large_Patch16_224", @@ -32,6 +36,10 @@ "DeiT3_Medium_Patch16_224", "DeiT3_Small_Patch16_224", "DeiT3_Small_Patch16_384", + "DeiT_Base_Patch16_224", + "DeiT_Base_Patch16_384", + "DeiT_Small_Patch16_224", + "DeiT_Tiny_Patch16_224", "DenseNet121", "DenseNet161", "DenseNet169", @@ -50,11 +58,20 @@ "EfficientNet_V2_L", "EfficientNet_V2_M", "EfficientNet_V2_S", + "EfficientViT_B3", + "EfficientViT_L1", + "EfficientViT_L2", + "EfficientViT_L3", "FlexiViT_Base", "FlexiViT_Large", "FlexiViT_Small", "GoogLeNet", "Inception_V3", + "LeViT_128", + "LeViT_128S", + "LeViT_192", + "LeViT_256", + "LeViT_384", "LeViT_Conv_128", "LeViT_Conv_128S", "LeViT_Conv_192", @@ -89,14 +106,14 @@ "RepViT_M2", "RepViT_M2_3", "RepViT_M3", + "ResNeXt50_32x4d", + "ResNeXt101_32x8d", + "ResNeXt101_64x4d", "ResNet18", "ResNet34", "ResNet50", "ResNet101", "ResNet152", - "ResNeXt50_32x4d", - "ResNeXt101_32x8d", - "ResNeXt101_64x4d", "ShuffleNet_V2_X0_5", "ShuffleNet_V2_X1_0", "ShuffleNet_V2_X1_5", @@ -106,34 +123,41 @@ "Swin_B", "Swin_S", "Swin_T", - "VGG11", - "VGG11_BN", - "VGG13", - "VGG13_BN", - "VGG16", - "VGG16_BN", - "VGG19", - "VGG19_BN", - "VisFormer_Small", - "VisFormer_Tiny", "ViT_B_16", "ViT_B_32", "ViT_Base_Patch8_224", "ViT_Base_Patch16_224", "ViT_Base_Patch16_384", + "ViT_Base_Patch16_CLIP_224", + "ViT_Base_Patch16_RPN_224", "ViT_Base_Patch32_224", "ViT_Base_Patch32_384", + "ViT_Base_Patch32_CLIP_224", + "ViT_Base_Patch32_CLIP_384", + "ViT_Base_Patch32_CLIP_448", + "ViT_Huge_Patch14_CLIP_224", "ViT_L_16", "ViT_L_32", + "ViT_Large_Patch14_CLIP_224", "ViT_Large_Patch16_224", "ViT_Large_Patch16_384", "ViT_Large_Patch32_384", + "ViT_Medium_Patch16_GAP_256", + "ViT_RelPos_Base_Patch16_224", + "ViT_RelPos_Base_Patch32_Plus_RPN_256", + "ViT_RelPos_Medium_Patch16_224", + "ViT_RelPos_Medium_Patch16_RPN_224", + "ViT_RelPos_Small_Patch16_224", + "ViT_SRelPos_Medium_Patch16_224", + "ViT_SRelPos_Small_Patch16_224", "ViT_Small_Patch16_224", "ViT_Small_Patch16_384", "ViT_Small_Patch32_224", "ViT_Small_Patch32_384", "ViT_Tiny_Patch16_224", "ViT_Tiny_Patch16_384", + "VisFormer_Small", + "VisFormer_Tiny", "Wide_ResNet50_2", "Wide_ResNet101_2", "YOLO11lCls", @@ -158,6 +182,30 @@ "YOLOv8xCls", ] +EfficientViT_B3 = create_model_class("EfficientViT_B3", __name__) +EfficientViT_L1 = create_model_class("EfficientViT_L1", __name__) +EfficientViT_L2 = create_model_class("EfficientViT_L2", __name__) +EfficientViT_L3 = create_model_class("EfficientViT_L3", __name__) +LeViT_128 = create_model_class("LeViT_128", __name__) +LeViT_128S = create_model_class("LeViT_128S", __name__) +LeViT_192 = create_model_class("LeViT_192", __name__) +LeViT_256 = create_model_class("LeViT_256", __name__) +LeViT_384 = create_model_class("LeViT_384", __name__) +ViT_RelPos_Medium_Patch16_RPN_224 = create_model_class("ViT_RelPos_Medium_Patch16_RPN_224", __name__) +ViT_RelPos_Base_Patch32_Plus_RPN_256 = create_model_class("ViT_RelPos_Base_Patch32_Plus_RPN_256", __name__) +ViT_Base_Patch16_RPN_224 = create_model_class("ViT_Base_Patch16_RPN_224", __name__) +ViT_Base_Patch16_CLIP_224 = create_model_class("ViT_Base_Patch16_CLIP_224", __name__) +ViT_Base_Patch32_CLIP_448 = create_model_class("ViT_Base_Patch32_CLIP_448", __name__) +ViT_Medium_Patch16_GAP_256 = create_model_class("ViT_Medium_Patch16_GAP_256", __name__) +ViT_Large_Patch14_CLIP_224 = create_model_class("ViT_Large_Patch14_CLIP_224", __name__) +ViT_Base_Patch32_CLIP_224 = create_model_class("ViT_Base_Patch32_CLIP_224", __name__) +ViT_RelPos_Small_Patch16_224 = create_model_class("ViT_RelPos_Small_Patch16_224", __name__) +ViT_Base_Patch32_CLIP_384 = create_model_class("ViT_Base_Patch32_CLIP_384", __name__) +ViT_SRelPos_Medium_Patch16_224 = create_model_class("ViT_SRelPos_Medium_Patch16_224", __name__) +ViT_Huge_Patch14_CLIP_224 = create_model_class("ViT_Huge_Patch14_CLIP_224", __name__) +ViT_RelPos_Base_Patch16_224 = create_model_class("ViT_RelPos_Base_Patch16_224", __name__) +ViT_RelPos_Medium_Patch16_224 = create_model_class("ViT_RelPos_Medium_Patch16_224", __name__) +ViT_SRelPos_Small_Patch16_224 = create_model_class("ViT_SRelPos_Small_Patch16_224", __name__) AlexNet = create_model_class("AlexNet", __name__) CAFormer_B36 = create_model_class("CAFormer_B36", __name__) CAFormer_M36 = create_model_class("CAFormer_M36", __name__) diff --git a/mblt_vision/models/EfficientViT_B3.yaml b/mblt_vision/models/EfficientViT_B3.yaml new file mode 100644 index 0000000..7f90dff --- /dev/null +++ b/mblt_vision/models/EfficientViT_B3.yaml @@ -0,0 +1,12 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/EfficientViT_B3/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/EfficientViT_B3 + filename: efficientvit_b3.mxq + revision: main + pre_cfg: + Resize: + interpolation: bicubic + size: 235 diff --git a/mblt_vision/models/EfficientViT_L1.yaml b/mblt_vision/models/EfficientViT_L1.yaml new file mode 100644 index 0000000..6dc9ab3 --- /dev/null +++ b/mblt_vision/models/EfficientViT_L1.yaml @@ -0,0 +1,8 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/EfficientViT_L1/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/EfficientViT_L1 + filename: efficientvit_l1.mxq + revision: main diff --git a/mblt_vision/models/EfficientViT_L2.yaml b/mblt_vision/models/EfficientViT_L2.yaml new file mode 100644 index 0000000..f09469d --- /dev/null +++ b/mblt_vision/models/EfficientViT_L2.yaml @@ -0,0 +1,8 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/EfficientViT_L2/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/EfficientViT_L2 + filename: efficientvit_l2.mxq + revision: main diff --git a/mblt_vision/models/EfficientViT_L3.yaml b/mblt_vision/models/EfficientViT_L3.yaml new file mode 100644 index 0000000..20f5c75 --- /dev/null +++ b/mblt_vision/models/EfficientViT_L3.yaml @@ -0,0 +1,8 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/EfficientViT_L3/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/EfficientViT_L3 + filename: efficientvit_l3.mxq + revision: main diff --git a/mblt_vision/models/LeViT_128.yaml b/mblt_vision/models/LeViT_128.yaml new file mode 100644 index 0000000..dd200ef --- /dev/null +++ b/mblt_vision/models/LeViT_128.yaml @@ -0,0 +1,12 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/LeViT_128/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/LeViT_128 + filename: levit_128.mxq + revision: main + pre_cfg: + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/LeViT_128S.yaml b/mblt_vision/models/LeViT_128S.yaml new file mode 100644 index 0000000..5b347aa --- /dev/null +++ b/mblt_vision/models/LeViT_128S.yaml @@ -0,0 +1,12 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/LeViT_128S/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/LeViT_128S + filename: levit_128s.mxq + revision: main + pre_cfg: + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/LeViT_192.yaml b/mblt_vision/models/LeViT_192.yaml new file mode 100644 index 0000000..536a983 --- /dev/null +++ b/mblt_vision/models/LeViT_192.yaml @@ -0,0 +1,12 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/LeViT_192/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/LeViT_192 + filename: levit_192.mxq + revision: main + pre_cfg: + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/LeViT_256.yaml b/mblt_vision/models/LeViT_256.yaml new file mode 100644 index 0000000..b7dfb0a --- /dev/null +++ b/mblt_vision/models/LeViT_256.yaml @@ -0,0 +1,12 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/LeViT_256/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/LeViT_256 + filename: levit_256.mxq + revision: main + pre_cfg: + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/LeViT_384.yaml b/mblt_vision/models/LeViT_384.yaml new file mode 100644 index 0000000..e0502d2 --- /dev/null +++ b/mblt_vision/models/LeViT_384.yaml @@ -0,0 +1,12 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/LeViT_384/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/LeViT_384 + filename: levit_384.mxq + revision: main + pre_cfg: + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/ViT_Base_Patch16_CLIP_224.yaml b/mblt_vision/models/ViT_Base_Patch16_CLIP_224.yaml new file mode 100644 index 0000000..7f31c24 --- /dev/null +++ b/mblt_vision/models/ViT_Base_Patch16_CLIP_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_Base_Patch16_CLIP_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_Base_Patch16_CLIP_224 + filename: vit_base_patch16_clip_224.mxq + revision: main + pre_cfg: + Normalize: + style: openai + Resize: + interpolation: bicubic + size: 235 diff --git a/mblt_vision/models/ViT_Base_Patch16_RPN_224.yaml b/mblt_vision/models/ViT_Base_Patch16_RPN_224.yaml new file mode 100644 index 0000000..013fe25 --- /dev/null +++ b/mblt_vision/models/ViT_Base_Patch16_RPN_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_Base_Patch16_RPN_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_Base_Patch16_RPN_224 + filename: vit_base_patch16_rpn_224.mxq + revision: main + pre_cfg: + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/ViT_Base_Patch32_CLIP_224.yaml b/mblt_vision/models/ViT_Base_Patch32_CLIP_224.yaml new file mode 100644 index 0000000..5c6464a --- /dev/null +++ b/mblt_vision/models/ViT_Base_Patch32_CLIP_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_Base_Patch32_CLIP_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_Base_Patch32_CLIP_224 + filename: vit_base_patch32_clip_224.mxq + revision: main + pre_cfg: + Normalize: + style: openai + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/ViT_Base_Patch32_CLIP_384.yaml b/mblt_vision/models/ViT_Base_Patch32_CLIP_384.yaml new file mode 100644 index 0000000..8086fc2 --- /dev/null +++ b/mblt_vision/models/ViT_Base_Patch32_CLIP_384.yaml @@ -0,0 +1,18 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_Base_Patch32_CLIP_384/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_Base_Patch32_CLIP_384 + filename: vit_base_patch32_clip_384.mxq + revision: main + pre_cfg: + CenterCrop: + size: + - 384 + - 384 + Normalize: + style: openai + Resize: + interpolation: bicubic + size: 384 diff --git a/mblt_vision/models/ViT_Base_Patch32_CLIP_448.yaml b/mblt_vision/models/ViT_Base_Patch32_CLIP_448.yaml new file mode 100644 index 0000000..c29252b --- /dev/null +++ b/mblt_vision/models/ViT_Base_Patch32_CLIP_448.yaml @@ -0,0 +1,18 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_Base_Patch32_CLIP_448/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_Base_Patch32_CLIP_448 + filename: vit_base_patch32_clip_448.mxq + revision: main + pre_cfg: + CenterCrop: + size: + - 448 + - 448 + Normalize: + style: openai + Resize: + interpolation: bicubic + size: 448 diff --git a/mblt_vision/models/ViT_Huge_Patch14_CLIP_224.yaml b/mblt_vision/models/ViT_Huge_Patch14_CLIP_224.yaml new file mode 100644 index 0000000..8fbe652 --- /dev/null +++ b/mblt_vision/models/ViT_Huge_Patch14_CLIP_224.yaml @@ -0,0 +1,11 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_Huge_Patch14_CLIP_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_Huge_Patch14_CLIP_224 + filename: vit_huge_patch14_clip_224.mxq + revision: main + pre_cfg: + Normalize: + style: openai diff --git a/mblt_vision/models/ViT_Large_Patch14_CLIP_224.yaml b/mblt_vision/models/ViT_Large_Patch14_CLIP_224.yaml new file mode 100644 index 0000000..d30d8c2 --- /dev/null +++ b/mblt_vision/models/ViT_Large_Patch14_CLIP_224.yaml @@ -0,0 +1,11 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_Large_Patch14_CLIP_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_Large_Patch14_CLIP_224 + filename: vit_large_patch14_clip_224.mxq + revision: main + pre_cfg: + Normalize: + style: tf diff --git a/mblt_vision/models/ViT_Medium_Patch16_GAP_256.yaml b/mblt_vision/models/ViT_Medium_Patch16_GAP_256.yaml new file mode 100644 index 0000000..d26adc3 --- /dev/null +++ b/mblt_vision/models/ViT_Medium_Patch16_GAP_256.yaml @@ -0,0 +1,18 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_Medium_Patch16_GAP_256/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_Medium_Patch16_GAP_256 + filename: vit_medium_patch16_gap_256.mxq + revision: main + pre_cfg: + CenterCrop: + size: + - 256 + - 256 + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 269 diff --git a/mblt_vision/models/ViT_RelPos_Base_Patch16_224.yaml b/mblt_vision/models/ViT_RelPos_Base_Patch16_224.yaml new file mode 100644 index 0000000..d5d0831 --- /dev/null +++ b/mblt_vision/models/ViT_RelPos_Base_Patch16_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_RelPos_Base_Patch16_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_RelPos_Base_Patch16_224 + filename: vit_relpos_base_patch16_224.mxq + revision: main + pre_cfg: + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/ViT_RelPos_Base_Patch32_Plus_RPN_256.yaml b/mblt_vision/models/ViT_RelPos_Base_Patch32_Plus_RPN_256.yaml new file mode 100644 index 0000000..2549da1 --- /dev/null +++ b/mblt_vision/models/ViT_RelPos_Base_Patch32_Plus_RPN_256.yaml @@ -0,0 +1,18 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_RelPos_Base_Patch32_Plus_RPN_256/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_RelPos_Base_Patch32_Plus_RPN_256 + filename: vit_relpos_base_patch32_plus_rpn_256.mxq + revision: main + pre_cfg: + CenterCrop: + size: + - 256 + - 256 + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 284 diff --git a/mblt_vision/models/ViT_RelPos_Medium_Patch16_224.yaml b/mblt_vision/models/ViT_RelPos_Medium_Patch16_224.yaml new file mode 100644 index 0000000..033ebef --- /dev/null +++ b/mblt_vision/models/ViT_RelPos_Medium_Patch16_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_RelPos_Medium_Patch16_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_RelPos_Medium_Patch16_224 + filename: vit_relpos_medium_patch16_224.mxq + revision: main + pre_cfg: + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/ViT_RelPos_Medium_Patch16_RPN_224.yaml b/mblt_vision/models/ViT_RelPos_Medium_Patch16_RPN_224.yaml new file mode 100644 index 0000000..4674b70 --- /dev/null +++ b/mblt_vision/models/ViT_RelPos_Medium_Patch16_RPN_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_RelPos_Medium_Patch16_RPN_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_RelPos_Medium_Patch16_RPN_224 + filename: vit_relpos_medium_patch16_rpn_224.mxq + revision: main + pre_cfg: + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/ViT_RelPos_Small_Patch16_224.yaml b/mblt_vision/models/ViT_RelPos_Small_Patch16_224.yaml new file mode 100644 index 0000000..ba1f738 --- /dev/null +++ b/mblt_vision/models/ViT_RelPos_Small_Patch16_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_RelPos_Small_Patch16_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_RelPos_Small_Patch16_224 + filename: vit_relpos_small_patch16_224.mxq + revision: main + pre_cfg: + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/ViT_SRelPos_Medium_Patch16_224.yaml b/mblt_vision/models/ViT_SRelPos_Medium_Patch16_224.yaml new file mode 100644 index 0000000..74e32bf --- /dev/null +++ b/mblt_vision/models/ViT_SRelPos_Medium_Patch16_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_SRelPos_Medium_Patch16_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_SRelPos_Medium_Patch16_224 + filename: vit_srelpos_medium_patch16_224.mxq + revision: main + pre_cfg: + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/ViT_SRelPos_Small_Patch16_224.yaml b/mblt_vision/models/ViT_SRelPos_Small_Patch16_224.yaml new file mode 100644 index 0000000..809b7c0 --- /dev/null +++ b/mblt_vision/models/ViT_SRelPos_Small_Patch16_224.yaml @@ -0,0 +1,14 @@ +# Approval HF commit: not recorded. Do not substitute upstream source-model commits. +# Preprocessing: mblt-model-ops models/image_classification/ViT_SRelPos_Small_Patch16_224/_source/predict_float.input.json +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/ViT_SRelPos_Small_Patch16_224 + filename: vit_srelpos_small_patch16_224.mxq + revision: main + pre_cfg: + Normalize: + style: tf + Resize: + interpolation: bicubic + size: 248 diff --git a/mblt_vision/models/bases/imagenet_classifier.yaml b/mblt_vision/models/bases/imagenet_classifier.yaml new file mode 100644 index 0000000..bc4b08b --- /dev/null +++ b/mblt_vision/models/bases/imagenet_classifier.yaml @@ -0,0 +1,19 @@ +# Shared ordered ImageNet preprocessing; model-specific values follow recorded ops inputs. +DEFAULT: + pre_cfg: + Reader: + style: pil + Resize: + size: 224 + interpolation: bicubic + CenterCrop: + size: + - 224 + - 224 + SetOrder: + shape: HWC + Normalize: + style: torch + post_cfg: + task: image_classification + dataset: imagenet diff --git a/tests/test_config_inheritance.py b/tests/test_config_inheritance.py index a36907b..7b561d4 100644 --- a/tests/test_config_inheritance.py +++ b/tests/test_config_inheritance.py @@ -54,7 +54,9 @@ def legacy_config(document: dict, variant: str) -> dict: CONFIG_VARIANTS = [ (path, variant) for path in sorted(MODEL_CONFIG_DIR.glob("*.yaml")) - for variant in yaml.safe_load(path.read_text(encoding="utf-8")) + for variant, definition in yaml.safe_load(path.read_text(encoding="utf-8")).items() + # Only pre-existing alias/update configurations have a legacy counterpart. + if not isinstance(definition, dict) or "extends" not in definition ] diff --git a/tests/test_published_classifiers.py b/tests/test_published_classifiers.py new file mode 100644 index 0000000..27d23b3 --- /dev/null +++ b/tests/test_published_classifiers.py @@ -0,0 +1,112 @@ +"""CPU processing and discovery contracts for additional published classifiers.""" + +from __future__ import annotations + +import numpy as np +import pytest +from PIL import Image + +import mblt_vision +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.wrapper import resolve_model_config + +# Model-specific values from the recorded ops predict_float inputs and distribution +# contracts at 6fdbf471a3bd6ec22035a42c8730fe88a68881e4. Keep the surprising TF +# normalization of ViT_Large_Patch14_CLIP_224: names do not define transforms. +SPECS = { + "EfficientViT_B3": (235, 224, "torch"), + "EfficientViT_L1": (224, 224, "torch"), + "EfficientViT_L2": (224, 224, "torch"), + "EfficientViT_L3": (224, 224, "torch"), + "LeViT_128": (248, 224, "torch"), + "LeViT_128S": (248, 224, "torch"), + "LeViT_192": (248, 224, "torch"), + "LeViT_256": (248, 224, "torch"), + "LeViT_384": (248, 224, "torch"), + "ViT_RelPos_Medium_Patch16_RPN_224": (248, 224, "tf"), + "ViT_RelPos_Base_Patch32_Plus_RPN_256": (284, 256, "tf"), + "ViT_Base_Patch16_RPN_224": (248, 224, "tf"), + "ViT_Base_Patch16_CLIP_224": (235, 224, "openai"), + "ViT_Base_Patch32_CLIP_448": (448, 448, "openai"), + "ViT_Medium_Patch16_GAP_256": (269, 256, "tf"), + "ViT_Large_Patch14_CLIP_224": (224, 224, "tf"), + "ViT_Base_Patch32_CLIP_224": (248, 224, "openai"), + "ViT_RelPos_Small_Patch16_224": (248, 224, "tf"), + "ViT_Base_Patch32_CLIP_384": (384, 384, "openai"), + "ViT_SRelPos_Medium_Patch16_224": (248, 224, "tf"), + "ViT_Huge_Patch14_CLIP_224": (224, 224, "openai"), + "ViT_RelPos_Base_Patch16_224": (248, 224, "tf"), + "ViT_RelPos_Medium_Patch16_224": (248, 224, "tf"), + "ViT_SRelPos_Small_Patch16_224": (248, 224, "tf"), +} +STATS = { + "torch": ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + "tf": ([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]), + "openai": ( + [0.48145466, 0.4578275, 0.40821073], + [0.26862954, 0.26130258, 0.27577711], + ), +} + + +@pytest.mark.parametrize("model", SPECS) +def test_configs_match_recorded_ops_inputs_and_discover(model: str) -> None: + resize, crop, style = SPECS[model] + config = resolve_model_config(model) + assert config["pre_cfg"] == { + "Reader": {"style": "pil"}, + "Resize": {"size": resize, "interpolation": "bicubic"}, + "CenterCrop": {"size": [crop, crop]}, + "SetOrder": {"shape": "HWC"}, + "Normalize": {"style": style}, + } + assert list(config["pre_cfg"]) == [ + "Reader", + "Resize", + "CenterCrop", + "SetOrder", + "Normalize", + ] + assert config["post_cfg"] == {"task": "image_classification", "dataset": "imagenet"} + assert config["file_cfg"]["repo_id"] == f"mobilint/{model}" + assert config["file_cfg"]["revision"] == "main" + assert ( + model in mblt_vision.list_models("image_classification")["image_classification"] + ) + assert getattr(mblt_vision, model) is getattr( + mblt_vision.image_classification, model + ) + + +@pytest.mark.parametrize("model", SPECS) +@pytest.mark.parametrize("shape", [(277, 391), (399, 271), (257, 257)]) +def test_cpu_numeric_parity_with_ops_adapter_math( + model: str, shape: tuple[int, int], tmp_path +) -> None: + resize, crop, style = SPECS[model] + image = Image.fromarray( + np.random.default_rng(0).integers(0, 256, (*shape, 3), dtype=np.uint8) + ) + source = tmp_path / "input.png" + image.save(source) + # Independent equivalent of ops customer_preprocessing_imagenet_torch.py; + # TF/OpenAI adapters use the same resize/crop with their own recorded stats. + width, height = image.size + target = ( + (resize, int(resize * height / width)) + if width <= height + else (int(resize * width / height), resize) + ) + image = image.resize(target, Image.Resampling.BICUBIC) + left, top = round((image.width - crop) / 2), round((image.height - crop) / 2) + reference = ( + np.asarray(image.crop((left, top, left + crop, top + crop)), dtype=np.float32) + / 255 + ) + mean, std = STATS[style] + reference = (reference - np.asarray(mean, dtype=np.float32)) / np.asarray( + std, dtype=np.float32 + ) + actual = build_preprocess(resolve_model_config(model)["pre_cfg"])(str(source)) + assert actual.dtype == np.float32 + np.testing.assert_allclose(actual, reference, rtol=0, atol=5e-7) From b1dbd45926940fca4edec730ca07efe245d2bf8e Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:42:12 +0900 Subject: [PATCH 05/13] Require local ONNX for MXQ-only published classifier repositories Keep existing marker-free configs unchanged and preserve Hub MXQ loading. Skip only marked ONNX in metadata verification, with positive and negative CPU guard tests. Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 4 +- AGENTS.md | 2 + mblt_vision/README.md | 5 ++ mblt_vision/_hub_validation.py | 16 +++- .../models/bases/imagenet_classifier.yaml | 2 + mblt_vision/wrapper.py | 5 ++ tests/test_onnx_local_artifact.py | 76 +++++++++++++++++++ tests/test_published_classifiers.py | 1 + 8 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 tests/test_onnx_local_artifact.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 623d61b..5de5a20 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -30,7 +30,9 @@ description: >- - Every post_cfg declares dataset; resolve output taxonomy from the dataset/task pair. - Additional published EfficientViT/LeViT/ViT classifiers use `bases/imagenet_classifier.yaml`. Preserve the recorded ops transforms checked by `tests/test_published_classifiers.py`, - including the TF-normalized `ViT_Large_Patch14_CLIP_224` exception. + including the TF-normalized `ViT_Large_Patch14_CLIP_224` exception. Their MXQ-only + repositories use `onnx_local_artifact_only: true`: require trusted local ONNX and + retain automatic MXQ loading; metadata validation skips only the marked framework. - For Hub linkage changes, run `tests/test_hub_validation.py` as described in `tests/TEST.md`. The metadata checker follows runtime board paths without downloading weights; live commits are diagnostics, not substitutes for missing historical approval provenance. diff --git a/AGENTS.md b/AGENTS.md index 05bd139..411570e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,8 @@ The current ownership boundary is deliberate: `bases/imagenet_classifier.yaml`. Their exact recorded preprocessing and dataset connection are pinned by `tests/test_published_classifiers.py`; preserve TF normalization for `ViT_Large_Patch14_CLIP_224` rather than inferring normalization from its name. + These 24 repositories are MXQ-only: their base uses `onnx_local_artifact_only: true`. + Keep automatic MXQ loading enabled, require trusted local ONNX, and test both guard paths. - To share processing defaults without changing legacy models, opt new family YAMLs into `extends: bases/family.yaml#DEFAULT`. See the configuration-inheritance section of `mblt_vision/README.md` for ordering and merge rules. Keep legacy `update` shallow and diff --git a/mblt_vision/README.md b/mblt_vision/README.md index f3dfecd..3e3c8bf 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -118,6 +118,11 @@ All 24 additions retain the existing `revision: main` convention because their historical approved artifact HF commits are not recorded; current Hub tips and upstream source-model commits are not substituted as approval pins. This is a configuration/CPU processing parity addition, not a new inference accuracy claim. +All 24 Hub repositories expose MXQ only, with no published ONNX file. Their base +sets `file_cfg.onnx_local_artifact_only: true`: MXQ Hub loading remains enabled, +but ONNX requires a trusted local `model_path` or `onnx_path`. Automatic ONNX +loading fails with a specific message, and metadata verification skips only that +framework. Existing configurations without this marker retain their behavior. | Model | Input Size
(H,W,C) | AccTop1
(NPU) | AccTop1
(GPU) | FLOPs (B) | params (M) | Source | Note | | --- | --- | --- | --- | --- | --- | --- | --- | diff --git a/mblt_vision/_hub_validation.py b/mblt_vision/_hub_validation.py index d1f136d..901373b 100644 --- a/mblt_vision/_hub_validation.py +++ b/mblt_vision/_hub_validation.py @@ -69,7 +69,13 @@ def check_hub_artifacts( paths.append( (framework, f"{board}/{filename}" if framework == "mxq" else filename) ) - if files.get("local_artifact_only"): + + def is_local_only(framework: str) -> bool: + return bool(files.get("local_artifact_only")) or ( + framework == "onnx" and bool(files.get("onnx_local_artifact_only")) + ) + + if all(is_local_only(framework) for framework, _ in paths): return tuple( HubArtifactCheck( repo_id, revision, None, framework, path, "local_artifact_only" @@ -89,10 +95,14 @@ def check_hub_artifacts( HubArtifactCheck( repo_id, revision, - info.sha, + None if is_local_only(framework) else info.sha, framework, path, - "resolved" if path in available else "missing", + "local_artifact_only" + if is_local_only(framework) + else "resolved" + if path in available + else "missing", ) for framework, path in paths ) diff --git a/mblt_vision/models/bases/imagenet_classifier.yaml b/mblt_vision/models/bases/imagenet_classifier.yaml index bc4b08b..ab0129e 100644 --- a/mblt_vision/models/bases/imagenet_classifier.yaml +++ b/mblt_vision/models/bases/imagenet_classifier.yaml @@ -1,5 +1,7 @@ # Shared ordered ImageNet preprocessing; model-specific values follow recorded ops inputs. DEFAULT: + file_cfg: + onnx_local_artifact_only: true pre_cfg: Reader: style: pil diff --git a/mblt_vision/wrapper.py b/mblt_vision/wrapper.py index d451284..fa5adb3 100644 --- a/mblt_vision/wrapper.py +++ b/mblt_vision/wrapper.py @@ -739,6 +739,11 @@ def file_config_cleansing(self) -> None: ) if onnx_filename and framework == "onnx" and not self.file_cfg.get("onnx_path"): + if self.file_cfg.get("onnx_local_artifact_only"): + raise RuntimeError( + f"'{repo_id}' publishes MXQ artifacts only. ONNX requires a trusted local " + "model_path or onnx_path; automatic ONNX downloads are disabled." + ) self.file_cfg["onnx_path"] = self._download_hub_artifact( repo_id=repo_id, filename=onnx_filename, diff --git a/tests/test_onnx_local_artifact.py b/tests/test_onnx_local_artifact.py new file mode 100644 index 0000000..342dcaf --- /dev/null +++ b/tests/test_onnx_local_artifact.py @@ -0,0 +1,76 @@ +"""Positive and negative CPU tests for framework-specific local-only artifacts.""" + +from __future__ import annotations + +from unittest.mock import Mock + +import pytest + +from mblt_vision._hub_validation import check_hub_artifacts +from mblt_vision.wrapper import MBLT_Engine, resolve_model_config + + +def engine_config(framework): + engine = object.__new__(MBLT_Engine) + engine.framework = framework + engine.file_cfg = resolve_model_config("EfficientViT_B3")["file_cfg"] + engine._download_hub_artifact = Mock(return_value="/downloaded/artifact") + return engine + + +def test_marked_remote_onnx_fails_without_download() -> None: + engine = engine_config("onnx") + with pytest.raises(RuntimeError, match="trusted local"): + engine.file_config_cleansing() + engine._download_hub_artifact.assert_not_called() + + +def test_marked_local_onnx_still_works(tmp_path) -> None: + engine = engine_config("onnx") + source = tmp_path / "trusted.onnx" + source.write_bytes(b"local fixture; runtime not constructed") + engine.file_cfg["onnx_path"] = str(source) + engine.file_config_cleansing() + assert engine.file_cfg["onnx_path"] == str(source) + engine._download_hub_artifact.assert_not_called() + + +def test_marked_mxq_keeps_hub_download() -> None: + engine = engine_config("mxq") + engine.file_config_cleansing() + engine._download_hub_artifact.assert_called_once_with( + repo_id="mobilint/EfficientViT_B3", + filename="efficientvit_b3.mxq", + revision="main", + subfolders=["aries-rb"], + ) + + +def test_unmarked_remote_onnx_keeps_hub_download() -> None: + engine = engine_config("onnx") + engine.file_cfg.pop("onnx_local_artifact_only") + engine.file_config_cleansing() + engine._download_hub_artifact.assert_called_once_with( + repo_id="mobilint/EfficientViT_B3", + filename="efficientvit_b3.onnx", + revision="main", + ) + + +def test_verifier_skips_only_marked_framework() -> None: + api = Mock() + api.model_info.return_value.sha = "observed" + api.list_repo_files.return_value = ["aries-rb/efficientvit_b3.mxq"] + checks = check_hub_artifacts(resolve_model_config("EfficientViT_B3"), api=api) + assert [item.status for item in checks] == ["resolved", "local_artifact_only"] + assert checks[1].commit is None + assert api.model_info.call_count == 1 + + +def test_onnx_only_verification_stays_offline() -> None: + api = Mock() + checks = check_hub_artifacts( + resolve_model_config("EfficientViT_B3"), frameworks=("onnx",), api=api + ) + assert checks[0].status == "local_artifact_only" + assert not api.mock_calls diff --git a/tests/test_published_classifiers.py b/tests/test_published_classifiers.py index 27d23b3..c1439c3 100644 --- a/tests/test_published_classifiers.py +++ b/tests/test_published_classifiers.py @@ -70,6 +70,7 @@ def test_configs_match_recorded_ops_inputs_and_discover(model: str) -> None: assert config["post_cfg"] == {"task": "image_classification", "dataset": "imagenet"} assert config["file_cfg"]["repo_id"] == f"mobilint/{model}" assert config["file_cfg"]["revision"] == "main" + assert config["file_cfg"]["onnx_local_artifact_only"] is True assert ( model in mblt_vision.list_models("image_classification")["image_classification"] ) From 81a3727893009bbb65a667e6d10ba5e6f30bd148 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:46:47 +0900 Subject: [PATCH 06/13] Connect retrieval embeddings to engine results and dataset readiness Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/README.md | 7 ++- mblt_vision/person_reid/__init__.py | 3 +- mblt_vision/utils/datasets/readiness.py | 26 +++++++++++ mblt_vision/utils/results.py | 27 ++++++++++- tests/test_reid.py | 60 +++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 3 deletions(-) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 6249bc9..25bccab 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -514,7 +514,12 @@ by default; caller-owned subsets must explicitly set `validate_counts=False`. Only Market-1501 drops pid -1 junk; pid 0 background distractors remain. Duke loaders reject pid -1. Root defaults use `~/.mblt_model_zoo/datasets//eval`. -`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`. It validates one +`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`; both are exported +from `mblt_vision.person_reid`. Engine postprocessing returns a `Results` object +whose `embeddings` and `output` retain the validated `(N,D)` matrix. +`dataset_ready(root, "person_reid", dataset)` checks source counts/metadata and +decodes every retained query/gallery image (default dataset: Market-1501). +`PersonReIDPost` It validates one floating-point `(N,D)` output or singleton-spatial NCHW/NHWC tensor, optionally checks `embedding_dim`, and uses `normalize: true` for row-wise L2 normalization (zero vectors remain zero). Return type is a float32 torch matrix, never class diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py index 07fecad..e5e6574 100644 --- a/mblt_vision/person_reid/__init__.py +++ b/mblt_vision/person_reid/__init__.py @@ -15,11 +15,12 @@ RetrievalResult, extract_embeddings, ) -from ..utils.postprocess.embedding_post import PersonReIDPost +from ..utils.postprocess.embedding_post import EmbeddingPost, PersonReIDPost __all__ = [ "DukeMTMCReIDDataset", "DukeMTMCReIDEvaluator", + "EmbeddingPost", "Market1501Dataset", "Market1501Evaluator", "OccludedDukeDataset", diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index 7fd18b4..8426389 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -20,6 +20,7 @@ from ..._tasks import normalize_vision_task from ...datasets import get_dataset_category_ids from .cityscapes import CITYSCAPES_SOURCE_TO_TRAIN_ID +from .reid import DukeMTMCReIDDataset, Market1501Dataset, OccludedDukeDataset IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} IMAGENET_CLASS_COUNT = 1000 @@ -1044,6 +1045,29 @@ def dense_dataset_ready(data_path: str | Path, dataset: str) -> bool: return False +def _reid_ready(root: Path, dataset: str) -> bool: + """Require source counts, camera/name metadata and decodable query/gallery crops.""" + loader = { + "market1501": Market1501Dataset, + "dukemtmc-reid": DukeMTMCReIDDataset, + "occluded-duke": OccludedDukeDataset, + }.get(dataset) + if loader is None: + return False + try: + samples = loader(root).samples + for sample in samples: + if not sample.image.is_file() or sample.image.is_symlink(): + return False + with Image.open(sample.image) as image: + image.load() + if min(image.size) <= 0: + return False + except (OSError, ValueError): + return False + return True + + def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) -> bool: """Return whether an organized dataset matches its task, taxonomy, and full validation split. @@ -1069,6 +1093,8 @@ def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) }.get(normalized_task) normalized_dataset = (dataset or expected_dataset or "").lower() + if normalized_task == "person_reid": + return _reid_ready(root, normalized_dataset or "market1501") if normalized_task == "semantic_segmentation": return dense_dataset_ready(root, normalized_dataset or "ade20k") if expected_dataset is None or normalized_dataset != expected_dataset: diff --git a/mblt_vision/utils/results.py b/mblt_vision/utils/results.py index 8bbed17..7a5b7f6 100644 --- a/mblt_vision/utils/results.py +++ b/mblt_vision/utils/results.py @@ -73,6 +73,7 @@ def __init__( self.task = normalize_vision_task(post_cfg["task"]) self.conf_thres = kwargs.get("conf_thres", 0.25) self.acc: torch.Tensor | np.ndarray | None = None + self.embeddings: torch.Tensor | np.ndarray | None = None self.box_cls: torch.Tensor | np.ndarray | None = None self.mask: torch.Tensor | np.ndarray | None = None self.depth: torch.Tensor | np.ndarray | list[TensorLike] | None = None @@ -142,11 +143,35 @@ def set_output( NotImplementedError: If the task is not supported. """ self.acc = None + self.embeddings = None self.box_cls = None self.mask = None self.depth = None self.semantic_mask = None - if self.task == "image_classification": + if self.task == "person_reid": + if not isinstance(output, (np.ndarray, torch.Tensor)): + raise TypeError("Person ReID results require an embedding matrix") + finite = ( + torch.isfinite(output).all() + if isinstance(output, torch.Tensor) + else np.isfinite(output).all() + ) + floating = ( + output.is_floating_point() + if isinstance(output, torch.Tensor) + else np.issubdtype(output.dtype, np.floating) + ) + if ( + output.ndim != 2 + or min(output.shape) <= 0 + or not floating + or not bool(finite) + ): + raise ValueError( + "Person ReID results require a finite floating-point (N,D) matrix" + ) + self.embeddings = output + elif self.task == "image_classification": if not isinstance(output, (np.ndarray, torch.Tensor)): raise TypeError( f"Expected tensor output for task {self.task}, got {type(output).__name__}." diff --git a/tests/test_reid.py b/tests/test_reid.py index a6c0fcc..bfbab10 100644 --- a/tests/test_reid.py +++ b/tests/test_reid.py @@ -214,3 +214,63 @@ def test_family_base_inheritance(tmp_path): build_postprocess(cfg["pre_cfg"], cfg["post_cfg"]), PersonReIDPost ) assert build_preprocess(cfg["pre_cfg"]) + + +def test_public_base_and_engine_result_path(): + from mblt_vision.person_reid import EmbeddingPost as PublicEmbeddingPost + from mblt_vision.utils.results import Results + from mblt_vision.wrapper import MBLT_Engine + + assert PublicEmbeddingPost is EmbeddingPost + engine = MBLT_Engine.__new__(MBLT_Engine) + engine.pre_cfg = {} + engine.post_cfg = { + "task": "person_reid", + "dataset": "market1501", + "normalize": True, + } + engine.postprocessor = build_postprocess(engine.pre_cfg, engine.post_cfg) + result = engine.postprocess([np.array([[3.0, 4.0]], dtype=np.float32)]) + assert isinstance(result, Results) + torch.testing.assert_close(result.embeddings, torch.tensor([[0.6, 0.8]])) + assert result.output is result.embeddings + assert result.acc is None and result.box_cls is None + replacement = torch.tensor([[1.0, 0.0]]) + result.set_output(replacement) + assert result.embeddings is replacement + for invalid in ( + torch.ones(2), + torch.ones((1, 2), dtype=torch.int64), + torch.tensor([[float("nan")]]), + ): + with pytest.raises(ValueError): + result.set_output(invalid) + with pytest.raises(TypeError): + result.set_output([replacement]) + + +@pytest.mark.parametrize("name", ["market1501", "dukemtmc-reid", "occluded-duke"]) +def test_reid_readiness(tmp_path, monkeypatch, name): + from mblt_vision.utils.datasets import readiness, reid + from PIL import Image + + original = reid.get_dataset_config_for_task + + def tiny_config(task, dataset): + cfg = original(task, dataset) + for split in cfg["retrieval_splits"].values(): + split["count"] = 1 + return cfg + + monkeypatch.setattr(reid, "get_dataset_config_for_task", tiny_config) + assert not readiness.dataset_ready(tmp_path, "person_reid", name) + for folder in ("query", "bounding_box_test"): + (tmp_path / folder).mkdir() + Image.new("RGB", (4, 8)).save(tmp_path / folder / "0001_c1_sample.jpg") + assert readiness.dataset_ready(tmp_path, "person_reid", name) + if name == "market1501": + assert readiness.dataset_ready(tmp_path, "person_reid") + assert not readiness.dataset_ready(tmp_path, "person_reid", "coco") + image = tmp_path / "query" / "0001_c1_sample.jpg" + image.write_bytes(b"not an image") + assert not readiness.dataset_ready(tmp_path, "person_reid", name) From 31ddef1a103793455476efbb0b082e00169b16be Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:49:53 +0900 Subject: [PATCH 07/13] Honor ONNX-only policy across backend and compile paths Exclude configuration-only marker from MXQ runtime kwargs, guard compilation downloads and cover local/unmarked positive paths. Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/compile/vision.py | 6 ++++++ mblt_vision/wrapper.py | 1 + tests/test_onnx_local_artifact.py | 35 +++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/mblt_vision/compile/vision.py b/mblt_vision/compile/vision.py index 50560b2..fb2fb98 100644 --- a/mblt_vision/compile/vision.py +++ b/mblt_vision/compile/vision.py @@ -891,6 +891,12 @@ def _resolve_compile_onnx_path( "have a repository-pinned immutable revision and SHA-256 digest. Supply a " "trusted local model_path or onnx_path instead." ) + if file_cfg.get("onnx_local_artifact_only"): + repository = repo_id if isinstance(repo_id, str) and repo_id else "this model" + raise RuntimeError( + f"'{repository}' publishes MXQ artifacts only. ONNX compilation requires a trusted " + "local model_path or onnx_path; automatic ONNX downloads are disabled." + ) if isinstance(repo_id, str) and isinstance(revision, str) and filename: downloaded_path = Path( hf_hub_download( diff --git a/mblt_vision/wrapper.py b/mblt_vision/wrapper.py index fa5adb3..39f8dd0 100644 --- a/mblt_vision/wrapper.py +++ b/mblt_vision/wrapper.py @@ -628,6 +628,7 @@ def _mxq_backend_kwargs(self) -> dict[str, Any]: "revision", "onnx_filename", "onnx_path", + "onnx_local_artifact_only", } return { key: value diff --git a/tests/test_onnx_local_artifact.py b/tests/test_onnx_local_artifact.py index 342dcaf..a26a5d1 100644 --- a/tests/test_onnx_local_artifact.py +++ b/tests/test_onnx_local_artifact.py @@ -7,6 +7,7 @@ import pytest from mblt_vision._hub_validation import check_hub_artifacts +from mblt_vision.compile import vision as compile_module from mblt_vision.wrapper import MBLT_Engine, resolve_model_config @@ -44,6 +45,9 @@ def test_marked_mxq_keeps_hub_download() -> None: revision="main", subfolders=["aries-rb"], ) + kwargs = engine._mxq_backend_kwargs() + assert "onnx_local_artifact_only" not in kwargs + assert kwargs["mxq_path"] == "/downloaded/artifact" def test_unmarked_remote_onnx_keeps_hub_download() -> None: @@ -74,3 +78,34 @@ def test_onnx_only_verification_stays_offline() -> None: ) assert checks[0].status == "local_artifact_only" assert not api.mock_calls + + +def test_compile_resolver_requires_local_onnx(monkeypatch, tmp_path) -> None: + download = Mock() + monkeypatch.setattr(compile_module, "hf_hub_download", download) + config = resolve_model_config("EfficientViT_B3")["file_cfg"] + with pytest.raises(RuntimeError, match="trusted local"): + compile_module._resolve_compile_onnx_path(config, None) + download.assert_not_called() + local = tmp_path / "trusted.onnx" + local.write_bytes(b"fixture") + assert compile_module._resolve_compile_onnx_path(config, local) == local.resolve() + assert ( + compile_module._resolve_compile_onnx_path( + {**config, "onnx_path": str(local)}, None + ) + == local.resolve() + ) + download.assert_not_called() + + +def test_compile_unmarked_config_retains_download(monkeypatch, tmp_path) -> None: + local = tmp_path / "downloaded.onnx" + local.write_bytes(b"fixture") + download = Mock(return_value=str(local)) + monkeypatch.setattr(compile_module, "hf_hub_download", download) + monkeypatch.setattr(compile_module, "get_mobilint_cache_dir", lambda: str(tmp_path)) + config = resolve_model_config("EfficientViT_B3")["file_cfg"] + config.pop("onnx_local_artifact_only") + assert compile_module._resolve_compile_onnx_path(config, None) == local.resolve() + download.assert_called_once() From 636c39058b3e70b9955de6e6f7f980b217f806b6 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:56:38 +0900 Subject: [PATCH 08/13] Add aligned face pair datasets and source-exact fold verification Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 4 + AGENTS.md | 4 + benchmark/benchmark_vision_models.py | 2 +- mblt_vision/README.md | 43 ++++ mblt_vision/__init__.py | 2 + mblt_vision/_tasks.py | 1 + mblt_vision/datasets/agedb-30.yaml | 34 +++ mblt_vision/datasets/cfp-fp.yaml | 36 +++ mblt_vision/datasets/lfw.yaml | 34 +++ mblt_vision/face_recognition/__init__.py | 40 ++++ .../models/bases/face_recognition.yaml | 18 ++ mblt_vision/utils/datasets/face_pairs.py | 131 +++++++++++ mblt_vision/utils/datasets/readiness.py | 22 ++ mblt_vision/utils/evaluation/verification.py | 160 +++++++++++++ mblt_vision/utils/postprocess/__init__.py | 11 +- mblt_vision/utils/postprocess/build_post.py | 4 +- .../utils/postprocess/embedding_post.py | 11 + mblt_vision/utils/results.py | 2 +- tests/reference/ops_face_preprocess.py | 57 +++++ tests/reference/ops_face_verification.py | 75 ++++++ tests/test_api.py | 1 + tests/test_face_verification.py | 218 ++++++++++++++++++ 22 files changed, 905 insertions(+), 5 deletions(-) create mode 100644 mblt_vision/datasets/agedb-30.yaml create mode 100644 mblt_vision/datasets/cfp-fp.yaml create mode 100644 mblt_vision/datasets/lfw.yaml create mode 100644 mblt_vision/face_recognition/__init__.py create mode 100644 mblt_vision/models/bases/face_recognition.yaml create mode 100644 mblt_vision/utils/datasets/face_pairs.py create mode 100644 mblt_vision/utils/evaluation/verification.py create mode 100644 tests/reference/ops_face_preprocess.py create mode 100644 tests/reference/ops_face_verification.py create mode 100644 tests/test_face_verification.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 7d563ab..1d40ab8 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -114,6 +114,10 @@ description: >- and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. +- Face verification uses prepared 112x112 crops without realignment or implicit flip fusion. + Keep pair/fold order, source threshold search, Results embeddings and readiness connected; + require exact metric and preprocessing parity (face verification API section). + ## Runtime and Packaging - Route NPU runtime access through mblt-npu-python; do not copy backend classes into Vision. diff --git a/AGENTS.md b/AGENTS.md index 5c3c614..7e45efd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -186,6 +186,10 @@ The current ownership boundary is deliberate: stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. Named Normalize styles retain legacy arithmetic; explicit mean/std uses source float32 arithmetic. +- Face verification utilities preserve prepared alignment and official pair/fold order. + Use dataset-bound `FaceRecognitionPost`, `Results.embeddings`, and face readiness checks. + Protocol details and source-parity requirements are in the face verification API section. + ## Python-First Architecture - Keep the public layer independent from a particular backend. Define small internal interfaces diff --git a/benchmark/benchmark_vision_models.py b/benchmark/benchmark_vision_models.py index 06f86e3..62fb7ec 100644 --- a/benchmark/benchmark_vision_models.py +++ b/benchmark/benchmark_vision_models.py @@ -36,7 +36,7 @@ tuple[CoreMode, ...], core_modes_for_target_device("aries-rb") ) # Utility-only embedding tasks have no benchmark engine/CLI route yet. -TASK_CHOICES = tuple(task for task in VISION_TASKS if task != "person_reid") +TASK_CHOICES = tuple(task for task in VISION_TASKS if task not in {"person_reid", "face_recognition"}) SUPPORTED_TARGET_DEVICES = frozenset({"aries-rb", "regulus-ra", "regulus-rb"}) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 25bccab..9faee01 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -543,3 +543,46 @@ Offline tests vendor the inspected ops functions at distance/CMC/mAP equality; synthetic RGB preprocessing requires byte equality (`atol=0`) against the ops adapter for explicit source mean/std profiles. These are protocol tests, not measured model accuracy claims. + +## Face verification utilities + +`mblt_vision.face_recognition` exports `FaceRecognitionPost(EmbeddingPost)`, +`LFWDataset`, `AgeDB30Dataset`, `CFPFPDataset`, and their `LFWEvaluator`, +`AgeDB30Evaluator`, `CFPEvaluator` subclasses of `VerificationEvaluator`. +The processing-only `bases/face_recognition.yaml` profile is 112x112 PIL bilinear, +explicit float32 0.5 mean/std, 512 dimensions, row-wise L2 normalization. Engine +postprocessing stores dense vectors in `Results.embeddings` and `output`. Model +wrappers/artifact YAMLs are a separate stage; CLI/benchmark routing is unchanged. + +Pair loaders read prepared `pairs.csv` columns `pair_id,fold_id,is_same,image_a,image_b`, +retain pair order and official one-based fold labels, and enumerate unique image +basenames in sorted order. LFW uses `aligned/`; AgeDB-30 and CFP-FP use `images/`. +They require existing 112x112 crops and never detect or realign faces. Source pair +counts are checked by default; caller-owned subsets explicitly opt out with +`validate_counts=False`. Default cache roots end in `/eval`. +`dataset_ready(root, "face_recognition", dataset)` additionally requires all ten +folds and decodable crops; its default dataset is LFW. + +```python +from mblt_vision.face_recognition import LFWDataset, LFWEvaluator + +data = LFWDataset("/path/to/lfw/eval") +# embeddings: one row per data.samples, from your inference callback. +# result = LFWEvaluator().evaluate(embeddings, data.samples, data.pairs) +# print(result.as_dict()) +``` + +Ops at `ec537764e36ca10da963fcb7afab157ea60ca022` owns the protocol: one +normalized embedding per crop, **no flip fusion**, float64 cosine scores, threshold +grid `linspace(-1,1,4001)`, inclusive `score >= threshold`, strictest threshold on +training ties, leave-one-fold-out threshold fitting, mean accuracy and population +standard deviation over folds. `extract_embeddings` shares the ordered bounded +inference helper with retrieval. There is no implicit fusion in scoring. + +Offline tests compare exact cosine scores, fold accuracies, thresholds and aggregate +metrics against a verbatim source snapshot over five synthetic seeds. Three RGB +fixtures (including no-op resize) require preprocessing byte equality (`atol=0`). +Dataset YAMLs record exact source paths/revision. CFP-FP specifically refers to +the prepared cat-claws mirror: ops warns it was not checked against the original +`cfp_fp.bin` and records a training-log discrepancy. This package preserves that +provenance and makes no new accuracy claim. diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index 9cf69d2..1f40160 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -9,6 +9,7 @@ from . import depth_estimation as depth_estimation from . import face_detection as face_detection +from . import face_recognition as face_recognition from . import image_classification as image_classification from . import instance_segmentation as instance_segmentation from . import obb as obb @@ -54,6 +55,7 @@ "obb", "pose_estimation", "person_reid", + "face_recognition", "semantic_segmentation", ] + sorted(_LEGACY_MODEL_EXPORTS) # Keep legacy compatibility exports synchronized with their task packages. diff --git a/mblt_vision/_tasks.py b/mblt_vision/_tasks.py index c8a4b6d..310e895 100644 --- a/mblt_vision/_tasks.py +++ b/mblt_vision/_tasks.py @@ -14,6 +14,7 @@ "pose_estimation", "face_detection", "person_reid", + "face_recognition", ) diff --git a/mblt_vision/datasets/agedb-30.yaml b/mblt_vision/datasets/agedb-30.yaml new file mode 100644 index 0000000..f17c8c8 --- /dev/null +++ b/mblt_vision/datasets/agedb-30.yaml @@ -0,0 +1,34 @@ +name: agedb-30 +path: ~/.mblt_model_zoo/datasets/agedb-30 +tasks: +- face_recognition +val: eval +image_directory: images +pairs: 6000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/agedb_30/agedb_30.yaml + name: agedb-30 + dataloader_path: datasets/agedb_30/_private/dataloader.py +eval_protocol: + id: agedb-30-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: agedb-30-ten-fold-verification.v1 + primary_metric: agedb_30_accuracy + secondary_metrics: + - agedb_30_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/agedb_30/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/cfp-fp.yaml b/mblt_vision/datasets/cfp-fp.yaml new file mode 100644 index 0000000..1fb614c --- /dev/null +++ b/mblt_vision/datasets/cfp-fp.yaml @@ -0,0 +1,36 @@ +name: cfp-fp +path: ~/.mblt_model_zoo/datasets/cfp-fp +tasks: +- face_recognition +val: eval +image_directory: images +pairs: 7000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/cfp_fp/cfp_fp.yaml + name: cfp-fp + dataloader_path: datasets/cfp_fp/_private/dataloader.py + caveat: Prepared cat-claws mirror is not checked against original cfp_fp.bin; ops + records lower CFP-FP accuracy than training logs. +eval_protocol: + id: cfp-fp-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: cfp-fp-ten-fold-verification.v1 + primary_metric: cfp_fp_accuracy + secondary_metrics: + - cfp_fp_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/cfp_fp/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/lfw.yaml b/mblt_vision/datasets/lfw.yaml new file mode 100644 index 0000000..7e83377 --- /dev/null +++ b/mblt_vision/datasets/lfw.yaml @@ -0,0 +1,34 @@ +name: lfw +path: ~/.mblt_model_zoo/datasets/lfw +tasks: +- face_recognition +val: eval +image_directory: aligned +pairs: 6000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/lfw/lfw.yaml + name: lfw + dataloader_path: datasets/lfw/_private/dataloader.py +eval_protocol: + id: lfw-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: lfw-ten-fold-verification.v1 + primary_metric: lfw_accuracy + secondary_metrics: + - lfw_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/lfw/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/face_recognition/__init__.py b/mblt_vision/face_recognition/__init__.py new file mode 100644 index 0000000..2d9df56 --- /dev/null +++ b/mblt_vision/face_recognition/__init__.py @@ -0,0 +1,40 @@ +"""Aligned face pair verification utilities; no implicit face detection/alignment.""" + +from ..utils.datasets.face_pairs import ( + AgeDB30Dataset, + CFPFPDataset, + FacePair, + FacePairDataset, + LFWDataset, + PairImage, +) +from ..utils.evaluation.verification import ( + AgeDB30Evaluator, + CFPEvaluator, + LFWEvaluator, + VerificationEvaluator, + VerificationResult, + cosine_scores, + extract_embeddings, + fold_accuracies, +) +from ..utils.postprocess.embedding_post import EmbeddingPost, FaceRecognitionPost + +__all__ = [ + "AgeDB30Dataset", + "AgeDB30Evaluator", + "CFPEvaluator", + "CFPFPDataset", + "EmbeddingPost", + "FacePair", + "FacePairDataset", + "FaceRecognitionPost", + "LFWDataset", + "LFWEvaluator", + "PairImage", + "VerificationEvaluator", + "VerificationResult", + "cosine_scores", + "extract_embeddings", + "fold_accuracies", +] diff --git a/mblt_vision/models/bases/face_recognition.yaml b/mblt_vision/models/bases/face_recognition.yaml new file mode 100644 index 0000000..6545407 --- /dev/null +++ b/mblt_vision/models/bases/face_recognition.yaml @@ -0,0 +1,18 @@ +# Processing only; concrete approved HF artifacts are added separately. +# ops ec537764e36ca10da963fcb7afab157ea60ca022 +# models/face_recognition/ArcFace_IResNet_100/distribution.yaml +DEFAULT: + pre_cfg: + Reader: + style: pil + Resize: + size: [112, 112] + interpolation: bilinear + Normalize: + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + post_cfg: + task: face_recognition + dataset: lfw + embedding_dim: 512 + normalize: true diff --git a/mblt_vision/utils/datasets/face_pairs.py b/mblt_vision/utils/datasets/face_pairs.py new file mode 100644 index 0000000..a72bd66 --- /dev/null +++ b/mblt_vision/utils/datasets/face_pairs.py @@ -0,0 +1,131 @@ +"""Prepared aligned face pairs, preserving source file order and fold labels.""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +from pathlib import Path + +from PIL import Image + +from ...datasets import get_dataset_config_for_task + + +@dataclass(frozen=True) +class FacePair: + """Source basenames, same/different label and official one-based fold.""" + + image_a: str + image_b: str + is_same: bool + fold: int + + +@dataclass(frozen=True) +class PairImage: + """One prepared 112x112 aligned crop, keyed by its source basename.""" + + image: Path + name: str + + +class FacePairDataset: + """Read prepared pairs.csv and already aligned crops without realignment. + + Root contains pairs.csv and the configured aligned/images directory. + Source pair counts are enforced unless validate_counts=False explicitly + selects a caller-owned subset. Basenames, labels and folds are always checked. + """ + + name: str + + def __init__( + self, root: str | Path | None = None, *, validate_counts: bool = True + ) -> None: + self.config = get_dataset_config_for_task("face_recognition", self.name) + self.root = ( + Path(root).expanduser() + if root is not None + else Path(self.config["path"]) / self.config["val"] + ) + path = self.root / "pairs.csv" + if not path.is_file(): + raise FileNotFoundError(f"Pair file is missing: {path}") + pairs = [] + ids = set() + with path.open(newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + columns = {"pair_id", "fold_id", "is_same", "image_a", "image_b"} + if reader.fieldnames is None or not columns <= set(reader.fieldnames): + raise ValueError(f"{path} lacks the pair columns {sorted(columns)}") + for row in reader: + for key in ("image_a", "image_b"): + name = row[key] + if ( + not name + or name in {".", ".."} + or any(c in name for c in "/\\:") + ): + raise ValueError(f"Invalid pair image basename: {name!r}") + if row["is_same"] not in {"0", "1"}: + raise ValueError("Pair is_same must be 0 or 1") + try: + fold, pair_id = int(row["fold_id"]), int(row["pair_id"]) + except (ValueError, TypeError) as exc: + raise ValueError( + "Pair fold_id and pair_id must be integers" + ) from exc + if not 1 <= fold <= 10 or pair_id in ids: + raise ValueError( + "Pair fold must be 1..10 and pair_id must be unique" + ) + ids.add(pair_id) + pairs.append( + FacePair( + row["image_a"], row["image_b"], row["is_same"] == "1", fold + ) + ) + if not pairs or (validate_counts and len(pairs) != self.config["pairs"]): + raise ValueError( + f"Pair file holds {len(pairs)} pairs, expected {self.config['pairs']}" + ) + self.pairs = tuple(pairs) + image_dir = self.root / self.config["image_directory"] + if not image_dir.is_dir(): + raise FileNotFoundError(f"Image directory is missing: {image_dir}") + names = sorted({p.image_a for p in pairs} | {p.image_b for p in pairs}) + images = [] + for name in names: + path = image_dir / name + if not path.is_file(): + raise FileNotFoundError(f"Pair image is missing: {path}") + # Alignment is a dataset preparation contract, never re-estimated here. + with Image.open(path) as image: + if image.size != (112, 112): + raise ValueError(f"Prepared aligned crop must be 112x112: {path}") + images.append(PairImage(path, name)) + self.samples = tuple(images) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> PairImage: + return self.samples[index] + + +class LFWDataset(FacePairDataset): + """LFW official 6000 prepared aligned pairs.""" + + name = "lfw" + + +class AgeDB30Dataset(FacePairDataset): + """Prepared AgeDB-30 mirror pair protocol, without changing its folds.""" + + name = "agedb-30" + + +class CFPFPDataset(FacePairDataset): + """Prepared CFP-FP mirror protocol; source caveats remain in the YAML.""" + + name = "cfp-fp" diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index 8426389..7bda3f7 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -20,6 +20,7 @@ from ..._tasks import normalize_vision_task from ...datasets import get_dataset_category_ids from .cityscapes import CITYSCAPES_SOURCE_TO_TRAIN_ID +from .face_pairs import LFWDataset, AgeDB30Dataset, CFPFPDataset from .reid import DukeMTMCReIDDataset, Market1501Dataset, OccludedDukeDataset IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} @@ -1068,6 +1069,25 @@ def _reid_ready(root: Path, dataset: str) -> bool: return True +def _face_ready(root: Path, dataset: str) -> bool: + """Validate prepared aligned pairs and image decoding, never realign crops.""" + loader = {"lfw": LFWDataset, "agedb-30": AgeDB30Dataset, "cfp-fp": CFPFPDataset}.get(dataset) + if loader is None: + return False + try: + data = loader(root) + if len({pair.fold for pair in data.pairs}) != 10: + return False + for sample in data.samples: + if sample.image.is_symlink(): + return False + with Image.open(sample.image) as image: + image.load() + except (OSError, ValueError): + return False + return True + + def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) -> bool: """Return whether an organized dataset matches its task, taxonomy, and full validation split. @@ -1093,6 +1113,8 @@ def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) }.get(normalized_task) normalized_dataset = (dataset or expected_dataset or "").lower() + if normalized_task == "face_recognition": + return _face_ready(root, normalized_dataset or "lfw") if normalized_task == "person_reid": return _reid_ready(root, normalized_dataset or "market1501") if normalized_task == "semantic_segmentation": diff --git a/mblt_vision/utils/evaluation/verification.py b/mblt_vision/utils/evaluation/verification.py new file mode 100644 index 0000000..8889c5f --- /dev/null +++ b/mblt_vision/utils/evaluation/verification.py @@ -0,0 +1,160 @@ +"""Fold-based face verification, pinned to ops ec537764 protocol arithmetic.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np + +from ..datasets.face_pairs import FacePair, PairImage +from .retrieval import extract_embeddings + +# Source grid includes both endpoints. Predictions use >= and ties choose highest t. +THRESHOLDS = np.linspace(-1.0, 1.0, 4001) + + +def cosine_scores(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Source float64 cosine scores; the norm-product floor is 1e-12.""" + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + if a.ndim != 2 or b.shape != a.shape or min(a.shape) == 0: + raise ValueError("Cosine inputs require matching nonempty (N,D) matrices") + if not np.isfinite(a).all() or not np.isfinite(b).all(): + raise ValueError("Cosine embeddings must be finite") + norms = np.linalg.norm(a, axis=1) * np.linalg.norm(b, axis=1) + return np.sum(a * b, axis=1) / np.maximum(norms, 1e-12) + + +def fold_accuracies( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Fit threshold on all other folds and score the held-out official fold.""" + scores = np.asarray(scores, dtype=np.float64) + labels = np.asarray(labels) + folds = np.asarray(folds) + if ( + scores.ndim != 1 + or scores.size == 0 + or labels.shape != scores.shape + or folds.shape != scores.shape + ): + raise ValueError("Scores, labels and folds must be matching nonempty vectors") + if not np.isfinite(scores).all() or not np.isin(labels, [0, 1]).all(): + raise ValueError("Verification requires finite scores and binary labels") + if ( + not np.issubdtype(folds.dtype, np.integer) + or not np.isin(folds, np.arange(1, 11)).all() + ): + raise ValueError("Verification fold IDs must be integers in 1..10") + labels = np.asarray(labels, dtype=bool) + fold_ids = np.unique(folds) + if len(fold_ids) < 2: + raise ValueError("Verification protocol needs at least two folds") + accuracies = [] + thresholds = [] + for fold in fold_ids: + test = folds == fold + train_scores, train_labels = scores[~test], labels[~test] + # Keep the source accumulation, threshold grid and strictest-tie policy. + fitted = np.array( + [float(np.mean((train_scores >= t) == train_labels)) for t in THRESHOLDS] + ) + best = float(THRESHOLDS[len(fitted) - 1 - int(np.argmax(fitted[::-1]))]) + thresholds.append(best) + accuracies.append(float(np.mean((scores[test] >= best) == labels[test]))) + return np.asarray(accuracies), np.asarray(thresholds) + + +@dataclass(frozen=True) +class VerificationResult: + """Unrounded fold results, with the source benchmark's rounded score view.""" + + accuracies: np.ndarray + thresholds: np.ndarray + sample_count: int + metric: str + + @property + def primary_score(self) -> float: + return float(np.mean(self.accuracies)) + + @property + def secondary_score(self) -> float: + return float(np.std(self.accuracies)) + + @property + def threshold(self) -> float: + return float(np.mean(self.thresholds)) + + def as_dict(self) -> dict: + return { + "primary": {self.metric: round(self.primary_score, 5)}, + "secondary": {self.metric + "_std": round(self.secondary_score, 5)}, + "sample_count": self.sample_count, + } + + +class VerificationEvaluator: + """One embedding per aligned image, no implicit flip/fusion or alignment. + + Source evaluate_embeddings decodes one L2-normalized vector per crop; + source score_pairs then computes cosine similarity, without flip fusion. + ``extract_embeddings`` is reused unchanged from retrieval evaluation. + """ + + metric = "accuracy" + extract_embeddings = staticmethod(extract_embeddings) + + def evaluate( + self, + embeddings: np.ndarray, + images: Sequence[PairImage], + pairs: Sequence[FacePair], + ) -> VerificationResult: + """Score embeddings in canonical image manifest order against source pairs.""" + values = np.asarray(embeddings) + if ( + values.ndim != 2 + or min(values.shape) == 0 + or values.shape[0] != len(images) + or not np.isfinite(values).all() + ): + raise ValueError("Embeddings require one finite nonempty row per image") + names = [image.name for image in images] + if len(set(names)) != len(names): + raise ValueError("Image manifest names must be unique") + if not pairs: + raise ValueError("No verification pairs supplied") + by_name = dict(zip(names, values)) + try: + a = np.stack([by_name[p.image_a] for p in pairs]) + b = np.stack([by_name[p.image_b] for p in pairs]) + except KeyError as exc: + raise ValueError( + f"Pair image missing from embedding manifest: {exc}" + ) from exc + accuracy, thresholds = fold_accuracies( + cosine_scores(a, b), + np.asarray([p.is_same for p in pairs]), + np.asarray([p.fold for p in pairs]), + ) + return VerificationResult(accuracy, thresholds, len(images), self.metric) + + +class LFWEvaluator(VerificationEvaluator): + """LFW official fold accuracy.""" + + metric = "lfw_accuracy" + + +class AgeDB30Evaluator(VerificationEvaluator): + """AgeDB-30 prepared mirror fold accuracy.""" + + metric = "agedb_30_accuracy" + + +class CFPEvaluator(VerificationEvaluator): + """CFP-FP prepared mirror fold accuracy (see source YAML caveat).""" + + metric = "cfp_fp_accuracy" diff --git a/mblt_vision/utils/postprocess/__init__.py b/mblt_vision/utils/postprocess/__init__.py index 879a06f..1455c4b 100644 --- a/mblt_vision/utils/postprocess/__init__.py +++ b/mblt_vision/utils/postprocess/__init__.py @@ -4,7 +4,14 @@ from .build_post import build_postprocess from .depth_post import DepthPost -from .embedding_post import EmbeddingPost, PersonReIDPost +from .embedding_post import EmbeddingPost, FaceRecognitionPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost -__all__ = ["DepthPost", "EmbeddingPost", "PersonReIDPost", "SemanticSegPost", "build_postprocess"] +__all__ = [ + "DepthPost", + "EmbeddingPost", + "FaceRecognitionPost", + "PersonReIDPost", + "SemanticSegPost", + "build_postprocess", +] diff --git a/mblt_vision/utils/postprocess/build_post.py b/mblt_vision/utils/postprocess/build_post.py index eb569ac..203818f 100644 --- a/mblt_vision/utils/postprocess/build_post.py +++ b/mblt_vision/utils/postprocess/build_post.py @@ -9,7 +9,7 @@ from .cls_post import ClsPost from .damoyolo_post import DAMOYOLODetectionPost from .depth_post import DepthPost -from .embedding_post import PersonReIDPost +from .embedding_post import FaceRecognitionPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost from .yolo_anchor_post import ( YOLOAnchorDetectionPost, @@ -71,6 +71,8 @@ def build_postprocess( f"Unsupported post_cfg.head {head!r}; expected one of {sorted(DETECTION_HEADS)}." ) return DETECTION_HEADS[head](pre_cfg, post_cfg, **kwargs) + if task == "face_recognition": + return FaceRecognitionPost(pre_cfg, post_cfg) if task == "person_reid": return PersonReIDPost(pre_cfg, post_cfg) if task == "image_classification": diff --git a/mblt_vision/utils/postprocess/embedding_post.py b/mblt_vision/utils/postprocess/embedding_post.py index 8d55353..4a449a1 100644 --- a/mblt_vision/utils/postprocess/embedding_post.py +++ b/mblt_vision/utils/postprocess/embedding_post.py @@ -83,3 +83,14 @@ def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: if not isinstance(dataset, str) or not dataset: raise ValueError("Person ReID postprocessing requires post_cfg.dataset") self.dataset = get_dataset_config_for_task("person_reid", dataset)["name"] + + +class FaceRecognitionPost(EmbeddingPost): + """One identity embedding per prepared aligned crop; no implicit flip fusion.""" + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__(pre_cfg, post_cfg) + dataset = post_cfg.get("dataset") + if not isinstance(dataset, str) or not dataset: + raise ValueError("Face recognition postprocessing requires post_cfg.dataset") + self.dataset = get_dataset_config_for_task("face_recognition", dataset)["name"] diff --git a/mblt_vision/utils/results.py b/mblt_vision/utils/results.py index 7a5b7f6..0b9e3a2 100644 --- a/mblt_vision/utils/results.py +++ b/mblt_vision/utils/results.py @@ -148,7 +148,7 @@ def set_output( self.mask = None self.depth = None self.semantic_mask = None - if self.task == "person_reid": + if self.task in {"person_reid", "face_recognition"}: if not isinstance(output, (np.ndarray, torch.Tensor)): raise TypeError("Person ReID results require an embedding matrix") finite = ( diff --git a/tests/reference/ops_face_preprocess.py b/tests/reference/ops_face_preprocess.py new file mode 100644 index 0000000..62848d1 --- /dev/null +++ b/tests/reference/ops_face_preprocess.py @@ -0,0 +1,57 @@ +# Verbatim models/face_recognition/_shared/preprocess.py at ec537764e36ca10da963fcb7afab157ea60ca022 +"""Default face_recognition preprocess: an aligned face crop, normalized. + +The benchmarks of this task hand the Model already aligned 112x112 crops (five-point +similarity alignment is part of the prepared evaluation layout), so the Model-side +transform is a guard resize to the network input and an explicit ``mean``/``std`` +normalization, in HWC order. Operations run in canonical order regardless of the +sorted-key order of ``pre_cfg`` on disk. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +PIPELINE_ORDER = ("Reader", "Resize", "SetOrder", "Normalize") +REQUIRED = ("Resize", "Normalize") +_RESAMPLING = { + "bilinear": Image.Resampling.BILINEAR, + "bicubic": Image.Resampling.BICUBIC, +} + + +def preprocess(source: Path, config: dict[str, Any]) -> tuple[Path, np.ndarray, None]: + """Return ``(source, HWC float32 tensor, None)``.""" + unknown = set(config) - set(PIPELINE_ORDER) + if unknown: + raise ValueError(f"unsupported preprocess operations: {sorted(unknown)}") + missing = [name for name in REQUIRED if name not in config] + if missing: + raise ValueError(f"pre_cfg is missing required operations: {missing}") + source = Path(source) + value: Any = Image.open(source).convert("RGB") + for operation in PIPELINE_ORDER: + if operation not in config: + continue + options = config[operation] + if operation == "Reader": + continue + if operation == "Resize": + height, width = (int(v) for v in options["size"]) + if value.size != (width, height): + mode = _RESAMPLING[str(options.get("interpolation", "bilinear"))] + value = value.resize((width, height), mode) + elif operation == "SetOrder": + if options.get("shape", "HWC") != "HWC": + raise ValueError( + "face_recognition preprocess produces HWC tensors only" + ) + elif operation == "Normalize": + mean = np.asarray(options["mean"], dtype=np.float32) + std = np.asarray(options["std"], dtype=np.float32) + value = (np.asarray(value, dtype=np.float32) / 255.0 - mean) / std + return source, np.ascontiguousarray(value, dtype=np.float32), None diff --git a/tests/reference/ops_face_verification.py b/tests/reference/ops_face_verification.py new file mode 100644 index 0000000..73f0eab --- /dev/null +++ b/tests/reference/ops_face_verification.py @@ -0,0 +1,75 @@ +# Verbatim datasets/_shared/face_verification.py at ec537764e36ca10da963fcb7afab157ea60ca022 +"""Pair-based face verification protocol shared by LFW-style benchmarks. + +A benchmark lists pairs of images with a same/different label and a fold id. The Model +provides one L2-normalized embedding per image. For every fold, the decision threshold on +cosine similarity is chosen to maximize accuracy on the other folds and then applied to +the held-out fold; the benchmark reports the mean and standard deviation over folds +(Huang et al., "Labeled Faces in the Wild", unrestricted protocol as implemented by +InsightFace / facenet ``evaluate``). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +THRESHOLDS = np.linspace(-1.0, 1.0, 4001) + + +@dataclass(frozen=True) +class Pair: + image_a: str + image_b: str + is_same: bool + fold: int + + +def cosine_scores(a: np.ndarray, b: np.ndarray) -> np.ndarray: + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + norms = np.linalg.norm(a, axis=1) * np.linalg.norm(b, axis=1) + return np.sum(a * b, axis=1) / np.maximum(norms, 1e-12) + + +def _accuracy(scores: np.ndarray, labels: np.ndarray, threshold: float) -> float: + return float(np.mean((scores >= threshold) == labels)) + + +def fold_accuracies( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Held-out accuracy per fold with the threshold fitted on the remaining folds.""" + scores = np.asarray(scores, dtype=np.float64) + labels = np.asarray(labels, dtype=bool) + folds = np.asarray(folds) + fold_ids = np.unique(folds) + if len(fold_ids) < 2: + raise ValueError("Verification protocol needs at least two folds") + accuracies = [] + thresholds = [] + for fold in fold_ids: + test = folds == fold + train_scores, train_labels = scores[~test], labels[~test] + fitted = np.array( + [_accuracy(train_scores, train_labels, t) for t in THRESHOLDS] + ) + # Ties on training accuracy resolve to the strictest threshold (the highest + # similarity), matching the reference implementations that search distance + # thresholds in ascending order and keep the first maximum. + best = float(THRESHOLDS[len(fitted) - 1 - int(np.argmax(fitted[::-1]))]) + thresholds.append(best) + accuracies.append(_accuracy(scores[test], labels[test], best)) + return np.asarray(accuracies), np.asarray(thresholds) + + +def verification_result( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> dict[str, float]: + accuracies, thresholds = fold_accuracies(scores, labels, folds) + return { + "accuracy": float(np.mean(accuracies)), + "accuracy_std": float(np.std(accuracies)), + "threshold": float(np.mean(thresholds)), + } diff --git a/tests/test_api.py b/tests/test_api.py index ce184aa..924192f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -22,6 +22,7 @@ def test_public_discovery_exposes_all_supported_tasks() -> None: "pose_estimation", "face_detection", "person_reid", + "face_recognition", ] assert list_models("obb")["obb"] diff --git a/tests/test_face_verification.py b/tests/test_face_verification.py new file mode 100644 index 0000000..2d1e60a --- /dev/null +++ b/tests/test_face_verification.py @@ -0,0 +1,218 @@ +"""CPU prepared face-pair, engine and source-exact numeric contracts.""" + +import csv +import importlib.util +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch +from mblt_vision._config import load_model_variant +from mblt_vision.face_recognition import ( + AgeDB30Dataset, + AgeDB30Evaluator, + CFPEvaluator, + CFPFPDataset, + EmbeddingPost, + FacePair, + FaceRecognitionPost, + LFWDataset, + LFWEvaluator, + PairImage, + cosine_scores, + fold_accuracies, +) +from mblt_vision.utils.datasets import face_pairs, readiness +from mblt_vision.utils.postprocess import build_postprocess +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.wrapper import MBLT_Engine +from PIL import Image + + +def reference(name): + path = Path(__file__).parent / "reference" / f"ops_face_{name}.py" + spec = importlib.util.spec_from_file_location(f"ops_face_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def write_pairs(root, folder, rows): + (root / folder).mkdir(exist_ok=True) + for name in {r[3] for r in rows} | {r[4] for r in rows}: + if "/" not in name: + Image.new("RGB", (112, 112)).save(root / folder / name) + with (root / "pairs.csv").open("w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["pair_id", "fold_id", "is_same", "image_a", "image_b"]) + writer.writerows(rows) + + +@pytest.mark.parametrize( + "loader,folder", + [(LFWDataset, "aligned"), (AgeDB30Dataset, "images"), (CFPFPDataset, "images")], +) +def test_prepared_pairs_preserve_order_folds_and_alignment(tmp_path, loader, folder): + rows = [(9, 3, 0, "b.png", "a.png"), (2, 1, 1, "a.png", "a.png")] + write_pairs(tmp_path, folder, rows) + data = loader(tmp_path, validate_counts=False) + assert [p.fold for p in data.pairs] == [3, 1] + assert [p.is_same for p in data.pairs] == [False, True] + assert [s.name for s in data.samples] == ["a.png", "b.png"] + assert isinstance(data[0], PairImage) and len(data) == 2 + with pytest.raises(ValueError, match="expected"): + loader(tmp_path) + Image.new("RGB", (113, 112)).save(tmp_path / folder / "a.png") + with pytest.raises(ValueError, match="112x112"): + loader(tmp_path, validate_counts=False) + + +@pytest.mark.parametrize( + "bad", + [ + (1, 0, 1, "a.png", "b.png"), + (1, 11, 1, "a.png", "b.png"), + (1, 1, 2, "a.png", "b.png"), + (1, 1, 1, "../a.png", "b.png"), + ], +) +def test_invalid_pair_metadata(tmp_path, bad): + write_pairs(tmp_path, "aligned", [bad]) + with pytest.raises(ValueError): + LFWDataset(tmp_path, validate_counts=False) + + +def test_duplicate_missing_and_empty_pairs(tmp_path): + with pytest.raises(FileNotFoundError): + LFWDataset(tmp_path) + write_pairs(tmp_path, "aligned", [(1, 1, 1, "a.png", "b.png")] * 2) + with pytest.raises(ValueError, match="unique"): + LFWDataset(tmp_path, validate_counts=False) + write_pairs(tmp_path, "aligned", [(1, 1, 1, "a.png", "b.png")]) + (tmp_path / "aligned/b.png").unlink() + with pytest.raises(FileNotFoundError): + LFWDataset(tmp_path, validate_counts=False) + + +def test_engine_embedding_path(): + assert issubclass(FaceRecognitionPost, EmbeddingPost) + engine = MBLT_Engine.__new__(MBLT_Engine) + engine.pre_cfg = {} + engine.post_cfg = { + "task": "face_recognition", + "dataset": "lfw", + "normalize": True, + "embedding_dim": 2, + } + engine.postprocessor = build_postprocess(engine.pre_cfg, engine.post_cfg) + assert isinstance(engine.postprocessor, FaceRecognitionPost) + result = engine.postprocess([np.array([[3.0, 4.0]], dtype=np.float32)]) + torch.testing.assert_close(result.embeddings, torch.tensor([[0.6, 0.8]])) + assert result.output is result.embeddings and result.acc is None + with pytest.raises(ValueError): + FaceRecognitionPost({}, {"dataset": "market1501"}) + with pytest.raises(ValueError): + FaceRecognitionPost({}, {}) + + +@pytest.mark.parametrize( + "loader,name,folder", + [ + (LFWDataset, "lfw", "aligned"), + (AgeDB30Dataset, "agedb-30", "images"), + (CFPFPDataset, "cfp-fp", "images"), + ], +) +def test_readiness(tmp_path, monkeypatch, loader, name, folder): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 10 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + rows = [(i, i, 1, "a.png", "b.png") for i in range(1, 11)] + write_pairs(tmp_path, folder, rows) + assert readiness.dataset_ready(tmp_path, "face_recognition", name) + assert not readiness.dataset_ready(tmp_path, "face_recognition", "market1501") + (tmp_path / folder / "a.png").write_bytes(b"broken") + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + + +@pytest.mark.parametrize("seed", range(5)) +def test_ops_metric_exact_parity(seed): + ops = reference("verification") + rng = np.random.default_rng(seed) + a = rng.integers(-2, 3, (80, 8)).astype(np.float32) + b = rng.integers(-2, 3, (80, 8)).astype(np.float32) + labels = rng.integers(0, 2, 80).astype(bool) + folds = np.tile(np.arange(1, 11), 8) + expected_scores = ops.cosine_scores(a, b) + scores = cosine_scores(a, b) + np.testing.assert_array_equal(scores, expected_scores) + expected_acc, expected_thresholds = ops.fold_accuracies( + expected_scores, labels, folds + ) + acc, thresholds = fold_accuracies(scores, labels, folds) + np.testing.assert_array_equal(acc, expected_acc) + np.testing.assert_array_equal(thresholds, expected_thresholds) + expected = ops.verification_result(scores, labels, folds) + assert float(np.mean(acc)) == expected["accuracy"] + assert float(np.std(acc)) == expected["accuracy_std"] + assert float(np.mean(thresholds)) == expected["threshold"] + + +def test_threshold_ties_and_inclusive_boundary(): + acc, thresholds = fold_accuracies( + np.zeros(4), np.array([0, 1, 0, 1]), np.array([1, 1, 2, 2]) + ) + np.testing.assert_array_equal(thresholds, [1.0, 1.0]) + np.testing.assert_array_equal(acc, [0.5, 0.5]) + # Exactly equal to threshold counts as same. + scores = cosine_scores(np.ones((2, 2)), np.ones((2, 2))) + np.testing.assert_allclose(scores, 1.0) + with pytest.raises(ValueError): + fold_accuracies(np.ones(2), np.ones(2), np.ones(2, dtype=int)) + + +@pytest.mark.parametrize("evaluator", [LFWEvaluator, AgeDB30Evaluator, CFPEvaluator]) +def test_evaluator_mapping_and_no_implicit_fusion(evaluator): + images = [PairImage(Path(n), n) for n in ["a", "b", "c"]] + pairs = [ + FacePair("a", "b", True, 1), + FacePair("a", "c", False, 1), + FacePair("a", "b", True, 2), + FacePair("a", "c", False, 2), + ] + embeddings = np.array([[1.0, 0.0], [1.0, 0.0], [-1.0, 0.0]], dtype=np.float32) + result = evaluator().evaluate(embeddings, images, pairs) + assert result.primary_score == 1.0 and result.secondary_score == 0.0 + assert result.as_dict()["primary"][evaluator.metric] == 1.0 + assert result.sample_count == 3 + with pytest.raises(ValueError): + evaluator().evaluate(embeddings[:2], images, pairs) + with pytest.raises(ValueError): + evaluator().evaluate(embeddings, images, [FacePair("a", "missing", True, 1)]) + + +@pytest.mark.parametrize("shape", [(112, 112), (35, 51), (241, 179)]) +def test_ops_preprocessing_byte_parity(tmp_path, shape): + ops = reference("preprocess") + cfg = load_model_variant( + Path(__file__).parents[1] / "mblt_vision/models/bases/face_recognition.yaml", + "DEFAULT", + ) + path = tmp_path / "synthetic.png" + Image.fromarray( + np.random.default_rng(0).integers(0, 256, (*shape, 3), dtype=np.uint8) + ).save(path) + expected = ops.preprocess(path, cfg["pre_cfg"])[1] + actual = build_preprocess(cfg["pre_cfg"])(path) + np.testing.assert_array_equal( + actual, expected + ) # atol=0, including no-op 112x112 resize + assert np.max(np.abs(actual - expected)) == 0.0 From aaf0c14c2453e28c01be221744b56332f42dbe7d Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:57:23 +0900 Subject: [PATCH 09/13] Add pending EfficientViT B0 B1 B2 family sibling configs Use official timm defaults and no-network pending status for new sibling configurations only. Preserve all pre-existing YAML bytes and allow trusted local artifacts without inventing HF publication metadata. Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 3 + AGENTS.md | 3 + mblt_vision/README.md | 11 +++ mblt_vision/_hub_validation.py | 8 +++ mblt_vision/compile/vision.py | 11 ++- mblt_vision/image_classification/__init__.py | 6 ++ mblt_vision/models/EfficientViT_B0.yaml | 13 ++++ mblt_vision/models/EfficientViT_B1.yaml | 13 ++++ mblt_vision/models/EfficientViT_B2.yaml | 13 ++++ mblt_vision/wrapper.py | 6 ++ tests/test_pending_family_siblings.py | 70 ++++++++++++++++++++ 11 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 mblt_vision/models/EfficientViT_B0.yaml create mode 100644 mblt_vision/models/EfficientViT_B1.yaml create mode 100644 mblt_vision/models/EfficientViT_B2.yaml create mode 100644 tests/test_pending_family_siblings.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 5de5a20..f8d3055 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -33,6 +33,9 @@ description: >- including the TF-normalized `ViT_Large_Patch14_CLIP_224` exception. Their MXQ-only repositories use `onnx_local_artifact_only: true`: require trusted local ONNX and retain automatic MXQ loading; metadata validation skips only the marked framework. +- New pending family siblings use `file_cfg.artifact_status: pending_hf_family_sibling` + and require trusted local artifacts for runtime/compile. Report them as pending without + network; retain byte-identical existing sibling YAMLs. - For Hub linkage changes, run `tests/test_hub_validation.py` as described in `tests/TEST.md`. The metadata checker follows runtime board paths without downloading weights; live commits are diagnostics, not substitutes for missing historical approval provenance. diff --git a/AGENTS.md b/AGENTS.md index 411570e..93e02fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,6 +78,9 @@ The current ownership boundary is deliberate: `ViT_Large_Patch14_CLIP_224` rather than inferring normalization from its name. These 24 repositories are MXQ-only: their base uses `onnx_local_artifact_only: true`. Keep automatic MXQ loading enabled, require trusted local ONNX, and test both guard paths. + Newly added EfficientViT B0/B1/B2 sibling configs carry + `artifact_status: pending_hf_family_sibling`: verify as pending without network and + require trusted local runtime/compile inputs. Existing sibling configs stay unchanged. - To share processing defaults without changing legacy models, opt new family YAMLs into `extends: bases/family.yaml#DEFAULT`. See the configuration-inheritance section of `mblt_vision/README.md` for ordering and merge rules. Keep legacy `update` shallow and diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 3e3c8bf..da1334e 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -124,6 +124,17 @@ but ONNX requires a trusted local `model_path` or `onnx_path`. Automatic ONNX loading fails with a specific message, and metadata verification skips only that framework. Existing configurations without this marker retain their behavior. +`EfficientViT_B0`, `EfficientViT_B1` and `EfficientViT_B2` are intentional family +sibling configurations using official timm 1.0.14 `r224_in1k` defaults (224 crop, +bicubic resize to `floor(224 / 0.95) = 235`, ImageNet normalization). They are +**pending HF (family sibling)**, not published-model claims. Their +`file_cfg.artifact_status: pending_hf_family_sibling` makes metadata verification +skip both frameworks without network access and allows runtime/compile use only +with trusted local artifacts. Existing sibling configurations are not rewritten. +EfficientViT M0–M5 is a distinct architecture and is not added here. Alternative +ViT CLIP/GAP resolutions require exact upstream squash-mode preprocessing support; +this change does not approximate them with center cropping. + | Model | Input Size
(H,W,C) | AccTop1
(NPU) | AccTop1
(GPU) | FLOPs (B) | params (M) | Source | Note | | --- | --- | --- | --- | --- | --- | --- | --- | | AlexNet | (224,224,3) | 56.084 | 56.556 | 1.43 | 61.10 | [Link](https://docs.pytorch.org/vision/stable/models/generated/torchvision.models.alexnet.html) | | diff --git a/mblt_vision/_hub_validation.py b/mblt_vision/_hub_validation.py index 901373b..04844d6 100644 --- a/mblt_vision/_hub_validation.py +++ b/mblt_vision/_hub_validation.py @@ -70,6 +70,14 @@ def check_hub_artifacts( (framework, f"{board}/{filename}" if framework == "mxq" else filename) ) + if files.get("artifact_status") == "pending_hf_family_sibling": + return tuple( + HubArtifactCheck( + repo_id, revision, None, framework, path, "pending_hf_family_sibling" + ) + for framework, path in paths + ) + def is_local_only(framework: str) -> bool: return bool(files.get("local_artifact_only")) or ( framework == "onnx" and bool(files.get("onnx_local_artifact_only")) diff --git a/mblt_vision/compile/vision.py b/mblt_vision/compile/vision.py index fb2fb98..d57cbf8 100644 --- a/mblt_vision/compile/vision.py +++ b/mblt_vision/compile/vision.py @@ -786,8 +786,10 @@ def resolve_quantization_values( _validate_ratio("topk_ratio", topk_ratio) if topk_ratio is not None else None ) hosted_config: dict[str, Any] | None = None - if (resolved_percentile is None or resolved_topk is None) and not file_cfg.get( - "local_artifact_only" + if ( + (resolved_percentile is None or resolved_topk is None) + and not file_cfg.get("local_artifact_only") + and file_cfg.get("artifact_status") != "pending_hf_family_sibling" ): repo_id = file_cfg.get("repo_id") revision = file_cfg.get("revision", "main") @@ -884,6 +886,11 @@ def _resolve_compile_onnx_path( repo_id = file_cfg.get("repo_id") revision = file_cfg.get("revision") filename = _configured_onnx_filename(file_cfg) + if file_cfg.get("artifact_status") == "pending_hf_family_sibling": + raise RuntimeError( + "This configuration is pending HF (family sibling). Supply a trusted local " + "model_path or onnx_path for compilation; automatic downloads are disabled." + ) if file_cfg.get("local_artifact_only"): repository = repo_id if isinstance(repo_id, str) and repo_id else "this model" raise RuntimeError( diff --git a/mblt_vision/image_classification/__init__.py b/mblt_vision/image_classification/__init__.py index 085defb..4bcab7c 100644 --- a/mblt_vision/image_classification/__init__.py +++ b/mblt_vision/image_classification/__init__.py @@ -58,6 +58,9 @@ "EfficientNet_V2_L", "EfficientNet_V2_M", "EfficientNet_V2_S", + "EfficientViT_B0", + "EfficientViT_B1", + "EfficientViT_B2", "EfficientViT_B3", "EfficientViT_L1", "EfficientViT_L2", @@ -182,6 +185,9 @@ "YOLOv8xCls", ] +EfficientViT_B0 = create_model_class("EfficientViT_B0", __name__) +EfficientViT_B1 = create_model_class("EfficientViT_B1", __name__) +EfficientViT_B2 = create_model_class("EfficientViT_B2", __name__) EfficientViT_B3 = create_model_class("EfficientViT_B3", __name__) EfficientViT_L1 = create_model_class("EfficientViT_L1", __name__) EfficientViT_L2 = create_model_class("EfficientViT_L2", __name__) diff --git a/mblt_vision/models/EfficientViT_B0.yaml b/mblt_vision/models/EfficientViT_B0.yaml new file mode 100644 index 0000000..406152d --- /dev/null +++ b/mblt_vision/models/EfficientViT_B0.yaml @@ -0,0 +1,13 @@ +# Official timm 1.0.14 efficientvit_b0.r224_in1k pretrained_cfg: +# 224 crop, bicubic, crop_pct 0.95 -> floor(224 / 0.95) = 235, ImageNet torch stats. +# Family sibling configuration; no published artifact or approval is claimed. +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/EfficientViT_B0 + filename: efficientvit_b0.mxq + revision: main + artifact_status: pending_hf_family_sibling + pre_cfg: + Resize: + size: 235 diff --git a/mblt_vision/models/EfficientViT_B1.yaml b/mblt_vision/models/EfficientViT_B1.yaml new file mode 100644 index 0000000..9bda945 --- /dev/null +++ b/mblt_vision/models/EfficientViT_B1.yaml @@ -0,0 +1,13 @@ +# Official timm 1.0.14 efficientvit_b1.r224_in1k pretrained_cfg: +# 224 crop, bicubic, crop_pct 0.95 -> floor(224 / 0.95) = 235, ImageNet torch stats. +# Family sibling configuration; no published artifact or approval is claimed. +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/EfficientViT_B1 + filename: efficientvit_b1.mxq + revision: main + artifact_status: pending_hf_family_sibling + pre_cfg: + Resize: + size: 235 diff --git a/mblt_vision/models/EfficientViT_B2.yaml b/mblt_vision/models/EfficientViT_B2.yaml new file mode 100644 index 0000000..879995c --- /dev/null +++ b/mblt_vision/models/EfficientViT_B2.yaml @@ -0,0 +1,13 @@ +# Official timm 1.0.14 efficientvit_b2.r224_in1k pretrained_cfg: +# 224 crop, bicubic, crop_pct 0.95 -> floor(224 / 0.95) = 235, ImageNet torch stats. +# Family sibling configuration; no published artifact or approval is claimed. +DEFAULT: + extends: bases/imagenet_classifier.yaml + file_cfg: + repo_id: mobilint/EfficientViT_B2 + filename: efficientvit_b2.mxq + revision: main + artifact_status: pending_hf_family_sibling + pre_cfg: + Resize: + size: 235 diff --git a/mblt_vision/wrapper.py b/mblt_vision/wrapper.py index 39f8dd0..9f99a43 100644 --- a/mblt_vision/wrapper.py +++ b/mblt_vision/wrapper.py @@ -629,6 +629,7 @@ def _mxq_backend_kwargs(self) -> dict[str, Any]: "onnx_filename", "onnx_path", "onnx_local_artifact_only", + "artifact_status", } return { key: value @@ -723,6 +724,11 @@ def file_config_cleansing(self) -> None: revision = self.file_cfg.pop("revision", None) if not repo_id or not revision: return + if self.file_cfg.get("artifact_status") == "pending_hf_family_sibling": + raise RuntimeError( + f"'{repo_id}' is pending HF (family sibling). Supply a trusted local " + "model_path, mxq_path, or onnx_path; automatic downloads are disabled." + ) if local_artifact_only: raise RuntimeError( f"Remote artifacts for '{repo_id}' are disabled because this model does not " diff --git a/tests/test_pending_family_siblings.py b/tests/test_pending_family_siblings.py new file mode 100644 index 0000000..b33dec5 --- /dev/null +++ b/tests/test_pending_family_siblings.py @@ -0,0 +1,70 @@ +"""Intentional family-sibling configs are discoverable but never auto-download.""" + +from __future__ import annotations + +from unittest.mock import Mock + +import pytest + +import mblt_vision +from mblt_vision._hub_validation import check_hub_artifacts +from mblt_vision.compile import vision as compile_module +from mblt_vision.wrapper import MBLT_Engine, resolve_model_config + + +@pytest.mark.parametrize( + "name", ["EfficientViT_B0", "EfficientViT_B1", "EfficientViT_B2"] +) +def test_official_sibling_pipeline_and_pending_metadata(name) -> None: + config = resolve_model_config(name) + reference = resolve_model_config("EfficientViT_B3") + assert config["pre_cfg"] == reference["pre_cfg"] + assert config["pre_cfg"]["Resize"] == {"size": 235, "interpolation": "bicubic"} + assert config["file_cfg"]["artifact_status"] == "pending_hf_family_sibling" + assert ( + name in mblt_vision.list_models("image_classification")["image_classification"] + ) + assert getattr(mblt_vision, name) is getattr(mblt_vision.image_classification, name) + api = Mock() + checks = check_hub_artifacts(config, api=api) + assert all(item.status == "pending_hf_family_sibling" for item in checks) + assert not api.mock_calls + + +@pytest.mark.parametrize("framework", ["mxq", "onnx"]) +def test_pending_engine_never_downloads(framework) -> None: + engine = object.__new__(MBLT_Engine) + engine.framework = framework + engine.file_cfg = resolve_model_config("EfficientViT_B0")["file_cfg"] + engine._download_hub_artifact = Mock() + with pytest.raises(RuntimeError, match="pending HF"): + engine.file_config_cleansing() + engine._download_hub_artifact.assert_not_called() + + +@pytest.mark.parametrize("framework", ["mxq", "onnx"]) +def test_pending_engine_accepts_trusted_local_artifact(framework, tmp_path) -> None: + engine = object.__new__(MBLT_Engine) + engine.framework = framework + engine.file_cfg = resolve_model_config("EfficientViT_B0")["file_cfg"] + source = tmp_path / f"trusted.{framework}" + source.write_bytes(b"fixture") + engine.file_cfg[f"{framework}_path"] = str(source) + engine._download_hub_artifact = Mock() + engine.file_config_cleansing() + assert "artifact_status" not in engine._mxq_backend_kwargs() + engine._download_hub_artifact.assert_not_called() + + +def test_pending_compile_never_downloads_and_accepts_local( + monkeypatch, tmp_path +) -> None: + config = resolve_model_config("EfficientViT_B0")["file_cfg"] + download = Mock() + monkeypatch.setattr(compile_module, "hf_hub_download", download) + with pytest.raises(RuntimeError, match="pending HF"): + compile_module._resolve_compile_onnx_path(config, None) + local = tmp_path / "trusted.onnx" + local.write_bytes(b"fixture") + assert compile_module._resolve_compile_onnx_path(config, local) == local.resolve() + download.assert_not_called() From 77478b4d3eda47524e08319f881e5fe00f6077cc Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 17:02:18 +0900 Subject: [PATCH 10/13] Validate official face pair counts independently for every fold Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/utils/datasets/face_pairs.py | 10 +++++++ tests/test_face_verification.py | 36 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/mblt_vision/utils/datasets/face_pairs.py b/mblt_vision/utils/datasets/face_pairs.py index a72bd66..52e3010 100644 --- a/mblt_vision/utils/datasets/face_pairs.py +++ b/mblt_vision/utils/datasets/face_pairs.py @@ -89,6 +89,16 @@ def __init__( raise ValueError( f"Pair file holds {len(pairs)} pairs, expected {self.config['pairs']}" ) + if validate_counts: + per_fold = self.config["pairs"] // self.config["folds"] + counts = { + fold: sum(pair.fold == fold for pair in pairs) + for fold in range(1, self.config["folds"] + 1) + } + if any(count != per_fold for count in counts.values()): + raise ValueError( + f"Each official fold requires {per_fold} pairs; found {counts}" + ) self.pairs = tuple(pairs) image_dir = self.root / self.config["image_directory"] if not image_dir.is_dir(): diff --git a/tests/test_face_verification.py b/tests/test_face_verification.py index 2d1e60a..62cb9ac 100644 --- a/tests/test_face_verification.py +++ b/tests/test_face_verification.py @@ -216,3 +216,39 @@ def test_ops_preprocessing_byte_parity(tmp_path, shape): actual, expected ) # atol=0, including no-op 112x112 resize assert np.max(np.abs(actual - expected)) == 0.0 + + +@pytest.mark.parametrize( + "name,loader,folder", + [ + ("lfw", LFWDataset, "aligned"), + ("agedb-30", AgeDB30Dataset, "images"), + ("cfp-fp", CFPFPDataset, "images"), + ], +) +def test_rejects_correct_total_with_unbalanced_official_folds( + tmp_path, monkeypatch, name, loader, folder +): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 20 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + rows = [(i, i // 2 + 1, i % 2, "a.png", "b.png") for i in range(20)] + write_pairs(tmp_path, folder, rows) + assert readiness.dataset_ready(tmp_path, "face_recognition", name) + rows[0] = ( + 0, + 2, + 0, + "a.png", + "b.png", + ) # 20 total, all ten folds, but 1/3 instead of 2/2. + write_pairs(tmp_path, folder, rows) + with pytest.raises(ValueError, match="official fold"): + loader(tmp_path) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + assert len(loader(tmp_path, validate_counts=False).pairs) == 20 From 6bc2db8f41241ab30551b0ffa66e1297aab79f26 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 17:10:38 +0900 Subject: [PATCH 11/13] Add twelve published CLIP-ReID model configs and registry exports Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 3 +- AGENTS.md | 4 +- mblt_vision/README.md | 21 +++++ mblt_vision/__init__.py | 1 + .../CLIP_ReID_RN50_Baseline_DukeMTMC.yaml | 17 ++++ .../CLIP_ReID_RN50_Baseline_Market1501.yaml | 17 ++++ .../CLIP_ReID_RN50_Baseline_OccDuke.yaml | 17 ++++ .../models/CLIP_ReID_RN50_DukeMTMC.yaml | 17 ++++ .../models/CLIP_ReID_RN50_Market1501.yaml | 17 ++++ .../models/CLIP_ReID_RN50_OccDuke.yaml | 17 ++++ .../CLIP_ReID_ViT_B_16_Baseline_DukeMTMC.yaml | 17 ++++ ...LIP_ReID_ViT_B_16_Baseline_Market1501.yaml | 17 ++++ .../CLIP_ReID_ViT_B_16_Baseline_OccDuke.yaml | 17 ++++ .../models/CLIP_ReID_ViT_B_16_DukeMTMC.yaml | 17 ++++ .../models/CLIP_ReID_ViT_B_16_Market1501.yaml | 17 ++++ .../models/CLIP_ReID_ViT_B_16_OccDuke.yaml | 17 ++++ mblt_vision/models/bases/person_reid.yaml | 6 ++ mblt_vision/person_reid/__init__.py | 46 +++++++++++ tests/test_model_metadata.py | 1 + tests/test_published_reid.py | 81 +++++++++++++++++++ 20 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 mblt_vision/models/CLIP_ReID_RN50_Baseline_DukeMTMC.yaml create mode 100644 mblt_vision/models/CLIP_ReID_RN50_Baseline_Market1501.yaml create mode 100644 mblt_vision/models/CLIP_ReID_RN50_Baseline_OccDuke.yaml create mode 100644 mblt_vision/models/CLIP_ReID_RN50_DukeMTMC.yaml create mode 100644 mblt_vision/models/CLIP_ReID_RN50_Market1501.yaml create mode 100644 mblt_vision/models/CLIP_ReID_RN50_OccDuke.yaml create mode 100644 mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_DukeMTMC.yaml create mode 100644 mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_Market1501.yaml create mode 100644 mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_OccDuke.yaml create mode 100644 mblt_vision/models/CLIP_ReID_ViT_B_16_DukeMTMC.yaml create mode 100644 mblt_vision/models/CLIP_ReID_ViT_B_16_Market1501.yaml create mode 100644 mblt_vision/models/CLIP_ReID_ViT_B_16_OccDuke.yaml create mode 100644 tests/test_published_reid.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 5d2b3bc..c3ddf57 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -120,7 +120,8 @@ description: >- metric). Load difficult objects as ordinary targets, as Ultralytics' validation does. - Run YOLOv7 models at `iou_thres: 0.65`, WongKinYiu/yolov7's own `test.py` default. -- ReID is utility-only until published wrappers and routing are added. Reuse +- Published CLIP-ReID wrappers use RN50/ViT processing bases and matching retrieval datasets; + keep their MXQ-only ONNX runtime/compile guards. CLI/benchmark routing remains unavailable. Reuse `EmbeddingPost` for dense vectors, explicit float32 mean/std for source preprocessing, and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. diff --git a/AGENTS.md b/AGENTS.md index 70b666a..66b97e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -192,7 +192,9 @@ The current ownership boundary is deliberate: any future model needing more than one Hub artifact, rather than duplicating Hub-resolution logic. -- ReID utilities live in `mblt_vision.person_reid`; concrete published wrappers are separate. +- ReID utilities and twelve published CLIP-ReID wrappers live in `mblt_vision.person_reid`. + Model YAMLs inherit the RN50/ViT processing base and explicitly bind their dataset/dimension. + Published ReID artifacts are MXQ-only; preserve runtime/compile trusted-local ONNX guards. Use `EmbeddingPost` for validated dense vectors and `PersonReIDPost` for dataset binding. Retrieval protocols and prepared loaders are pinned to ops source in dataset YAMLs; keep stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. diff --git a/mblt_vision/README.md b/mblt_vision/README.md index c349bbd..52336f1 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -621,3 +621,24 @@ Dataset YAMLs record exact source paths/revision. CFP-FP specifically refers to the prepared cat-claws mirror: ops warns it was not checked against the original `cfp_fp.bin` and records a training-log discrepancy. This package preserves that provenance and makes no new accuracy claim. + +### Published ReID models + +The twelve published `CLIP_ReID_*` wrappers are available from +`mblt_vision.person_reid`, lazy top-level imports, and `list_models("person_reid")`. +They cover RN50 / ViT-B-16, baseline / fine-tuned, each for Market-1501, DukeMTMC-reID +and Occluded-Duke. Every YAML inherits `bases/person_reid.yaml` and binds the +matching retrieval dataset. RN50 uses 3072 dimensions and torch mean/std; ViT uses +1280 dimensions and 0.5 mean/std. Both take 256x128 PIL bilinear crops and one +image input, confirmed from ops distribution contracts and recorded preprocessing. + +HF metadata confirms one canonical `aries-rb/.mxq` per repository, +with no published ONNX. These configurations retain `revision: main` by project +policy; the approval HF commit is explicitly **not recorded**, while the ops source +commit and exact source paths are included under `source`. Live metadata SHAs are +audit diagnostics, not approval pins. `onnx_local_artifact_only: true` preserves +automatic MXQ loading and requires trusted local ONNX for runtime and compile. +The guard is inherited from the PR31 dependency, not reimplemented here. + +ArcFace/CosFace eight-model publication is pending. Those models remain +inventory-only: no guessed HF repository identity or downloadable YAML is shipped. diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index 1f40160..a259d92 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -31,6 +31,7 @@ object_detection, obb, pose_estimation, + person_reid, semantic_segmentation, ) diff --git a/mblt_vision/models/CLIP_ReID_RN50_Baseline_DukeMTMC.yaml b/mblt_vision/models/CLIP_ReID_RN50_Baseline_DukeMTMC.yaml new file mode 100644 index 0000000..93f46ed --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_RN50_Baseline_DukeMTMC.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#DEFAULT + file_cfg: + repo_id: mobilint/CLIP_ReID_RN50_Baseline_DukeMTMC + filename: clip_reid_rn50_baseline_dukemtmc.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: dukemtmc-reid + embedding_dim: 3072 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_RN50_Baseline_DukeMTMC/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_RN50_Baseline_DukeMTMC/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_RN50_Baseline_Market1501.yaml b/mblt_vision/models/CLIP_ReID_RN50_Baseline_Market1501.yaml new file mode 100644 index 0000000..762082b --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_RN50_Baseline_Market1501.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#DEFAULT + file_cfg: + repo_id: mobilint/CLIP_ReID_RN50_Baseline_Market1501 + filename: clip_reid_rn50_baseline_market1501.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: market1501 + embedding_dim: 3072 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_RN50_Baseline_Market1501/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_RN50_Baseline_Market1501/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_RN50_Baseline_OccDuke.yaml b/mblt_vision/models/CLIP_ReID_RN50_Baseline_OccDuke.yaml new file mode 100644 index 0000000..7df7104 --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_RN50_Baseline_OccDuke.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#DEFAULT + file_cfg: + repo_id: mobilint/CLIP_ReID_RN50_Baseline_OccDuke + filename: clip_reid_rn50_baseline_occduke.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: occluded-duke + embedding_dim: 3072 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_RN50_Baseline_OccDuke/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_RN50_Baseline_OccDuke/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_RN50_DukeMTMC.yaml b/mblt_vision/models/CLIP_ReID_RN50_DukeMTMC.yaml new file mode 100644 index 0000000..a0edfff --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_RN50_DukeMTMC.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#DEFAULT + file_cfg: + repo_id: mobilint/CLIP_ReID_RN50_DukeMTMC + filename: clip_reid_rn50_dukemtmc.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: dukemtmc-reid + embedding_dim: 3072 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_RN50_DukeMTMC/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_RN50_DukeMTMC/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_RN50_Market1501.yaml b/mblt_vision/models/CLIP_ReID_RN50_Market1501.yaml new file mode 100644 index 0000000..54cc350 --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_RN50_Market1501.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#DEFAULT + file_cfg: + repo_id: mobilint/CLIP_ReID_RN50_Market1501 + filename: clip_reid_rn50_market1501.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: market1501 + embedding_dim: 3072 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_RN50_Market1501/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_RN50_Market1501/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_RN50_OccDuke.yaml b/mblt_vision/models/CLIP_ReID_RN50_OccDuke.yaml new file mode 100644 index 0000000..8e5ff73 --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_RN50_OccDuke.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#DEFAULT + file_cfg: + repo_id: mobilint/CLIP_ReID_RN50_OccDuke + filename: clip_reid_rn50_occduke.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: occluded-duke + embedding_dim: 3072 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_RN50_OccDuke/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_RN50_OccDuke/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_DukeMTMC.yaml b/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_DukeMTMC.yaml new file mode 100644 index 0000000..478aa35 --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_DukeMTMC.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#VIT + file_cfg: + repo_id: mobilint/CLIP_ReID_ViT_B_16_Baseline_DukeMTMC + filename: clip_reid_vit_b_16_baseline_dukemtmc.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: dukemtmc-reid + embedding_dim: 1280 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_ViT_B_16_Baseline_DukeMTMC/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_ViT_B_16_Baseline_DukeMTMC/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_Market1501.yaml b/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_Market1501.yaml new file mode 100644 index 0000000..7ea0b0b --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_Market1501.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#VIT + file_cfg: + repo_id: mobilint/CLIP_ReID_ViT_B_16_Baseline_Market1501 + filename: clip_reid_vit_b_16_baseline_market1501.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: market1501 + embedding_dim: 1280 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_ViT_B_16_Baseline_Market1501/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_ViT_B_16_Baseline_Market1501/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_OccDuke.yaml b/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_OccDuke.yaml new file mode 100644 index 0000000..c655b3c --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_ViT_B_16_Baseline_OccDuke.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#VIT + file_cfg: + repo_id: mobilint/CLIP_ReID_ViT_B_16_Baseline_OccDuke + filename: clip_reid_vit_b_16_baseline_occduke.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: occluded-duke + embedding_dim: 1280 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_ViT_B_16_Baseline_OccDuke/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_ViT_B_16_Baseline_OccDuke/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_ViT_B_16_DukeMTMC.yaml b/mblt_vision/models/CLIP_ReID_ViT_B_16_DukeMTMC.yaml new file mode 100644 index 0000000..b6462e4 --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_ViT_B_16_DukeMTMC.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#VIT + file_cfg: + repo_id: mobilint/CLIP_ReID_ViT_B_16_DukeMTMC + filename: clip_reid_vit_b_16_dukemtmc.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: dukemtmc-reid + embedding_dim: 1280 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_ViT_B_16_DukeMTMC/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_ViT_B_16_DukeMTMC/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_ViT_B_16_Market1501.yaml b/mblt_vision/models/CLIP_ReID_ViT_B_16_Market1501.yaml new file mode 100644 index 0000000..f9b601c --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_ViT_B_16_Market1501.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#VIT + file_cfg: + repo_id: mobilint/CLIP_ReID_ViT_B_16_Market1501 + filename: clip_reid_vit_b_16_market1501.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: market1501 + embedding_dim: 1280 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_ViT_B_16_Market1501/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_ViT_B_16_Market1501/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/CLIP_ReID_ViT_B_16_OccDuke.yaml b/mblt_vision/models/CLIP_ReID_ViT_B_16_OccDuke.yaml new file mode 100644 index 0000000..a7fabfb --- /dev/null +++ b/mblt_vision/models/CLIP_ReID_ViT_B_16_OccDuke.yaml @@ -0,0 +1,17 @@ +# Approval HF commit: not recorded; main retained by explicit project policy. +DEFAULT: + extends: bases/person_reid.yaml#VIT + file_cfg: + repo_id: mobilint/CLIP_ReID_ViT_B_16_OccDuke + filename: clip_reid_vit_b_16_occduke.mxq + revision: main + onnx_local_artifact_only: true + post_cfg: + dataset: occluded-duke + embedding_dim: 1280 + source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + distribution_path: models/person_reid/CLIP_ReID_ViT_B_16_OccDuke/distribution.yaml + pre_cfg_path: models/person_reid/CLIP_ReID_ViT_B_16_OccDuke/_source/predict_float.input.json + approved_hf_revision: not recorded diff --git a/mblt_vision/models/bases/person_reid.yaml b/mblt_vision/models/bases/person_reid.yaml index cafeaf3..1d75d79 100644 --- a/mblt_vision/models/bases/person_reid.yaml +++ b/mblt_vision/models/bases/person_reid.yaml @@ -24,3 +24,9 @@ OCCLUDED_DUKE: extends: person_reid.yaml#DEFAULT post_cfg: dataset: occluded-duke +VIT: + extends: person_reid.yaml#DEFAULT + pre_cfg: + Normalize: + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py index e5e6574..19fc50f 100644 --- a/mblt_vision/person_reid/__init__.py +++ b/mblt_vision/person_reid/__init__.py @@ -1,5 +1,6 @@ """Person retrieval utilities; published model wrappers are added separately.""" +from .._compat import create_model_class from ..utils.datasets.reid import ( DukeMTMCReIDDataset, Market1501Dataset, @@ -32,3 +33,48 @@ "RetrievalResult", "extract_embeddings", ] + +# Concrete published single-input models; utilities above remain explicit exports. +CLIP_ReID_RN50_Baseline_DukeMTMC = create_model_class( + "CLIP_ReID_RN50_Baseline_DukeMTMC", __name__ +) +CLIP_ReID_RN50_Baseline_Market1501 = create_model_class( + "CLIP_ReID_RN50_Baseline_Market1501", __name__ +) +CLIP_ReID_RN50_Baseline_OccDuke = create_model_class( + "CLIP_ReID_RN50_Baseline_OccDuke", __name__ +) +CLIP_ReID_RN50_DukeMTMC = create_model_class("CLIP_ReID_RN50_DukeMTMC", __name__) +CLIP_ReID_RN50_Market1501 = create_model_class("CLIP_ReID_RN50_Market1501", __name__) +CLIP_ReID_RN50_OccDuke = create_model_class("CLIP_ReID_RN50_OccDuke", __name__) +CLIP_ReID_ViT_B_16_Baseline_DukeMTMC = create_model_class( + "CLIP_ReID_ViT_B_16_Baseline_DukeMTMC", __name__ +) +CLIP_ReID_ViT_B_16_Baseline_Market1501 = create_model_class( + "CLIP_ReID_ViT_B_16_Baseline_Market1501", __name__ +) +CLIP_ReID_ViT_B_16_Baseline_OccDuke = create_model_class( + "CLIP_ReID_ViT_B_16_Baseline_OccDuke", __name__ +) +CLIP_ReID_ViT_B_16_DukeMTMC = create_model_class( + "CLIP_ReID_ViT_B_16_DukeMTMC", __name__ +) +CLIP_ReID_ViT_B_16_Market1501 = create_model_class( + "CLIP_ReID_ViT_B_16_Market1501", __name__ +) +CLIP_ReID_ViT_B_16_OccDuke = create_model_class("CLIP_ReID_ViT_B_16_OccDuke", __name__) + +__all__ += [ + "CLIP_ReID_RN50_Baseline_DukeMTMC", + "CLIP_ReID_RN50_Baseline_Market1501", + "CLIP_ReID_RN50_Baseline_OccDuke", + "CLIP_ReID_RN50_DukeMTMC", + "CLIP_ReID_RN50_Market1501", + "CLIP_ReID_RN50_OccDuke", + "CLIP_ReID_ViT_B_16_Baseline_DukeMTMC", + "CLIP_ReID_ViT_B_16_Baseline_Market1501", + "CLIP_ReID_ViT_B_16_Baseline_OccDuke", + "CLIP_ReID_ViT_B_16_DukeMTMC", + "CLIP_ReID_ViT_B_16_Market1501", + "CLIP_ReID_ViT_B_16_OccDuke", +] diff --git a/tests/test_model_metadata.py b/tests/test_model_metadata.py index de3fb68..d45bcc4 100644 --- a/tests/test_model_metadata.py +++ b/tests/test_model_metadata.py @@ -19,6 +19,7 @@ "object_detection": {"coco"}, "obb": {"dotav1"}, "pose_estimation": {"coco"}, + "person_reid": {"market1501", "dukemtmc-reid", "occluded-duke"}, "semantic_segmentation": {"ade20k", "cityscapes"}, } diff --git a/tests/test_published_reid.py b/tests/test_published_reid.py new file mode 100644 index 0000000..731c697 --- /dev/null +++ b/tests/test_published_reid.py @@ -0,0 +1,81 @@ +"""Published ReID metadata, offline processing and source parity (no weights).""" + +from pathlib import Path + +import mblt_vision +import numpy as np +import pytest +import torch +from mblt_vision.utils.postprocess import PersonReIDPost, build_postprocess +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.wrapper import resolve_model_config +from PIL import Image + +MODELS = [ + f"CLIP_ReID_{backbone}{baseline}_{dataset}" + for backbone in ("RN50", "ViT_B_16") + for baseline in ("", "_Baseline") + for dataset in ("Market1501", "DukeMTMC", "OccDuke") +] +DATASETS = { + "Market1501": "market1501", + "DukeMTMC": "dukemtmc-reid", + "OccDuke": "occluded-duke", +} + + +@pytest.mark.parametrize("model", MODELS) +def test_published_reid_config_and_discovery(model): + cfg = resolve_model_config(model) + dim = 1280 if "ViT" in model else 3072 + assert cfg["file_cfg"] == { + "repo_id": "mobilint/" + model, + "filename": model.lower() + ".mxq", + "revision": "main", + "onnx_local_artifact_only": True, + "onnx_filename": model.lower() + ".onnx", + } + assert cfg["post_cfg"] == { + "task": "person_reid", + "dataset": DATASETS[model.split("_")[-1]], + "normalize": True, + "embedding_dim": dim, + } + assert cfg["source"]["approved_hf_revision"] == "not recorded" + assert cfg["source"]["revision"] == "ec537764e36ca10da963fcb7afab157ea60ca022" + assert model in mblt_vision.list_models("person_reid")["person_reid"] + assert getattr(mblt_vision, model) is getattr(mblt_vision.person_reid, model) + post = build_postprocess(cfg["pre_cfg"], cfg["post_cfg"]) + assert isinstance(post, PersonReIDPost) + raw = np.ones((1, dim), dtype=np.float32) + out = post([raw]) + assert tuple(out.shape) == (1, dim) + torch.testing.assert_close(torch.linalg.vector_norm(out, dim=1), torch.ones(1)) + + +@pytest.mark.parametrize("model", MODELS) +def test_published_reid_preprocess_source_numeric_parity(tmp_path, model): + cfg = resolve_model_config(model) + rng = np.random.default_rng(0) + pixels = rng.integers(0, 256, (53, 31, 3), dtype=np.uint8) + path = tmp_path / "0001_c1_synthetic.png" + Image.fromarray(pixels).save(path) + actual = build_preprocess(cfg["pre_cfg"])(path) + if "ViT" in model: + mean, std = [0.5] * 3, [0.5] * 3 + else: + mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] + # ops models/person_reid/_shared/preprocess.py at ec537764, canonical order. + with Image.open(path) as image: + resized = image.convert("RGB").resize((128, 256), Image.Resampling.BILINEAR) + expected = ( + np.asarray(resized, dtype=np.float32) / 255.0 + - np.asarray(mean, dtype=np.float32) + ) / np.asarray(std, dtype=np.float32) + np.testing.assert_array_equal(actual, expected) # atol=0 for every concrete config + + +def test_face_publish_pending_models_remain_inventory_only(): + root = Path(__file__).parents[1] / "mblt_vision/models" + assert not list(root.glob("ArcFace_IResNet_*.yaml")) + assert not list(root.glob("CosFace_IResNet_*.yaml")) From 6a4957dd3fed0b6289832f053fd4f73c34dfbf05 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 18:05:05 +0900 Subject: [PATCH 12/13] Limit ReID model exports and clarify unsupported compilation Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 2 +- AGENTS.md | 3 ++- mblt_vision/README.md | 4 +++- mblt_vision/__init__.py | 4 +++- mblt_vision/person_reid/__init__.py | 4 +++- tests/test_published_reid.py | 21 +++++++++++++++++++++ 6 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index c3ddf57..0f1d62b 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -121,7 +121,7 @@ description: >- - Run YOLOv7 models at `iou_thres: 0.65`, WongKinYiu/yolov7's own `test.py` default. - Published CLIP-ReID wrappers use RN50/ViT processing bases and matching retrieval datasets; - keep their MXQ-only ONNX runtime/compile guards. CLI/benchmark routing remains unavailable. Reuse + keep their MXQ-only ONNX runtime guard. ReID compile, CLI/benchmark routing remain unavailable. Reuse `EmbeddingPost` for dense vectors, explicit float32 mean/std for source preprocessing, and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. diff --git a/AGENTS.md b/AGENTS.md index 66b97e1..da8085b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -194,7 +194,8 @@ The current ownership boundary is deliberate: - ReID utilities and twelve published CLIP-ReID wrappers live in `mblt_vision.person_reid`. Model YAMLs inherit the RN50/ViT processing base and explicitly bind their dataset/dimension. - Published ReID artifacts are MXQ-only; preserve runtime/compile trusted-local ONNX guards. + Published ReID artifacts are MXQ-only; preserve the trusted-local ONNX runtime guard. + ReID compilation/calibration is not supported by compile_vision_model in this stage. Use `EmbeddingPost` for validated dense vectors and `PersonReIDPost` for dataset binding. Retrieval protocols and prepared loaders are pinned to ops source in dataset YAMLs; keep stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 52336f1..d4f5ae4 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -637,7 +637,9 @@ with no published ONNX. These configurations retain `revision: main` by project policy; the approval HF commit is explicitly **not recorded**, while the ops source commit and exact source paths are included under `source`. Live metadata SHAs are audit diagnostics, not approval pins. `onnx_local_artifact_only: true` preserves -automatic MXQ loading and requires trusted local ONNX for runtime and compile. +automatic MXQ loading and requires trusted local ONNX for runtime. +`compile_vision_model` does not yet support `person_reid`; this stage does not +add a ReID compilation/calibration path. The guard is inherited from the PR31 dependency, not reimplemented here. ArcFace/CosFace eight-model publication is pending. Those models remain diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index a259d92..03ae878 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -37,7 +37,9 @@ _LEGACY_MODEL_EXPORTS: dict[str, object] = {} for _task_module in _TASK_MODULES: - for _export_name in getattr(_task_module, "__all__", ()): + for _export_name in getattr( + _task_module, "MODEL_EXPORTS", getattr(_task_module, "__all__", ()) + ): if _export_name in _LEGACY_MODEL_EXPORTS: raise RuntimeError( f"Duplicate vision export detected for '{_export_name}'." diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py index 19fc50f..0abfca2 100644 --- a/mblt_vision/person_reid/__init__.py +++ b/mblt_vision/person_reid/__init__.py @@ -64,7 +64,7 @@ ) CLIP_ReID_ViT_B_16_OccDuke = create_model_class("CLIP_ReID_ViT_B_16_OccDuke", __name__) -__all__ += [ +MODEL_EXPORTS = [ "CLIP_ReID_RN50_Baseline_DukeMTMC", "CLIP_ReID_RN50_Baseline_Market1501", "CLIP_ReID_RN50_Baseline_OccDuke", @@ -78,3 +78,5 @@ "CLIP_ReID_ViT_B_16_Market1501", "CLIP_ReID_ViT_B_16_OccDuke", ] + +__all__ += MODEL_EXPORTS diff --git a/tests/test_published_reid.py b/tests/test_published_reid.py index 731c697..bb5dfa7 100644 --- a/tests/test_published_reid.py +++ b/tests/test_published_reid.py @@ -79,3 +79,24 @@ def test_face_publish_pending_models_remain_inventory_only(): root = Path(__file__).parents[1] / "mblt_vision/models" assert not list(root.glob("ArcFace_IResNet_*.yaml")) assert not list(root.glob("CosFace_IResNet_*.yaml")) + + +def test_top_level_reid_exports_are_model_only(): + assert set(mblt_vision.person_reid.MODEL_EXPORTS) == set(MODELS) + for name in ( + "EmbeddingPost", + "ReIDSample", + "Market1501Dataset", + "RetrievalEvaluator", + "extract_embeddings", + ): + assert name not in mblt_vision.__all__ + assert name not in mblt_vision._LEGACY_MODEL_EXPORTS + assert hasattr(mblt_vision.person_reid, name) + + +def test_reid_compile_is_explicitly_unsupported(): + from mblt_vision.compile.vision import _normalize_task + + with pytest.raises(ValueError, match="Unsupported Vision task"): + _normalize_task("person_reid") From 097c810c4d324947ee235d8ec674c787f3213562 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 18:33:36 +0900 Subject: [PATCH 13/13] Reject face-cache symlink roots images and pair manifests Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/utils/datasets/readiness.py | 8 +++++-- tests/test_face_verification.py | 31 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index b1c9d5d..f405ffa 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -1072,14 +1072,18 @@ def _reid_ready(root: Path, dataset: str) -> bool: def _face_ready(root: Path, dataset: str) -> bool: """Validate prepared aligned pairs and image decoding, never realign crops.""" loader = {"lfw": LFWDataset, "agedb-30": AgeDB30Dataset, "cfp-fp": CFPFPDataset}.get(dataset) - if loader is None: + if ( + loader is None + or _path_has_symlink_component(root) + or _path_has_symlink_component(root / "pairs.csv") + ): return False try: data = loader(root) if len({pair.fold for pair in data.pairs}) != 10: return False for sample in data.samples: - if sample.image.is_symlink(): + if _path_has_symlink_component(sample.image): return False with Image.open(sample.image) as image: image.load() diff --git a/tests/test_face_verification.py b/tests/test_face_verification.py index 62cb9ac..bfa0786 100644 --- a/tests/test_face_verification.py +++ b/tests/test_face_verification.py @@ -143,6 +143,37 @@ def tiny(task, dataset): assert not readiness.dataset_ready(tmp_path, "face_recognition", name) +@pytest.mark.parametrize( + "folder,name", [("aligned", "lfw"), ("images", "agedb-30"), ("images", "cfp-fp")] +) +@pytest.mark.parametrize("component", ["root", "images", "manifest"]) +def test_readiness_rejects_symlink_ancestors( + tmp_path, monkeypatch, folder, name, component +): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 10 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + root = tmp_path / "cache" + root.mkdir() + write_pairs(root, folder, [(i, i, 1, "a.png", "b.png") for i in range(1, 11)]) + assert readiness.dataset_ready(root, "face_recognition", name) + if component == "root": + alias = tmp_path / "alias" + alias.symlink_to(root, target_is_directory=True) + root = alias + else: + source = root / (folder if component == "images" else "pairs.csv") + target = tmp_path / "outside" + source.rename(target) + source.symlink_to(target, target_is_directory=component == "images") + assert not readiness.dataset_ready(root, "face_recognition", name) + + @pytest.mark.parametrize("seed", range(5)) def test_ops_metric_exact_parity(seed): ops = reference("verification")