From 281b8683f42342e6fdd57764b76ba1f39ded2356 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:22:52 +0900 Subject: [PATCH 1/4] Add opt-in model YAML family inheritance with identity proofs Preserve existing alias and update semantics across 334 configs and 595 variants. Add ordered resolution and CPU preprocessing regression tests without changing shipped model definitions. Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 3 + AGENTS.md | 4 + mblt_vision/README.md | 34 ++++++ mblt_vision/_config.py | 120 ++++++++++++++++++++ mblt_vision/wrapper.py | 62 +---------- pyproject.toml | 1 + tests/test_config_inheritance.py | 164 ++++++++++++++++++++++++++++ 7 files changed, 329 insertions(+), 59 deletions(-) create mode 100644 mblt_vision/_config.py create mode 100644 tests/test_config_inheritance.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 1b9b9c5..b695725 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -28,6 +28,9 @@ description: >- Use file_cfg.filename for MXQ and derive the same-stem ONNX artifact unless onnx_filename is required. - Every post_cfg declares dataset; resolve output taxonomy from the dataset/task pair. +- For opt-in YAML family inheritance, follow `mblt_vision/README.md`'s configuration + inheritance rules. Keep legacy `update` shallow and run `tests/test_config_inheritance.py` + to prove every existing variant's values and processing order remain identical. - Reuse wrapper.download_hub_artifact for any model that needs more than one Hub artifact rather than duplicating Hub-resolution logic. - Registering a new dataset requires readiness (`_*_ready` + `dataset_ready` diff --git a/AGENTS.md b/AGENTS.md index 5fd0aa6..06a243b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,6 +69,10 @@ The current ownership boundary is deliberate: floors, and rounding differs by a pixel at 224/0.9 and 224/0.95. Several names (ConvNext_Base) resolve in timm yet carry torchvision's transform here, so read the expected value from that model's source.yaml provenance rather than from the name. +- To share processing defaults without changing legacy models, opt new family YAMLs into + `extends: bases/family.yaml#DEFAULT`. See the configuration-inheritance section of + `mblt_vision/README.md` for ordering and merge rules. Keep legacy `update` shallow and + prove every existing variant order-identical with `tests/test_config_inheritance.py`. - Require post_cfg.dataset in every model YAML and resolve output class counts using the dataset/task pair. Do not assume one output taxonomy for every model in a task. - Preserve automatic .mxq/.onnx framework detection and the fail-fast error when a local diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 30bbddc..0187f74 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -60,6 +60,40 @@ and `list_models()` are the preferred discovery and loading APIs for new code. Top-level model imports remain available for compatibility. `obb` is the sole oriented-bounding-box task name. +## Model configuration inheritance + +Model variants may opt into local family YAMLs with `extends`. This keeps the +artifact, preprocessing and dataset connection together without duplicating a +family's processing defaults. Existing `update: VARIANT` and alias behavior is +unchanged; no shipped model YAML has been migrated by this loader refactor. + +```yaml +DEFAULT: + extends: bases/classifier.yaml#DEFAULT + file_cfg: + repo_id: mobilint/YourPublishedModel + filename: model.mxq + revision: approved-hf-commit + pre_cfg: + Resize: + size: 256 +``` + +The example is a schema illustration, not a downloadable model registration. +The base path is relative to the declaring YAML; omit `#DEFAULT` to select the +same default variant. Family bases belong in `models/bases/`, outside top-level +model discovery. `extends` recursively merges mappings and replaces lists and +scalars. Inherited mapping order is preserved: existing preprocessing operations +stay in place and new operations append. Use a standalone pipeline when operation +order must change. In contrast, `update` retains its historical one-section merge, +so replacing `pre_cfg.Resize` replaces that whole operation's parameters. A variant +cannot combine `update` and `extends`; an `update` may reference another variant +that uses `extends`. References are local only and cycles fail before inference. + +`tests/test_config_inheritance.py` compares every existing YAML variant with the +legacy resolver, including ordered processing mappings and derived ONNX filenames, +and checks exact CPU preprocessing equality for representative model families. + ## Pre-Trained Vision Models This section lists the publicly pre-trained models supported by the vision framework. diff --git a/mblt_vision/_config.py b/mblt_vision/_config.py new file mode 100644 index 0000000..ab2fee2 --- /dev/null +++ b/mblt_vision/_config.py @@ -0,0 +1,120 @@ +"""Local YAML model inheritance, independent of inference backends.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any + +import yaml + + +def _merge( + base: dict[str, Any], child: dict[str, Any], *, recursive: bool +) -> dict[str, Any]: + """Merge independent mappings while retaining preprocessing operation order.""" + + result = copy.deepcopy(base) + for key, value in child.items(): + if isinstance(result.get(key), dict) and isinstance(value, dict): + if recursive: + result[key] = _merge(result[key], value, recursive=True) + else: + # Historical `update` merges exactly one section level. + result[key].update(copy.deepcopy(value)) + else: + result[key] = copy.deepcopy(value) + return result + + +def load_model_variant( + config_path: str | Path, variant: str = "DEFAULT" +) -> dict[str, Any]: + """Resolve a local YAML variant, its aliases, and optional family inheritance. + + ``update`` retains the legacy same-file, one-section merge semantics. + ``extends: bases/family.yaml#DEFAULT`` opts into recursive mapping inheritance; + the fragment defaults to DEFAULT and paths are relative to the declaring file. + Lists and scalars replace their base values. Mapping key order is retained, so + inherited preprocessing operations retain their order. New operations append; + replace a complete operation sequence with a standalone config when reordering. + Only local files are read; each result is independently deep-copied. + """ + + documents: dict[Path, dict[str, Any]] = {} + resolving: set[tuple[Path, str]] = set() + + def document(path: Path) -> dict[str, Any]: + if path not in documents: + with path.open(encoding="utf-8") as stream: + parsed = yaml.safe_load(stream) + if not isinstance(parsed, dict): + raise TypeError( + f"Model configuration '{path}' should define a dictionary." + ) + documents[path] = parsed + return documents[path] + + def resolve(path: Path, key: str) -> dict[str, Any]: + identity = (path, key) + if identity in resolving: + raise ValueError( + f"Circular model configuration reference detected for '{path}#{key}'." + ) + resolving.add(identity) + try: + full = document(path) + part = full.get(key) + if part is None: + raise ValueError(f"Model type '{key}' not found in configuration.") + if isinstance(part, str): + if part not in full: + raise ValueError( + f"Model alias '{part}' not found in configuration." + ) + return resolve(path, part) + if not isinstance(part, dict): + raise TypeError( + f"Resolved model configuration for '{key}' is not a dictionary." + ) + child = copy.deepcopy(part) + base_key = child.pop("update", None) + has_extends = "extends" in child + reference = child.pop("extends", None) + if base_key is not None and has_extends: + raise ValueError( + "A model variant cannot combine 'update' and 'extends'." + ) + if base_key is not None: + if not isinstance(base_key, str) or base_key not in full: + raise ValueError( + f"Base configuration '{base_key}' not found for update." + ) + return _merge(resolve(path, base_key), child, recursive=False) + if has_extends: + if not isinstance(reference, str) or not reference: + raise ValueError( + "Model 'extends' must be a nonempty local YAML reference." + ) + filename, separator, base_variant = reference.partition("#") + if ( + not filename + or "://" in filename + or (separator and not base_variant) + ): + raise ValueError( + f"Invalid local model inheritance reference: {reference!r}." + ) + if "#" in base_variant: + raise ValueError( + f"Invalid local model inheritance reference: {reference!r}." + ) + base_path = (path.parent / filename).resolve() + return _merge( + resolve(base_path, base_variant or "DEFAULT"), child, recursive=True + ) + return child + finally: + resolving.remove(identity) + + return resolve(Path(config_path).resolve(), variant) diff --git a/mblt_vision/wrapper.py b/mblt_vision/wrapper.py index a15296f..d451284 100644 --- a/mblt_vision/wrapper.py +++ b/mblt_vision/wrapper.py @@ -16,11 +16,11 @@ import numpy as np import torch -import yaml from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError - from mblt_npu import MobilintNPUBackend, ONNXBackend, normalize_target_device + +from ._config import load_model_variant from ._model_paths import resolve_framework as _resolve_framework from ._model_paths import split_model_paths as _split_model_paths from ._model_paths import ( @@ -365,63 +365,7 @@ def resolve_model_config( if not config_path.is_file(): config_path = MODEL_CONFIG_DIR / _model_name_aliasing(model_cls) - with config_path.open(encoding="utf-8") as config_file: - full_config = yaml.safe_load(config_file) - if not isinstance(full_config, dict): - raise TypeError( - f"Model configuration '{config_path}' should define a dictionary." - ) - - resolving: set[str] = set() - - def _resolve_variant(variant: str) -> dict[str, Any]: - if variant in resolving: - raise ValueError( - f"Circular model configuration reference detected for '{variant}'." - ) - resolving.add(variant) - try: - model_config_part = full_config.get(variant) - if model_config_part is None: - raise ValueError(f"Model type '{variant}' not found in configuration.") - if isinstance(model_config_part, str): - if model_config_part not in full_config: - raise ValueError( - f"Model alias '{model_config_part}' not found in configuration." - ) - return _resolve_variant(model_config_part) - if not isinstance(model_config_part, dict): - raise TypeError( - f"Resolved model configuration for '{variant}' is not a dictionary." - ) - - resolved = copy.deepcopy(model_config_part) - base_config_key = resolved.pop("update", None) - if base_config_key is None: - return resolved - if ( - not isinstance(base_config_key, str) - or base_config_key not in full_config - ): - raise ValueError( - f"Base configuration '{base_config_key}' not found for update." - ) - - merged_config = _resolve_variant(base_config_key) - for key, value in resolved.items(): - if ( - key in merged_config - and isinstance(merged_config[key], dict) - and isinstance(value, dict) - ): - merged_config[key].update(value) - else: - merged_config[key] = value - return merged_config - finally: - resolving.remove(variant) - - return _normalize_model_artifacts(_resolve_variant(model_type)) + return _normalize_model_artifacts(load_model_variant(config_path, model_type)) class MBLT_Engine: diff --git a/pyproject.toml b/pyproject.toml index 5514221..9850c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ version = { attr = "mblt_vision.__version__" } "mblt_vision" = [ "py.typed", "models/*.yaml", + "models/bases/*.yaml", "datasets/*.yaml", "datasets/*.txt", ] diff --git a/tests/test_config_inheritance.py b/tests/test_config_inheritance.py new file mode 100644 index 0000000..a36907b --- /dev/null +++ b/tests/test_config_inheritance.py @@ -0,0 +1,164 @@ +"""CPU proofs for legacy resolution and opt-in local YAML inheritance.""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import numpy as np +import pytest +import yaml + +from mblt_vision._config import load_model_variant +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.wrapper import MODEL_CONFIG_DIR, resolve_model_config + + +def legacy_resolve(document: dict, variant: str) -> dict: + """Independent pre-refactor resolution, including ONNX artifact derivation.""" + + part = document[variant] + if isinstance(part, str): + return legacy_resolve(document, part) + child = copy.deepcopy(part) + base = child.pop("update", None) + if base is not None: + resolved = legacy_resolve(document, base) + for key, value in child.items(): + if isinstance(resolved.get(key), dict) and isinstance(value, dict): + resolved[key].update(value) + else: + resolved[key] = value + return resolved + return child + + +def ordered(value): + """Deep equality alone misses insertion order, which defines pre-op order.""" + + if isinstance(value, dict): + return [(key, ordered(item)) for key, item in value.items()] + if isinstance(value, list): + return [ordered(item) for item in value] + return value + + +def legacy_config(document: dict, variant: str) -> dict: + config = legacy_resolve(document, variant) + files = config.get("file_cfg", {}) + if files.get("filename") and "onnx_filename" not in files: + files["onnx_filename"] = str(Path(files["filename"]).with_suffix(".onnx")) + return config + + +CONFIG_VARIANTS = [ + (path, variant) + for path in sorted(MODEL_CONFIG_DIR.glob("*.yaml")) + for variant in yaml.safe_load(path.read_text(encoding="utf-8")) +] + + +@pytest.mark.parametrize("path,variant", CONFIG_VARIANTS, ids=lambda value: str(value)) +def test_every_legacy_variant_is_order_identical(path: Path, variant: str) -> None: + expected = legacy_config(yaml.safe_load(path.read_text(encoding="utf-8")), variant) + actual = resolve_model_config(str(path), variant) + assert actual == expected + assert ordered(actual) == ordered(expected) + + +def write_config(path: Path, config: dict) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + return path + + +def test_recursive_extends_keeps_order_and_independent_results(tmp_path: Path) -> None: + write_config( + tmp_path / "bases/base.yaml", + { + "DEFAULT": { + "pre_cfg": { + "Reader": {"style": "pil", "color_mode": "RGB"}, + "Resize": {"size": [32, 32], "interpolation": "bilinear"}, + }, + "post_cfg": {"nested": {"labels": [1, 2], "other": True}}, + } + }, + ) + child = write_config( + tmp_path / "child.yaml", + { + "DEFAULT": { + "extends": "bases/base.yaml", + "pre_cfg": {"Resize": {"size": [16, 16]}, "SetOrder": {"shape": "HWC"}}, + "post_cfg": {"nested": {"labels": [3]}}, + } + }, + ) + result = load_model_variant(child) + assert list(result["pre_cfg"]) == ["Reader", "Resize", "SetOrder"] + assert result["pre_cfg"]["Resize"] == { + "size": [16, 16], + "interpolation": "bilinear", + } + assert result["post_cfg"]["nested"] == {"labels": [3], "other": True} + result["post_cfg"]["nested"]["labels"].append(4) + assert load_model_variant(child)["post_cfg"]["nested"]["labels"] == [3] + + +def test_update_stays_shallow_with_extends_base(tmp_path: Path) -> None: + write_config( + tmp_path / "base.yaml", + {"DEFAULT": {"pre_cfg": {"Resize": {"size": 32, "interpolation": "bilinear"}}}}, + ) + path = write_config( + tmp_path / "child.yaml", + { + "BASE": {"extends": "base.yaml"}, + "DEFAULT": {"update": "BASE", "pre_cfg": {"Resize": {"size": 16}}}, + }, + ) + assert load_model_variant(path)["pre_cfg"]["Resize"] == {"size": 16} + + +@pytest.mark.parametrize( + "reference", + [None, "", 123, "#DEFAULT", "base.yaml#", "https://host/base.yaml", "a#b#c"], +) +def test_invalid_extends(tmp_path: Path, reference) -> None: + path = write_config(tmp_path / "child.yaml", {"DEFAULT": {"extends": reference}}) + with pytest.raises(ValueError): + load_model_variant(path) + + +def test_cross_file_cycle(tmp_path: Path) -> None: + a = write_config(tmp_path / "a.yaml", {"DEFAULT": {"extends": "b.yaml"}}) + write_config(tmp_path / "b.yaml", {"DEFAULT": {"extends": "./a.yaml"}}) + with pytest.raises(ValueError, match="Circular"): + load_model_variant(a) + + +def test_missing_base_and_conflicting_modes(tmp_path: Path) -> None: + path = write_config( + tmp_path / "child.yaml", {"DEFAULT": {"extends": "absent.yaml"}} + ) + with pytest.raises(FileNotFoundError): + load_model_variant(path) + write_config( + path, {"BASE": {}, "DEFAULT": {"extends": "absent.yaml", "update": "BASE"}} + ) + with pytest.raises(ValueError, match="cannot combine"): + load_model_variant(path) + + +@pytest.mark.parametrize("model", ["ResNet50", "YOLOv8n", "YOLOX-s", "DAMO-YOLO-T"]) +def test_cpu_preprocessing_numeric_identity(model: str) -> None: + path = MODEL_CONFIG_DIR / f"{model}.yaml" + expected = legacy_config( + yaml.safe_load(path.read_text(encoding="utf-8")), "DEFAULT" + ) + actual = resolve_model_config(str(path)) + image = np.random.default_rng(0).integers(0, 256, (71, 103, 3), dtype=np.uint8) + before = build_preprocess(expected["pre_cfg"])(image.copy()) + after = build_preprocess(actual["pre_cfg"])(image.copy()) + np.testing.assert_array_equal(np.asarray(after), np.asarray(before)) From c1289eb095bdb4852ca62eaaeee2ee28e8e05f83 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:37:33 +0900 Subject: [PATCH 2/4] Add person retrieval datasets, embedding postprocessing and exact protocol scoring Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 5 + AGENTS.md | 6 + benchmark/benchmark_vision_models.py | 3 +- mblt_vision/README.md | 49 ++++ mblt_vision/__init__.py | 2 + mblt_vision/_tasks.py | 1 + mblt_vision/datasets/dukemtmc-reid.yaml | 35 +++ mblt_vision/datasets/market1501.yaml | 35 +++ mblt_vision/datasets/occluded-duke.yaml | 35 +++ mblt_vision/models/bases/person_reid.yaml | 26 +++ mblt_vision/person_reid/__init__.py | 33 +++ mblt_vision/utils/datasets/reid.py | 96 ++++++++ mblt_vision/utils/evaluation/retrieval.py | 208 +++++++++++++++++ mblt_vision/utils/postprocess/__init__.py | 3 +- mblt_vision/utils/postprocess/build_post.py | 3 + .../utils/postprocess/embedding_post.py | 85 +++++++ mblt_vision/utils/preprocess/normalize.py | 30 ++- tests/reference/ops_reid_metrics.py | 53 +++++ tests/reference/ops_reid_preprocess.py | 92 ++++++++ tests/test_api.py | 1 + tests/test_reid.py | 216 ++++++++++++++++++ tests/test_reid_parity.py | 69 ++++++ 22 files changed, 1082 insertions(+), 4 deletions(-) create mode 100644 mblt_vision/datasets/dukemtmc-reid.yaml create mode 100644 mblt_vision/datasets/market1501.yaml create mode 100644 mblt_vision/datasets/occluded-duke.yaml create mode 100644 mblt_vision/models/bases/person_reid.yaml create mode 100644 mblt_vision/person_reid/__init__.py create mode 100644 mblt_vision/utils/datasets/reid.py create mode 100644 mblt_vision/utils/evaluation/retrieval.py create mode 100644 mblt_vision/utils/postprocess/embedding_post.py create mode 100644 tests/reference/ops_reid_metrics.py create mode 100644 tests/reference/ops_reid_preprocess.py create mode 100644 tests/test_reid.py create mode 100644 tests/test_reid_parity.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index b695725..7d563ab 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -109,6 +109,11 @@ description: >- metric). Load difficult objects as ordinary targets, as Ultralytics' validation does. - Run YOLOv7 models at `iou_thres: 0.65`, WongKinYiu/yolov7's own `test.py` default. +- ReID is utility-only until published wrappers and routing are added. Reuse + `EmbeddingPost` for dense vectors, explicit float32 mean/std for source preprocessing, + and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions + and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. + ## Runtime and Packaging - Route NPU runtime access through mblt-npu-python; do not copy backend classes into Vision. diff --git a/AGENTS.md b/AGENTS.md index 06a243b..5c3c614 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,6 +180,12 @@ The current ownership boundary is deliberate: any future model needing more than one Hub artifact, rather than duplicating Hub-resolution logic. +- ReID utilities live in `mblt_vision.person_reid`; concrete published wrappers are separate. + Use `EmbeddingPost` for validated dense vectors and `PersonReIDPost` for dataset binding. + Retrieval protocols and prepared loaders are pinned to ops source in dataset YAMLs; keep + stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. + Named Normalize styles retain legacy arithmetic; explicit mean/std uses source float32 arithmetic. + ## Python-First Architecture - Keep the public layer independent from a particular backend. Define small internal interfaces diff --git a/benchmark/benchmark_vision_models.py b/benchmark/benchmark_vision_models.py index a5c086d..06f86e3 100644 --- a/benchmark/benchmark_vision_models.py +++ b/benchmark/benchmark_vision_models.py @@ -35,7 +35,8 @@ CORE_MODES: tuple[CoreMode, ...] = cast( tuple[CoreMode, ...], core_modes_for_target_device("aries-rb") ) -TASK_CHOICES = VISION_TASKS +# Utility-only embedding tasks have no benchmark engine/CLI route yet. +TASK_CHOICES = tuple(task for task in VISION_TASKS if task != "person_reid") SUPPORTED_TARGET_DEVICES = frozenset({"aries-rb", "regulus-ra", "regulus-rb"}) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 0187f74..6249bc9 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -489,3 +489,52 @@ validation split, following Ultralytics' depth-validation convention. - Difficult objects count as ordinary targets, as in Ultralytics' validation. + +## Person re-identification utilities + +`mblt_vision.person_reid` provides prepared query/gallery loaders, embedding +postprocessing and retrieval scoring. Published model wrappers/HF artifacts are +not added in this stage; no weights or datasets are downloaded by these utilities. +The CLI/benchmark router is unchanged. + +```python +from mblt_vision.person_reid import Market1501Dataset, Market1501Evaluator + +# root contains query/ and bounding_box_test/; embeddings have one row per sample. +dataset = Market1501Dataset("/path/to/market1501/eval") +# result = Market1501Evaluator().evaluate(embeddings, dataset.samples) +# print(result.as_dict()) +``` + +Also available: `DukeMTMCReIDDataset`, `OccludedDukeDataset` and matching +`DukeMTMCReIDEvaluator`, `OccludedDukeEvaluator`. Loaders emit immutable +`ReIDSample(image, pid, camid, split)` records, sorted query-first then gallery; +cameras are zero-based, identities are not relabeled. Counts are source-checked +by default; caller-owned subsets must explicitly set `validate_counts=False`. +Only Market-1501 drops pid -1 junk; pid 0 background distractors remain. +Duke loaders reject pid -1. Root defaults use `~/.mblt_model_zoo/datasets//eval`. + +`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`. It validates one +floating-point `(N,D)` output or singleton-spatial NCHW/NHWC tensor, optionally +checks `embedding_dim`, and uses `normalize: true` for row-wise L2 normalization +(zero vectors remain zero). Return type is a float32 torch matrix, never class +probabilities. An explicit `post_cfg.dataset` is required. Processing-only base +profiles in `models/bases/person_reid.yaml` can be selected using `extends`; +concrete models must supply their own HF `file_cfg`, dataset and embedding dimension. +`Normalize` accepts either a historical named style or explicit three-element +`mean` and `std` lists; named styles retain unchanged legacy arithmetic. + +`extract_embeddings(batches, infer, postprocess)` runs prepared batches through +the shared bounded inference pipeline, preserving order. Evaluators consume +these embeddings without adding normalization, camera inputs, re-ranking or flip +fusion. Same-ID/same-camera gallery rows are excluded; unmatched queries are +skipped. Squared float32 Euclidean distances use stable ties in gallery order. +Ranks beyond the gallery saturate; mAP is primary, rank-1/5/10 secondary, all +ratios. `as_dict()` rounds to five decimals like ops; raw `mean_ap`/`cmc` remain +unrounded. Dataset YAMLs record exact ops config, loader, evaluator and revision. + +Offline tests vendor the inspected ops functions at +`ec537764e36ca10da963fcb7afab157ea60ca022`. Synthetic embeddings require exact +distance/CMC/mAP equality; synthetic RGB preprocessing requires byte equality +(`atol=0`) against the ops adapter for explicit source mean/std profiles. These +are protocol tests, not measured model accuracy claims. diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index e56c12a..9cf69d2 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -13,6 +13,7 @@ from . import instance_segmentation as instance_segmentation from . import obb as obb from . import object_detection as object_detection +from . import person_reid as person_reid from . import pose_estimation as pose_estimation from . import semantic_segmentation as semantic_segmentation from ._api import list_models as list_models @@ -52,6 +53,7 @@ "object_detection", "obb", "pose_estimation", + "person_reid", "semantic_segmentation", ] + sorted(_LEGACY_MODEL_EXPORTS) # Keep legacy compatibility exports synchronized with their task packages. diff --git a/mblt_vision/_tasks.py b/mblt_vision/_tasks.py index df9a462..c8a4b6d 100644 --- a/mblt_vision/_tasks.py +++ b/mblt_vision/_tasks.py @@ -13,6 +13,7 @@ "obb", "pose_estimation", "face_detection", + "person_reid", ) diff --git a/mblt_vision/datasets/dukemtmc-reid.yaml b/mblt_vision/datasets/dukemtmc-reid.yaml new file mode 100644 index 0000000..c1fb241 --- /dev/null +++ b/mblt_vision/datasets/dukemtmc-reid.yaml @@ -0,0 +1,35 @@ +name: dukemtmc-reid +path: ~/.mblt_model_zoo/datasets/dukemtmc-reid +tasks: +- person_reid +val: eval +cameras: 8 +skip_junk: false +retrieval_splits: + query: + directory: query + count: 2228 + gallery: + directory: bounding_box_test + count: 17661 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/dukemtmc_reid/dukemtmc_reid.yaml + name: dukemtmc-reid + dataloader_path: datasets/dukemtmc_reid/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: dukemtmc-reid-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: dukemtmc-reid-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/dukemtmc_reid/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/market1501.yaml b/mblt_vision/datasets/market1501.yaml new file mode 100644 index 0000000..b638134 --- /dev/null +++ b/mblt_vision/datasets/market1501.yaml @@ -0,0 +1,35 @@ +name: market1501 +path: ~/.mblt_model_zoo/datasets/market1501 +tasks: +- person_reid +val: eval +cameras: 6 +skip_junk: true +retrieval_splits: + query: + directory: query + count: 3368 + gallery: + directory: bounding_box_test + count: 15913 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/market1501/market1501.yaml + name: market1501 + dataloader_path: datasets/market1501/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: market1501-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: market1501-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/market1501/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/occluded-duke.yaml b/mblt_vision/datasets/occluded-duke.yaml new file mode 100644 index 0000000..dc039b2 --- /dev/null +++ b/mblt_vision/datasets/occluded-duke.yaml @@ -0,0 +1,35 @@ +name: occluded-duke +path: ~/.mblt_model_zoo/datasets/occluded-duke +tasks: +- person_reid +val: eval +cameras: 8 +skip_junk: false +retrieval_splits: + query: + directory: query + count: 2210 + gallery: + directory: bounding_box_test + count: 17661 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/occluded_duke/occluded_duke.yaml + name: occluded-duke + dataloader_path: datasets/occluded_duke/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: occluded-duke-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: occluded-duke-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/occluded_duke/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/models/bases/person_reid.yaml b/mblt_vision/models/bases/person_reid.yaml new file mode 100644 index 0000000..cafeaf3 --- /dev/null +++ b/mblt_vision/models/bases/person_reid.yaml @@ -0,0 +1,26 @@ +# Processing only: published HF artifact identities belong to concrete stage-d2 YAMLs. +# Source: mblt-model-ops ec537764e36ca10da963fcb7afab157ea60ca022 +# models/person_reid/CLIP_ReID_RN50_Baseline_DukeMTMC/distribution.yaml +DEFAULT: + pre_cfg: + Reader: + style: pil + Resize: + size: [256, 128] + interpolation: bilinear + Normalize: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + post_cfg: + task: person_reid + dataset: dukemtmc-reid + normalize: true +# Inherit this processing profile and override dataset/dimension/artifacts per model. +MARKET1501: + extends: person_reid.yaml#DEFAULT + post_cfg: + dataset: market1501 +OCCLUDED_DUKE: + extends: person_reid.yaml#DEFAULT + post_cfg: + dataset: occluded-duke diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py new file mode 100644 index 0000000..07fecad --- /dev/null +++ b/mblt_vision/person_reid/__init__.py @@ -0,0 +1,33 @@ +"""Person retrieval utilities; published model wrappers are added separately.""" + +from ..utils.datasets.reid import ( + DukeMTMCReIDDataset, + Market1501Dataset, + OccludedDukeDataset, + ReIDDataset, + ReIDSample, +) +from ..utils.evaluation.retrieval import ( + DukeMTMCReIDEvaluator, + Market1501Evaluator, + OccludedDukeEvaluator, + RetrievalEvaluator, + RetrievalResult, + extract_embeddings, +) +from ..utils.postprocess.embedding_post import PersonReIDPost + +__all__ = [ + "DukeMTMCReIDDataset", + "DukeMTMCReIDEvaluator", + "Market1501Dataset", + "Market1501Evaluator", + "OccludedDukeDataset", + "OccludedDukeEvaluator", + "PersonReIDPost", + "ReIDDataset", + "ReIDSample", + "RetrievalEvaluator", + "RetrievalResult", + "extract_embeddings", +] diff --git a/mblt_vision/utils/datasets/reid.py b/mblt_vision/utils/datasets/reid.py new file mode 100644 index 0000000..af3e68d --- /dev/null +++ b/mblt_vision/utils/datasets/reid.py @@ -0,0 +1,96 @@ +"""Prepared query/gallery person crops; no download or identity relabeling.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from ...datasets import get_dataset_config_for_task + +_FILENAME = re.compile(r"^(-?\d+)_c(\d+)(?:s\d+)?(?:_|\.)") + + +@dataclass(frozen=True) +class ReIDSample: + """One crop with source identity, zero-based camera and query/gallery split.""" + + image: Path + pid: int + camid: int + split: Literal["query", "gallery"] + + +class ReIDDataset: + """Read a prepared benchmark root in source-defined deterministic order. + + ``root`` is the directory containing ``query`` and ``bounding_box_test`` + (the ops prepared ``eval`` directory). Counts are checked by default; + ``validate_counts=False`` explicitly supports caller-owned small subsets. + """ + + name: str + + def __init__( + self, root: str | Path | None = None, *, validate_counts: bool = True + ) -> None: + self.config = get_dataset_config_for_task("person_reid", self.name) + self.root = ( + Path(root).expanduser() + if root is not None + else Path(self.config["path"]) / self.config["val"] + ) + samples = [] + for split in ("query", "gallery"): + options = self.config["retrieval_splits"][split] + directory = self.root / options["directory"] + if not directory.is_dir(): + raise FileNotFoundError(f"ReID split directory is missing: {directory}") + rows = [] + for path in sorted(directory.glob("*.jpg")): + match = _FILENAME.match(path.name) + if match is None: + raise ValueError( + f"ReID file name carries no _c token: {path}" + ) + pid, camera = map(int, match.groups()) + if pid == -1: + if self.config["skip_junk"]: + continue + raise ValueError(f"Unexpected junk identity: {path}") + if not 1 <= camera <= self.config["cameras"]: + raise ValueError(f"Camera {camera} outside benchmark range: {path}") + rows.append(ReIDSample(path, pid, camera - 1, split)) + if not rows: + raise ValueError(f"ReID split directory holds no images: {directory}") + if validate_counts and len(rows) != options["count"]: + raise ValueError( + f"{split} has {len(rows)} crops, expected {options['count']}" + ) + samples.extend(rows) + self.samples = tuple(samples) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> ReIDSample: + return self.samples[index] + + +class Market1501Dataset(ReIDDataset): + """Market-1501: discard pid -1; retain pid 0 background distractors.""" + + name = "market1501" + + +class DukeMTMCReIDDataset(ReIDDataset): + """DukeMTMC-reID prepared held-out query/gallery crops.""" + + name = "dukemtmc-reid" + + +class OccludedDukeDataset(ReIDDataset): + """Occluded-Duke prepared query split and Duke gallery.""" + + name = "occluded-duke" diff --git a/mblt_vision/utils/evaluation/retrieval.py b/mblt_vision/utils/evaluation/retrieval.py new file mode 100644 index 0000000..e3d8ffa --- /dev/null +++ b/mblt_vision/utils/evaluation/retrieval.py @@ -0,0 +1,208 @@ +"""Source-exact Market protocol retrieval metrics (ops ec537764).""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch + +from ..datasets.reid import ReIDSample +from ._pipeline import map_batched_inference + + +def extract_embeddings( + batches: Iterable[Any], infer: Callable, postprocess: Callable +) -> np.ndarray: + """Decode prepared batches in input order through the shared bounded pipeline. + + ``infer`` receives a prepared batch; ``postprocess`` receives raw outputs and + returns a validated ``(N,D)`` tensor. No implicit normalization or flip fusion. + """ + + def decode(batch: Any, output: Any) -> np.ndarray: + value = postprocess(output) + if isinstance(value, torch.Tensor): + value = value.detach().cpu().numpy() + value = np.asarray(value, dtype=np.float32) + if value.ndim != 2 or min(value.shape) == 0 or not np.isfinite(value).all(): + raise ValueError( + "Decoded embeddings require a nonempty finite (N,D) matrix" + ) + return value + + rows = list(map_batched_inference(batches, infer, decode)) + if not rows: + raise ValueError("No embedding batches supplied") + return np.concatenate(rows, axis=0) + + +def euclidean_distance(query: np.ndarray, gallery: np.ndarray) -> np.ndarray: + """Source float32 squared Euclidean distance (not cosine or square root).""" + query = np.asarray(query, dtype=np.float32) + gallery = np.asarray(gallery, dtype=np.float32) + return ( + np.sum(query**2, axis=1, keepdims=True) + + np.sum(gallery**2, axis=1, keepdims=True).T + - 2.0 * (query @ gallery.T) + ) + + +@dataclass(frozen=True) +class RetrievalResult: + """Unrounded mAP and CMC curve; ``as_dict`` matches ops score rounding.""" + + cmc: np.ndarray + mean_ap: float + sample_count: int + + @property + def primary_score(self) -> float: + return self.mean_ap + + @property + def secondary_score(self) -> float: + return self.rank(1) + + def rank(self, k: int) -> float: + """Saturate ranks beyond the gallery length, as in the source scorer.""" + if not isinstance(k, int) or isinstance(k, bool) or k <= 0: + raise ValueError("Rank must be a positive integer") + return float(self.cmc[min(k, len(self.cmc)) - 1]) + + def as_dict(self) -> dict: + """Return source-compatible primary/secondary metrics in ratio units.""" + return { + "primary": {"map": round(self.mean_ap, 5)}, + "secondary": {f"rank{k}": round(self.rank(k), 5) for k in (1, 5, 10)}, + "sample_count": self.sample_count, + } + + +class RetrievalEvaluator: + """Market retrieval protocol shared by three dataset-specific evaluators. + + Stable distance ties retain gallery manifest order. Exclude only same-ID, + same-camera rows; pid 0 distractors remain. Dataset loaders own pid -1 junk + filtering. Queries without cross-camera matches do not enter either mean. + """ + + dataset: str | None = None + + def evaluate( + self, + embeddings: np.ndarray, + samples: Sequence[ReIDSample], + *, + max_rank: int = 50, + ) -> RetrievalResult: + """Score one embedding row per sample in query-then-gallery order.""" + values = np.asarray(embeddings, dtype=np.float32) + if ( + values.ndim != 2 + or min(values.shape) == 0 + or values.shape[0] != len(samples) + or not np.isfinite(values).all() + ): + raise ValueError("Embeddings require one finite nonempty vector per sample") + if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: + raise ValueError("max_rank must be a positive integer") + if any(s.split not in {"query", "gallery"} or s.pid == -1 for s in samples): + raise ValueError( + "Samples require query/gallery splits with loader-filtered junk" + ) + query = [i for i, s in enumerate(samples) if s.split == "query"] + gallery = [i for i, s in enumerate(samples) if s.split == "gallery"] + if not query or not gallery: + raise ValueError("Benchmark needs both a query and a gallery split") + cmc, mean_ap = self.evaluate_distances( + euclidean_distance(values[query], values[gallery]), + np.asarray([samples[i].pid for i in query]), + np.asarray([samples[i].pid for i in gallery]), + np.asarray([samples[i].camid for i in query]), + np.asarray([samples[i].camid for i in gallery]), + max_rank, + ) + return RetrievalResult(cmc, mean_ap, len(samples)) + + @staticmethod + def evaluate_distances( + distmat: np.ndarray, + q_pids: np.ndarray, + g_pids: np.ndarray, + q_camids: np.ndarray, + g_camids: np.ndarray, + max_rank: int = 50, + ) -> tuple[np.ndarray, float]: + """CMC and mAP with the exact source accumulation and padding order.""" + distmat = np.asarray(distmat) + q_pids, g_pids = np.asarray(q_pids), np.asarray(g_pids) + q_camids, g_camids = np.asarray(q_camids), np.asarray(g_camids) + if ( + distmat.ndim != 2 + or min(distmat.shape) == 0 + or not np.isfinite(distmat).all() + ): + raise ValueError("Distances require a nonempty finite matrix") + num_q, num_g = distmat.shape + if any( + v.shape != (n,) + for v, n in ( + (q_pids, num_q), + (g_pids, num_g), + (q_camids, num_q), + (g_camids, num_g), + ) + ): + raise ValueError( + "Identity/camera metadata does not match distance dimensions" + ) + if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: + raise ValueError("max_rank must be a positive integer") + max_rank = min(max_rank, num_g) + indices = np.argsort(distmat, axis=1, kind="stable") + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + all_cmc = [] + all_ap = [] + for q_idx in range(num_q): + order = indices[q_idx] + remove = (g_pids[order] == q_pids[q_idx]) & ( + g_camids[order] == q_camids[q_idx] + ) + orig_cmc = matches[q_idx][~remove] + if not np.any(orig_cmc): + continue + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + all_cmc.append( + np.pad( + cmc[:max_rank], (0, max_rank - min(len(cmc), max_rank)), mode="edge" + ) + ) + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() / np.arange(1, orig_cmc.shape[0] + 1) + all_ap.append(float((tmp_cmc * orig_cmc).sum() / num_rel)) + if not all_cmc: + raise ValueError("No query identity appears in the gallery") + cmc_curve = np.asarray(all_cmc, dtype=np.float64).sum(0) / len(all_cmc) + return cmc_curve, float(np.mean(all_ap)) + + +class Market1501Evaluator(RetrievalEvaluator): + """Market-1501 held-out retrieval protocol v1.""" + + dataset = "market1501" + + +class DukeMTMCReIDEvaluator(RetrievalEvaluator): + """DukeMTMC-reID held-out retrieval protocol v1.""" + + dataset = "dukemtmc-reid" + + +class OccludedDukeEvaluator(RetrievalEvaluator): + """Occluded-Duke held-out retrieval protocol v1.""" + + dataset = "occluded-duke" diff --git a/mblt_vision/utils/postprocess/__init__.py b/mblt_vision/utils/postprocess/__init__.py index e5090d0..879a06f 100644 --- a/mblt_vision/utils/postprocess/__init__.py +++ b/mblt_vision/utils/postprocess/__init__.py @@ -4,6 +4,7 @@ from .build_post import build_postprocess from .depth_post import DepthPost +from .embedding_post import EmbeddingPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost -__all__ = ["DepthPost", "SemanticSegPost", "build_postprocess"] +__all__ = ["DepthPost", "EmbeddingPost", "PersonReIDPost", "SemanticSegPost", "build_postprocess"] diff --git a/mblt_vision/utils/postprocess/build_post.py b/mblt_vision/utils/postprocess/build_post.py index dc89192..eb569ac 100644 --- a/mblt_vision/utils/postprocess/build_post.py +++ b/mblt_vision/utils/postprocess/build_post.py @@ -9,6 +9,7 @@ from .cls_post import ClsPost from .damoyolo_post import DAMOYOLODetectionPost from .depth_post import DepthPost +from .embedding_post import PersonReIDPost from .semantic_seg_post import SemanticSegPost from .yolo_anchor_post import ( YOLOAnchorDetectionPost, @@ -70,6 +71,8 @@ def build_postprocess( f"Unsupported post_cfg.head {head!r}; expected one of {sorted(DETECTION_HEADS)}." ) return DETECTION_HEADS[head](pre_cfg, post_cfg, **kwargs) + if task == "person_reid": + return PersonReIDPost(pre_cfg, post_cfg) if task == "image_classification": return ClsPost(pre_cfg, post_cfg) if task == "depth_estimation": diff --git a/mblt_vision/utils/postprocess/embedding_post.py b/mblt_vision/utils/postprocess/embedding_post.py new file mode 100644 index 0000000..8d55353 --- /dev/null +++ b/mblt_vision/utils/postprocess/embedding_post.py @@ -0,0 +1,85 @@ +"""Validated appearance embeddings, shared by retrieval and verification tasks.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np +import torch +import torch.nn.functional as F + +from ...datasets import get_dataset_config_for_task +from ..types import ListTensorLike, TensorLike +from .base import PostBase + + +class EmbeddingPost(PostBase): + """Return floating-point ``(batch, dimension)`` embeddings without argmax. + + Accept a single matrix or singleton-spatial NCHW/NHWC output (also wrapped + in a one-element runtime output list). L2 normalization is opt-in; zero + vectors stay zero, matching the source embedding decoder. + """ + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__() + self.normalize = post_cfg.get("normalize", False) + if not isinstance(self.normalize, bool): + raise TypeError("post_cfg.normalize must be a boolean") + self.embedding_dim = post_cfg.get("embedding_dim") + if self.embedding_dim is not None and ( + not isinstance(self.embedding_dim, int) + or isinstance(self.embedding_dim, bool) + or self.embedding_dim <= 0 + ): + raise ValueError("post_cfg.embedding_dim must be a positive integer") + + def __call__(self, x: TensorLike | ListTensorLike) -> torch.Tensor: + """Validate one dense output and optionally L2-normalize each row.""" + if isinstance(x, Sequence): + if len(x) != 1: + raise ValueError( + "Embedding postprocessing expects exactly one output tensor" + ) + x = x[0] + if isinstance(x, np.ndarray): + x = torch.from_numpy(x) + if not isinstance(x, torch.Tensor): + raise TypeError("Embedding output must be a NumPy array or torch tensor") + if not x.is_floating_point(): + raise TypeError("Embedding output must have a floating-point dtype") + x = x.to(self.device) + if x.ndim == 4 and (x.shape[2:] == (1, 1) or x.shape[1:3] == (1, 1)): + x = x.reshape(x.shape[0], -1) + if x.ndim != 2 or min(x.shape) <= 0: + raise ValueError( + f"Embedding output requires nonempty (N, D), got {tuple(x.shape)}" + ) + if self.embedding_dim is not None and x.shape[1] != self.embedding_dim: + raise ValueError( + f"Embedding dimension {x.shape[1]} does not match {self.embedding_dim}" + ) + if not bool(torch.isfinite(x).all()): + raise ValueError("Embedding output must contain only finite values") + # Float32 agrees with the ops decoder, including its 1e-12 norm floor. + x = x.float() + if not bool(torch.isfinite(x).all()): + raise ValueError("Embedding values overflow float32") + if self.normalize and not bool( + torch.isfinite(torch.linalg.vector_norm(x, dim=1)).all() + ): + raise ValueError("Embedding norms overflow float32") + if self.normalize: + x = F.normalize(x, p=2, dim=1, eps=1e-12) + return x + + +class PersonReIDPost(EmbeddingPost): + """Appearance embeddings tied to an explicitly configured ReID dataset.""" + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__(pre_cfg, post_cfg) + dataset = post_cfg.get("dataset") + if not isinstance(dataset, str) or not dataset: + raise ValueError("Person ReID postprocessing requires post_cfg.dataset") + self.dataset = get_dataset_config_for_task("person_reid", dataset)["name"] diff --git a/mblt_vision/utils/preprocess/normalize.py b/mblt_vision/utils/preprocess/normalize.py index be7dd1b..ad19c3d 100644 --- a/mblt_vision/utils/preprocess/normalize.py +++ b/mblt_vision/utils/preprocess/normalize.py @@ -30,14 +30,40 @@ class Normalize(PreOps): std: Array of standard deviation values for normalization. """ - def __init__(self, style: str) -> None: + def __init__( + self, + style: str | None = None, + *, + mean: list[float] | None = None, + std: list[float] | None = None, + ) -> None: """Initializes the Normalize layer with a specific style. Args: - style: The preprocessing style to use. Must be one of STYLE_LIST. + style: Historical named style; mutually exclusive with mean/std. + mean: Optional explicit three-element RGB mean, using float32 arithmetic. + std: Optional explicit positive RGB standard deviations. """ super().__init__() + self.explicit_stats = mean is not None or std is not None + if self.explicit_stats: + if style is not None or mean is None or std is None: + raise ValueError("Provide either style or both mean and std") + self.mean = np.asarray(mean, dtype=np.float32) + self.std = np.asarray(std, dtype=np.float32) + if ( + self.mean.shape != (3,) + or self.std.shape != (3,) + or not np.isfinite(self.mean).all() + or not np.isfinite(self.std).all() + or not (self.std > 0).all() + ): + raise ValueError( + "mean/std require three finite RGB values and positive std" + ) + self.style = "explicit" + return if not isinstance(style, str): raise TypeError( f"Normalize style must be a string, got {type(style).__name__}." diff --git a/tests/reference/ops_reid_metrics.py b/tests/reference/ops_reid_metrics.py new file mode 100644 index 0000000..78f10fa --- /dev/null +++ b/tests/reference/ops_reid_metrics.py @@ -0,0 +1,53 @@ +"""Verbatim functions from datasets/_shared/reid.py at +ec537764e36ca10da963fcb7afab157ea60ca022 (mblt-model-ops). +Snapshot permits ordinary offline differential tests without sibling repos. +""" +import numpy as np +MAX_RANK = 50 + +def euclidean_distance(query: np.ndarray, gallery: np.ndarray) -> np.ndarray: + """Squared euclidean distance, the reference ``euclidean_distance`` without the sqrt. + + Ranking is monotone in the square, so CMC and mAP are unchanged. + """ + query = np.asarray(query, dtype=np.float32) + gallery = np.asarray(gallery, dtype=np.float32) + q2 = np.sum(query**2, axis=1, keepdims=True) + g2 = np.sum(gallery**2, axis=1, keepdims=True).T + return q2 + g2 - 2.0 * (query @ gallery.T) + +def evaluate( + distmat: np.ndarray, + q_pids: np.ndarray, + g_pids: np.ndarray, + q_camids: np.ndarray, + g_camids: np.ndarray, + max_rank: int = MAX_RANK, +) -> tuple[np.ndarray, float]: + """Market-1501 protocol CMC curve and mAP (reference ``eval_func``).""" + num_q, num_g = distmat.shape + max_rank = min(max_rank, num_g) + indices = np.argsort(distmat, axis=1, kind="stable") + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + all_cmc = [] + all_ap = [] + for q_idx in range(num_q): + order = indices[q_idx] + remove = (g_pids[order] == q_pids[q_idx]) & (g_camids[order] == q_camids[q_idx]) + orig_cmc = matches[q_idx][~remove] + if not np.any(orig_cmc): + continue # the query identity never appears in the gallery + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + # Same-identity/same-camera removal leaves different gallery lengths. + # Saturate missing ranks with the final CMC value for this query. + all_cmc.append( + np.pad(cmc[:max_rank], (0, max_rank - min(len(cmc), max_rank)), mode="edge") + ) + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() / np.arange(1, orig_cmc.shape[0] + 1) + all_ap.append(float((tmp_cmc * orig_cmc).sum() / num_rel)) + if not all_cmc: + raise ValueError("No query identity appears in the gallery") + cmc_curve = np.asarray(all_cmc, dtype=np.float64).sum(0) / len(all_cmc) + return cmc_curve, float(np.mean(all_ap)) diff --git a/tests/reference/ops_reid_preprocess.py b/tests/reference/ops_reid_preprocess.py new file mode 100644 index 0000000..b2b4ce0 --- /dev/null +++ b/tests/reference/ops_reid_preprocess.py @@ -0,0 +1,92 @@ +# Verbatim source: models/person_reid/_shared/preprocess.py +# mblt-model-ops ec537764e36ca10da963fcb7afab157ea60ca022 +"""Default person_reid preprocess: the CLIP-ReID / TransReID test transform. + +Upstream evaluates with ``Resize((H, W)) -> ToTensor -> Normalize(mean, std)``; there is no +crop, and the two backbone families differ only in the normalization statistics, so the +statistics are explicit ``mean``/``std`` lists in ``pre_cfg`` rather than a named style. + +Operations are applied in a fixed canonical order, not in ``config`` iteration order: +``predict.input.json`` is written with sorted keys, so relying on dict order would run +``Normalize`` before ``Resize``. + +A camera-aware Model (SIE) declares ``CameraOneHot``. The camera index is the ``_c`` +token of the crop's file name, which is how every ReID dataset and the upstream loader +encode it; it is a Model input, not ground truth. The network then takes two inputs, and +the returned tensor is a tuple ``(image, camera_one_hot)`` in that order. + +The return value is ``(source, network_input, camera_index)`` so that the shared +``inference.tensor_of`` finds the network input in the second slot. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +PIPELINE_ORDER = ("Reader", "Resize", "SetOrder", "Normalize", "CameraOneHot") +REQUIRED = ("Resize", "Normalize") +_RESAMPLING = { + "bilinear": Image.Resampling.BILINEAR, + "bicubic": Image.Resampling.BICUBIC, +} +DEFAULT_CAMERA_PATTERN = r"_c(\d+)" + +NetworkInput = np.ndarray | tuple[np.ndarray, np.ndarray] + + +def camera_index(source: Path, config: dict[str, Any]) -> int: + """The 0-based camera index of a crop, from its ``_c`` file-name token.""" + pattern = str(config.get("pattern", DEFAULT_CAMERA_PATTERN)) + match = re.search(pattern, Path(source).name) + if match is None: + raise ValueError(f"{source}: file name carries no camera token {pattern!r}") + cameras = int(config["num_cameras"]) + camera = int(match.group(1)) - 1 + if not 0 <= camera < cameras: + raise ValueError(f"{source}: camera {camera + 1} is outside 1..{cameras}") + return camera + + +def preprocess( + source: Path, config: dict[str, Any] +) -> tuple[Path, NetworkInput, int | None]: + """Return ``(source, network input, camera index or None)``.""" + unknown = set(config) - set(PIPELINE_ORDER) + if unknown: + raise ValueError(f"unsupported preprocess operations: {sorted(unknown)}") + missing = [name for name in REQUIRED if name not in config] + if missing: + raise ValueError(f"pre_cfg is missing required operations: {missing}") + source = Path(source) + value: Any = Image.open(source).convert("RGB") + camera: int | None = None + for operation in PIPELINE_ORDER: + if operation not in config: + continue + options = config[operation] + if operation == "Reader": + continue + if operation == "Resize": + height, width = (int(v) for v in options["size"]) + mode = _RESAMPLING[str(options.get("interpolation", "bilinear"))] + value = value.resize((width, height), mode) + elif operation == "SetOrder": + if options.get("shape", "HWC") != "HWC": + raise ValueError("person_reid preprocess produces HWC tensors only") + elif operation == "Normalize": + mean = np.asarray(options["mean"], dtype=np.float32) + std = np.asarray(options["std"], dtype=np.float32) + value = (np.asarray(value, dtype=np.float32) / 255.0 - mean) / std + elif operation == "CameraOneHot": + camera = camera_index(source, options) + tensor = np.ascontiguousarray(value, dtype=np.float32) + if camera is None: + return source, tensor, None + one_hot = np.zeros(int(config["CameraOneHot"]["num_cameras"]), dtype=np.float32) + one_hot[camera] = 1.0 + return source, (tensor, one_hot), camera diff --git a/tests/test_api.py b/tests/test_api.py index 40b4182..ce184aa 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -21,6 +21,7 @@ def test_public_discovery_exposes_all_supported_tasks() -> None: "obb", "pose_estimation", "face_detection", + "person_reid", ] assert list_models("obb")["obb"] diff --git a/tests/test_reid.py b/tests/test_reid.py new file mode 100644 index 0000000..a6c0fcc --- /dev/null +++ b/tests/test_reid.py @@ -0,0 +1,216 @@ +"""CPU-only embedding, prepared loader and retrieval protocol contracts.""" + +from pathlib import Path + +import numpy as np +import pytest +import torch +from mblt_vision._config import load_model_variant +from mblt_vision.datasets import get_dataset_config +from mblt_vision.utils.datasets.reid import ( + DukeMTMCReIDDataset, + Market1501Dataset, + OccludedDukeDataset, + ReIDSample, +) +from mblt_vision.utils.evaluation.retrieval import ( + DukeMTMCReIDEvaluator, + Market1501Evaluator, + OccludedDukeEvaluator, + RetrievalEvaluator, + extract_embeddings, +) +from mblt_vision.utils.postprocess import build_postprocess +from mblt_vision.utils.postprocess.embedding_post import EmbeddingPost, PersonReIDPost +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.utils.preprocess.normalize import Normalize + + +@pytest.mark.parametrize("shape", [(2, 3), (2, 3, 1, 1), (2, 1, 1, 3)]) +def test_embedding_shapes(shape): + raw = np.array([[3.0, 4.0, 0.0], [0.0, 0.0, 0.0]], dtype=np.float32).reshape(shape) + post = build_postprocess( + {}, + { + "task": "person_reid", + "dataset": "market1501", + "normalize": True, + "embedding_dim": 3, + }, + ) + assert isinstance(post, (EmbeddingPost, PersonReIDPost)) + np.testing.assert_array_equal( + post([raw]), np.array([[0.6, 0.8, 0.0], [0.0, 0.0, 0.0]], dtype=np.float32) + ) + np.testing.assert_array_equal(raw.reshape(2, 3)[0], [3.0, 4.0, 0.0]) + + +@pytest.mark.parametrize( + "raw", + [ + np.empty((0, 3)), + np.ones(3), + np.ones((2, 3, 2, 2)), + np.ones((2, 3, 1)), + np.array([[np.nan]]), + np.array([[np.inf]]), + ], +) +def test_embedding_rejects_invalid(raw): + with pytest.raises(ValueError): + EmbeddingPost({}, {})(raw) + + +def test_embedding_validation_and_optional_normalization(): + post = EmbeddingPost({}, {}) + raw = torch.tensor([[3.0, 4.0]]) + assert torch.equal(post(raw), raw) + with pytest.raises(TypeError): + post(np.ones((1, 2), dtype=np.int32)) + with pytest.raises(ValueError): + post([raw, raw]) + with pytest.raises(ValueError): + EmbeddingPost({}, {"embedding_dim": 3})(raw) + with pytest.raises(ValueError): + PersonReIDPost({}, {"dataset": "coco"}) + with pytest.raises(ValueError): + PersonReIDPost({}, {}) + with pytest.raises(TypeError): + EmbeddingPost({}, {"normalize": "yes"}) + + +@pytest.mark.parametrize( + "dataset,loader", + [ + ("market1501", Market1501Dataset), + ("dukemtmc-reid", DukeMTMCReIDDataset), + ("occluded-duke", OccludedDukeDataset), + ], +) +def test_loader_metadata_and_source(tmp_path, dataset, loader): + for folder in ("query", "bounding_box_test"): + (tmp_path / folder).mkdir() + for filename in ("0002_c2_b.jpg", "0001_c1s1_a.jpg"): + (tmp_path / "query" / filename).touch() + (tmp_path / "bounding_box_test" / "0001_c2_a.jpg").touch() + if dataset == "market1501": + (tmp_path / "query" / "-1_c1_junk.jpg").touch() + (tmp_path / "bounding_box_test" / "0000_c1_background.jpg").touch() + data = loader(tmp_path, validate_counts=False) + assert [(s.pid, s.camid, s.split) for s in data][:2] == [ + (1, 0, "query"), + (2, 1, "query"), + ] + assert data[-1].split == "gallery" + assert all(s.pid != -1 for s in data) + if dataset == "market1501": + assert data[2].pid == 0 + with pytest.raises(ValueError, match="expected"): + loader(tmp_path) + cfg = get_dataset_config(dataset) + assert cfg["source"]["revision"] == "ec537764e36ca10da963fcb7afab157ea60ca022" + assert cfg["eval_protocol"]["id"] == cfg["evaluation"]["protocol"] + + +@pytest.mark.parametrize("filename", ["bad.jpg", "1_c9_bad.jpg", "-1_c1_junk.jpg"]) +def test_duke_invalid_names(tmp_path, filename): + (tmp_path / "query").mkdir() + (tmp_path / "bounding_box_test").mkdir() + (tmp_path / "query" / filename).touch() + with pytest.raises(ValueError): + DukeMTMCReIDDataset(tmp_path, validate_counts=False) + + +def test_missing_and_empty_splits(tmp_path): + with pytest.raises(FileNotFoundError): + Market1501Dataset(tmp_path) + (tmp_path / "query").mkdir() + with pytest.raises(ValueError, match="no images"): + Market1501Dataset(tmp_path) + + +def test_handbuilt_exclusions_ties_and_missing_identity(): + # same-camera positive is removed; tied background stays before the positive. + distances = np.array([[0.0, 1.0, 1.0, 2.0], [4.0, 3.0, 2.0, 1.0]]) + cmc, mean_ap = RetrievalEvaluator.evaluate_distances( + distances, + np.array([1, 99]), + np.array([1, 0, 1, 1]), + np.array([0, 0]), + np.array([0, 1, 1, 2]), + ) + np.testing.assert_array_equal(cmc, [0.0, 1.0, 1.0, 1.0]) + assert mean_ap == (1 / 2 + 2 / 3) / 2 + # Stable ties are manifest order, not identity order. + cmc, mean_ap = RetrievalEvaluator.evaluate_distances( + np.zeros((1, 2)), + np.array([1]), + np.array([1, 0]), + np.array([0]), + np.array([1, 1]), + ) + assert mean_ap == 1.0 + np.testing.assert_array_equal(cmc, [1.0, 1.0]) + + +@pytest.mark.parametrize( + "evaluator", [Market1501Evaluator, DukeMTMCReIDEvaluator, OccludedDukeEvaluator] +) +def test_score_and_rank_saturation(evaluator): + samples = [ + ReIDSample(Path("q"), 1, 0, "query"), + ReIDSample(Path("g"), 1, 1, "gallery"), + ] + result = evaluator().evaluate(np.array([[1.0, 0.0], [1.0, 0.0]]), samples) + assert result.as_dict() == { + "primary": {"map": 1.0}, + "secondary": {"rank1": 1.0, "rank5": 1.0, "rank10": 1.0}, + "sample_count": 2, + } + assert result.primary_score == result.secondary_score == result.rank(50) == 1.0 + with pytest.raises(ValueError): + result.rank(0) + + +def test_no_valid_queries(): + with pytest.raises(ValueError, match="No query"): + RetrievalEvaluator.evaluate_distances( + np.zeros((1, 1)), np.array([1]), np.array([1]), np.array([0]), np.array([0]) + ) + + +def test_extraction_order(): + batches = [np.full((2, 3), i, dtype=np.float32) for i in range(5)] + actual = extract_embeddings(batches, lambda x: [x], EmbeddingPost({}, {})) + np.testing.assert_array_equal(actual, np.concatenate(batches)) + with pytest.raises(ValueError): + extract_embeddings([], lambda x: x, lambda x: x) + + +@pytest.mark.parametrize( + "options", + [ + {"mean": [0, 0, 0]}, + {"mean": [0, 0, 0], "std": [1, 0, 1]}, + {"mean": [0, 0], "std": [1, 1]}, + {"style": "torch", "mean": [0, 0, 0], "std": [1, 1, 1]}, + ], +) +def test_explicit_normalization_validation(options): + with pytest.raises(ValueError): + Normalize(**options) + + +def test_family_base_inheritance(tmp_path): + base = Path(__file__).parents[1] / "mblt_vision/models/bases/person_reid.yaml" + for variant, name in [ + ("DEFAULT", "dukemtmc-reid"), + ("MARKET1501", "market1501"), + ("OCCLUDED_DUKE", "occluded-duke"), + ]: + cfg = load_model_variant(base, variant) + assert cfg["post_cfg"]["dataset"] == name + assert isinstance( + build_postprocess(cfg["pre_cfg"], cfg["post_cfg"]), PersonReIDPost + ) + assert build_preprocess(cfg["pre_cfg"]) diff --git a/tests/test_reid_parity.py b/tests/test_reid_parity.py new file mode 100644 index 0000000..84bdfa6 --- /dev/null +++ b/tests/test_reid_parity.py @@ -0,0 +1,69 @@ +"""Offline numeric differential tests against pinned, verbatim ops functions.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from mblt_vision._config import load_model_variant +from mblt_vision.utils.evaluation.retrieval import ( + RetrievalEvaluator, + euclidean_distance, +) +from mblt_vision.utils.preprocess import build_preprocess +from PIL import Image + + +def reference(name): + path = Path(__file__).parent / "reference" / f"ops_reid_{name}.py" + spec = importlib.util.spec_from_file_location(f"ops_reid_{name}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("seed", range(10)) +def test_ops_evaluator_exact_numeric_parity(seed): + ops = reference("metrics") + rng = np.random.default_rng(seed) + # Quantized integer vectors produce numerous exact distance ties. + query = rng.integers(-2, 3, size=(17, 8)).astype(np.float32) + gallery = rng.integers(-2, 3, size=(61, 8)).astype(np.float32) + q_ids = rng.integers(1, 6, 17) + g_ids = rng.integers(0, 6, 61) # zero-valued distractors remain ordinary rows + q_cam = rng.integers(0, 3, 17) + g_cam = rng.integers(0, 3, 61) + expected_distances = ops.euclidean_distance(query, gallery) + actual_distances = euclidean_distance(query, gallery) + np.testing.assert_array_equal(actual_distances, expected_distances) + for max_rank in (1, 5, 50, 100): + expected_cmc, expected_map = ops.evaluate( + expected_distances, q_ids, g_ids, q_cam, g_cam, max_rank + ) + actual_cmc, actual_map = RetrievalEvaluator.evaluate_distances( + actual_distances, q_ids, g_ids, q_cam, g_cam, max_rank + ) + np.testing.assert_array_equal(actual_cmc, expected_cmc) + assert actual_map == expected_map + + +@pytest.mark.parametrize("shape", [(19, 11), (128, 256), (383, 191)]) +@pytest.mark.parametrize("style", ["torch", "tf"]) +def test_ops_preprocessing_parity(tmp_path, shape, style): + ops = reference("preprocess") + cfg = load_model_variant( + Path(__file__).parents[1] / "mblt_vision/models/bases/person_reid.yaml", + "DEFAULT", + ) + config = cfg["pre_cfg"] + if style == "tf": + # CLIP_ReID_ViT_B_16_Baseline_DukeMTMC/distribution.yaml at ec537764. + config["Normalize"] = {"mean": [0.5, 0.5, 0.5], "std": [0.5, 0.5, 0.5]} + rng = np.random.default_rng(0) + path = tmp_path / "0001_c2_synthetic.png" + Image.fromarray(rng.integers(0, 256, size=(*shape, 3), dtype=np.uint8)).save(path) + expected = ops.preprocess(path, config)[1] + actual = build_preprocess(config)(path) + # Both use PIL bilinear + explicit float32 stats; no tolerance needed. + np.testing.assert_array_equal(actual, expected) + assert np.max(np.abs(actual - expected)) == 0.0 From 81a3727893009bbb65a667e6d10ba5e6f30bd148 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:46:47 +0900 Subject: [PATCH 3/4] Connect retrieval embeddings to engine results and dataset readiness Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/README.md | 7 ++- mblt_vision/person_reid/__init__.py | 3 +- mblt_vision/utils/datasets/readiness.py | 26 +++++++++++ mblt_vision/utils/results.py | 27 ++++++++++- tests/test_reid.py | 60 +++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 3 deletions(-) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 6249bc9..25bccab 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -514,7 +514,12 @@ by default; caller-owned subsets must explicitly set `validate_counts=False`. Only Market-1501 drops pid -1 junk; pid 0 background distractors remain. Duke loaders reject pid -1. Root defaults use `~/.mblt_model_zoo/datasets//eval`. -`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`. It validates one +`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`; both are exported +from `mblt_vision.person_reid`. Engine postprocessing returns a `Results` object +whose `embeddings` and `output` retain the validated `(N,D)` matrix. +`dataset_ready(root, "person_reid", dataset)` checks source counts/metadata and +decodes every retained query/gallery image (default dataset: Market-1501). +`PersonReIDPost` It validates one floating-point `(N,D)` output or singleton-spatial NCHW/NHWC tensor, optionally checks `embedding_dim`, and uses `normalize: true` for row-wise L2 normalization (zero vectors remain zero). Return type is a float32 torch matrix, never class diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py index 07fecad..e5e6574 100644 --- a/mblt_vision/person_reid/__init__.py +++ b/mblt_vision/person_reid/__init__.py @@ -15,11 +15,12 @@ RetrievalResult, extract_embeddings, ) -from ..utils.postprocess.embedding_post import PersonReIDPost +from ..utils.postprocess.embedding_post import EmbeddingPost, PersonReIDPost __all__ = [ "DukeMTMCReIDDataset", "DukeMTMCReIDEvaluator", + "EmbeddingPost", "Market1501Dataset", "Market1501Evaluator", "OccludedDukeDataset", diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index 7fd18b4..8426389 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -20,6 +20,7 @@ from ..._tasks import normalize_vision_task from ...datasets import get_dataset_category_ids from .cityscapes import CITYSCAPES_SOURCE_TO_TRAIN_ID +from .reid import DukeMTMCReIDDataset, Market1501Dataset, OccludedDukeDataset IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} IMAGENET_CLASS_COUNT = 1000 @@ -1044,6 +1045,29 @@ def dense_dataset_ready(data_path: str | Path, dataset: str) -> bool: return False +def _reid_ready(root: Path, dataset: str) -> bool: + """Require source counts, camera/name metadata and decodable query/gallery crops.""" + loader = { + "market1501": Market1501Dataset, + "dukemtmc-reid": DukeMTMCReIDDataset, + "occluded-duke": OccludedDukeDataset, + }.get(dataset) + if loader is None: + return False + try: + samples = loader(root).samples + for sample in samples: + if not sample.image.is_file() or sample.image.is_symlink(): + return False + with Image.open(sample.image) as image: + image.load() + if min(image.size) <= 0: + return False + except (OSError, ValueError): + return False + return True + + def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) -> bool: """Return whether an organized dataset matches its task, taxonomy, and full validation split. @@ -1069,6 +1093,8 @@ def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) }.get(normalized_task) normalized_dataset = (dataset or expected_dataset or "").lower() + if normalized_task == "person_reid": + return _reid_ready(root, normalized_dataset or "market1501") if normalized_task == "semantic_segmentation": return dense_dataset_ready(root, normalized_dataset or "ade20k") if expected_dataset is None or normalized_dataset != expected_dataset: diff --git a/mblt_vision/utils/results.py b/mblt_vision/utils/results.py index 8bbed17..7a5b7f6 100644 --- a/mblt_vision/utils/results.py +++ b/mblt_vision/utils/results.py @@ -73,6 +73,7 @@ def __init__( self.task = normalize_vision_task(post_cfg["task"]) self.conf_thres = kwargs.get("conf_thres", 0.25) self.acc: torch.Tensor | np.ndarray | None = None + self.embeddings: torch.Tensor | np.ndarray | None = None self.box_cls: torch.Tensor | np.ndarray | None = None self.mask: torch.Tensor | np.ndarray | None = None self.depth: torch.Tensor | np.ndarray | list[TensorLike] | None = None @@ -142,11 +143,35 @@ def set_output( NotImplementedError: If the task is not supported. """ self.acc = None + self.embeddings = None self.box_cls = None self.mask = None self.depth = None self.semantic_mask = None - if self.task == "image_classification": + if self.task == "person_reid": + if not isinstance(output, (np.ndarray, torch.Tensor)): + raise TypeError("Person ReID results require an embedding matrix") + finite = ( + torch.isfinite(output).all() + if isinstance(output, torch.Tensor) + else np.isfinite(output).all() + ) + floating = ( + output.is_floating_point() + if isinstance(output, torch.Tensor) + else np.issubdtype(output.dtype, np.floating) + ) + if ( + output.ndim != 2 + or min(output.shape) <= 0 + or not floating + or not bool(finite) + ): + raise ValueError( + "Person ReID results require a finite floating-point (N,D) matrix" + ) + self.embeddings = output + elif self.task == "image_classification": if not isinstance(output, (np.ndarray, torch.Tensor)): raise TypeError( f"Expected tensor output for task {self.task}, got {type(output).__name__}." diff --git a/tests/test_reid.py b/tests/test_reid.py index a6c0fcc..bfbab10 100644 --- a/tests/test_reid.py +++ b/tests/test_reid.py @@ -214,3 +214,63 @@ def test_family_base_inheritance(tmp_path): build_postprocess(cfg["pre_cfg"], cfg["post_cfg"]), PersonReIDPost ) assert build_preprocess(cfg["pre_cfg"]) + + +def test_public_base_and_engine_result_path(): + from mblt_vision.person_reid import EmbeddingPost as PublicEmbeddingPost + from mblt_vision.utils.results import Results + from mblt_vision.wrapper import MBLT_Engine + + assert PublicEmbeddingPost is EmbeddingPost + engine = MBLT_Engine.__new__(MBLT_Engine) + engine.pre_cfg = {} + engine.post_cfg = { + "task": "person_reid", + "dataset": "market1501", + "normalize": True, + } + engine.postprocessor = build_postprocess(engine.pre_cfg, engine.post_cfg) + result = engine.postprocess([np.array([[3.0, 4.0]], dtype=np.float32)]) + assert isinstance(result, Results) + torch.testing.assert_close(result.embeddings, torch.tensor([[0.6, 0.8]])) + assert result.output is result.embeddings + assert result.acc is None and result.box_cls is None + replacement = torch.tensor([[1.0, 0.0]]) + result.set_output(replacement) + assert result.embeddings is replacement + for invalid in ( + torch.ones(2), + torch.ones((1, 2), dtype=torch.int64), + torch.tensor([[float("nan")]]), + ): + with pytest.raises(ValueError): + result.set_output(invalid) + with pytest.raises(TypeError): + result.set_output([replacement]) + + +@pytest.mark.parametrize("name", ["market1501", "dukemtmc-reid", "occluded-duke"]) +def test_reid_readiness(tmp_path, monkeypatch, name): + from mblt_vision.utils.datasets import readiness, reid + from PIL import Image + + original = reid.get_dataset_config_for_task + + def tiny_config(task, dataset): + cfg = original(task, dataset) + for split in cfg["retrieval_splits"].values(): + split["count"] = 1 + return cfg + + monkeypatch.setattr(reid, "get_dataset_config_for_task", tiny_config) + assert not readiness.dataset_ready(tmp_path, "person_reid", name) + for folder in ("query", "bounding_box_test"): + (tmp_path / folder).mkdir() + Image.new("RGB", (4, 8)).save(tmp_path / folder / "0001_c1_sample.jpg") + assert readiness.dataset_ready(tmp_path, "person_reid", name) + if name == "market1501": + assert readiness.dataset_ready(tmp_path, "person_reid") + assert not readiness.dataset_ready(tmp_path, "person_reid", "coco") + image = tmp_path / "query" / "0001_c1_sample.jpg" + image.write_bytes(b"not an image") + assert not readiness.dataset_ready(tmp_path, "person_reid", name) From 1e70398bc39a0b2e0175fd1bff060242edde19b2 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 18:19:02 +0900 Subject: [PATCH 4/4] Validate retrieval cutoffs and prepared ReID cache boundaries Omit uncomputed rank metrics for truncated CMC, reject negative IDs below junk marker and reject symlinked roots or split ancestors. Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/utils/datasets/readiness.py | 4 +- mblt_vision/utils/datasets/reid.py | 2 + mblt_vision/utils/evaluation/retrieval.py | 15 ++++-- tests/test_reid.py | 2 +- tests/test_reid_review_regressions.py | 56 +++++++++++++++++++++++ 5 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 tests/test_reid_review_regressions.py diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index 8426389..c846aca 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -1052,12 +1052,12 @@ def _reid_ready(root: Path, dataset: str) -> bool: "dukemtmc-reid": DukeMTMCReIDDataset, "occluded-duke": OccludedDukeDataset, }.get(dataset) - if loader is None: + if loader is None or _path_has_symlink_component(root): return False try: samples = loader(root).samples for sample in samples: - if not sample.image.is_file() or sample.image.is_symlink(): + if not sample.image.is_file() or _path_has_symlink_component(sample.image): return False with Image.open(sample.image) as image: image.load() diff --git a/mblt_vision/utils/datasets/reid.py b/mblt_vision/utils/datasets/reid.py index af3e68d..c5bca16 100644 --- a/mblt_vision/utils/datasets/reid.py +++ b/mblt_vision/utils/datasets/reid.py @@ -55,6 +55,8 @@ def __init__( f"ReID file name carries no _c token: {path}" ) pid, camera = map(int, match.groups()) + if pid < -1: + raise ValueError(f"Unsupported negative person identity {pid}: {path}") if pid == -1: if self.config["skip_junk"]: continue diff --git a/mblt_vision/utils/evaluation/retrieval.py b/mblt_vision/utils/evaluation/retrieval.py index e3d8ffa..59f5982 100644 --- a/mblt_vision/utils/evaluation/retrieval.py +++ b/mblt_vision/utils/evaluation/retrieval.py @@ -57,6 +57,7 @@ class RetrievalResult: cmc: np.ndarray mean_ap: float sample_count: int + gallery_count: int | None = None @property def primary_score(self) -> float: @@ -67,16 +68,22 @@ def secondary_score(self) -> float: return self.rank(1) def rank(self, k: int) -> float: - """Saturate ranks beyond the gallery length, as in the source scorer.""" + """Saturate only a complete gallery curve, never a caller-truncated one.""" if not isinstance(k, int) or isinstance(k, bool) or k <= 0: raise ValueError("Rank must be a positive integer") + if k > len(self.cmc) and self.gallery_count != len(self.cmc): + raise ValueError("Requested rank is beyond the computed CMC cutoff") return float(self.cmc[min(k, len(self.cmc)) - 1]) def as_dict(self) -> dict: """Return source-compatible primary/secondary metrics in ratio units.""" return { "primary": {"map": round(self.mean_ap, 5)}, - "secondary": {f"rank{k}": round(self.rank(k), 5) for k in (1, 5, 10)}, + "secondary": { + f"rank{k}": round(self.rank(k), 5) + for k in (1, 5, 10) + if k <= len(self.cmc) or self.gallery_count == len(self.cmc) + }, "sample_count": self.sample_count, } @@ -109,7 +116,7 @@ def evaluate( raise ValueError("Embeddings require one finite nonempty vector per sample") if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: raise ValueError("max_rank must be a positive integer") - if any(s.split not in {"query", "gallery"} or s.pid == -1 for s in samples): + if any(s.split not in {"query", "gallery"} or s.pid < 0 for s in samples): raise ValueError( "Samples require query/gallery splits with loader-filtered junk" ) @@ -125,7 +132,7 @@ def evaluate( np.asarray([samples[i].camid for i in gallery]), max_rank, ) - return RetrievalResult(cmc, mean_ap, len(samples)) + return RetrievalResult(cmc, mean_ap, len(samples), len(gallery)) @staticmethod def evaluate_distances( diff --git a/tests/test_reid.py b/tests/test_reid.py index bfbab10..f0cd0f1 100644 --- a/tests/test_reid.py +++ b/tests/test_reid.py @@ -112,7 +112,7 @@ def test_loader_metadata_and_source(tmp_path, dataset, loader): assert cfg["eval_protocol"]["id"] == cfg["evaluation"]["protocol"] -@pytest.mark.parametrize("filename", ["bad.jpg", "1_c9_bad.jpg", "-1_c1_junk.jpg"]) +@pytest.mark.parametrize("filename", ["bad.jpg", "1_c9_bad.jpg", "-1_c1_junk.jpg", "-2_c1_bad.jpg"]) def test_duke_invalid_names(tmp_path, filename): (tmp_path / "query").mkdir() (tmp_path / "bounding_box_test").mkdir() diff --git a/tests/test_reid_review_regressions.py b/tests/test_reid_review_regressions.py new file mode 100644 index 0000000..66ddbbd --- /dev/null +++ b/tests/test_reid_review_regressions.py @@ -0,0 +1,56 @@ +"""Retrieval cutoffs and prepared-cache boundary regressions.""" + +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +from mblt_vision.utils.datasets import readiness, reid +from mblt_vision.utils.datasets.reid import ReIDSample +from mblt_vision.utils.evaluation.retrieval import RetrievalEvaluator + + +def test_truncated_cmc_omits_uncomputed_rank_metrics(): + samples = [ReIDSample(Path("query"), 1, 0, "query")] + samples += [ + ReIDSample(Path(str(i)), pid, 1, "gallery") + for i, pid in enumerate([2, 1, 3, 4, 5]) + ] + vectors = np.array( + [[0, 0], [0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], dtype=np.float32 + ) + result = RetrievalEvaluator().evaluate(vectors, samples, max_rank=1) + assert result.as_dict()["secondary"] == {"rank1": 0.0} + with pytest.raises(ValueError, match="cutoff"): + result.rank(5) + complete = RetrievalEvaluator().evaluate(vectors, samples, max_rank=50) + assert complete.as_dict()["secondary"]["rank5"] == 1.0 + assert complete.as_dict()["secondary"]["rank10"] == 1.0 + + +@pytest.mark.parametrize("component", ["root", "query", "bounding_box_test"]) +def test_reid_readiness_rejects_symlink_components(tmp_path, monkeypatch, component): + original = reid.get_dataset_config_for_task + + def tiny_config(task, dataset): + config = original(task, dataset) + for split in config["retrieval_splits"].values(): + split["count"] = 1 + return config + + monkeypatch.setattr(reid, "get_dataset_config_for_task", tiny_config) + root = tmp_path / "cache" + for folder in ("query", "bounding_box_test"): + (root / folder).mkdir(parents=True) + Image.new("RGB", (4, 8)).save(root / folder / "0001_c1_sample.jpg") + assert readiness.dataset_ready(root, "person_reid", "market1501") + if component == "root": + alias = tmp_path / "alias" + alias.symlink_to(root, target_is_directory=True) + root = alias + else: + target = tmp_path / "outside" + (root / component).rename(target) + (root / component).symlink_to(target, target_is_directory=True) + assert not readiness.dataset_ready(root, "person_reid", "market1501")