diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 643c86b..23cad94 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -112,6 +112,11 @@ description: >- metric). Load difficult objects as ordinary targets, as Ultralytics' validation does. - Run YOLOv7 models at `iou_thres: 0.65`, WongKinYiu/yolov7's own `test.py` default. +- ReID is utility-only until published wrappers and routing are added. Reuse + `EmbeddingPost` for dense vectors, explicit float32 mean/std for source preprocessing, + and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions + and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. + ## Runtime and Packaging - Route NPU runtime access through mblt-npu-python; do not copy backend classes into Vision. diff --git a/AGENTS.md b/AGENTS.md index a6f72b3..c8f4cc1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -183,6 +183,12 @@ The current ownership boundary is deliberate: any future model needing more than one Hub artifact, rather than duplicating Hub-resolution logic. +- ReID utilities live in `mblt_vision.person_reid`; concrete published wrappers are separate. + Use `EmbeddingPost` for validated dense vectors and `PersonReIDPost` for dataset binding. + Retrieval protocols and prepared loaders are pinned to ops source in dataset YAMLs; keep + stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. + Named Normalize styles retain legacy arithmetic; explicit mean/std uses source float32 arithmetic. + ## Python-First Architecture - Keep the public layer independent from a particular backend. Define small internal interfaces diff --git a/benchmark/benchmark_vision_models.py b/benchmark/benchmark_vision_models.py index a5c086d..06f86e3 100644 --- a/benchmark/benchmark_vision_models.py +++ b/benchmark/benchmark_vision_models.py @@ -35,7 +35,8 @@ CORE_MODES: tuple[CoreMode, ...] = cast( tuple[CoreMode, ...], core_modes_for_target_device("aries-rb") ) -TASK_CHOICES = VISION_TASKS +# Utility-only embedding tasks have no benchmark engine/CLI route yet. +TASK_CHOICES = tuple(task for task in VISION_TASKS if task != "person_reid") SUPPORTED_TARGET_DEVICES = frozenset({"aries-rb", "regulus-ra", "regulus-rb"}) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 0187f74..25bccab 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -489,3 +489,57 @@ validation split, following Ultralytics' depth-validation convention. - Difficult objects count as ordinary targets, as in Ultralytics' validation. + +## Person re-identification utilities + +`mblt_vision.person_reid` provides prepared query/gallery loaders, embedding +postprocessing and retrieval scoring. Published model wrappers/HF artifacts are +not added in this stage; no weights or datasets are downloaded by these utilities. +The CLI/benchmark router is unchanged. + +```python +from mblt_vision.person_reid import Market1501Dataset, Market1501Evaluator + +# root contains query/ and bounding_box_test/; embeddings have one row per sample. +dataset = Market1501Dataset("/path/to/market1501/eval") +# result = Market1501Evaluator().evaluate(embeddings, dataset.samples) +# print(result.as_dict()) +``` + +Also available: `DukeMTMCReIDDataset`, `OccludedDukeDataset` and matching +`DukeMTMCReIDEvaluator`, `OccludedDukeEvaluator`. Loaders emit immutable +`ReIDSample(image, pid, camid, split)` records, sorted query-first then gallery; +cameras are zero-based, identities are not relabeled. Counts are source-checked +by default; caller-owned subsets must explicitly set `validate_counts=False`. +Only Market-1501 drops pid -1 junk; pid 0 background distractors remain. +Duke loaders reject pid -1. Root defaults use `~/.mblt_model_zoo/datasets//eval`. + +`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`; both are exported +from `mblt_vision.person_reid`. Engine postprocessing returns a `Results` object +whose `embeddings` and `output` retain the validated `(N,D)` matrix. +`dataset_ready(root, "person_reid", dataset)` checks source counts/metadata and +decodes every retained query/gallery image (default dataset: Market-1501). +`PersonReIDPost` It validates one +floating-point `(N,D)` output or singleton-spatial NCHW/NHWC tensor, optionally +checks `embedding_dim`, and uses `normalize: true` for row-wise L2 normalization +(zero vectors remain zero). Return type is a float32 torch matrix, never class +probabilities. An explicit `post_cfg.dataset` is required. Processing-only base +profiles in `models/bases/person_reid.yaml` can be selected using `extends`; +concrete models must supply their own HF `file_cfg`, dataset and embedding dimension. +`Normalize` accepts either a historical named style or explicit three-element +`mean` and `std` lists; named styles retain unchanged legacy arithmetic. + +`extract_embeddings(batches, infer, postprocess)` runs prepared batches through +the shared bounded inference pipeline, preserving order. Evaluators consume +these embeddings without adding normalization, camera inputs, re-ranking or flip +fusion. Same-ID/same-camera gallery rows are excluded; unmatched queries are +skipped. Squared float32 Euclidean distances use stable ties in gallery order. +Ranks beyond the gallery saturate; mAP is primary, rank-1/5/10 secondary, all +ratios. `as_dict()` rounds to five decimals like ops; raw `mean_ap`/`cmc` remain +unrounded. Dataset YAMLs record exact ops config, loader, evaluator and revision. + +Offline tests vendor the inspected ops functions at +`ec537764e36ca10da963fcb7afab157ea60ca022`. Synthetic embeddings require exact +distance/CMC/mAP equality; synthetic RGB preprocessing requires byte equality +(`atol=0`) against the ops adapter for explicit source mean/std profiles. These +are protocol tests, not measured model accuracy claims. diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index e56c12a..9cf69d2 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -13,6 +13,7 @@ from . import instance_segmentation as instance_segmentation from . import obb as obb from . import object_detection as object_detection +from . import person_reid as person_reid from . import pose_estimation as pose_estimation from . import semantic_segmentation as semantic_segmentation from ._api import list_models as list_models @@ -52,6 +53,7 @@ "object_detection", "obb", "pose_estimation", + "person_reid", "semantic_segmentation", ] + sorted(_LEGACY_MODEL_EXPORTS) # Keep legacy compatibility exports synchronized with their task packages. diff --git a/mblt_vision/_tasks.py b/mblt_vision/_tasks.py index df9a462..c8a4b6d 100644 --- a/mblt_vision/_tasks.py +++ b/mblt_vision/_tasks.py @@ -13,6 +13,7 @@ "obb", "pose_estimation", "face_detection", + "person_reid", ) diff --git a/mblt_vision/datasets/dukemtmc-reid.yaml b/mblt_vision/datasets/dukemtmc-reid.yaml new file mode 100644 index 0000000..c1fb241 --- /dev/null +++ b/mblt_vision/datasets/dukemtmc-reid.yaml @@ -0,0 +1,35 @@ +name: dukemtmc-reid +path: ~/.mblt_model_zoo/datasets/dukemtmc-reid +tasks: +- person_reid +val: eval +cameras: 8 +skip_junk: false +retrieval_splits: + query: + directory: query + count: 2228 + gallery: + directory: bounding_box_test + count: 17661 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/dukemtmc_reid/dukemtmc_reid.yaml + name: dukemtmc-reid + dataloader_path: datasets/dukemtmc_reid/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: dukemtmc-reid-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: dukemtmc-reid-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/dukemtmc_reid/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/market1501.yaml b/mblt_vision/datasets/market1501.yaml new file mode 100644 index 0000000..b638134 --- /dev/null +++ b/mblt_vision/datasets/market1501.yaml @@ -0,0 +1,35 @@ +name: market1501 +path: ~/.mblt_model_zoo/datasets/market1501 +tasks: +- person_reid +val: eval +cameras: 6 +skip_junk: true +retrieval_splits: + query: + directory: query + count: 3368 + gallery: + directory: bounding_box_test + count: 15913 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/market1501/market1501.yaml + name: market1501 + dataloader_path: datasets/market1501/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: market1501-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: market1501-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/market1501/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/occluded-duke.yaml b/mblt_vision/datasets/occluded-duke.yaml new file mode 100644 index 0000000..dc039b2 --- /dev/null +++ b/mblt_vision/datasets/occluded-duke.yaml @@ -0,0 +1,35 @@ +name: occluded-duke +path: ~/.mblt_model_zoo/datasets/occluded-duke +tasks: +- person_reid +val: eval +cameras: 8 +skip_junk: false +retrieval_splits: + query: + directory: query + count: 2210 + gallery: + directory: bounding_box_test + count: 17661 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/occluded_duke/occluded_duke.yaml + name: occluded-duke + dataloader_path: datasets/occluded_duke/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: occluded-duke-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: occluded-duke-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/occluded_duke/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/models/bases/person_reid.yaml b/mblt_vision/models/bases/person_reid.yaml new file mode 100644 index 0000000..cafeaf3 --- /dev/null +++ b/mblt_vision/models/bases/person_reid.yaml @@ -0,0 +1,26 @@ +# Processing only: published HF artifact identities belong to concrete stage-d2 YAMLs. +# Source: mblt-model-ops ec537764e36ca10da963fcb7afab157ea60ca022 +# models/person_reid/CLIP_ReID_RN50_Baseline_DukeMTMC/distribution.yaml +DEFAULT: + pre_cfg: + Reader: + style: pil + Resize: + size: [256, 128] + interpolation: bilinear + Normalize: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + post_cfg: + task: person_reid + dataset: dukemtmc-reid + normalize: true +# Inherit this processing profile and override dataset/dimension/artifacts per model. +MARKET1501: + extends: person_reid.yaml#DEFAULT + post_cfg: + dataset: market1501 +OCCLUDED_DUKE: + extends: person_reid.yaml#DEFAULT + post_cfg: + dataset: occluded-duke diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py new file mode 100644 index 0000000..e5e6574 --- /dev/null +++ b/mblt_vision/person_reid/__init__.py @@ -0,0 +1,34 @@ +"""Person retrieval utilities; published model wrappers are added separately.""" + +from ..utils.datasets.reid import ( + DukeMTMCReIDDataset, + Market1501Dataset, + OccludedDukeDataset, + ReIDDataset, + ReIDSample, +) +from ..utils.evaluation.retrieval import ( + DukeMTMCReIDEvaluator, + Market1501Evaluator, + OccludedDukeEvaluator, + RetrievalEvaluator, + RetrievalResult, + extract_embeddings, +) +from ..utils.postprocess.embedding_post import EmbeddingPost, PersonReIDPost + +__all__ = [ + "DukeMTMCReIDDataset", + "DukeMTMCReIDEvaluator", + "EmbeddingPost", + "Market1501Dataset", + "Market1501Evaluator", + "OccludedDukeDataset", + "OccludedDukeEvaluator", + "PersonReIDPost", + "ReIDDataset", + "ReIDSample", + "RetrievalEvaluator", + "RetrievalResult", + "extract_embeddings", +] diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index 7fd18b4..c846aca 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -20,6 +20,7 @@ from ..._tasks import normalize_vision_task from ...datasets import get_dataset_category_ids from .cityscapes import CITYSCAPES_SOURCE_TO_TRAIN_ID +from .reid import DukeMTMCReIDDataset, Market1501Dataset, OccludedDukeDataset IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} IMAGENET_CLASS_COUNT = 1000 @@ -1044,6 +1045,29 @@ def dense_dataset_ready(data_path: str | Path, dataset: str) -> bool: return False +def _reid_ready(root: Path, dataset: str) -> bool: + """Require source counts, camera/name metadata and decodable query/gallery crops.""" + loader = { + "market1501": Market1501Dataset, + "dukemtmc-reid": DukeMTMCReIDDataset, + "occluded-duke": OccludedDukeDataset, + }.get(dataset) + if loader is None or _path_has_symlink_component(root): + return False + try: + samples = loader(root).samples + for sample in samples: + if not sample.image.is_file() or _path_has_symlink_component(sample.image): + return False + with Image.open(sample.image) as image: + image.load() + if min(image.size) <= 0: + return False + except (OSError, ValueError): + return False + return True + + def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) -> bool: """Return whether an organized dataset matches its task, taxonomy, and full validation split. @@ -1069,6 +1093,8 @@ def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) }.get(normalized_task) normalized_dataset = (dataset or expected_dataset or "").lower() + if normalized_task == "person_reid": + return _reid_ready(root, normalized_dataset or "market1501") if normalized_task == "semantic_segmentation": return dense_dataset_ready(root, normalized_dataset or "ade20k") if expected_dataset is None or normalized_dataset != expected_dataset: diff --git a/mblt_vision/utils/datasets/reid.py b/mblt_vision/utils/datasets/reid.py new file mode 100644 index 0000000..c5bca16 --- /dev/null +++ b/mblt_vision/utils/datasets/reid.py @@ -0,0 +1,98 @@ +"""Prepared query/gallery person crops; no download or identity relabeling.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from ...datasets import get_dataset_config_for_task + +_FILENAME = re.compile(r"^(-?\d+)_c(\d+)(?:s\d+)?(?:_|\.)") + + +@dataclass(frozen=True) +class ReIDSample: + """One crop with source identity, zero-based camera and query/gallery split.""" + + image: Path + pid: int + camid: int + split: Literal["query", "gallery"] + + +class ReIDDataset: + """Read a prepared benchmark root in source-defined deterministic order. + + ``root`` is the directory containing ``query`` and ``bounding_box_test`` + (the ops prepared ``eval`` directory). Counts are checked by default; + ``validate_counts=False`` explicitly supports caller-owned small subsets. + """ + + name: str + + def __init__( + self, root: str | Path | None = None, *, validate_counts: bool = True + ) -> None: + self.config = get_dataset_config_for_task("person_reid", self.name) + self.root = ( + Path(root).expanduser() + if root is not None + else Path(self.config["path"]) / self.config["val"] + ) + samples = [] + for split in ("query", "gallery"): + options = self.config["retrieval_splits"][split] + directory = self.root / options["directory"] + if not directory.is_dir(): + raise FileNotFoundError(f"ReID split directory is missing: {directory}") + rows = [] + for path in sorted(directory.glob("*.jpg")): + match = _FILENAME.match(path.name) + if match is None: + raise ValueError( + f"ReID file name carries no _c token: {path}" + ) + pid, camera = map(int, match.groups()) + if pid < -1: + raise ValueError(f"Unsupported negative person identity {pid}: {path}") + if pid == -1: + if self.config["skip_junk"]: + continue + raise ValueError(f"Unexpected junk identity: {path}") + if not 1 <= camera <= self.config["cameras"]: + raise ValueError(f"Camera {camera} outside benchmark range: {path}") + rows.append(ReIDSample(path, pid, camera - 1, split)) + if not rows: + raise ValueError(f"ReID split directory holds no images: {directory}") + if validate_counts and len(rows) != options["count"]: + raise ValueError( + f"{split} has {len(rows)} crops, expected {options['count']}" + ) + samples.extend(rows) + self.samples = tuple(samples) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> ReIDSample: + return self.samples[index] + + +class Market1501Dataset(ReIDDataset): + """Market-1501: discard pid -1; retain pid 0 background distractors.""" + + name = "market1501" + + +class DukeMTMCReIDDataset(ReIDDataset): + """DukeMTMC-reID prepared held-out query/gallery crops.""" + + name = "dukemtmc-reid" + + +class OccludedDukeDataset(ReIDDataset): + """Occluded-Duke prepared query split and Duke gallery.""" + + name = "occluded-duke" diff --git a/mblt_vision/utils/evaluation/retrieval.py b/mblt_vision/utils/evaluation/retrieval.py new file mode 100644 index 0000000..59f5982 --- /dev/null +++ b/mblt_vision/utils/evaluation/retrieval.py @@ -0,0 +1,215 @@ +"""Source-exact Market protocol retrieval metrics (ops ec537764).""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch + +from ..datasets.reid import ReIDSample +from ._pipeline import map_batched_inference + + +def extract_embeddings( + batches: Iterable[Any], infer: Callable, postprocess: Callable +) -> np.ndarray: + """Decode prepared batches in input order through the shared bounded pipeline. + + ``infer`` receives a prepared batch; ``postprocess`` receives raw outputs and + returns a validated ``(N,D)`` tensor. No implicit normalization or flip fusion. + """ + + def decode(batch: Any, output: Any) -> np.ndarray: + value = postprocess(output) + if isinstance(value, torch.Tensor): + value = value.detach().cpu().numpy() + value = np.asarray(value, dtype=np.float32) + if value.ndim != 2 or min(value.shape) == 0 or not np.isfinite(value).all(): + raise ValueError( + "Decoded embeddings require a nonempty finite (N,D) matrix" + ) + return value + + rows = list(map_batched_inference(batches, infer, decode)) + if not rows: + raise ValueError("No embedding batches supplied") + return np.concatenate(rows, axis=0) + + +def euclidean_distance(query: np.ndarray, gallery: np.ndarray) -> np.ndarray: + """Source float32 squared Euclidean distance (not cosine or square root).""" + query = np.asarray(query, dtype=np.float32) + gallery = np.asarray(gallery, dtype=np.float32) + return ( + np.sum(query**2, axis=1, keepdims=True) + + np.sum(gallery**2, axis=1, keepdims=True).T + - 2.0 * (query @ gallery.T) + ) + + +@dataclass(frozen=True) +class RetrievalResult: + """Unrounded mAP and CMC curve; ``as_dict`` matches ops score rounding.""" + + cmc: np.ndarray + mean_ap: float + sample_count: int + gallery_count: int | None = None + + @property + def primary_score(self) -> float: + return self.mean_ap + + @property + def secondary_score(self) -> float: + return self.rank(1) + + def rank(self, k: int) -> float: + """Saturate only a complete gallery curve, never a caller-truncated one.""" + if not isinstance(k, int) or isinstance(k, bool) or k <= 0: + raise ValueError("Rank must be a positive integer") + if k > len(self.cmc) and self.gallery_count != len(self.cmc): + raise ValueError("Requested rank is beyond the computed CMC cutoff") + return float(self.cmc[min(k, len(self.cmc)) - 1]) + + def as_dict(self) -> dict: + """Return source-compatible primary/secondary metrics in ratio units.""" + return { + "primary": {"map": round(self.mean_ap, 5)}, + "secondary": { + f"rank{k}": round(self.rank(k), 5) + for k in (1, 5, 10) + if k <= len(self.cmc) or self.gallery_count == len(self.cmc) + }, + "sample_count": self.sample_count, + } + + +class RetrievalEvaluator: + """Market retrieval protocol shared by three dataset-specific evaluators. + + Stable distance ties retain gallery manifest order. Exclude only same-ID, + same-camera rows; pid 0 distractors remain. Dataset loaders own pid -1 junk + filtering. Queries without cross-camera matches do not enter either mean. + """ + + dataset: str | None = None + + def evaluate( + self, + embeddings: np.ndarray, + samples: Sequence[ReIDSample], + *, + max_rank: int = 50, + ) -> RetrievalResult: + """Score one embedding row per sample in query-then-gallery order.""" + values = np.asarray(embeddings, dtype=np.float32) + if ( + values.ndim != 2 + or min(values.shape) == 0 + or values.shape[0] != len(samples) + or not np.isfinite(values).all() + ): + raise ValueError("Embeddings require one finite nonempty vector per sample") + if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: + raise ValueError("max_rank must be a positive integer") + if any(s.split not in {"query", "gallery"} or s.pid < 0 for s in samples): + raise ValueError( + "Samples require query/gallery splits with loader-filtered junk" + ) + query = [i for i, s in enumerate(samples) if s.split == "query"] + gallery = [i for i, s in enumerate(samples) if s.split == "gallery"] + if not query or not gallery: + raise ValueError("Benchmark needs both a query and a gallery split") + cmc, mean_ap = self.evaluate_distances( + euclidean_distance(values[query], values[gallery]), + np.asarray([samples[i].pid for i in query]), + np.asarray([samples[i].pid for i in gallery]), + np.asarray([samples[i].camid for i in query]), + np.asarray([samples[i].camid for i in gallery]), + max_rank, + ) + return RetrievalResult(cmc, mean_ap, len(samples), len(gallery)) + + @staticmethod + def evaluate_distances( + distmat: np.ndarray, + q_pids: np.ndarray, + g_pids: np.ndarray, + q_camids: np.ndarray, + g_camids: np.ndarray, + max_rank: int = 50, + ) -> tuple[np.ndarray, float]: + """CMC and mAP with the exact source accumulation and padding order.""" + distmat = np.asarray(distmat) + q_pids, g_pids = np.asarray(q_pids), np.asarray(g_pids) + q_camids, g_camids = np.asarray(q_camids), np.asarray(g_camids) + if ( + distmat.ndim != 2 + or min(distmat.shape) == 0 + or not np.isfinite(distmat).all() + ): + raise ValueError("Distances require a nonempty finite matrix") + num_q, num_g = distmat.shape + if any( + v.shape != (n,) + for v, n in ( + (q_pids, num_q), + (g_pids, num_g), + (q_camids, num_q), + (g_camids, num_g), + ) + ): + raise ValueError( + "Identity/camera metadata does not match distance dimensions" + ) + if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: + raise ValueError("max_rank must be a positive integer") + max_rank = min(max_rank, num_g) + indices = np.argsort(distmat, axis=1, kind="stable") + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + all_cmc = [] + all_ap = [] + for q_idx in range(num_q): + order = indices[q_idx] + remove = (g_pids[order] == q_pids[q_idx]) & ( + g_camids[order] == q_camids[q_idx] + ) + orig_cmc = matches[q_idx][~remove] + if not np.any(orig_cmc): + continue + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + all_cmc.append( + np.pad( + cmc[:max_rank], (0, max_rank - min(len(cmc), max_rank)), mode="edge" + ) + ) + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() / np.arange(1, orig_cmc.shape[0] + 1) + all_ap.append(float((tmp_cmc * orig_cmc).sum() / num_rel)) + if not all_cmc: + raise ValueError("No query identity appears in the gallery") + cmc_curve = np.asarray(all_cmc, dtype=np.float64).sum(0) / len(all_cmc) + return cmc_curve, float(np.mean(all_ap)) + + +class Market1501Evaluator(RetrievalEvaluator): + """Market-1501 held-out retrieval protocol v1.""" + + dataset = "market1501" + + +class DukeMTMCReIDEvaluator(RetrievalEvaluator): + """DukeMTMC-reID held-out retrieval protocol v1.""" + + dataset = "dukemtmc-reid" + + +class OccludedDukeEvaluator(RetrievalEvaluator): + """Occluded-Duke held-out retrieval protocol v1.""" + + dataset = "occluded-duke" diff --git a/mblt_vision/utils/postprocess/__init__.py b/mblt_vision/utils/postprocess/__init__.py index e5090d0..879a06f 100644 --- a/mblt_vision/utils/postprocess/__init__.py +++ b/mblt_vision/utils/postprocess/__init__.py @@ -4,6 +4,7 @@ from .build_post import build_postprocess from .depth_post import DepthPost +from .embedding_post import EmbeddingPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost -__all__ = ["DepthPost", "SemanticSegPost", "build_postprocess"] +__all__ = ["DepthPost", "EmbeddingPost", "PersonReIDPost", "SemanticSegPost", "build_postprocess"] diff --git a/mblt_vision/utils/postprocess/build_post.py b/mblt_vision/utils/postprocess/build_post.py index dc89192..eb569ac 100644 --- a/mblt_vision/utils/postprocess/build_post.py +++ b/mblt_vision/utils/postprocess/build_post.py @@ -9,6 +9,7 @@ from .cls_post import ClsPost from .damoyolo_post import DAMOYOLODetectionPost from .depth_post import DepthPost +from .embedding_post import PersonReIDPost from .semantic_seg_post import SemanticSegPost from .yolo_anchor_post import ( YOLOAnchorDetectionPost, @@ -70,6 +71,8 @@ def build_postprocess( f"Unsupported post_cfg.head {head!r}; expected one of {sorted(DETECTION_HEADS)}." ) return DETECTION_HEADS[head](pre_cfg, post_cfg, **kwargs) + if task == "person_reid": + return PersonReIDPost(pre_cfg, post_cfg) if task == "image_classification": return ClsPost(pre_cfg, post_cfg) if task == "depth_estimation": diff --git a/mblt_vision/utils/postprocess/embedding_post.py b/mblt_vision/utils/postprocess/embedding_post.py new file mode 100644 index 0000000..8d55353 --- /dev/null +++ b/mblt_vision/utils/postprocess/embedding_post.py @@ -0,0 +1,85 @@ +"""Validated appearance embeddings, shared by retrieval and verification tasks.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np +import torch +import torch.nn.functional as F + +from ...datasets import get_dataset_config_for_task +from ..types import ListTensorLike, TensorLike +from .base import PostBase + + +class EmbeddingPost(PostBase): + """Return floating-point ``(batch, dimension)`` embeddings without argmax. + + Accept a single matrix or singleton-spatial NCHW/NHWC output (also wrapped + in a one-element runtime output list). L2 normalization is opt-in; zero + vectors stay zero, matching the source embedding decoder. + """ + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__() + self.normalize = post_cfg.get("normalize", False) + if not isinstance(self.normalize, bool): + raise TypeError("post_cfg.normalize must be a boolean") + self.embedding_dim = post_cfg.get("embedding_dim") + if self.embedding_dim is not None and ( + not isinstance(self.embedding_dim, int) + or isinstance(self.embedding_dim, bool) + or self.embedding_dim <= 0 + ): + raise ValueError("post_cfg.embedding_dim must be a positive integer") + + def __call__(self, x: TensorLike | ListTensorLike) -> torch.Tensor: + """Validate one dense output and optionally L2-normalize each row.""" + if isinstance(x, Sequence): + if len(x) != 1: + raise ValueError( + "Embedding postprocessing expects exactly one output tensor" + ) + x = x[0] + if isinstance(x, np.ndarray): + x = torch.from_numpy(x) + if not isinstance(x, torch.Tensor): + raise TypeError("Embedding output must be a NumPy array or torch tensor") + if not x.is_floating_point(): + raise TypeError("Embedding output must have a floating-point dtype") + x = x.to(self.device) + if x.ndim == 4 and (x.shape[2:] == (1, 1) or x.shape[1:3] == (1, 1)): + x = x.reshape(x.shape[0], -1) + if x.ndim != 2 or min(x.shape) <= 0: + raise ValueError( + f"Embedding output requires nonempty (N, D), got {tuple(x.shape)}" + ) + if self.embedding_dim is not None and x.shape[1] != self.embedding_dim: + raise ValueError( + f"Embedding dimension {x.shape[1]} does not match {self.embedding_dim}" + ) + if not bool(torch.isfinite(x).all()): + raise ValueError("Embedding output must contain only finite values") + # Float32 agrees with the ops decoder, including its 1e-12 norm floor. + x = x.float() + if not bool(torch.isfinite(x).all()): + raise ValueError("Embedding values overflow float32") + if self.normalize and not bool( + torch.isfinite(torch.linalg.vector_norm(x, dim=1)).all() + ): + raise ValueError("Embedding norms overflow float32") + if self.normalize: + x = F.normalize(x, p=2, dim=1, eps=1e-12) + return x + + +class PersonReIDPost(EmbeddingPost): + """Appearance embeddings tied to an explicitly configured ReID dataset.""" + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__(pre_cfg, post_cfg) + dataset = post_cfg.get("dataset") + if not isinstance(dataset, str) or not dataset: + raise ValueError("Person ReID postprocessing requires post_cfg.dataset") + self.dataset = get_dataset_config_for_task("person_reid", dataset)["name"] diff --git a/mblt_vision/utils/preprocess/normalize.py b/mblt_vision/utils/preprocess/normalize.py index be7dd1b..ad19c3d 100644 --- a/mblt_vision/utils/preprocess/normalize.py +++ b/mblt_vision/utils/preprocess/normalize.py @@ -30,14 +30,40 @@ class Normalize(PreOps): std: Array of standard deviation values for normalization. """ - def __init__(self, style: str) -> None: + def __init__( + self, + style: str | None = None, + *, + mean: list[float] | None = None, + std: list[float] | None = None, + ) -> None: """Initializes the Normalize layer with a specific style. Args: - style: The preprocessing style to use. Must be one of STYLE_LIST. + style: Historical named style; mutually exclusive with mean/std. + mean: Optional explicit three-element RGB mean, using float32 arithmetic. + std: Optional explicit positive RGB standard deviations. """ super().__init__() + self.explicit_stats = mean is not None or std is not None + if self.explicit_stats: + if style is not None or mean is None or std is None: + raise ValueError("Provide either style or both mean and std") + self.mean = np.asarray(mean, dtype=np.float32) + self.std = np.asarray(std, dtype=np.float32) + if ( + self.mean.shape != (3,) + or self.std.shape != (3,) + or not np.isfinite(self.mean).all() + or not np.isfinite(self.std).all() + or not (self.std > 0).all() + ): + raise ValueError( + "mean/std require three finite RGB values and positive std" + ) + self.style = "explicit" + return if not isinstance(style, str): raise TypeError( f"Normalize style must be a string, got {type(style).__name__}." diff --git a/mblt_vision/utils/results.py b/mblt_vision/utils/results.py index 8bbed17..7a5b7f6 100644 --- a/mblt_vision/utils/results.py +++ b/mblt_vision/utils/results.py @@ -73,6 +73,7 @@ def __init__( self.task = normalize_vision_task(post_cfg["task"]) self.conf_thres = kwargs.get("conf_thres", 0.25) self.acc: torch.Tensor | np.ndarray | None = None + self.embeddings: torch.Tensor | np.ndarray | None = None self.box_cls: torch.Tensor | np.ndarray | None = None self.mask: torch.Tensor | np.ndarray | None = None self.depth: torch.Tensor | np.ndarray | list[TensorLike] | None = None @@ -142,11 +143,35 @@ def set_output( NotImplementedError: If the task is not supported. """ self.acc = None + self.embeddings = None self.box_cls = None self.mask = None self.depth = None self.semantic_mask = None - if self.task == "image_classification": + if self.task == "person_reid": + if not isinstance(output, (np.ndarray, torch.Tensor)): + raise TypeError("Person ReID results require an embedding matrix") + finite = ( + torch.isfinite(output).all() + if isinstance(output, torch.Tensor) + else np.isfinite(output).all() + ) + floating = ( + output.is_floating_point() + if isinstance(output, torch.Tensor) + else np.issubdtype(output.dtype, np.floating) + ) + if ( + output.ndim != 2 + or min(output.shape) <= 0 + or not floating + or not bool(finite) + ): + raise ValueError( + "Person ReID results require a finite floating-point (N,D) matrix" + ) + self.embeddings = output + elif self.task == "image_classification": if not isinstance(output, (np.ndarray, torch.Tensor)): raise TypeError( f"Expected tensor output for task {self.task}, got {type(output).__name__}." diff --git a/tests/reference/ops_reid_metrics.py b/tests/reference/ops_reid_metrics.py new file mode 100644 index 0000000..78f10fa --- /dev/null +++ b/tests/reference/ops_reid_metrics.py @@ -0,0 +1,53 @@ +"""Verbatim functions from datasets/_shared/reid.py at +ec537764e36ca10da963fcb7afab157ea60ca022 (mblt-model-ops). +Snapshot permits ordinary offline differential tests without sibling repos. +""" +import numpy as np +MAX_RANK = 50 + +def euclidean_distance(query: np.ndarray, gallery: np.ndarray) -> np.ndarray: + """Squared euclidean distance, the reference ``euclidean_distance`` without the sqrt. + + Ranking is monotone in the square, so CMC and mAP are unchanged. + """ + query = np.asarray(query, dtype=np.float32) + gallery = np.asarray(gallery, dtype=np.float32) + q2 = np.sum(query**2, axis=1, keepdims=True) + g2 = np.sum(gallery**2, axis=1, keepdims=True).T + return q2 + g2 - 2.0 * (query @ gallery.T) + +def evaluate( + distmat: np.ndarray, + q_pids: np.ndarray, + g_pids: np.ndarray, + q_camids: np.ndarray, + g_camids: np.ndarray, + max_rank: int = MAX_RANK, +) -> tuple[np.ndarray, float]: + """Market-1501 protocol CMC curve and mAP (reference ``eval_func``).""" + num_q, num_g = distmat.shape + max_rank = min(max_rank, num_g) + indices = np.argsort(distmat, axis=1, kind="stable") + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + all_cmc = [] + all_ap = [] + for q_idx in range(num_q): + order = indices[q_idx] + remove = (g_pids[order] == q_pids[q_idx]) & (g_camids[order] == q_camids[q_idx]) + orig_cmc = matches[q_idx][~remove] + if not np.any(orig_cmc): + continue # the query identity never appears in the gallery + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + # Same-identity/same-camera removal leaves different gallery lengths. + # Saturate missing ranks with the final CMC value for this query. + all_cmc.append( + np.pad(cmc[:max_rank], (0, max_rank - min(len(cmc), max_rank)), mode="edge") + ) + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() / np.arange(1, orig_cmc.shape[0] + 1) + all_ap.append(float((tmp_cmc * orig_cmc).sum() / num_rel)) + if not all_cmc: + raise ValueError("No query identity appears in the gallery") + cmc_curve = np.asarray(all_cmc, dtype=np.float64).sum(0) / len(all_cmc) + return cmc_curve, float(np.mean(all_ap)) diff --git a/tests/reference/ops_reid_preprocess.py b/tests/reference/ops_reid_preprocess.py new file mode 100644 index 0000000..b2b4ce0 --- /dev/null +++ b/tests/reference/ops_reid_preprocess.py @@ -0,0 +1,92 @@ +# Verbatim source: models/person_reid/_shared/preprocess.py +# mblt-model-ops ec537764e36ca10da963fcb7afab157ea60ca022 +"""Default person_reid preprocess: the CLIP-ReID / TransReID test transform. + +Upstream evaluates with ``Resize((H, W)) -> ToTensor -> Normalize(mean, std)``; there is no +crop, and the two backbone families differ only in the normalization statistics, so the +statistics are explicit ``mean``/``std`` lists in ``pre_cfg`` rather than a named style. + +Operations are applied in a fixed canonical order, not in ``config`` iteration order: +``predict.input.json`` is written with sorted keys, so relying on dict order would run +``Normalize`` before ``Resize``. + +A camera-aware Model (SIE) declares ``CameraOneHot``. The camera index is the ``_c`` +token of the crop's file name, which is how every ReID dataset and the upstream loader +encode it; it is a Model input, not ground truth. The network then takes two inputs, and +the returned tensor is a tuple ``(image, camera_one_hot)`` in that order. + +The return value is ``(source, network_input, camera_index)`` so that the shared +``inference.tensor_of`` finds the network input in the second slot. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +PIPELINE_ORDER = ("Reader", "Resize", "SetOrder", "Normalize", "CameraOneHot") +REQUIRED = ("Resize", "Normalize") +_RESAMPLING = { + "bilinear": Image.Resampling.BILINEAR, + "bicubic": Image.Resampling.BICUBIC, +} +DEFAULT_CAMERA_PATTERN = r"_c(\d+)" + +NetworkInput = np.ndarray | tuple[np.ndarray, np.ndarray] + + +def camera_index(source: Path, config: dict[str, Any]) -> int: + """The 0-based camera index of a crop, from its ``_c`` file-name token.""" + pattern = str(config.get("pattern", DEFAULT_CAMERA_PATTERN)) + match = re.search(pattern, Path(source).name) + if match is None: + raise ValueError(f"{source}: file name carries no camera token {pattern!r}") + cameras = int(config["num_cameras"]) + camera = int(match.group(1)) - 1 + if not 0 <= camera < cameras: + raise ValueError(f"{source}: camera {camera + 1} is outside 1..{cameras}") + return camera + + +def preprocess( + source: Path, config: dict[str, Any] +) -> tuple[Path, NetworkInput, int | None]: + """Return ``(source, network input, camera index or None)``.""" + unknown = set(config) - set(PIPELINE_ORDER) + if unknown: + raise ValueError(f"unsupported preprocess operations: {sorted(unknown)}") + missing = [name for name in REQUIRED if name not in config] + if missing: + raise ValueError(f"pre_cfg is missing required operations: {missing}") + source = Path(source) + value: Any = Image.open(source).convert("RGB") + camera: int | None = None + for operation in PIPELINE_ORDER: + if operation not in config: + continue + options = config[operation] + if operation == "Reader": + continue + if operation == "Resize": + height, width = (int(v) for v in options["size"]) + mode = _RESAMPLING[str(options.get("interpolation", "bilinear"))] + value = value.resize((width, height), mode) + elif operation == "SetOrder": + if options.get("shape", "HWC") != "HWC": + raise ValueError("person_reid preprocess produces HWC tensors only") + elif operation == "Normalize": + mean = np.asarray(options["mean"], dtype=np.float32) + std = np.asarray(options["std"], dtype=np.float32) + value = (np.asarray(value, dtype=np.float32) / 255.0 - mean) / std + elif operation == "CameraOneHot": + camera = camera_index(source, options) + tensor = np.ascontiguousarray(value, dtype=np.float32) + if camera is None: + return source, tensor, None + one_hot = np.zeros(int(config["CameraOneHot"]["num_cameras"]), dtype=np.float32) + one_hot[camera] = 1.0 + return source, (tensor, one_hot), camera diff --git a/tests/test_api.py b/tests/test_api.py index 40b4182..ce184aa 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -21,6 +21,7 @@ def test_public_discovery_exposes_all_supported_tasks() -> None: "obb", "pose_estimation", "face_detection", + "person_reid", ] assert list_models("obb")["obb"] diff --git a/tests/test_reid.py b/tests/test_reid.py new file mode 100644 index 0000000..f0cd0f1 --- /dev/null +++ b/tests/test_reid.py @@ -0,0 +1,276 @@ +"""CPU-only embedding, prepared loader and retrieval protocol contracts.""" + +from pathlib import Path + +import numpy as np +import pytest +import torch +from mblt_vision._config import load_model_variant +from mblt_vision.datasets import get_dataset_config +from mblt_vision.utils.datasets.reid import ( + DukeMTMCReIDDataset, + Market1501Dataset, + OccludedDukeDataset, + ReIDSample, +) +from mblt_vision.utils.evaluation.retrieval import ( + DukeMTMCReIDEvaluator, + Market1501Evaluator, + OccludedDukeEvaluator, + RetrievalEvaluator, + extract_embeddings, +) +from mblt_vision.utils.postprocess import build_postprocess +from mblt_vision.utils.postprocess.embedding_post import EmbeddingPost, PersonReIDPost +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.utils.preprocess.normalize import Normalize + + +@pytest.mark.parametrize("shape", [(2, 3), (2, 3, 1, 1), (2, 1, 1, 3)]) +def test_embedding_shapes(shape): + raw = np.array([[3.0, 4.0, 0.0], [0.0, 0.0, 0.0]], dtype=np.float32).reshape(shape) + post = build_postprocess( + {}, + { + "task": "person_reid", + "dataset": "market1501", + "normalize": True, + "embedding_dim": 3, + }, + ) + assert isinstance(post, (EmbeddingPost, PersonReIDPost)) + np.testing.assert_array_equal( + post([raw]), np.array([[0.6, 0.8, 0.0], [0.0, 0.0, 0.0]], dtype=np.float32) + ) + np.testing.assert_array_equal(raw.reshape(2, 3)[0], [3.0, 4.0, 0.0]) + + +@pytest.mark.parametrize( + "raw", + [ + np.empty((0, 3)), + np.ones(3), + np.ones((2, 3, 2, 2)), + np.ones((2, 3, 1)), + np.array([[np.nan]]), + np.array([[np.inf]]), + ], +) +def test_embedding_rejects_invalid(raw): + with pytest.raises(ValueError): + EmbeddingPost({}, {})(raw) + + +def test_embedding_validation_and_optional_normalization(): + post = EmbeddingPost({}, {}) + raw = torch.tensor([[3.0, 4.0]]) + assert torch.equal(post(raw), raw) + with pytest.raises(TypeError): + post(np.ones((1, 2), dtype=np.int32)) + with pytest.raises(ValueError): + post([raw, raw]) + with pytest.raises(ValueError): + EmbeddingPost({}, {"embedding_dim": 3})(raw) + with pytest.raises(ValueError): + PersonReIDPost({}, {"dataset": "coco"}) + with pytest.raises(ValueError): + PersonReIDPost({}, {}) + with pytest.raises(TypeError): + EmbeddingPost({}, {"normalize": "yes"}) + + +@pytest.mark.parametrize( + "dataset,loader", + [ + ("market1501", Market1501Dataset), + ("dukemtmc-reid", DukeMTMCReIDDataset), + ("occluded-duke", OccludedDukeDataset), + ], +) +def test_loader_metadata_and_source(tmp_path, dataset, loader): + for folder in ("query", "bounding_box_test"): + (tmp_path / folder).mkdir() + for filename in ("0002_c2_b.jpg", "0001_c1s1_a.jpg"): + (tmp_path / "query" / filename).touch() + (tmp_path / "bounding_box_test" / "0001_c2_a.jpg").touch() + if dataset == "market1501": + (tmp_path / "query" / "-1_c1_junk.jpg").touch() + (tmp_path / "bounding_box_test" / "0000_c1_background.jpg").touch() + data = loader(tmp_path, validate_counts=False) + assert [(s.pid, s.camid, s.split) for s in data][:2] == [ + (1, 0, "query"), + (2, 1, "query"), + ] + assert data[-1].split == "gallery" + assert all(s.pid != -1 for s in data) + if dataset == "market1501": + assert data[2].pid == 0 + with pytest.raises(ValueError, match="expected"): + loader(tmp_path) + cfg = get_dataset_config(dataset) + assert cfg["source"]["revision"] == "ec537764e36ca10da963fcb7afab157ea60ca022" + assert cfg["eval_protocol"]["id"] == cfg["evaluation"]["protocol"] + + +@pytest.mark.parametrize("filename", ["bad.jpg", "1_c9_bad.jpg", "-1_c1_junk.jpg", "-2_c1_bad.jpg"]) +def test_duke_invalid_names(tmp_path, filename): + (tmp_path / "query").mkdir() + (tmp_path / "bounding_box_test").mkdir() + (tmp_path / "query" / filename).touch() + with pytest.raises(ValueError): + DukeMTMCReIDDataset(tmp_path, validate_counts=False) + + +def test_missing_and_empty_splits(tmp_path): + with pytest.raises(FileNotFoundError): + Market1501Dataset(tmp_path) + (tmp_path / "query").mkdir() + with pytest.raises(ValueError, match="no images"): + Market1501Dataset(tmp_path) + + +def test_handbuilt_exclusions_ties_and_missing_identity(): + # same-camera positive is removed; tied background stays before the positive. + distances = np.array([[0.0, 1.0, 1.0, 2.0], [4.0, 3.0, 2.0, 1.0]]) + cmc, mean_ap = RetrievalEvaluator.evaluate_distances( + distances, + np.array([1, 99]), + np.array([1, 0, 1, 1]), + np.array([0, 0]), + np.array([0, 1, 1, 2]), + ) + np.testing.assert_array_equal(cmc, [0.0, 1.0, 1.0, 1.0]) + assert mean_ap == (1 / 2 + 2 / 3) / 2 + # Stable ties are manifest order, not identity order. + cmc, mean_ap = RetrievalEvaluator.evaluate_distances( + np.zeros((1, 2)), + np.array([1]), + np.array([1, 0]), + np.array([0]), + np.array([1, 1]), + ) + assert mean_ap == 1.0 + np.testing.assert_array_equal(cmc, [1.0, 1.0]) + + +@pytest.mark.parametrize( + "evaluator", [Market1501Evaluator, DukeMTMCReIDEvaluator, OccludedDukeEvaluator] +) +def test_score_and_rank_saturation(evaluator): + samples = [ + ReIDSample(Path("q"), 1, 0, "query"), + ReIDSample(Path("g"), 1, 1, "gallery"), + ] + result = evaluator().evaluate(np.array([[1.0, 0.0], [1.0, 0.0]]), samples) + assert result.as_dict() == { + "primary": {"map": 1.0}, + "secondary": {"rank1": 1.0, "rank5": 1.0, "rank10": 1.0}, + "sample_count": 2, + } + assert result.primary_score == result.secondary_score == result.rank(50) == 1.0 + with pytest.raises(ValueError): + result.rank(0) + + +def test_no_valid_queries(): + with pytest.raises(ValueError, match="No query"): + RetrievalEvaluator.evaluate_distances( + np.zeros((1, 1)), np.array([1]), np.array([1]), np.array([0]), np.array([0]) + ) + + +def test_extraction_order(): + batches = [np.full((2, 3), i, dtype=np.float32) for i in range(5)] + actual = extract_embeddings(batches, lambda x: [x], EmbeddingPost({}, {})) + np.testing.assert_array_equal(actual, np.concatenate(batches)) + with pytest.raises(ValueError): + extract_embeddings([], lambda x: x, lambda x: x) + + +@pytest.mark.parametrize( + "options", + [ + {"mean": [0, 0, 0]}, + {"mean": [0, 0, 0], "std": [1, 0, 1]}, + {"mean": [0, 0], "std": [1, 1]}, + {"style": "torch", "mean": [0, 0, 0], "std": [1, 1, 1]}, + ], +) +def test_explicit_normalization_validation(options): + with pytest.raises(ValueError): + Normalize(**options) + + +def test_family_base_inheritance(tmp_path): + base = Path(__file__).parents[1] / "mblt_vision/models/bases/person_reid.yaml" + for variant, name in [ + ("DEFAULT", "dukemtmc-reid"), + ("MARKET1501", "market1501"), + ("OCCLUDED_DUKE", "occluded-duke"), + ]: + cfg = load_model_variant(base, variant) + assert cfg["post_cfg"]["dataset"] == name + assert isinstance( + build_postprocess(cfg["pre_cfg"], cfg["post_cfg"]), PersonReIDPost + ) + assert build_preprocess(cfg["pre_cfg"]) + + +def test_public_base_and_engine_result_path(): + from mblt_vision.person_reid import EmbeddingPost as PublicEmbeddingPost + from mblt_vision.utils.results import Results + from mblt_vision.wrapper import MBLT_Engine + + assert PublicEmbeddingPost is EmbeddingPost + engine = MBLT_Engine.__new__(MBLT_Engine) + engine.pre_cfg = {} + engine.post_cfg = { + "task": "person_reid", + "dataset": "market1501", + "normalize": True, + } + engine.postprocessor = build_postprocess(engine.pre_cfg, engine.post_cfg) + result = engine.postprocess([np.array([[3.0, 4.0]], dtype=np.float32)]) + assert isinstance(result, Results) + torch.testing.assert_close(result.embeddings, torch.tensor([[0.6, 0.8]])) + assert result.output is result.embeddings + assert result.acc is None and result.box_cls is None + replacement = torch.tensor([[1.0, 0.0]]) + result.set_output(replacement) + assert result.embeddings is replacement + for invalid in ( + torch.ones(2), + torch.ones((1, 2), dtype=torch.int64), + torch.tensor([[float("nan")]]), + ): + with pytest.raises(ValueError): + result.set_output(invalid) + with pytest.raises(TypeError): + result.set_output([replacement]) + + +@pytest.mark.parametrize("name", ["market1501", "dukemtmc-reid", "occluded-duke"]) +def test_reid_readiness(tmp_path, monkeypatch, name): + from mblt_vision.utils.datasets import readiness, reid + from PIL import Image + + original = reid.get_dataset_config_for_task + + def tiny_config(task, dataset): + cfg = original(task, dataset) + for split in cfg["retrieval_splits"].values(): + split["count"] = 1 + return cfg + + monkeypatch.setattr(reid, "get_dataset_config_for_task", tiny_config) + assert not readiness.dataset_ready(tmp_path, "person_reid", name) + for folder in ("query", "bounding_box_test"): + (tmp_path / folder).mkdir() + Image.new("RGB", (4, 8)).save(tmp_path / folder / "0001_c1_sample.jpg") + assert readiness.dataset_ready(tmp_path, "person_reid", name) + if name == "market1501": + assert readiness.dataset_ready(tmp_path, "person_reid") + assert not readiness.dataset_ready(tmp_path, "person_reid", "coco") + image = tmp_path / "query" / "0001_c1_sample.jpg" + image.write_bytes(b"not an image") + assert not readiness.dataset_ready(tmp_path, "person_reid", name) diff --git a/tests/test_reid_parity.py b/tests/test_reid_parity.py new file mode 100644 index 0000000..84bdfa6 --- /dev/null +++ b/tests/test_reid_parity.py @@ -0,0 +1,69 @@ +"""Offline numeric differential tests against pinned, verbatim ops functions.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from mblt_vision._config import load_model_variant +from mblt_vision.utils.evaluation.retrieval import ( + RetrievalEvaluator, + euclidean_distance, +) +from mblt_vision.utils.preprocess import build_preprocess +from PIL import Image + + +def reference(name): + path = Path(__file__).parent / "reference" / f"ops_reid_{name}.py" + spec = importlib.util.spec_from_file_location(f"ops_reid_{name}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("seed", range(10)) +def test_ops_evaluator_exact_numeric_parity(seed): + ops = reference("metrics") + rng = np.random.default_rng(seed) + # Quantized integer vectors produce numerous exact distance ties. + query = rng.integers(-2, 3, size=(17, 8)).astype(np.float32) + gallery = rng.integers(-2, 3, size=(61, 8)).astype(np.float32) + q_ids = rng.integers(1, 6, 17) + g_ids = rng.integers(0, 6, 61) # zero-valued distractors remain ordinary rows + q_cam = rng.integers(0, 3, 17) + g_cam = rng.integers(0, 3, 61) + expected_distances = ops.euclidean_distance(query, gallery) + actual_distances = euclidean_distance(query, gallery) + np.testing.assert_array_equal(actual_distances, expected_distances) + for max_rank in (1, 5, 50, 100): + expected_cmc, expected_map = ops.evaluate( + expected_distances, q_ids, g_ids, q_cam, g_cam, max_rank + ) + actual_cmc, actual_map = RetrievalEvaluator.evaluate_distances( + actual_distances, q_ids, g_ids, q_cam, g_cam, max_rank + ) + np.testing.assert_array_equal(actual_cmc, expected_cmc) + assert actual_map == expected_map + + +@pytest.mark.parametrize("shape", [(19, 11), (128, 256), (383, 191)]) +@pytest.mark.parametrize("style", ["torch", "tf"]) +def test_ops_preprocessing_parity(tmp_path, shape, style): + ops = reference("preprocess") + cfg = load_model_variant( + Path(__file__).parents[1] / "mblt_vision/models/bases/person_reid.yaml", + "DEFAULT", + ) + config = cfg["pre_cfg"] + if style == "tf": + # CLIP_ReID_ViT_B_16_Baseline_DukeMTMC/distribution.yaml at ec537764. + config["Normalize"] = {"mean": [0.5, 0.5, 0.5], "std": [0.5, 0.5, 0.5]} + rng = np.random.default_rng(0) + path = tmp_path / "0001_c2_synthetic.png" + Image.fromarray(rng.integers(0, 256, size=(*shape, 3), dtype=np.uint8)).save(path) + expected = ops.preprocess(path, config)[1] + actual = build_preprocess(config)(path) + # Both use PIL bilinear + explicit float32 stats; no tolerance needed. + np.testing.assert_array_equal(actual, expected) + assert np.max(np.abs(actual - expected)) == 0.0 diff --git a/tests/test_reid_review_regressions.py b/tests/test_reid_review_regressions.py new file mode 100644 index 0000000..66ddbbd --- /dev/null +++ b/tests/test_reid_review_regressions.py @@ -0,0 +1,56 @@ +"""Retrieval cutoffs and prepared-cache boundary regressions.""" + +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +from mblt_vision.utils.datasets import readiness, reid +from mblt_vision.utils.datasets.reid import ReIDSample +from mblt_vision.utils.evaluation.retrieval import RetrievalEvaluator + + +def test_truncated_cmc_omits_uncomputed_rank_metrics(): + samples = [ReIDSample(Path("query"), 1, 0, "query")] + samples += [ + ReIDSample(Path(str(i)), pid, 1, "gallery") + for i, pid in enumerate([2, 1, 3, 4, 5]) + ] + vectors = np.array( + [[0, 0], [0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], dtype=np.float32 + ) + result = RetrievalEvaluator().evaluate(vectors, samples, max_rank=1) + assert result.as_dict()["secondary"] == {"rank1": 0.0} + with pytest.raises(ValueError, match="cutoff"): + result.rank(5) + complete = RetrievalEvaluator().evaluate(vectors, samples, max_rank=50) + assert complete.as_dict()["secondary"]["rank5"] == 1.0 + assert complete.as_dict()["secondary"]["rank10"] == 1.0 + + +@pytest.mark.parametrize("component", ["root", "query", "bounding_box_test"]) +def test_reid_readiness_rejects_symlink_components(tmp_path, monkeypatch, component): + original = reid.get_dataset_config_for_task + + def tiny_config(task, dataset): + config = original(task, dataset) + for split in config["retrieval_splits"].values(): + split["count"] = 1 + return config + + monkeypatch.setattr(reid, "get_dataset_config_for_task", tiny_config) + root = tmp_path / "cache" + for folder in ("query", "bounding_box_test"): + (root / folder).mkdir(parents=True) + Image.new("RGB", (4, 8)).save(root / folder / "0001_c1_sample.jpg") + assert readiness.dataset_ready(root, "person_reid", "market1501") + if component == "root": + alias = tmp_path / "alias" + alias.symlink_to(root, target_is_directory=True) + root = alias + else: + target = tmp_path / "outside" + (root / component).rename(target) + (root / component).symlink_to(target, target_is_directory=True) + assert not readiness.dataset_ready(root, "person_reid", "market1501")