From 281b8683f42342e6fdd57764b76ba1f39ded2356 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:22:52 +0900 Subject: [PATCH 1/7] Add opt-in model YAML family inheritance with identity proofs Preserve existing alias and update semantics across 334 configs and 595 variants. Add ordered resolution and CPU preprocessing regression tests without changing shipped model definitions. Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 3 + AGENTS.md | 4 + mblt_vision/README.md | 34 ++++++ mblt_vision/_config.py | 120 ++++++++++++++++++++ mblt_vision/wrapper.py | 62 +---------- pyproject.toml | 1 + tests/test_config_inheritance.py | 164 ++++++++++++++++++++++++++++ 7 files changed, 329 insertions(+), 59 deletions(-) create mode 100644 mblt_vision/_config.py create mode 100644 tests/test_config_inheritance.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 1b9b9c5..b695725 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -28,6 +28,9 @@ description: >- Use file_cfg.filename for MXQ and derive the same-stem ONNX artifact unless onnx_filename is required. - Every post_cfg declares dataset; resolve output taxonomy from the dataset/task pair. +- For opt-in YAML family inheritance, follow `mblt_vision/README.md`'s configuration + inheritance rules. Keep legacy `update` shallow and run `tests/test_config_inheritance.py` + to prove every existing variant's values and processing order remain identical. - Reuse wrapper.download_hub_artifact for any model that needs more than one Hub artifact rather than duplicating Hub-resolution logic. - Registering a new dataset requires readiness (`_*_ready` + `dataset_ready` diff --git a/AGENTS.md b/AGENTS.md index 5fd0aa6..06a243b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,6 +69,10 @@ The current ownership boundary is deliberate: floors, and rounding differs by a pixel at 224/0.9 and 224/0.95. Several names (ConvNext_Base) resolve in timm yet carry torchvision's transform here, so read the expected value from that model's source.yaml provenance rather than from the name. +- To share processing defaults without changing legacy models, opt new family YAMLs into + `extends: bases/family.yaml#DEFAULT`. See the configuration-inheritance section of + `mblt_vision/README.md` for ordering and merge rules. Keep legacy `update` shallow and + prove every existing variant order-identical with `tests/test_config_inheritance.py`. - Require post_cfg.dataset in every model YAML and resolve output class counts using the dataset/task pair. Do not assume one output taxonomy for every model in a task. - Preserve automatic .mxq/.onnx framework detection and the fail-fast error when a local diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 30bbddc..0187f74 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -60,6 +60,40 @@ and `list_models()` are the preferred discovery and loading APIs for new code. Top-level model imports remain available for compatibility. `obb` is the sole oriented-bounding-box task name. +## Model configuration inheritance + +Model variants may opt into local family YAMLs with `extends`. This keeps the +artifact, preprocessing and dataset connection together without duplicating a +family's processing defaults. Existing `update: VARIANT` and alias behavior is +unchanged; no shipped model YAML has been migrated by this loader refactor. + +```yaml +DEFAULT: + extends: bases/classifier.yaml#DEFAULT + file_cfg: + repo_id: mobilint/YourPublishedModel + filename: model.mxq + revision: approved-hf-commit + pre_cfg: + Resize: + size: 256 +``` + +The example is a schema illustration, not a downloadable model registration. +The base path is relative to the declaring YAML; omit `#DEFAULT` to select the +same default variant. Family bases belong in `models/bases/`, outside top-level +model discovery. `extends` recursively merges mappings and replaces lists and +scalars. Inherited mapping order is preserved: existing preprocessing operations +stay in place and new operations append. Use a standalone pipeline when operation +order must change. In contrast, `update` retains its historical one-section merge, +so replacing `pre_cfg.Resize` replaces that whole operation's parameters. A variant +cannot combine `update` and `extends`; an `update` may reference another variant +that uses `extends`. References are local only and cycles fail before inference. + +`tests/test_config_inheritance.py` compares every existing YAML variant with the +legacy resolver, including ordered processing mappings and derived ONNX filenames, +and checks exact CPU preprocessing equality for representative model families. + ## Pre-Trained Vision Models This section lists the publicly pre-trained models supported by the vision framework. diff --git a/mblt_vision/_config.py b/mblt_vision/_config.py new file mode 100644 index 0000000..ab2fee2 --- /dev/null +++ b/mblt_vision/_config.py @@ -0,0 +1,120 @@ +"""Local YAML model inheritance, independent of inference backends.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any + +import yaml + + +def _merge( + base: dict[str, Any], child: dict[str, Any], *, recursive: bool +) -> dict[str, Any]: + """Merge independent mappings while retaining preprocessing operation order.""" + + result = copy.deepcopy(base) + for key, value in child.items(): + if isinstance(result.get(key), dict) and isinstance(value, dict): + if recursive: + result[key] = _merge(result[key], value, recursive=True) + else: + # Historical `update` merges exactly one section level. + result[key].update(copy.deepcopy(value)) + else: + result[key] = copy.deepcopy(value) + return result + + +def load_model_variant( + config_path: str | Path, variant: str = "DEFAULT" +) -> dict[str, Any]: + """Resolve a local YAML variant, its aliases, and optional family inheritance. + + ``update`` retains the legacy same-file, one-section merge semantics. + ``extends: bases/family.yaml#DEFAULT`` opts into recursive mapping inheritance; + the fragment defaults to DEFAULT and paths are relative to the declaring file. + Lists and scalars replace their base values. Mapping key order is retained, so + inherited preprocessing operations retain their order. New operations append; + replace a complete operation sequence with a standalone config when reordering. + Only local files are read; each result is independently deep-copied. + """ + + documents: dict[Path, dict[str, Any]] = {} + resolving: set[tuple[Path, str]] = set() + + def document(path: Path) -> dict[str, Any]: + if path not in documents: + with path.open(encoding="utf-8") as stream: + parsed = yaml.safe_load(stream) + if not isinstance(parsed, dict): + raise TypeError( + f"Model configuration '{path}' should define a dictionary." + ) + documents[path] = parsed + return documents[path] + + def resolve(path: Path, key: str) -> dict[str, Any]: + identity = (path, key) + if identity in resolving: + raise ValueError( + f"Circular model configuration reference detected for '{path}#{key}'." + ) + resolving.add(identity) + try: + full = document(path) + part = full.get(key) + if part is None: + raise ValueError(f"Model type '{key}' not found in configuration.") + if isinstance(part, str): + if part not in full: + raise ValueError( + f"Model alias '{part}' not found in configuration." + ) + return resolve(path, part) + if not isinstance(part, dict): + raise TypeError( + f"Resolved model configuration for '{key}' is not a dictionary." + ) + child = copy.deepcopy(part) + base_key = child.pop("update", None) + has_extends = "extends" in child + reference = child.pop("extends", None) + if base_key is not None and has_extends: + raise ValueError( + "A model variant cannot combine 'update' and 'extends'." + ) + if base_key is not None: + if not isinstance(base_key, str) or base_key not in full: + raise ValueError( + f"Base configuration '{base_key}' not found for update." + ) + return _merge(resolve(path, base_key), child, recursive=False) + if has_extends: + if not isinstance(reference, str) or not reference: + raise ValueError( + "Model 'extends' must be a nonempty local YAML reference." + ) + filename, separator, base_variant = reference.partition("#") + if ( + not filename + or "://" in filename + or (separator and not base_variant) + ): + raise ValueError( + f"Invalid local model inheritance reference: {reference!r}." + ) + if "#" in base_variant: + raise ValueError( + f"Invalid local model inheritance reference: {reference!r}." + ) + base_path = (path.parent / filename).resolve() + return _merge( + resolve(base_path, base_variant or "DEFAULT"), child, recursive=True + ) + return child + finally: + resolving.remove(identity) + + return resolve(Path(config_path).resolve(), variant) diff --git a/mblt_vision/wrapper.py b/mblt_vision/wrapper.py index a15296f..d451284 100644 --- a/mblt_vision/wrapper.py +++ b/mblt_vision/wrapper.py @@ -16,11 +16,11 @@ import numpy as np import torch -import yaml from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError - from mblt_npu import MobilintNPUBackend, ONNXBackend, normalize_target_device + +from ._config import load_model_variant from ._model_paths import resolve_framework as _resolve_framework from ._model_paths import split_model_paths as _split_model_paths from ._model_paths import ( @@ -365,63 +365,7 @@ def resolve_model_config( if not config_path.is_file(): config_path = MODEL_CONFIG_DIR / _model_name_aliasing(model_cls) - with config_path.open(encoding="utf-8") as config_file: - full_config = yaml.safe_load(config_file) - if not isinstance(full_config, dict): - raise TypeError( - f"Model configuration '{config_path}' should define a dictionary." - ) - - resolving: set[str] = set() - - def _resolve_variant(variant: str) -> dict[str, Any]: - if variant in resolving: - raise ValueError( - f"Circular model configuration reference detected for '{variant}'." - ) - resolving.add(variant) - try: - model_config_part = full_config.get(variant) - if model_config_part is None: - raise ValueError(f"Model type '{variant}' not found in configuration.") - if isinstance(model_config_part, str): - if model_config_part not in full_config: - raise ValueError( - f"Model alias '{model_config_part}' not found in configuration." - ) - return _resolve_variant(model_config_part) - if not isinstance(model_config_part, dict): - raise TypeError( - f"Resolved model configuration for '{variant}' is not a dictionary." - ) - - resolved = copy.deepcopy(model_config_part) - base_config_key = resolved.pop("update", None) - if base_config_key is None: - return resolved - if ( - not isinstance(base_config_key, str) - or base_config_key not in full_config - ): - raise ValueError( - f"Base configuration '{base_config_key}' not found for update." - ) - - merged_config = _resolve_variant(base_config_key) - for key, value in resolved.items(): - if ( - key in merged_config - and isinstance(merged_config[key], dict) - and isinstance(value, dict) - ): - merged_config[key].update(value) - else: - merged_config[key] = value - return merged_config - finally: - resolving.remove(variant) - - return _normalize_model_artifacts(_resolve_variant(model_type)) + return _normalize_model_artifacts(load_model_variant(config_path, model_type)) class MBLT_Engine: diff --git a/pyproject.toml b/pyproject.toml index 5514221..9850c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ version = { attr = "mblt_vision.__version__" } "mblt_vision" = [ "py.typed", "models/*.yaml", + "models/bases/*.yaml", "datasets/*.yaml", "datasets/*.txt", ] diff --git a/tests/test_config_inheritance.py b/tests/test_config_inheritance.py new file mode 100644 index 0000000..a36907b --- /dev/null +++ b/tests/test_config_inheritance.py @@ -0,0 +1,164 @@ +"""CPU proofs for legacy resolution and opt-in local YAML inheritance.""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import numpy as np +import pytest +import yaml + +from mblt_vision._config import load_model_variant +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.wrapper import MODEL_CONFIG_DIR, resolve_model_config + + +def legacy_resolve(document: dict, variant: str) -> dict: + """Independent pre-refactor resolution, including ONNX artifact derivation.""" + + part = document[variant] + if isinstance(part, str): + return legacy_resolve(document, part) + child = copy.deepcopy(part) + base = child.pop("update", None) + if base is not None: + resolved = legacy_resolve(document, base) + for key, value in child.items(): + if isinstance(resolved.get(key), dict) and isinstance(value, dict): + resolved[key].update(value) + else: + resolved[key] = value + return resolved + return child + + +def ordered(value): + """Deep equality alone misses insertion order, which defines pre-op order.""" + + if isinstance(value, dict): + return [(key, ordered(item)) for key, item in value.items()] + if isinstance(value, list): + return [ordered(item) for item in value] + return value + + +def legacy_config(document: dict, variant: str) -> dict: + config = legacy_resolve(document, variant) + files = config.get("file_cfg", {}) + if files.get("filename") and "onnx_filename" not in files: + files["onnx_filename"] = str(Path(files["filename"]).with_suffix(".onnx")) + return config + + +CONFIG_VARIANTS = [ + (path, variant) + for path in sorted(MODEL_CONFIG_DIR.glob("*.yaml")) + for variant in yaml.safe_load(path.read_text(encoding="utf-8")) +] + + +@pytest.mark.parametrize("path,variant", CONFIG_VARIANTS, ids=lambda value: str(value)) +def test_every_legacy_variant_is_order_identical(path: Path, variant: str) -> None: + expected = legacy_config(yaml.safe_load(path.read_text(encoding="utf-8")), variant) + actual = resolve_model_config(str(path), variant) + assert actual == expected + assert ordered(actual) == ordered(expected) + + +def write_config(path: Path, config: dict) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + return path + + +def test_recursive_extends_keeps_order_and_independent_results(tmp_path: Path) -> None: + write_config( + tmp_path / "bases/base.yaml", + { + "DEFAULT": { + "pre_cfg": { + "Reader": {"style": "pil", "color_mode": "RGB"}, + "Resize": {"size": [32, 32], "interpolation": "bilinear"}, + }, + "post_cfg": {"nested": {"labels": [1, 2], "other": True}}, + } + }, + ) + child = write_config( + tmp_path / "child.yaml", + { + "DEFAULT": { + "extends": "bases/base.yaml", + "pre_cfg": {"Resize": {"size": [16, 16]}, "SetOrder": {"shape": "HWC"}}, + "post_cfg": {"nested": {"labels": [3]}}, + } + }, + ) + result = load_model_variant(child) + assert list(result["pre_cfg"]) == ["Reader", "Resize", "SetOrder"] + assert result["pre_cfg"]["Resize"] == { + "size": [16, 16], + "interpolation": "bilinear", + } + assert result["post_cfg"]["nested"] == {"labels": [3], "other": True} + result["post_cfg"]["nested"]["labels"].append(4) + assert load_model_variant(child)["post_cfg"]["nested"]["labels"] == [3] + + +def test_update_stays_shallow_with_extends_base(tmp_path: Path) -> None: + write_config( + tmp_path / "base.yaml", + {"DEFAULT": {"pre_cfg": {"Resize": {"size": 32, "interpolation": "bilinear"}}}}, + ) + path = write_config( + tmp_path / "child.yaml", + { + "BASE": {"extends": "base.yaml"}, + "DEFAULT": {"update": "BASE", "pre_cfg": {"Resize": {"size": 16}}}, + }, + ) + assert load_model_variant(path)["pre_cfg"]["Resize"] == {"size": 16} + + +@pytest.mark.parametrize( + "reference", + [None, "", 123, "#DEFAULT", "base.yaml#", "https://host/base.yaml", "a#b#c"], +) +def test_invalid_extends(tmp_path: Path, reference) -> None: + path = write_config(tmp_path / "child.yaml", {"DEFAULT": {"extends": reference}}) + with pytest.raises(ValueError): + load_model_variant(path) + + +def test_cross_file_cycle(tmp_path: Path) -> None: + a = write_config(tmp_path / "a.yaml", {"DEFAULT": {"extends": "b.yaml"}}) + write_config(tmp_path / "b.yaml", {"DEFAULT": {"extends": "./a.yaml"}}) + with pytest.raises(ValueError, match="Circular"): + load_model_variant(a) + + +def test_missing_base_and_conflicting_modes(tmp_path: Path) -> None: + path = write_config( + tmp_path / "child.yaml", {"DEFAULT": {"extends": "absent.yaml"}} + ) + with pytest.raises(FileNotFoundError): + load_model_variant(path) + write_config( + path, {"BASE": {}, "DEFAULT": {"extends": "absent.yaml", "update": "BASE"}} + ) + with pytest.raises(ValueError, match="cannot combine"): + load_model_variant(path) + + +@pytest.mark.parametrize("model", ["ResNet50", "YOLOv8n", "YOLOX-s", "DAMO-YOLO-T"]) +def test_cpu_preprocessing_numeric_identity(model: str) -> None: + path = MODEL_CONFIG_DIR / f"{model}.yaml" + expected = legacy_config( + yaml.safe_load(path.read_text(encoding="utf-8")), "DEFAULT" + ) + actual = resolve_model_config(str(path)) + image = np.random.default_rng(0).integers(0, 256, (71, 103, 3), dtype=np.uint8) + before = build_preprocess(expected["pre_cfg"])(image.copy()) + after = build_preprocess(actual["pre_cfg"])(image.copy()) + np.testing.assert_array_equal(np.asarray(after), np.asarray(before)) From c1289eb095bdb4852ca62eaaeee2ee28e8e05f83 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:37:33 +0900 Subject: [PATCH 2/7] Add person retrieval datasets, embedding postprocessing and exact protocol scoring Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 5 + AGENTS.md | 6 + benchmark/benchmark_vision_models.py | 3 +- mblt_vision/README.md | 49 ++++ mblt_vision/__init__.py | 2 + mblt_vision/_tasks.py | 1 + mblt_vision/datasets/dukemtmc-reid.yaml | 35 +++ mblt_vision/datasets/market1501.yaml | 35 +++ mblt_vision/datasets/occluded-duke.yaml | 35 +++ mblt_vision/models/bases/person_reid.yaml | 26 +++ mblt_vision/person_reid/__init__.py | 33 +++ mblt_vision/utils/datasets/reid.py | 96 ++++++++ mblt_vision/utils/evaluation/retrieval.py | 208 +++++++++++++++++ mblt_vision/utils/postprocess/__init__.py | 3 +- mblt_vision/utils/postprocess/build_post.py | 3 + .../utils/postprocess/embedding_post.py | 85 +++++++ mblt_vision/utils/preprocess/normalize.py | 30 ++- tests/reference/ops_reid_metrics.py | 53 +++++ tests/reference/ops_reid_preprocess.py | 92 ++++++++ tests/test_api.py | 1 + tests/test_reid.py | 216 ++++++++++++++++++ tests/test_reid_parity.py | 69 ++++++ 22 files changed, 1082 insertions(+), 4 deletions(-) create mode 100644 mblt_vision/datasets/dukemtmc-reid.yaml create mode 100644 mblt_vision/datasets/market1501.yaml create mode 100644 mblt_vision/datasets/occluded-duke.yaml create mode 100644 mblt_vision/models/bases/person_reid.yaml create mode 100644 mblt_vision/person_reid/__init__.py create mode 100644 mblt_vision/utils/datasets/reid.py create mode 100644 mblt_vision/utils/evaluation/retrieval.py create mode 100644 mblt_vision/utils/postprocess/embedding_post.py create mode 100644 tests/reference/ops_reid_metrics.py create mode 100644 tests/reference/ops_reid_preprocess.py create mode 100644 tests/test_reid.py create mode 100644 tests/test_reid_parity.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index b695725..7d563ab 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -109,6 +109,11 @@ description: >- metric). Load difficult objects as ordinary targets, as Ultralytics' validation does. - Run YOLOv7 models at `iou_thres: 0.65`, WongKinYiu/yolov7's own `test.py` default. +- ReID is utility-only until published wrappers and routing are added. Reuse + `EmbeddingPost` for dense vectors, explicit float32 mean/std for source preprocessing, + and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions + and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. + ## Runtime and Packaging - Route NPU runtime access through mblt-npu-python; do not copy backend classes into Vision. diff --git a/AGENTS.md b/AGENTS.md index 06a243b..5c3c614 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,6 +180,12 @@ The current ownership boundary is deliberate: any future model needing more than one Hub artifact, rather than duplicating Hub-resolution logic. +- ReID utilities live in `mblt_vision.person_reid`; concrete published wrappers are separate. + Use `EmbeddingPost` for validated dense vectors and `PersonReIDPost` for dataset binding. + Retrieval protocols and prepared loaders are pinned to ops source in dataset YAMLs; keep + stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. + Named Normalize styles retain legacy arithmetic; explicit mean/std uses source float32 arithmetic. + ## Python-First Architecture - Keep the public layer independent from a particular backend. Define small internal interfaces diff --git a/benchmark/benchmark_vision_models.py b/benchmark/benchmark_vision_models.py index a5c086d..06f86e3 100644 --- a/benchmark/benchmark_vision_models.py +++ b/benchmark/benchmark_vision_models.py @@ -35,7 +35,8 @@ CORE_MODES: tuple[CoreMode, ...] = cast( tuple[CoreMode, ...], core_modes_for_target_device("aries-rb") ) -TASK_CHOICES = VISION_TASKS +# Utility-only embedding tasks have no benchmark engine/CLI route yet. +TASK_CHOICES = tuple(task for task in VISION_TASKS if task != "person_reid") SUPPORTED_TARGET_DEVICES = frozenset({"aries-rb", "regulus-ra", "regulus-rb"}) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 0187f74..6249bc9 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -489,3 +489,52 @@ validation split, following Ultralytics' depth-validation convention. - Difficult objects count as ordinary targets, as in Ultralytics' validation. + +## Person re-identification utilities + +`mblt_vision.person_reid` provides prepared query/gallery loaders, embedding +postprocessing and retrieval scoring. Published model wrappers/HF artifacts are +not added in this stage; no weights or datasets are downloaded by these utilities. +The CLI/benchmark router is unchanged. + +```python +from mblt_vision.person_reid import Market1501Dataset, Market1501Evaluator + +# root contains query/ and bounding_box_test/; embeddings have one row per sample. +dataset = Market1501Dataset("/path/to/market1501/eval") +# result = Market1501Evaluator().evaluate(embeddings, dataset.samples) +# print(result.as_dict()) +``` + +Also available: `DukeMTMCReIDDataset`, `OccludedDukeDataset` and matching +`DukeMTMCReIDEvaluator`, `OccludedDukeEvaluator`. Loaders emit immutable +`ReIDSample(image, pid, camid, split)` records, sorted query-first then gallery; +cameras are zero-based, identities are not relabeled. Counts are source-checked +by default; caller-owned subsets must explicitly set `validate_counts=False`. +Only Market-1501 drops pid -1 junk; pid 0 background distractors remain. +Duke loaders reject pid -1. Root defaults use `~/.mblt_model_zoo/datasets//eval`. + +`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`. It validates one +floating-point `(N,D)` output or singleton-spatial NCHW/NHWC tensor, optionally +checks `embedding_dim`, and uses `normalize: true` for row-wise L2 normalization +(zero vectors remain zero). Return type is a float32 torch matrix, never class +probabilities. An explicit `post_cfg.dataset` is required. Processing-only base +profiles in `models/bases/person_reid.yaml` can be selected using `extends`; +concrete models must supply their own HF `file_cfg`, dataset and embedding dimension. +`Normalize` accepts either a historical named style or explicit three-element +`mean` and `std` lists; named styles retain unchanged legacy arithmetic. + +`extract_embeddings(batches, infer, postprocess)` runs prepared batches through +the shared bounded inference pipeline, preserving order. Evaluators consume +these embeddings without adding normalization, camera inputs, re-ranking or flip +fusion. Same-ID/same-camera gallery rows are excluded; unmatched queries are +skipped. Squared float32 Euclidean distances use stable ties in gallery order. +Ranks beyond the gallery saturate; mAP is primary, rank-1/5/10 secondary, all +ratios. `as_dict()` rounds to five decimals like ops; raw `mean_ap`/`cmc` remain +unrounded. Dataset YAMLs record exact ops config, loader, evaluator and revision. + +Offline tests vendor the inspected ops functions at +`ec537764e36ca10da963fcb7afab157ea60ca022`. Synthetic embeddings require exact +distance/CMC/mAP equality; synthetic RGB preprocessing requires byte equality +(`atol=0`) against the ops adapter for explicit source mean/std profiles. These +are protocol tests, not measured model accuracy claims. diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index e56c12a..9cf69d2 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -13,6 +13,7 @@ from . import instance_segmentation as instance_segmentation from . import obb as obb from . import object_detection as object_detection +from . import person_reid as person_reid from . import pose_estimation as pose_estimation from . import semantic_segmentation as semantic_segmentation from ._api import list_models as list_models @@ -52,6 +53,7 @@ "object_detection", "obb", "pose_estimation", + "person_reid", "semantic_segmentation", ] + sorted(_LEGACY_MODEL_EXPORTS) # Keep legacy compatibility exports synchronized with their task packages. diff --git a/mblt_vision/_tasks.py b/mblt_vision/_tasks.py index df9a462..c8a4b6d 100644 --- a/mblt_vision/_tasks.py +++ b/mblt_vision/_tasks.py @@ -13,6 +13,7 @@ "obb", "pose_estimation", "face_detection", + "person_reid", ) diff --git a/mblt_vision/datasets/dukemtmc-reid.yaml b/mblt_vision/datasets/dukemtmc-reid.yaml new file mode 100644 index 0000000..c1fb241 --- /dev/null +++ b/mblt_vision/datasets/dukemtmc-reid.yaml @@ -0,0 +1,35 @@ +name: dukemtmc-reid +path: ~/.mblt_model_zoo/datasets/dukemtmc-reid +tasks: +- person_reid +val: eval +cameras: 8 +skip_junk: false +retrieval_splits: + query: + directory: query + count: 2228 + gallery: + directory: bounding_box_test + count: 17661 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/dukemtmc_reid/dukemtmc_reid.yaml + name: dukemtmc-reid + dataloader_path: datasets/dukemtmc_reid/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: dukemtmc-reid-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: dukemtmc-reid-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/dukemtmc_reid/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/market1501.yaml b/mblt_vision/datasets/market1501.yaml new file mode 100644 index 0000000..b638134 --- /dev/null +++ b/mblt_vision/datasets/market1501.yaml @@ -0,0 +1,35 @@ +name: market1501 +path: ~/.mblt_model_zoo/datasets/market1501 +tasks: +- person_reid +val: eval +cameras: 6 +skip_junk: true +retrieval_splits: + query: + directory: query + count: 3368 + gallery: + directory: bounding_box_test + count: 15913 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/market1501/market1501.yaml + name: market1501 + dataloader_path: datasets/market1501/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: market1501-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: market1501-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/market1501/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/occluded-duke.yaml b/mblt_vision/datasets/occluded-duke.yaml new file mode 100644 index 0000000..dc039b2 --- /dev/null +++ b/mblt_vision/datasets/occluded-duke.yaml @@ -0,0 +1,35 @@ +name: occluded-duke +path: ~/.mblt_model_zoo/datasets/occluded-duke +tasks: +- person_reid +val: eval +cameras: 8 +skip_junk: false +retrieval_splits: + query: + directory: query + count: 2210 + gallery: + directory: bounding_box_test + count: 17661 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/occluded_duke/occluded_duke.yaml + name: occluded-duke + dataloader_path: datasets/occluded_duke/_private/dataloader.py + shared_path: datasets/_shared/reid.py +eval_protocol: + id: occluded-duke-reid-map.v1 +evaluation: + split: eval/query + eval/bounding_box_test + protocol: occluded-duke-reid-map.v1 + primary_metric: map + secondary_metrics: + - rank1 + - rank5 + - rank10 + implementation_reference: + path: datasets/occluded_duke/_private/evaluator.py + shared_path: datasets/_shared/reid.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/models/bases/person_reid.yaml b/mblt_vision/models/bases/person_reid.yaml new file mode 100644 index 0000000..cafeaf3 --- /dev/null +++ b/mblt_vision/models/bases/person_reid.yaml @@ -0,0 +1,26 @@ +# Processing only: published HF artifact identities belong to concrete stage-d2 YAMLs. +# Source: mblt-model-ops ec537764e36ca10da963fcb7afab157ea60ca022 +# models/person_reid/CLIP_ReID_RN50_Baseline_DukeMTMC/distribution.yaml +DEFAULT: + pre_cfg: + Reader: + style: pil + Resize: + size: [256, 128] + interpolation: bilinear + Normalize: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + post_cfg: + task: person_reid + dataset: dukemtmc-reid + normalize: true +# Inherit this processing profile and override dataset/dimension/artifacts per model. +MARKET1501: + extends: person_reid.yaml#DEFAULT + post_cfg: + dataset: market1501 +OCCLUDED_DUKE: + extends: person_reid.yaml#DEFAULT + post_cfg: + dataset: occluded-duke diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py new file mode 100644 index 0000000..07fecad --- /dev/null +++ b/mblt_vision/person_reid/__init__.py @@ -0,0 +1,33 @@ +"""Person retrieval utilities; published model wrappers are added separately.""" + +from ..utils.datasets.reid import ( + DukeMTMCReIDDataset, + Market1501Dataset, + OccludedDukeDataset, + ReIDDataset, + ReIDSample, +) +from ..utils.evaluation.retrieval import ( + DukeMTMCReIDEvaluator, + Market1501Evaluator, + OccludedDukeEvaluator, + RetrievalEvaluator, + RetrievalResult, + extract_embeddings, +) +from ..utils.postprocess.embedding_post import PersonReIDPost + +__all__ = [ + "DukeMTMCReIDDataset", + "DukeMTMCReIDEvaluator", + "Market1501Dataset", + "Market1501Evaluator", + "OccludedDukeDataset", + "OccludedDukeEvaluator", + "PersonReIDPost", + "ReIDDataset", + "ReIDSample", + "RetrievalEvaluator", + "RetrievalResult", + "extract_embeddings", +] diff --git a/mblt_vision/utils/datasets/reid.py b/mblt_vision/utils/datasets/reid.py new file mode 100644 index 0000000..af3e68d --- /dev/null +++ b/mblt_vision/utils/datasets/reid.py @@ -0,0 +1,96 @@ +"""Prepared query/gallery person crops; no download or identity relabeling.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from ...datasets import get_dataset_config_for_task + +_FILENAME = re.compile(r"^(-?\d+)_c(\d+)(?:s\d+)?(?:_|\.)") + + +@dataclass(frozen=True) +class ReIDSample: + """One crop with source identity, zero-based camera and query/gallery split.""" + + image: Path + pid: int + camid: int + split: Literal["query", "gallery"] + + +class ReIDDataset: + """Read a prepared benchmark root in source-defined deterministic order. + + ``root`` is the directory containing ``query`` and ``bounding_box_test`` + (the ops prepared ``eval`` directory). Counts are checked by default; + ``validate_counts=False`` explicitly supports caller-owned small subsets. + """ + + name: str + + def __init__( + self, root: str | Path | None = None, *, validate_counts: bool = True + ) -> None: + self.config = get_dataset_config_for_task("person_reid", self.name) + self.root = ( + Path(root).expanduser() + if root is not None + else Path(self.config["path"]) / self.config["val"] + ) + samples = [] + for split in ("query", "gallery"): + options = self.config["retrieval_splits"][split] + directory = self.root / options["directory"] + if not directory.is_dir(): + raise FileNotFoundError(f"ReID split directory is missing: {directory}") + rows = [] + for path in sorted(directory.glob("*.jpg")): + match = _FILENAME.match(path.name) + if match is None: + raise ValueError( + f"ReID file name carries no _c token: {path}" + ) + pid, camera = map(int, match.groups()) + if pid == -1: + if self.config["skip_junk"]: + continue + raise ValueError(f"Unexpected junk identity: {path}") + if not 1 <= camera <= self.config["cameras"]: + raise ValueError(f"Camera {camera} outside benchmark range: {path}") + rows.append(ReIDSample(path, pid, camera - 1, split)) + if not rows: + raise ValueError(f"ReID split directory holds no images: {directory}") + if validate_counts and len(rows) != options["count"]: + raise ValueError( + f"{split} has {len(rows)} crops, expected {options['count']}" + ) + samples.extend(rows) + self.samples = tuple(samples) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> ReIDSample: + return self.samples[index] + + +class Market1501Dataset(ReIDDataset): + """Market-1501: discard pid -1; retain pid 0 background distractors.""" + + name = "market1501" + + +class DukeMTMCReIDDataset(ReIDDataset): + """DukeMTMC-reID prepared held-out query/gallery crops.""" + + name = "dukemtmc-reid" + + +class OccludedDukeDataset(ReIDDataset): + """Occluded-Duke prepared query split and Duke gallery.""" + + name = "occluded-duke" diff --git a/mblt_vision/utils/evaluation/retrieval.py b/mblt_vision/utils/evaluation/retrieval.py new file mode 100644 index 0000000..e3d8ffa --- /dev/null +++ b/mblt_vision/utils/evaluation/retrieval.py @@ -0,0 +1,208 @@ +"""Source-exact Market protocol retrieval metrics (ops ec537764).""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch + +from ..datasets.reid import ReIDSample +from ._pipeline import map_batched_inference + + +def extract_embeddings( + batches: Iterable[Any], infer: Callable, postprocess: Callable +) -> np.ndarray: + """Decode prepared batches in input order through the shared bounded pipeline. + + ``infer`` receives a prepared batch; ``postprocess`` receives raw outputs and + returns a validated ``(N,D)`` tensor. No implicit normalization or flip fusion. + """ + + def decode(batch: Any, output: Any) -> np.ndarray: + value = postprocess(output) + if isinstance(value, torch.Tensor): + value = value.detach().cpu().numpy() + value = np.asarray(value, dtype=np.float32) + if value.ndim != 2 or min(value.shape) == 0 or not np.isfinite(value).all(): + raise ValueError( + "Decoded embeddings require a nonempty finite (N,D) matrix" + ) + return value + + rows = list(map_batched_inference(batches, infer, decode)) + if not rows: + raise ValueError("No embedding batches supplied") + return np.concatenate(rows, axis=0) + + +def euclidean_distance(query: np.ndarray, gallery: np.ndarray) -> np.ndarray: + """Source float32 squared Euclidean distance (not cosine or square root).""" + query = np.asarray(query, dtype=np.float32) + gallery = np.asarray(gallery, dtype=np.float32) + return ( + np.sum(query**2, axis=1, keepdims=True) + + np.sum(gallery**2, axis=1, keepdims=True).T + - 2.0 * (query @ gallery.T) + ) + + +@dataclass(frozen=True) +class RetrievalResult: + """Unrounded mAP and CMC curve; ``as_dict`` matches ops score rounding.""" + + cmc: np.ndarray + mean_ap: float + sample_count: int + + @property + def primary_score(self) -> float: + return self.mean_ap + + @property + def secondary_score(self) -> float: + return self.rank(1) + + def rank(self, k: int) -> float: + """Saturate ranks beyond the gallery length, as in the source scorer.""" + if not isinstance(k, int) or isinstance(k, bool) or k <= 0: + raise ValueError("Rank must be a positive integer") + return float(self.cmc[min(k, len(self.cmc)) - 1]) + + def as_dict(self) -> dict: + """Return source-compatible primary/secondary metrics in ratio units.""" + return { + "primary": {"map": round(self.mean_ap, 5)}, + "secondary": {f"rank{k}": round(self.rank(k), 5) for k in (1, 5, 10)}, + "sample_count": self.sample_count, + } + + +class RetrievalEvaluator: + """Market retrieval protocol shared by three dataset-specific evaluators. + + Stable distance ties retain gallery manifest order. Exclude only same-ID, + same-camera rows; pid 0 distractors remain. Dataset loaders own pid -1 junk + filtering. Queries without cross-camera matches do not enter either mean. + """ + + dataset: str | None = None + + def evaluate( + self, + embeddings: np.ndarray, + samples: Sequence[ReIDSample], + *, + max_rank: int = 50, + ) -> RetrievalResult: + """Score one embedding row per sample in query-then-gallery order.""" + values = np.asarray(embeddings, dtype=np.float32) + if ( + values.ndim != 2 + or min(values.shape) == 0 + or values.shape[0] != len(samples) + or not np.isfinite(values).all() + ): + raise ValueError("Embeddings require one finite nonempty vector per sample") + if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: + raise ValueError("max_rank must be a positive integer") + if any(s.split not in {"query", "gallery"} or s.pid == -1 for s in samples): + raise ValueError( + "Samples require query/gallery splits with loader-filtered junk" + ) + query = [i for i, s in enumerate(samples) if s.split == "query"] + gallery = [i for i, s in enumerate(samples) if s.split == "gallery"] + if not query or not gallery: + raise ValueError("Benchmark needs both a query and a gallery split") + cmc, mean_ap = self.evaluate_distances( + euclidean_distance(values[query], values[gallery]), + np.asarray([samples[i].pid for i in query]), + np.asarray([samples[i].pid for i in gallery]), + np.asarray([samples[i].camid for i in query]), + np.asarray([samples[i].camid for i in gallery]), + max_rank, + ) + return RetrievalResult(cmc, mean_ap, len(samples)) + + @staticmethod + def evaluate_distances( + distmat: np.ndarray, + q_pids: np.ndarray, + g_pids: np.ndarray, + q_camids: np.ndarray, + g_camids: np.ndarray, + max_rank: int = 50, + ) -> tuple[np.ndarray, float]: + """CMC and mAP with the exact source accumulation and padding order.""" + distmat = np.asarray(distmat) + q_pids, g_pids = np.asarray(q_pids), np.asarray(g_pids) + q_camids, g_camids = np.asarray(q_camids), np.asarray(g_camids) + if ( + distmat.ndim != 2 + or min(distmat.shape) == 0 + or not np.isfinite(distmat).all() + ): + raise ValueError("Distances require a nonempty finite matrix") + num_q, num_g = distmat.shape + if any( + v.shape != (n,) + for v, n in ( + (q_pids, num_q), + (g_pids, num_g), + (q_camids, num_q), + (g_camids, num_g), + ) + ): + raise ValueError( + "Identity/camera metadata does not match distance dimensions" + ) + if not isinstance(max_rank, int) or isinstance(max_rank, bool) or max_rank <= 0: + raise ValueError("max_rank must be a positive integer") + max_rank = min(max_rank, num_g) + indices = np.argsort(distmat, axis=1, kind="stable") + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + all_cmc = [] + all_ap = [] + for q_idx in range(num_q): + order = indices[q_idx] + remove = (g_pids[order] == q_pids[q_idx]) & ( + g_camids[order] == q_camids[q_idx] + ) + orig_cmc = matches[q_idx][~remove] + if not np.any(orig_cmc): + continue + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + all_cmc.append( + np.pad( + cmc[:max_rank], (0, max_rank - min(len(cmc), max_rank)), mode="edge" + ) + ) + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() / np.arange(1, orig_cmc.shape[0] + 1) + all_ap.append(float((tmp_cmc * orig_cmc).sum() / num_rel)) + if not all_cmc: + raise ValueError("No query identity appears in the gallery") + cmc_curve = np.asarray(all_cmc, dtype=np.float64).sum(0) / len(all_cmc) + return cmc_curve, float(np.mean(all_ap)) + + +class Market1501Evaluator(RetrievalEvaluator): + """Market-1501 held-out retrieval protocol v1.""" + + dataset = "market1501" + + +class DukeMTMCReIDEvaluator(RetrievalEvaluator): + """DukeMTMC-reID held-out retrieval protocol v1.""" + + dataset = "dukemtmc-reid" + + +class OccludedDukeEvaluator(RetrievalEvaluator): + """Occluded-Duke held-out retrieval protocol v1.""" + + dataset = "occluded-duke" diff --git a/mblt_vision/utils/postprocess/__init__.py b/mblt_vision/utils/postprocess/__init__.py index e5090d0..879a06f 100644 --- a/mblt_vision/utils/postprocess/__init__.py +++ b/mblt_vision/utils/postprocess/__init__.py @@ -4,6 +4,7 @@ from .build_post import build_postprocess from .depth_post import DepthPost +from .embedding_post import EmbeddingPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost -__all__ = ["DepthPost", "SemanticSegPost", "build_postprocess"] +__all__ = ["DepthPost", "EmbeddingPost", "PersonReIDPost", "SemanticSegPost", "build_postprocess"] diff --git a/mblt_vision/utils/postprocess/build_post.py b/mblt_vision/utils/postprocess/build_post.py index dc89192..eb569ac 100644 --- a/mblt_vision/utils/postprocess/build_post.py +++ b/mblt_vision/utils/postprocess/build_post.py @@ -9,6 +9,7 @@ from .cls_post import ClsPost from .damoyolo_post import DAMOYOLODetectionPost from .depth_post import DepthPost +from .embedding_post import PersonReIDPost from .semantic_seg_post import SemanticSegPost from .yolo_anchor_post import ( YOLOAnchorDetectionPost, @@ -70,6 +71,8 @@ def build_postprocess( f"Unsupported post_cfg.head {head!r}; expected one of {sorted(DETECTION_HEADS)}." ) return DETECTION_HEADS[head](pre_cfg, post_cfg, **kwargs) + if task == "person_reid": + return PersonReIDPost(pre_cfg, post_cfg) if task == "image_classification": return ClsPost(pre_cfg, post_cfg) if task == "depth_estimation": diff --git a/mblt_vision/utils/postprocess/embedding_post.py b/mblt_vision/utils/postprocess/embedding_post.py new file mode 100644 index 0000000..8d55353 --- /dev/null +++ b/mblt_vision/utils/postprocess/embedding_post.py @@ -0,0 +1,85 @@ +"""Validated appearance embeddings, shared by retrieval and verification tasks.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np +import torch +import torch.nn.functional as F + +from ...datasets import get_dataset_config_for_task +from ..types import ListTensorLike, TensorLike +from .base import PostBase + + +class EmbeddingPost(PostBase): + """Return floating-point ``(batch, dimension)`` embeddings without argmax. + + Accept a single matrix or singleton-spatial NCHW/NHWC output (also wrapped + in a one-element runtime output list). L2 normalization is opt-in; zero + vectors stay zero, matching the source embedding decoder. + """ + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__() + self.normalize = post_cfg.get("normalize", False) + if not isinstance(self.normalize, bool): + raise TypeError("post_cfg.normalize must be a boolean") + self.embedding_dim = post_cfg.get("embedding_dim") + if self.embedding_dim is not None and ( + not isinstance(self.embedding_dim, int) + or isinstance(self.embedding_dim, bool) + or self.embedding_dim <= 0 + ): + raise ValueError("post_cfg.embedding_dim must be a positive integer") + + def __call__(self, x: TensorLike | ListTensorLike) -> torch.Tensor: + """Validate one dense output and optionally L2-normalize each row.""" + if isinstance(x, Sequence): + if len(x) != 1: + raise ValueError( + "Embedding postprocessing expects exactly one output tensor" + ) + x = x[0] + if isinstance(x, np.ndarray): + x = torch.from_numpy(x) + if not isinstance(x, torch.Tensor): + raise TypeError("Embedding output must be a NumPy array or torch tensor") + if not x.is_floating_point(): + raise TypeError("Embedding output must have a floating-point dtype") + x = x.to(self.device) + if x.ndim == 4 and (x.shape[2:] == (1, 1) or x.shape[1:3] == (1, 1)): + x = x.reshape(x.shape[0], -1) + if x.ndim != 2 or min(x.shape) <= 0: + raise ValueError( + f"Embedding output requires nonempty (N, D), got {tuple(x.shape)}" + ) + if self.embedding_dim is not None and x.shape[1] != self.embedding_dim: + raise ValueError( + f"Embedding dimension {x.shape[1]} does not match {self.embedding_dim}" + ) + if not bool(torch.isfinite(x).all()): + raise ValueError("Embedding output must contain only finite values") + # Float32 agrees with the ops decoder, including its 1e-12 norm floor. + x = x.float() + if not bool(torch.isfinite(x).all()): + raise ValueError("Embedding values overflow float32") + if self.normalize and not bool( + torch.isfinite(torch.linalg.vector_norm(x, dim=1)).all() + ): + raise ValueError("Embedding norms overflow float32") + if self.normalize: + x = F.normalize(x, p=2, dim=1, eps=1e-12) + return x + + +class PersonReIDPost(EmbeddingPost): + """Appearance embeddings tied to an explicitly configured ReID dataset.""" + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__(pre_cfg, post_cfg) + dataset = post_cfg.get("dataset") + if not isinstance(dataset, str) or not dataset: + raise ValueError("Person ReID postprocessing requires post_cfg.dataset") + self.dataset = get_dataset_config_for_task("person_reid", dataset)["name"] diff --git a/mblt_vision/utils/preprocess/normalize.py b/mblt_vision/utils/preprocess/normalize.py index be7dd1b..ad19c3d 100644 --- a/mblt_vision/utils/preprocess/normalize.py +++ b/mblt_vision/utils/preprocess/normalize.py @@ -30,14 +30,40 @@ class Normalize(PreOps): std: Array of standard deviation values for normalization. """ - def __init__(self, style: str) -> None: + def __init__( + self, + style: str | None = None, + *, + mean: list[float] | None = None, + std: list[float] | None = None, + ) -> None: """Initializes the Normalize layer with a specific style. Args: - style: The preprocessing style to use. Must be one of STYLE_LIST. + style: Historical named style; mutually exclusive with mean/std. + mean: Optional explicit three-element RGB mean, using float32 arithmetic. + std: Optional explicit positive RGB standard deviations. """ super().__init__() + self.explicit_stats = mean is not None or std is not None + if self.explicit_stats: + if style is not None or mean is None or std is None: + raise ValueError("Provide either style or both mean and std") + self.mean = np.asarray(mean, dtype=np.float32) + self.std = np.asarray(std, dtype=np.float32) + if ( + self.mean.shape != (3,) + or self.std.shape != (3,) + or not np.isfinite(self.mean).all() + or not np.isfinite(self.std).all() + or not (self.std > 0).all() + ): + raise ValueError( + "mean/std require three finite RGB values and positive std" + ) + self.style = "explicit" + return if not isinstance(style, str): raise TypeError( f"Normalize style must be a string, got {type(style).__name__}." diff --git a/tests/reference/ops_reid_metrics.py b/tests/reference/ops_reid_metrics.py new file mode 100644 index 0000000..78f10fa --- /dev/null +++ b/tests/reference/ops_reid_metrics.py @@ -0,0 +1,53 @@ +"""Verbatim functions from datasets/_shared/reid.py at +ec537764e36ca10da963fcb7afab157ea60ca022 (mblt-model-ops). +Snapshot permits ordinary offline differential tests without sibling repos. +""" +import numpy as np +MAX_RANK = 50 + +def euclidean_distance(query: np.ndarray, gallery: np.ndarray) -> np.ndarray: + """Squared euclidean distance, the reference ``euclidean_distance`` without the sqrt. + + Ranking is monotone in the square, so CMC and mAP are unchanged. + """ + query = np.asarray(query, dtype=np.float32) + gallery = np.asarray(gallery, dtype=np.float32) + q2 = np.sum(query**2, axis=1, keepdims=True) + g2 = np.sum(gallery**2, axis=1, keepdims=True).T + return q2 + g2 - 2.0 * (query @ gallery.T) + +def evaluate( + distmat: np.ndarray, + q_pids: np.ndarray, + g_pids: np.ndarray, + q_camids: np.ndarray, + g_camids: np.ndarray, + max_rank: int = MAX_RANK, +) -> tuple[np.ndarray, float]: + """Market-1501 protocol CMC curve and mAP (reference ``eval_func``).""" + num_q, num_g = distmat.shape + max_rank = min(max_rank, num_g) + indices = np.argsort(distmat, axis=1, kind="stable") + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + all_cmc = [] + all_ap = [] + for q_idx in range(num_q): + order = indices[q_idx] + remove = (g_pids[order] == q_pids[q_idx]) & (g_camids[order] == q_camids[q_idx]) + orig_cmc = matches[q_idx][~remove] + if not np.any(orig_cmc): + continue # the query identity never appears in the gallery + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + # Same-identity/same-camera removal leaves different gallery lengths. + # Saturate missing ranks with the final CMC value for this query. + all_cmc.append( + np.pad(cmc[:max_rank], (0, max_rank - min(len(cmc), max_rank)), mode="edge") + ) + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() / np.arange(1, orig_cmc.shape[0] + 1) + all_ap.append(float((tmp_cmc * orig_cmc).sum() / num_rel)) + if not all_cmc: + raise ValueError("No query identity appears in the gallery") + cmc_curve = np.asarray(all_cmc, dtype=np.float64).sum(0) / len(all_cmc) + return cmc_curve, float(np.mean(all_ap)) diff --git a/tests/reference/ops_reid_preprocess.py b/tests/reference/ops_reid_preprocess.py new file mode 100644 index 0000000..b2b4ce0 --- /dev/null +++ b/tests/reference/ops_reid_preprocess.py @@ -0,0 +1,92 @@ +# Verbatim source: models/person_reid/_shared/preprocess.py +# mblt-model-ops ec537764e36ca10da963fcb7afab157ea60ca022 +"""Default person_reid preprocess: the CLIP-ReID / TransReID test transform. + +Upstream evaluates with ``Resize((H, W)) -> ToTensor -> Normalize(mean, std)``; there is no +crop, and the two backbone families differ only in the normalization statistics, so the +statistics are explicit ``mean``/``std`` lists in ``pre_cfg`` rather than a named style. + +Operations are applied in a fixed canonical order, not in ``config`` iteration order: +``predict.input.json`` is written with sorted keys, so relying on dict order would run +``Normalize`` before ``Resize``. + +A camera-aware Model (SIE) declares ``CameraOneHot``. The camera index is the ``_c`` +token of the crop's file name, which is how every ReID dataset and the upstream loader +encode it; it is a Model input, not ground truth. The network then takes two inputs, and +the returned tensor is a tuple ``(image, camera_one_hot)`` in that order. + +The return value is ``(source, network_input, camera_index)`` so that the shared +``inference.tensor_of`` finds the network input in the second slot. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +PIPELINE_ORDER = ("Reader", "Resize", "SetOrder", "Normalize", "CameraOneHot") +REQUIRED = ("Resize", "Normalize") +_RESAMPLING = { + "bilinear": Image.Resampling.BILINEAR, + "bicubic": Image.Resampling.BICUBIC, +} +DEFAULT_CAMERA_PATTERN = r"_c(\d+)" + +NetworkInput = np.ndarray | tuple[np.ndarray, np.ndarray] + + +def camera_index(source: Path, config: dict[str, Any]) -> int: + """The 0-based camera index of a crop, from its ``_c`` file-name token.""" + pattern = str(config.get("pattern", DEFAULT_CAMERA_PATTERN)) + match = re.search(pattern, Path(source).name) + if match is None: + raise ValueError(f"{source}: file name carries no camera token {pattern!r}") + cameras = int(config["num_cameras"]) + camera = int(match.group(1)) - 1 + if not 0 <= camera < cameras: + raise ValueError(f"{source}: camera {camera + 1} is outside 1..{cameras}") + return camera + + +def preprocess( + source: Path, config: dict[str, Any] +) -> tuple[Path, NetworkInput, int | None]: + """Return ``(source, network input, camera index or None)``.""" + unknown = set(config) - set(PIPELINE_ORDER) + if unknown: + raise ValueError(f"unsupported preprocess operations: {sorted(unknown)}") + missing = [name for name in REQUIRED if name not in config] + if missing: + raise ValueError(f"pre_cfg is missing required operations: {missing}") + source = Path(source) + value: Any = Image.open(source).convert("RGB") + camera: int | None = None + for operation in PIPELINE_ORDER: + if operation not in config: + continue + options = config[operation] + if operation == "Reader": + continue + if operation == "Resize": + height, width = (int(v) for v in options["size"]) + mode = _RESAMPLING[str(options.get("interpolation", "bilinear"))] + value = value.resize((width, height), mode) + elif operation == "SetOrder": + if options.get("shape", "HWC") != "HWC": + raise ValueError("person_reid preprocess produces HWC tensors only") + elif operation == "Normalize": + mean = np.asarray(options["mean"], dtype=np.float32) + std = np.asarray(options["std"], dtype=np.float32) + value = (np.asarray(value, dtype=np.float32) / 255.0 - mean) / std + elif operation == "CameraOneHot": + camera = camera_index(source, options) + tensor = np.ascontiguousarray(value, dtype=np.float32) + if camera is None: + return source, tensor, None + one_hot = np.zeros(int(config["CameraOneHot"]["num_cameras"]), dtype=np.float32) + one_hot[camera] = 1.0 + return source, (tensor, one_hot), camera diff --git a/tests/test_api.py b/tests/test_api.py index 40b4182..ce184aa 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -21,6 +21,7 @@ def test_public_discovery_exposes_all_supported_tasks() -> None: "obb", "pose_estimation", "face_detection", + "person_reid", ] assert list_models("obb")["obb"] diff --git a/tests/test_reid.py b/tests/test_reid.py new file mode 100644 index 0000000..a6c0fcc --- /dev/null +++ b/tests/test_reid.py @@ -0,0 +1,216 @@ +"""CPU-only embedding, prepared loader and retrieval protocol contracts.""" + +from pathlib import Path + +import numpy as np +import pytest +import torch +from mblt_vision._config import load_model_variant +from mblt_vision.datasets import get_dataset_config +from mblt_vision.utils.datasets.reid import ( + DukeMTMCReIDDataset, + Market1501Dataset, + OccludedDukeDataset, + ReIDSample, +) +from mblt_vision.utils.evaluation.retrieval import ( + DukeMTMCReIDEvaluator, + Market1501Evaluator, + OccludedDukeEvaluator, + RetrievalEvaluator, + extract_embeddings, +) +from mblt_vision.utils.postprocess import build_postprocess +from mblt_vision.utils.postprocess.embedding_post import EmbeddingPost, PersonReIDPost +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.utils.preprocess.normalize import Normalize + + +@pytest.mark.parametrize("shape", [(2, 3), (2, 3, 1, 1), (2, 1, 1, 3)]) +def test_embedding_shapes(shape): + raw = np.array([[3.0, 4.0, 0.0], [0.0, 0.0, 0.0]], dtype=np.float32).reshape(shape) + post = build_postprocess( + {}, + { + "task": "person_reid", + "dataset": "market1501", + "normalize": True, + "embedding_dim": 3, + }, + ) + assert isinstance(post, (EmbeddingPost, PersonReIDPost)) + np.testing.assert_array_equal( + post([raw]), np.array([[0.6, 0.8, 0.0], [0.0, 0.0, 0.0]], dtype=np.float32) + ) + np.testing.assert_array_equal(raw.reshape(2, 3)[0], [3.0, 4.0, 0.0]) + + +@pytest.mark.parametrize( + "raw", + [ + np.empty((0, 3)), + np.ones(3), + np.ones((2, 3, 2, 2)), + np.ones((2, 3, 1)), + np.array([[np.nan]]), + np.array([[np.inf]]), + ], +) +def test_embedding_rejects_invalid(raw): + with pytest.raises(ValueError): + EmbeddingPost({}, {})(raw) + + +def test_embedding_validation_and_optional_normalization(): + post = EmbeddingPost({}, {}) + raw = torch.tensor([[3.0, 4.0]]) + assert torch.equal(post(raw), raw) + with pytest.raises(TypeError): + post(np.ones((1, 2), dtype=np.int32)) + with pytest.raises(ValueError): + post([raw, raw]) + with pytest.raises(ValueError): + EmbeddingPost({}, {"embedding_dim": 3})(raw) + with pytest.raises(ValueError): + PersonReIDPost({}, {"dataset": "coco"}) + with pytest.raises(ValueError): + PersonReIDPost({}, {}) + with pytest.raises(TypeError): + EmbeddingPost({}, {"normalize": "yes"}) + + +@pytest.mark.parametrize( + "dataset,loader", + [ + ("market1501", Market1501Dataset), + ("dukemtmc-reid", DukeMTMCReIDDataset), + ("occluded-duke", OccludedDukeDataset), + ], +) +def test_loader_metadata_and_source(tmp_path, dataset, loader): + for folder in ("query", "bounding_box_test"): + (tmp_path / folder).mkdir() + for filename in ("0002_c2_b.jpg", "0001_c1s1_a.jpg"): + (tmp_path / "query" / filename).touch() + (tmp_path / "bounding_box_test" / "0001_c2_a.jpg").touch() + if dataset == "market1501": + (tmp_path / "query" / "-1_c1_junk.jpg").touch() + (tmp_path / "bounding_box_test" / "0000_c1_background.jpg").touch() + data = loader(tmp_path, validate_counts=False) + assert [(s.pid, s.camid, s.split) for s in data][:2] == [ + (1, 0, "query"), + (2, 1, "query"), + ] + assert data[-1].split == "gallery" + assert all(s.pid != -1 for s in data) + if dataset == "market1501": + assert data[2].pid == 0 + with pytest.raises(ValueError, match="expected"): + loader(tmp_path) + cfg = get_dataset_config(dataset) + assert cfg["source"]["revision"] == "ec537764e36ca10da963fcb7afab157ea60ca022" + assert cfg["eval_protocol"]["id"] == cfg["evaluation"]["protocol"] + + +@pytest.mark.parametrize("filename", ["bad.jpg", "1_c9_bad.jpg", "-1_c1_junk.jpg"]) +def test_duke_invalid_names(tmp_path, filename): + (tmp_path / "query").mkdir() + (tmp_path / "bounding_box_test").mkdir() + (tmp_path / "query" / filename).touch() + with pytest.raises(ValueError): + DukeMTMCReIDDataset(tmp_path, validate_counts=False) + + +def test_missing_and_empty_splits(tmp_path): + with pytest.raises(FileNotFoundError): + Market1501Dataset(tmp_path) + (tmp_path / "query").mkdir() + with pytest.raises(ValueError, match="no images"): + Market1501Dataset(tmp_path) + + +def test_handbuilt_exclusions_ties_and_missing_identity(): + # same-camera positive is removed; tied background stays before the positive. + distances = np.array([[0.0, 1.0, 1.0, 2.0], [4.0, 3.0, 2.0, 1.0]]) + cmc, mean_ap = RetrievalEvaluator.evaluate_distances( + distances, + np.array([1, 99]), + np.array([1, 0, 1, 1]), + np.array([0, 0]), + np.array([0, 1, 1, 2]), + ) + np.testing.assert_array_equal(cmc, [0.0, 1.0, 1.0, 1.0]) + assert mean_ap == (1 / 2 + 2 / 3) / 2 + # Stable ties are manifest order, not identity order. + cmc, mean_ap = RetrievalEvaluator.evaluate_distances( + np.zeros((1, 2)), + np.array([1]), + np.array([1, 0]), + np.array([0]), + np.array([1, 1]), + ) + assert mean_ap == 1.0 + np.testing.assert_array_equal(cmc, [1.0, 1.0]) + + +@pytest.mark.parametrize( + "evaluator", [Market1501Evaluator, DukeMTMCReIDEvaluator, OccludedDukeEvaluator] +) +def test_score_and_rank_saturation(evaluator): + samples = [ + ReIDSample(Path("q"), 1, 0, "query"), + ReIDSample(Path("g"), 1, 1, "gallery"), + ] + result = evaluator().evaluate(np.array([[1.0, 0.0], [1.0, 0.0]]), samples) + assert result.as_dict() == { + "primary": {"map": 1.0}, + "secondary": {"rank1": 1.0, "rank5": 1.0, "rank10": 1.0}, + "sample_count": 2, + } + assert result.primary_score == result.secondary_score == result.rank(50) == 1.0 + with pytest.raises(ValueError): + result.rank(0) + + +def test_no_valid_queries(): + with pytest.raises(ValueError, match="No query"): + RetrievalEvaluator.evaluate_distances( + np.zeros((1, 1)), np.array([1]), np.array([1]), np.array([0]), np.array([0]) + ) + + +def test_extraction_order(): + batches = [np.full((2, 3), i, dtype=np.float32) for i in range(5)] + actual = extract_embeddings(batches, lambda x: [x], EmbeddingPost({}, {})) + np.testing.assert_array_equal(actual, np.concatenate(batches)) + with pytest.raises(ValueError): + extract_embeddings([], lambda x: x, lambda x: x) + + +@pytest.mark.parametrize( + "options", + [ + {"mean": [0, 0, 0]}, + {"mean": [0, 0, 0], "std": [1, 0, 1]}, + {"mean": [0, 0], "std": [1, 1]}, + {"style": "torch", "mean": [0, 0, 0], "std": [1, 1, 1]}, + ], +) +def test_explicit_normalization_validation(options): + with pytest.raises(ValueError): + Normalize(**options) + + +def test_family_base_inheritance(tmp_path): + base = Path(__file__).parents[1] / "mblt_vision/models/bases/person_reid.yaml" + for variant, name in [ + ("DEFAULT", "dukemtmc-reid"), + ("MARKET1501", "market1501"), + ("OCCLUDED_DUKE", "occluded-duke"), + ]: + cfg = load_model_variant(base, variant) + assert cfg["post_cfg"]["dataset"] == name + assert isinstance( + build_postprocess(cfg["pre_cfg"], cfg["post_cfg"]), PersonReIDPost + ) + assert build_preprocess(cfg["pre_cfg"]) diff --git a/tests/test_reid_parity.py b/tests/test_reid_parity.py new file mode 100644 index 0000000..84bdfa6 --- /dev/null +++ b/tests/test_reid_parity.py @@ -0,0 +1,69 @@ +"""Offline numeric differential tests against pinned, verbatim ops functions.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from mblt_vision._config import load_model_variant +from mblt_vision.utils.evaluation.retrieval import ( + RetrievalEvaluator, + euclidean_distance, +) +from mblt_vision.utils.preprocess import build_preprocess +from PIL import Image + + +def reference(name): + path = Path(__file__).parent / "reference" / f"ops_reid_{name}.py" + spec = importlib.util.spec_from_file_location(f"ops_reid_{name}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("seed", range(10)) +def test_ops_evaluator_exact_numeric_parity(seed): + ops = reference("metrics") + rng = np.random.default_rng(seed) + # Quantized integer vectors produce numerous exact distance ties. + query = rng.integers(-2, 3, size=(17, 8)).astype(np.float32) + gallery = rng.integers(-2, 3, size=(61, 8)).astype(np.float32) + q_ids = rng.integers(1, 6, 17) + g_ids = rng.integers(0, 6, 61) # zero-valued distractors remain ordinary rows + q_cam = rng.integers(0, 3, 17) + g_cam = rng.integers(0, 3, 61) + expected_distances = ops.euclidean_distance(query, gallery) + actual_distances = euclidean_distance(query, gallery) + np.testing.assert_array_equal(actual_distances, expected_distances) + for max_rank in (1, 5, 50, 100): + expected_cmc, expected_map = ops.evaluate( + expected_distances, q_ids, g_ids, q_cam, g_cam, max_rank + ) + actual_cmc, actual_map = RetrievalEvaluator.evaluate_distances( + actual_distances, q_ids, g_ids, q_cam, g_cam, max_rank + ) + np.testing.assert_array_equal(actual_cmc, expected_cmc) + assert actual_map == expected_map + + +@pytest.mark.parametrize("shape", [(19, 11), (128, 256), (383, 191)]) +@pytest.mark.parametrize("style", ["torch", "tf"]) +def test_ops_preprocessing_parity(tmp_path, shape, style): + ops = reference("preprocess") + cfg = load_model_variant( + Path(__file__).parents[1] / "mblt_vision/models/bases/person_reid.yaml", + "DEFAULT", + ) + config = cfg["pre_cfg"] + if style == "tf": + # CLIP_ReID_ViT_B_16_Baseline_DukeMTMC/distribution.yaml at ec537764. + config["Normalize"] = {"mean": [0.5, 0.5, 0.5], "std": [0.5, 0.5, 0.5]} + rng = np.random.default_rng(0) + path = tmp_path / "0001_c2_synthetic.png" + Image.fromarray(rng.integers(0, 256, size=(*shape, 3), dtype=np.uint8)).save(path) + expected = ops.preprocess(path, config)[1] + actual = build_preprocess(config)(path) + # Both use PIL bilinear + explicit float32 stats; no tolerance needed. + np.testing.assert_array_equal(actual, expected) + assert np.max(np.abs(actual - expected)) == 0.0 From 81a3727893009bbb65a667e6d10ba5e6f30bd148 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:46:47 +0900 Subject: [PATCH 3/7] Connect retrieval embeddings to engine results and dataset readiness Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/README.md | 7 ++- mblt_vision/person_reid/__init__.py | 3 +- mblt_vision/utils/datasets/readiness.py | 26 +++++++++++ mblt_vision/utils/results.py | 27 ++++++++++- tests/test_reid.py | 60 +++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 3 deletions(-) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 6249bc9..25bccab 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -514,7 +514,12 @@ by default; caller-owned subsets must explicitly set `validate_counts=False`. Only Market-1501 drops pid -1 junk; pid 0 background distractors remain. Duke loaders reject pid -1. Root defaults use `~/.mblt_model_zoo/datasets//eval`. -`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`. It validates one +`PersonReIDPost(pre_cfg, post_cfg)` extends `EmbeddingPost`; both are exported +from `mblt_vision.person_reid`. Engine postprocessing returns a `Results` object +whose `embeddings` and `output` retain the validated `(N,D)` matrix. +`dataset_ready(root, "person_reid", dataset)` checks source counts/metadata and +decodes every retained query/gallery image (default dataset: Market-1501). +`PersonReIDPost` It validates one floating-point `(N,D)` output or singleton-spatial NCHW/NHWC tensor, optionally checks `embedding_dim`, and uses `normalize: true` for row-wise L2 normalization (zero vectors remain zero). Return type is a float32 torch matrix, never class diff --git a/mblt_vision/person_reid/__init__.py b/mblt_vision/person_reid/__init__.py index 07fecad..e5e6574 100644 --- a/mblt_vision/person_reid/__init__.py +++ b/mblt_vision/person_reid/__init__.py @@ -15,11 +15,12 @@ RetrievalResult, extract_embeddings, ) -from ..utils.postprocess.embedding_post import PersonReIDPost +from ..utils.postprocess.embedding_post import EmbeddingPost, PersonReIDPost __all__ = [ "DukeMTMCReIDDataset", "DukeMTMCReIDEvaluator", + "EmbeddingPost", "Market1501Dataset", "Market1501Evaluator", "OccludedDukeDataset", diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index 7fd18b4..8426389 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -20,6 +20,7 @@ from ..._tasks import normalize_vision_task from ...datasets import get_dataset_category_ids from .cityscapes import CITYSCAPES_SOURCE_TO_TRAIN_ID +from .reid import DukeMTMCReIDDataset, Market1501Dataset, OccludedDukeDataset IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} IMAGENET_CLASS_COUNT = 1000 @@ -1044,6 +1045,29 @@ def dense_dataset_ready(data_path: str | Path, dataset: str) -> bool: return False +def _reid_ready(root: Path, dataset: str) -> bool: + """Require source counts, camera/name metadata and decodable query/gallery crops.""" + loader = { + "market1501": Market1501Dataset, + "dukemtmc-reid": DukeMTMCReIDDataset, + "occluded-duke": OccludedDukeDataset, + }.get(dataset) + if loader is None: + return False + try: + samples = loader(root).samples + for sample in samples: + if not sample.image.is_file() or sample.image.is_symlink(): + return False + with Image.open(sample.image) as image: + image.load() + if min(image.size) <= 0: + return False + except (OSError, ValueError): + return False + return True + + def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) -> bool: """Return whether an organized dataset matches its task, taxonomy, and full validation split. @@ -1069,6 +1093,8 @@ def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) }.get(normalized_task) normalized_dataset = (dataset or expected_dataset or "").lower() + if normalized_task == "person_reid": + return _reid_ready(root, normalized_dataset or "market1501") if normalized_task == "semantic_segmentation": return dense_dataset_ready(root, normalized_dataset or "ade20k") if expected_dataset is None or normalized_dataset != expected_dataset: diff --git a/mblt_vision/utils/results.py b/mblt_vision/utils/results.py index 8bbed17..7a5b7f6 100644 --- a/mblt_vision/utils/results.py +++ b/mblt_vision/utils/results.py @@ -73,6 +73,7 @@ def __init__( self.task = normalize_vision_task(post_cfg["task"]) self.conf_thres = kwargs.get("conf_thres", 0.25) self.acc: torch.Tensor | np.ndarray | None = None + self.embeddings: torch.Tensor | np.ndarray | None = None self.box_cls: torch.Tensor | np.ndarray | None = None self.mask: torch.Tensor | np.ndarray | None = None self.depth: torch.Tensor | np.ndarray | list[TensorLike] | None = None @@ -142,11 +143,35 @@ def set_output( NotImplementedError: If the task is not supported. """ self.acc = None + self.embeddings = None self.box_cls = None self.mask = None self.depth = None self.semantic_mask = None - if self.task == "image_classification": + if self.task == "person_reid": + if not isinstance(output, (np.ndarray, torch.Tensor)): + raise TypeError("Person ReID results require an embedding matrix") + finite = ( + torch.isfinite(output).all() + if isinstance(output, torch.Tensor) + else np.isfinite(output).all() + ) + floating = ( + output.is_floating_point() + if isinstance(output, torch.Tensor) + else np.issubdtype(output.dtype, np.floating) + ) + if ( + output.ndim != 2 + or min(output.shape) <= 0 + or not floating + or not bool(finite) + ): + raise ValueError( + "Person ReID results require a finite floating-point (N,D) matrix" + ) + self.embeddings = output + elif self.task == "image_classification": if not isinstance(output, (np.ndarray, torch.Tensor)): raise TypeError( f"Expected tensor output for task {self.task}, got {type(output).__name__}." diff --git a/tests/test_reid.py b/tests/test_reid.py index a6c0fcc..bfbab10 100644 --- a/tests/test_reid.py +++ b/tests/test_reid.py @@ -214,3 +214,63 @@ def test_family_base_inheritance(tmp_path): build_postprocess(cfg["pre_cfg"], cfg["post_cfg"]), PersonReIDPost ) assert build_preprocess(cfg["pre_cfg"]) + + +def test_public_base_and_engine_result_path(): + from mblt_vision.person_reid import EmbeddingPost as PublicEmbeddingPost + from mblt_vision.utils.results import Results + from mblt_vision.wrapper import MBLT_Engine + + assert PublicEmbeddingPost is EmbeddingPost + engine = MBLT_Engine.__new__(MBLT_Engine) + engine.pre_cfg = {} + engine.post_cfg = { + "task": "person_reid", + "dataset": "market1501", + "normalize": True, + } + engine.postprocessor = build_postprocess(engine.pre_cfg, engine.post_cfg) + result = engine.postprocess([np.array([[3.0, 4.0]], dtype=np.float32)]) + assert isinstance(result, Results) + torch.testing.assert_close(result.embeddings, torch.tensor([[0.6, 0.8]])) + assert result.output is result.embeddings + assert result.acc is None and result.box_cls is None + replacement = torch.tensor([[1.0, 0.0]]) + result.set_output(replacement) + assert result.embeddings is replacement + for invalid in ( + torch.ones(2), + torch.ones((1, 2), dtype=torch.int64), + torch.tensor([[float("nan")]]), + ): + with pytest.raises(ValueError): + result.set_output(invalid) + with pytest.raises(TypeError): + result.set_output([replacement]) + + +@pytest.mark.parametrize("name", ["market1501", "dukemtmc-reid", "occluded-duke"]) +def test_reid_readiness(tmp_path, monkeypatch, name): + from mblt_vision.utils.datasets import readiness, reid + from PIL import Image + + original = reid.get_dataset_config_for_task + + def tiny_config(task, dataset): + cfg = original(task, dataset) + for split in cfg["retrieval_splits"].values(): + split["count"] = 1 + return cfg + + monkeypatch.setattr(reid, "get_dataset_config_for_task", tiny_config) + assert not readiness.dataset_ready(tmp_path, "person_reid", name) + for folder in ("query", "bounding_box_test"): + (tmp_path / folder).mkdir() + Image.new("RGB", (4, 8)).save(tmp_path / folder / "0001_c1_sample.jpg") + assert readiness.dataset_ready(tmp_path, "person_reid", name) + if name == "market1501": + assert readiness.dataset_ready(tmp_path, "person_reid") + assert not readiness.dataset_ready(tmp_path, "person_reid", "coco") + image = tmp_path / "query" / "0001_c1_sample.jpg" + image.write_bytes(b"not an image") + assert not readiness.dataset_ready(tmp_path, "person_reid", name) From 636c39058b3e70b9955de6e6f7f980b217f806b6 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 16:56:38 +0900 Subject: [PATCH 4/7] Add aligned face pair datasets and source-exact fold verification Co-Authored-By: Claude Haiku 5.5 --- .claude/skills/mblt-vision/SKILL.md | 4 + AGENTS.md | 4 + benchmark/benchmark_vision_models.py | 2 +- mblt_vision/README.md | 43 ++++ mblt_vision/__init__.py | 2 + mblt_vision/_tasks.py | 1 + mblt_vision/datasets/agedb-30.yaml | 34 +++ mblt_vision/datasets/cfp-fp.yaml | 36 +++ mblt_vision/datasets/lfw.yaml | 34 +++ mblt_vision/face_recognition/__init__.py | 40 ++++ .../models/bases/face_recognition.yaml | 18 ++ mblt_vision/utils/datasets/face_pairs.py | 131 +++++++++++ mblt_vision/utils/datasets/readiness.py | 22 ++ mblt_vision/utils/evaluation/verification.py | 160 +++++++++++++ mblt_vision/utils/postprocess/__init__.py | 11 +- mblt_vision/utils/postprocess/build_post.py | 4 +- .../utils/postprocess/embedding_post.py | 11 + mblt_vision/utils/results.py | 2 +- tests/reference/ops_face_preprocess.py | 57 +++++ tests/reference/ops_face_verification.py | 75 ++++++ tests/test_api.py | 1 + tests/test_face_verification.py | 218 ++++++++++++++++++ 22 files changed, 905 insertions(+), 5 deletions(-) create mode 100644 mblt_vision/datasets/agedb-30.yaml create mode 100644 mblt_vision/datasets/cfp-fp.yaml create mode 100644 mblt_vision/datasets/lfw.yaml create mode 100644 mblt_vision/face_recognition/__init__.py create mode 100644 mblt_vision/models/bases/face_recognition.yaml create mode 100644 mblt_vision/utils/datasets/face_pairs.py create mode 100644 mblt_vision/utils/evaluation/verification.py create mode 100644 tests/reference/ops_face_preprocess.py create mode 100644 tests/reference/ops_face_verification.py create mode 100644 tests/test_face_verification.py diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index 7d563ab..1d40ab8 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -114,6 +114,10 @@ description: >- and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. +- Face verification uses prepared 112x112 crops without realignment or implicit flip fusion. + Keep pair/fold order, source threshold search, Results embeddings and readiness connected; + require exact metric and preprocessing parity (face verification API section). + ## Runtime and Packaging - Route NPU runtime access through mblt-npu-python; do not copy backend classes into Vision. diff --git a/AGENTS.md b/AGENTS.md index 5c3c614..7e45efd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -186,6 +186,10 @@ The current ownership boundary is deliberate: stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. Named Normalize styles retain legacy arithmetic; explicit mean/std uses source float32 arithmetic. +- Face verification utilities preserve prepared alignment and official pair/fold order. + Use dataset-bound `FaceRecognitionPost`, `Results.embeddings`, and face readiness checks. + Protocol details and source-parity requirements are in the face verification API section. + ## Python-First Architecture - Keep the public layer independent from a particular backend. Define small internal interfaces diff --git a/benchmark/benchmark_vision_models.py b/benchmark/benchmark_vision_models.py index 06f86e3..62fb7ec 100644 --- a/benchmark/benchmark_vision_models.py +++ b/benchmark/benchmark_vision_models.py @@ -36,7 +36,7 @@ tuple[CoreMode, ...], core_modes_for_target_device("aries-rb") ) # Utility-only embedding tasks have no benchmark engine/CLI route yet. -TASK_CHOICES = tuple(task for task in VISION_TASKS if task != "person_reid") +TASK_CHOICES = tuple(task for task in VISION_TASKS if task not in {"person_reid", "face_recognition"}) SUPPORTED_TARGET_DEVICES = frozenset({"aries-rb", "regulus-ra", "regulus-rb"}) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 25bccab..9faee01 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -543,3 +543,46 @@ Offline tests vendor the inspected ops functions at distance/CMC/mAP equality; synthetic RGB preprocessing requires byte equality (`atol=0`) against the ops adapter for explicit source mean/std profiles. These are protocol tests, not measured model accuracy claims. + +## Face verification utilities + +`mblt_vision.face_recognition` exports `FaceRecognitionPost(EmbeddingPost)`, +`LFWDataset`, `AgeDB30Dataset`, `CFPFPDataset`, and their `LFWEvaluator`, +`AgeDB30Evaluator`, `CFPEvaluator` subclasses of `VerificationEvaluator`. +The processing-only `bases/face_recognition.yaml` profile is 112x112 PIL bilinear, +explicit float32 0.5 mean/std, 512 dimensions, row-wise L2 normalization. Engine +postprocessing stores dense vectors in `Results.embeddings` and `output`. Model +wrappers/artifact YAMLs are a separate stage; CLI/benchmark routing is unchanged. + +Pair loaders read prepared `pairs.csv` columns `pair_id,fold_id,is_same,image_a,image_b`, +retain pair order and official one-based fold labels, and enumerate unique image +basenames in sorted order. LFW uses `aligned/`; AgeDB-30 and CFP-FP use `images/`. +They require existing 112x112 crops and never detect or realign faces. Source pair +counts are checked by default; caller-owned subsets explicitly opt out with +`validate_counts=False`. Default cache roots end in `/eval`. +`dataset_ready(root, "face_recognition", dataset)` additionally requires all ten +folds and decodable crops; its default dataset is LFW. + +```python +from mblt_vision.face_recognition import LFWDataset, LFWEvaluator + +data = LFWDataset("/path/to/lfw/eval") +# embeddings: one row per data.samples, from your inference callback. +# result = LFWEvaluator().evaluate(embeddings, data.samples, data.pairs) +# print(result.as_dict()) +``` + +Ops at `ec537764e36ca10da963fcb7afab157ea60ca022` owns the protocol: one +normalized embedding per crop, **no flip fusion**, float64 cosine scores, threshold +grid `linspace(-1,1,4001)`, inclusive `score >= threshold`, strictest threshold on +training ties, leave-one-fold-out threshold fitting, mean accuracy and population +standard deviation over folds. `extract_embeddings` shares the ordered bounded +inference helper with retrieval. There is no implicit fusion in scoring. + +Offline tests compare exact cosine scores, fold accuracies, thresholds and aggregate +metrics against a verbatim source snapshot over five synthetic seeds. Three RGB +fixtures (including no-op resize) require preprocessing byte equality (`atol=0`). +Dataset YAMLs record exact source paths/revision. CFP-FP specifically refers to +the prepared cat-claws mirror: ops warns it was not checked against the original +`cfp_fp.bin` and records a training-log discrepancy. This package preserves that +provenance and makes no new accuracy claim. diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index 9cf69d2..1f40160 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -9,6 +9,7 @@ from . import depth_estimation as depth_estimation from . import face_detection as face_detection +from . import face_recognition as face_recognition from . import image_classification as image_classification from . import instance_segmentation as instance_segmentation from . import obb as obb @@ -54,6 +55,7 @@ "obb", "pose_estimation", "person_reid", + "face_recognition", "semantic_segmentation", ] + sorted(_LEGACY_MODEL_EXPORTS) # Keep legacy compatibility exports synchronized with their task packages. diff --git a/mblt_vision/_tasks.py b/mblt_vision/_tasks.py index c8a4b6d..310e895 100644 --- a/mblt_vision/_tasks.py +++ b/mblt_vision/_tasks.py @@ -14,6 +14,7 @@ "pose_estimation", "face_detection", "person_reid", + "face_recognition", ) diff --git a/mblt_vision/datasets/agedb-30.yaml b/mblt_vision/datasets/agedb-30.yaml new file mode 100644 index 0000000..f17c8c8 --- /dev/null +++ b/mblt_vision/datasets/agedb-30.yaml @@ -0,0 +1,34 @@ +name: agedb-30 +path: ~/.mblt_model_zoo/datasets/agedb-30 +tasks: +- face_recognition +val: eval +image_directory: images +pairs: 6000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/agedb_30/agedb_30.yaml + name: agedb-30 + dataloader_path: datasets/agedb_30/_private/dataloader.py +eval_protocol: + id: agedb-30-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: agedb-30-ten-fold-verification.v1 + primary_metric: agedb_30_accuracy + secondary_metrics: + - agedb_30_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/agedb_30/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/cfp-fp.yaml b/mblt_vision/datasets/cfp-fp.yaml new file mode 100644 index 0000000..1fb614c --- /dev/null +++ b/mblt_vision/datasets/cfp-fp.yaml @@ -0,0 +1,36 @@ +name: cfp-fp +path: ~/.mblt_model_zoo/datasets/cfp-fp +tasks: +- face_recognition +val: eval +image_directory: images +pairs: 7000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/cfp_fp/cfp_fp.yaml + name: cfp-fp + dataloader_path: datasets/cfp_fp/_private/dataloader.py + caveat: Prepared cat-claws mirror is not checked against original cfp_fp.bin; ops + records lower CFP-FP accuracy than training logs. +eval_protocol: + id: cfp-fp-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: cfp-fp-ten-fold-verification.v1 + primary_metric: cfp_fp_accuracy + secondary_metrics: + - cfp_fp_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/cfp_fp/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/lfw.yaml b/mblt_vision/datasets/lfw.yaml new file mode 100644 index 0000000..7e83377 --- /dev/null +++ b/mblt_vision/datasets/lfw.yaml @@ -0,0 +1,34 @@ +name: lfw +path: ~/.mblt_model_zoo/datasets/lfw +tasks: +- face_recognition +val: eval +image_directory: aligned +pairs: 6000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/lfw/lfw.yaml + name: lfw + dataloader_path: datasets/lfw/_private/dataloader.py +eval_protocol: + id: lfw-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: lfw-ten-fold-verification.v1 + primary_metric: lfw_accuracy + secondary_metrics: + - lfw_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/lfw/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/face_recognition/__init__.py b/mblt_vision/face_recognition/__init__.py new file mode 100644 index 0000000..2d9df56 --- /dev/null +++ b/mblt_vision/face_recognition/__init__.py @@ -0,0 +1,40 @@ +"""Aligned face pair verification utilities; no implicit face detection/alignment.""" + +from ..utils.datasets.face_pairs import ( + AgeDB30Dataset, + CFPFPDataset, + FacePair, + FacePairDataset, + LFWDataset, + PairImage, +) +from ..utils.evaluation.verification import ( + AgeDB30Evaluator, + CFPEvaluator, + LFWEvaluator, + VerificationEvaluator, + VerificationResult, + cosine_scores, + extract_embeddings, + fold_accuracies, +) +from ..utils.postprocess.embedding_post import EmbeddingPost, FaceRecognitionPost + +__all__ = [ + "AgeDB30Dataset", + "AgeDB30Evaluator", + "CFPEvaluator", + "CFPFPDataset", + "EmbeddingPost", + "FacePair", + "FacePairDataset", + "FaceRecognitionPost", + "LFWDataset", + "LFWEvaluator", + "PairImage", + "VerificationEvaluator", + "VerificationResult", + "cosine_scores", + "extract_embeddings", + "fold_accuracies", +] diff --git a/mblt_vision/models/bases/face_recognition.yaml b/mblt_vision/models/bases/face_recognition.yaml new file mode 100644 index 0000000..6545407 --- /dev/null +++ b/mblt_vision/models/bases/face_recognition.yaml @@ -0,0 +1,18 @@ +# Processing only; concrete approved HF artifacts are added separately. +# ops ec537764e36ca10da963fcb7afab157ea60ca022 +# models/face_recognition/ArcFace_IResNet_100/distribution.yaml +DEFAULT: + pre_cfg: + Reader: + style: pil + Resize: + size: [112, 112] + interpolation: bilinear + Normalize: + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + post_cfg: + task: face_recognition + dataset: lfw + embedding_dim: 512 + normalize: true diff --git a/mblt_vision/utils/datasets/face_pairs.py b/mblt_vision/utils/datasets/face_pairs.py new file mode 100644 index 0000000..a72bd66 --- /dev/null +++ b/mblt_vision/utils/datasets/face_pairs.py @@ -0,0 +1,131 @@ +"""Prepared aligned face pairs, preserving source file order and fold labels.""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +from pathlib import Path + +from PIL import Image + +from ...datasets import get_dataset_config_for_task + + +@dataclass(frozen=True) +class FacePair: + """Source basenames, same/different label and official one-based fold.""" + + image_a: str + image_b: str + is_same: bool + fold: int + + +@dataclass(frozen=True) +class PairImage: + """One prepared 112x112 aligned crop, keyed by its source basename.""" + + image: Path + name: str + + +class FacePairDataset: + """Read prepared pairs.csv and already aligned crops without realignment. + + Root contains pairs.csv and the configured aligned/images directory. + Source pair counts are enforced unless validate_counts=False explicitly + selects a caller-owned subset. Basenames, labels and folds are always checked. + """ + + name: str + + def __init__( + self, root: str | Path | None = None, *, validate_counts: bool = True + ) -> None: + self.config = get_dataset_config_for_task("face_recognition", self.name) + self.root = ( + Path(root).expanduser() + if root is not None + else Path(self.config["path"]) / self.config["val"] + ) + path = self.root / "pairs.csv" + if not path.is_file(): + raise FileNotFoundError(f"Pair file is missing: {path}") + pairs = [] + ids = set() + with path.open(newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + columns = {"pair_id", "fold_id", "is_same", "image_a", "image_b"} + if reader.fieldnames is None or not columns <= set(reader.fieldnames): + raise ValueError(f"{path} lacks the pair columns {sorted(columns)}") + for row in reader: + for key in ("image_a", "image_b"): + name = row[key] + if ( + not name + or name in {".", ".."} + or any(c in name for c in "/\\:") + ): + raise ValueError(f"Invalid pair image basename: {name!r}") + if row["is_same"] not in {"0", "1"}: + raise ValueError("Pair is_same must be 0 or 1") + try: + fold, pair_id = int(row["fold_id"]), int(row["pair_id"]) + except (ValueError, TypeError) as exc: + raise ValueError( + "Pair fold_id and pair_id must be integers" + ) from exc + if not 1 <= fold <= 10 or pair_id in ids: + raise ValueError( + "Pair fold must be 1..10 and pair_id must be unique" + ) + ids.add(pair_id) + pairs.append( + FacePair( + row["image_a"], row["image_b"], row["is_same"] == "1", fold + ) + ) + if not pairs or (validate_counts and len(pairs) != self.config["pairs"]): + raise ValueError( + f"Pair file holds {len(pairs)} pairs, expected {self.config['pairs']}" + ) + self.pairs = tuple(pairs) + image_dir = self.root / self.config["image_directory"] + if not image_dir.is_dir(): + raise FileNotFoundError(f"Image directory is missing: {image_dir}") + names = sorted({p.image_a for p in pairs} | {p.image_b for p in pairs}) + images = [] + for name in names: + path = image_dir / name + if not path.is_file(): + raise FileNotFoundError(f"Pair image is missing: {path}") + # Alignment is a dataset preparation contract, never re-estimated here. + with Image.open(path) as image: + if image.size != (112, 112): + raise ValueError(f"Prepared aligned crop must be 112x112: {path}") + images.append(PairImage(path, name)) + self.samples = tuple(images) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> PairImage: + return self.samples[index] + + +class LFWDataset(FacePairDataset): + """LFW official 6000 prepared aligned pairs.""" + + name = "lfw" + + +class AgeDB30Dataset(FacePairDataset): + """Prepared AgeDB-30 mirror pair protocol, without changing its folds.""" + + name = "agedb-30" + + +class CFPFPDataset(FacePairDataset): + """Prepared CFP-FP mirror protocol; source caveats remain in the YAML.""" + + name = "cfp-fp" diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index 8426389..7bda3f7 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -20,6 +20,7 @@ from ..._tasks import normalize_vision_task from ...datasets import get_dataset_category_ids from .cityscapes import CITYSCAPES_SOURCE_TO_TRAIN_ID +from .face_pairs import LFWDataset, AgeDB30Dataset, CFPFPDataset from .reid import DukeMTMCReIDDataset, Market1501Dataset, OccludedDukeDataset IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} @@ -1068,6 +1069,25 @@ def _reid_ready(root: Path, dataset: str) -> bool: return True +def _face_ready(root: Path, dataset: str) -> bool: + """Validate prepared aligned pairs and image decoding, never realign crops.""" + loader = {"lfw": LFWDataset, "agedb-30": AgeDB30Dataset, "cfp-fp": CFPFPDataset}.get(dataset) + if loader is None: + return False + try: + data = loader(root) + if len({pair.fold for pair in data.pairs}) != 10: + return False + for sample in data.samples: + if sample.image.is_symlink(): + return False + with Image.open(sample.image) as image: + image.load() + except (OSError, ValueError): + return False + return True + + def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) -> bool: """Return whether an organized dataset matches its task, taxonomy, and full validation split. @@ -1093,6 +1113,8 @@ def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) }.get(normalized_task) normalized_dataset = (dataset or expected_dataset or "").lower() + if normalized_task == "face_recognition": + return _face_ready(root, normalized_dataset or "lfw") if normalized_task == "person_reid": return _reid_ready(root, normalized_dataset or "market1501") if normalized_task == "semantic_segmentation": diff --git a/mblt_vision/utils/evaluation/verification.py b/mblt_vision/utils/evaluation/verification.py new file mode 100644 index 0000000..8889c5f --- /dev/null +++ b/mblt_vision/utils/evaluation/verification.py @@ -0,0 +1,160 @@ +"""Fold-based face verification, pinned to ops ec537764 protocol arithmetic.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np + +from ..datasets.face_pairs import FacePair, PairImage +from .retrieval import extract_embeddings + +# Source grid includes both endpoints. Predictions use >= and ties choose highest t. +THRESHOLDS = np.linspace(-1.0, 1.0, 4001) + + +def cosine_scores(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Source float64 cosine scores; the norm-product floor is 1e-12.""" + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + if a.ndim != 2 or b.shape != a.shape or min(a.shape) == 0: + raise ValueError("Cosine inputs require matching nonempty (N,D) matrices") + if not np.isfinite(a).all() or not np.isfinite(b).all(): + raise ValueError("Cosine embeddings must be finite") + norms = np.linalg.norm(a, axis=1) * np.linalg.norm(b, axis=1) + return np.sum(a * b, axis=1) / np.maximum(norms, 1e-12) + + +def fold_accuracies( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Fit threshold on all other folds and score the held-out official fold.""" + scores = np.asarray(scores, dtype=np.float64) + labels = np.asarray(labels) + folds = np.asarray(folds) + if ( + scores.ndim != 1 + or scores.size == 0 + or labels.shape != scores.shape + or folds.shape != scores.shape + ): + raise ValueError("Scores, labels and folds must be matching nonempty vectors") + if not np.isfinite(scores).all() or not np.isin(labels, [0, 1]).all(): + raise ValueError("Verification requires finite scores and binary labels") + if ( + not np.issubdtype(folds.dtype, np.integer) + or not np.isin(folds, np.arange(1, 11)).all() + ): + raise ValueError("Verification fold IDs must be integers in 1..10") + labels = np.asarray(labels, dtype=bool) + fold_ids = np.unique(folds) + if len(fold_ids) < 2: + raise ValueError("Verification protocol needs at least two folds") + accuracies = [] + thresholds = [] + for fold in fold_ids: + test = folds == fold + train_scores, train_labels = scores[~test], labels[~test] + # Keep the source accumulation, threshold grid and strictest-tie policy. + fitted = np.array( + [float(np.mean((train_scores >= t) == train_labels)) for t in THRESHOLDS] + ) + best = float(THRESHOLDS[len(fitted) - 1 - int(np.argmax(fitted[::-1]))]) + thresholds.append(best) + accuracies.append(float(np.mean((scores[test] >= best) == labels[test]))) + return np.asarray(accuracies), np.asarray(thresholds) + + +@dataclass(frozen=True) +class VerificationResult: + """Unrounded fold results, with the source benchmark's rounded score view.""" + + accuracies: np.ndarray + thresholds: np.ndarray + sample_count: int + metric: str + + @property + def primary_score(self) -> float: + return float(np.mean(self.accuracies)) + + @property + def secondary_score(self) -> float: + return float(np.std(self.accuracies)) + + @property + def threshold(self) -> float: + return float(np.mean(self.thresholds)) + + def as_dict(self) -> dict: + return { + "primary": {self.metric: round(self.primary_score, 5)}, + "secondary": {self.metric + "_std": round(self.secondary_score, 5)}, + "sample_count": self.sample_count, + } + + +class VerificationEvaluator: + """One embedding per aligned image, no implicit flip/fusion or alignment. + + Source evaluate_embeddings decodes one L2-normalized vector per crop; + source score_pairs then computes cosine similarity, without flip fusion. + ``extract_embeddings`` is reused unchanged from retrieval evaluation. + """ + + metric = "accuracy" + extract_embeddings = staticmethod(extract_embeddings) + + def evaluate( + self, + embeddings: np.ndarray, + images: Sequence[PairImage], + pairs: Sequence[FacePair], + ) -> VerificationResult: + """Score embeddings in canonical image manifest order against source pairs.""" + values = np.asarray(embeddings) + if ( + values.ndim != 2 + or min(values.shape) == 0 + or values.shape[0] != len(images) + or not np.isfinite(values).all() + ): + raise ValueError("Embeddings require one finite nonempty row per image") + names = [image.name for image in images] + if len(set(names)) != len(names): + raise ValueError("Image manifest names must be unique") + if not pairs: + raise ValueError("No verification pairs supplied") + by_name = dict(zip(names, values)) + try: + a = np.stack([by_name[p.image_a] for p in pairs]) + b = np.stack([by_name[p.image_b] for p in pairs]) + except KeyError as exc: + raise ValueError( + f"Pair image missing from embedding manifest: {exc}" + ) from exc + accuracy, thresholds = fold_accuracies( + cosine_scores(a, b), + np.asarray([p.is_same for p in pairs]), + np.asarray([p.fold for p in pairs]), + ) + return VerificationResult(accuracy, thresholds, len(images), self.metric) + + +class LFWEvaluator(VerificationEvaluator): + """LFW official fold accuracy.""" + + metric = "lfw_accuracy" + + +class AgeDB30Evaluator(VerificationEvaluator): + """AgeDB-30 prepared mirror fold accuracy.""" + + metric = "agedb_30_accuracy" + + +class CFPEvaluator(VerificationEvaluator): + """CFP-FP prepared mirror fold accuracy (see source YAML caveat).""" + + metric = "cfp_fp_accuracy" diff --git a/mblt_vision/utils/postprocess/__init__.py b/mblt_vision/utils/postprocess/__init__.py index 879a06f..1455c4b 100644 --- a/mblt_vision/utils/postprocess/__init__.py +++ b/mblt_vision/utils/postprocess/__init__.py @@ -4,7 +4,14 @@ from .build_post import build_postprocess from .depth_post import DepthPost -from .embedding_post import EmbeddingPost, PersonReIDPost +from .embedding_post import EmbeddingPost, FaceRecognitionPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost -__all__ = ["DepthPost", "EmbeddingPost", "PersonReIDPost", "SemanticSegPost", "build_postprocess"] +__all__ = [ + "DepthPost", + "EmbeddingPost", + "FaceRecognitionPost", + "PersonReIDPost", + "SemanticSegPost", + "build_postprocess", +] diff --git a/mblt_vision/utils/postprocess/build_post.py b/mblt_vision/utils/postprocess/build_post.py index eb569ac..203818f 100644 --- a/mblt_vision/utils/postprocess/build_post.py +++ b/mblt_vision/utils/postprocess/build_post.py @@ -9,7 +9,7 @@ from .cls_post import ClsPost from .damoyolo_post import DAMOYOLODetectionPost from .depth_post import DepthPost -from .embedding_post import PersonReIDPost +from .embedding_post import FaceRecognitionPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost from .yolo_anchor_post import ( YOLOAnchorDetectionPost, @@ -71,6 +71,8 @@ def build_postprocess( f"Unsupported post_cfg.head {head!r}; expected one of {sorted(DETECTION_HEADS)}." ) return DETECTION_HEADS[head](pre_cfg, post_cfg, **kwargs) + if task == "face_recognition": + return FaceRecognitionPost(pre_cfg, post_cfg) if task == "person_reid": return PersonReIDPost(pre_cfg, post_cfg) if task == "image_classification": diff --git a/mblt_vision/utils/postprocess/embedding_post.py b/mblt_vision/utils/postprocess/embedding_post.py index 8d55353..4a449a1 100644 --- a/mblt_vision/utils/postprocess/embedding_post.py +++ b/mblt_vision/utils/postprocess/embedding_post.py @@ -83,3 +83,14 @@ def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: if not isinstance(dataset, str) or not dataset: raise ValueError("Person ReID postprocessing requires post_cfg.dataset") self.dataset = get_dataset_config_for_task("person_reid", dataset)["name"] + + +class FaceRecognitionPost(EmbeddingPost): + """One identity embedding per prepared aligned crop; no implicit flip fusion.""" + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__(pre_cfg, post_cfg) + dataset = post_cfg.get("dataset") + if not isinstance(dataset, str) or not dataset: + raise ValueError("Face recognition postprocessing requires post_cfg.dataset") + self.dataset = get_dataset_config_for_task("face_recognition", dataset)["name"] diff --git a/mblt_vision/utils/results.py b/mblt_vision/utils/results.py index 7a5b7f6..0b9e3a2 100644 --- a/mblt_vision/utils/results.py +++ b/mblt_vision/utils/results.py @@ -148,7 +148,7 @@ def set_output( self.mask = None self.depth = None self.semantic_mask = None - if self.task == "person_reid": + if self.task in {"person_reid", "face_recognition"}: if not isinstance(output, (np.ndarray, torch.Tensor)): raise TypeError("Person ReID results require an embedding matrix") finite = ( diff --git a/tests/reference/ops_face_preprocess.py b/tests/reference/ops_face_preprocess.py new file mode 100644 index 0000000..62848d1 --- /dev/null +++ b/tests/reference/ops_face_preprocess.py @@ -0,0 +1,57 @@ +# Verbatim models/face_recognition/_shared/preprocess.py at ec537764e36ca10da963fcb7afab157ea60ca022 +"""Default face_recognition preprocess: an aligned face crop, normalized. + +The benchmarks of this task hand the Model already aligned 112x112 crops (five-point +similarity alignment is part of the prepared evaluation layout), so the Model-side +transform is a guard resize to the network input and an explicit ``mean``/``std`` +normalization, in HWC order. Operations run in canonical order regardless of the +sorted-key order of ``pre_cfg`` on disk. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +PIPELINE_ORDER = ("Reader", "Resize", "SetOrder", "Normalize") +REQUIRED = ("Resize", "Normalize") +_RESAMPLING = { + "bilinear": Image.Resampling.BILINEAR, + "bicubic": Image.Resampling.BICUBIC, +} + + +def preprocess(source: Path, config: dict[str, Any]) -> tuple[Path, np.ndarray, None]: + """Return ``(source, HWC float32 tensor, None)``.""" + unknown = set(config) - set(PIPELINE_ORDER) + if unknown: + raise ValueError(f"unsupported preprocess operations: {sorted(unknown)}") + missing = [name for name in REQUIRED if name not in config] + if missing: + raise ValueError(f"pre_cfg is missing required operations: {missing}") + source = Path(source) + value: Any = Image.open(source).convert("RGB") + for operation in PIPELINE_ORDER: + if operation not in config: + continue + options = config[operation] + if operation == "Reader": + continue + if operation == "Resize": + height, width = (int(v) for v in options["size"]) + if value.size != (width, height): + mode = _RESAMPLING[str(options.get("interpolation", "bilinear"))] + value = value.resize((width, height), mode) + elif operation == "SetOrder": + if options.get("shape", "HWC") != "HWC": + raise ValueError( + "face_recognition preprocess produces HWC tensors only" + ) + elif operation == "Normalize": + mean = np.asarray(options["mean"], dtype=np.float32) + std = np.asarray(options["std"], dtype=np.float32) + value = (np.asarray(value, dtype=np.float32) / 255.0 - mean) / std + return source, np.ascontiguousarray(value, dtype=np.float32), None diff --git a/tests/reference/ops_face_verification.py b/tests/reference/ops_face_verification.py new file mode 100644 index 0000000..73f0eab --- /dev/null +++ b/tests/reference/ops_face_verification.py @@ -0,0 +1,75 @@ +# Verbatim datasets/_shared/face_verification.py at ec537764e36ca10da963fcb7afab157ea60ca022 +"""Pair-based face verification protocol shared by LFW-style benchmarks. + +A benchmark lists pairs of images with a same/different label and a fold id. The Model +provides one L2-normalized embedding per image. For every fold, the decision threshold on +cosine similarity is chosen to maximize accuracy on the other folds and then applied to +the held-out fold; the benchmark reports the mean and standard deviation over folds +(Huang et al., "Labeled Faces in the Wild", unrestricted protocol as implemented by +InsightFace / facenet ``evaluate``). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +THRESHOLDS = np.linspace(-1.0, 1.0, 4001) + + +@dataclass(frozen=True) +class Pair: + image_a: str + image_b: str + is_same: bool + fold: int + + +def cosine_scores(a: np.ndarray, b: np.ndarray) -> np.ndarray: + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + norms = np.linalg.norm(a, axis=1) * np.linalg.norm(b, axis=1) + return np.sum(a * b, axis=1) / np.maximum(norms, 1e-12) + + +def _accuracy(scores: np.ndarray, labels: np.ndarray, threshold: float) -> float: + return float(np.mean((scores >= threshold) == labels)) + + +def fold_accuracies( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Held-out accuracy per fold with the threshold fitted on the remaining folds.""" + scores = np.asarray(scores, dtype=np.float64) + labels = np.asarray(labels, dtype=bool) + folds = np.asarray(folds) + fold_ids = np.unique(folds) + if len(fold_ids) < 2: + raise ValueError("Verification protocol needs at least two folds") + accuracies = [] + thresholds = [] + for fold in fold_ids: + test = folds == fold + train_scores, train_labels = scores[~test], labels[~test] + fitted = np.array( + [_accuracy(train_scores, train_labels, t) for t in THRESHOLDS] + ) + # Ties on training accuracy resolve to the strictest threshold (the highest + # similarity), matching the reference implementations that search distance + # thresholds in ascending order and keep the first maximum. + best = float(THRESHOLDS[len(fitted) - 1 - int(np.argmax(fitted[::-1]))]) + thresholds.append(best) + accuracies.append(_accuracy(scores[test], labels[test], best)) + return np.asarray(accuracies), np.asarray(thresholds) + + +def verification_result( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> dict[str, float]: + accuracies, thresholds = fold_accuracies(scores, labels, folds) + return { + "accuracy": float(np.mean(accuracies)), + "accuracy_std": float(np.std(accuracies)), + "threshold": float(np.mean(thresholds)), + } diff --git a/tests/test_api.py b/tests/test_api.py index ce184aa..924192f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -22,6 +22,7 @@ def test_public_discovery_exposes_all_supported_tasks() -> None: "pose_estimation", "face_detection", "person_reid", + "face_recognition", ] assert list_models("obb")["obb"] diff --git a/tests/test_face_verification.py b/tests/test_face_verification.py new file mode 100644 index 0000000..2d1e60a --- /dev/null +++ b/tests/test_face_verification.py @@ -0,0 +1,218 @@ +"""CPU prepared face-pair, engine and source-exact numeric contracts.""" + +import csv +import importlib.util +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch +from mblt_vision._config import load_model_variant +from mblt_vision.face_recognition import ( + AgeDB30Dataset, + AgeDB30Evaluator, + CFPEvaluator, + CFPFPDataset, + EmbeddingPost, + FacePair, + FaceRecognitionPost, + LFWDataset, + LFWEvaluator, + PairImage, + cosine_scores, + fold_accuracies, +) +from mblt_vision.utils.datasets import face_pairs, readiness +from mblt_vision.utils.postprocess import build_postprocess +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.wrapper import MBLT_Engine +from PIL import Image + + +def reference(name): + path = Path(__file__).parent / "reference" / f"ops_face_{name}.py" + spec = importlib.util.spec_from_file_location(f"ops_face_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def write_pairs(root, folder, rows): + (root / folder).mkdir(exist_ok=True) + for name in {r[3] for r in rows} | {r[4] for r in rows}: + if "/" not in name: + Image.new("RGB", (112, 112)).save(root / folder / name) + with (root / "pairs.csv").open("w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["pair_id", "fold_id", "is_same", "image_a", "image_b"]) + writer.writerows(rows) + + +@pytest.mark.parametrize( + "loader,folder", + [(LFWDataset, "aligned"), (AgeDB30Dataset, "images"), (CFPFPDataset, "images")], +) +def test_prepared_pairs_preserve_order_folds_and_alignment(tmp_path, loader, folder): + rows = [(9, 3, 0, "b.png", "a.png"), (2, 1, 1, "a.png", "a.png")] + write_pairs(tmp_path, folder, rows) + data = loader(tmp_path, validate_counts=False) + assert [p.fold for p in data.pairs] == [3, 1] + assert [p.is_same for p in data.pairs] == [False, True] + assert [s.name for s in data.samples] == ["a.png", "b.png"] + assert isinstance(data[0], PairImage) and len(data) == 2 + with pytest.raises(ValueError, match="expected"): + loader(tmp_path) + Image.new("RGB", (113, 112)).save(tmp_path / folder / "a.png") + with pytest.raises(ValueError, match="112x112"): + loader(tmp_path, validate_counts=False) + + +@pytest.mark.parametrize( + "bad", + [ + (1, 0, 1, "a.png", "b.png"), + (1, 11, 1, "a.png", "b.png"), + (1, 1, 2, "a.png", "b.png"), + (1, 1, 1, "../a.png", "b.png"), + ], +) +def test_invalid_pair_metadata(tmp_path, bad): + write_pairs(tmp_path, "aligned", [bad]) + with pytest.raises(ValueError): + LFWDataset(tmp_path, validate_counts=False) + + +def test_duplicate_missing_and_empty_pairs(tmp_path): + with pytest.raises(FileNotFoundError): + LFWDataset(tmp_path) + write_pairs(tmp_path, "aligned", [(1, 1, 1, "a.png", "b.png")] * 2) + with pytest.raises(ValueError, match="unique"): + LFWDataset(tmp_path, validate_counts=False) + write_pairs(tmp_path, "aligned", [(1, 1, 1, "a.png", "b.png")]) + (tmp_path / "aligned/b.png").unlink() + with pytest.raises(FileNotFoundError): + LFWDataset(tmp_path, validate_counts=False) + + +def test_engine_embedding_path(): + assert issubclass(FaceRecognitionPost, EmbeddingPost) + engine = MBLT_Engine.__new__(MBLT_Engine) + engine.pre_cfg = {} + engine.post_cfg = { + "task": "face_recognition", + "dataset": "lfw", + "normalize": True, + "embedding_dim": 2, + } + engine.postprocessor = build_postprocess(engine.pre_cfg, engine.post_cfg) + assert isinstance(engine.postprocessor, FaceRecognitionPost) + result = engine.postprocess([np.array([[3.0, 4.0]], dtype=np.float32)]) + torch.testing.assert_close(result.embeddings, torch.tensor([[0.6, 0.8]])) + assert result.output is result.embeddings and result.acc is None + with pytest.raises(ValueError): + FaceRecognitionPost({}, {"dataset": "market1501"}) + with pytest.raises(ValueError): + FaceRecognitionPost({}, {}) + + +@pytest.mark.parametrize( + "loader,name,folder", + [ + (LFWDataset, "lfw", "aligned"), + (AgeDB30Dataset, "agedb-30", "images"), + (CFPFPDataset, "cfp-fp", "images"), + ], +) +def test_readiness(tmp_path, monkeypatch, loader, name, folder): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 10 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + rows = [(i, i, 1, "a.png", "b.png") for i in range(1, 11)] + write_pairs(tmp_path, folder, rows) + assert readiness.dataset_ready(tmp_path, "face_recognition", name) + assert not readiness.dataset_ready(tmp_path, "face_recognition", "market1501") + (tmp_path / folder / "a.png").write_bytes(b"broken") + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + + +@pytest.mark.parametrize("seed", range(5)) +def test_ops_metric_exact_parity(seed): + ops = reference("verification") + rng = np.random.default_rng(seed) + a = rng.integers(-2, 3, (80, 8)).astype(np.float32) + b = rng.integers(-2, 3, (80, 8)).astype(np.float32) + labels = rng.integers(0, 2, 80).astype(bool) + folds = np.tile(np.arange(1, 11), 8) + expected_scores = ops.cosine_scores(a, b) + scores = cosine_scores(a, b) + np.testing.assert_array_equal(scores, expected_scores) + expected_acc, expected_thresholds = ops.fold_accuracies( + expected_scores, labels, folds + ) + acc, thresholds = fold_accuracies(scores, labels, folds) + np.testing.assert_array_equal(acc, expected_acc) + np.testing.assert_array_equal(thresholds, expected_thresholds) + expected = ops.verification_result(scores, labels, folds) + assert float(np.mean(acc)) == expected["accuracy"] + assert float(np.std(acc)) == expected["accuracy_std"] + assert float(np.mean(thresholds)) == expected["threshold"] + + +def test_threshold_ties_and_inclusive_boundary(): + acc, thresholds = fold_accuracies( + np.zeros(4), np.array([0, 1, 0, 1]), np.array([1, 1, 2, 2]) + ) + np.testing.assert_array_equal(thresholds, [1.0, 1.0]) + np.testing.assert_array_equal(acc, [0.5, 0.5]) + # Exactly equal to threshold counts as same. + scores = cosine_scores(np.ones((2, 2)), np.ones((2, 2))) + np.testing.assert_allclose(scores, 1.0) + with pytest.raises(ValueError): + fold_accuracies(np.ones(2), np.ones(2), np.ones(2, dtype=int)) + + +@pytest.mark.parametrize("evaluator", [LFWEvaluator, AgeDB30Evaluator, CFPEvaluator]) +def test_evaluator_mapping_and_no_implicit_fusion(evaluator): + images = [PairImage(Path(n), n) for n in ["a", "b", "c"]] + pairs = [ + FacePair("a", "b", True, 1), + FacePair("a", "c", False, 1), + FacePair("a", "b", True, 2), + FacePair("a", "c", False, 2), + ] + embeddings = np.array([[1.0, 0.0], [1.0, 0.0], [-1.0, 0.0]], dtype=np.float32) + result = evaluator().evaluate(embeddings, images, pairs) + assert result.primary_score == 1.0 and result.secondary_score == 0.0 + assert result.as_dict()["primary"][evaluator.metric] == 1.0 + assert result.sample_count == 3 + with pytest.raises(ValueError): + evaluator().evaluate(embeddings[:2], images, pairs) + with pytest.raises(ValueError): + evaluator().evaluate(embeddings, images, [FacePair("a", "missing", True, 1)]) + + +@pytest.mark.parametrize("shape", [(112, 112), (35, 51), (241, 179)]) +def test_ops_preprocessing_byte_parity(tmp_path, shape): + ops = reference("preprocess") + cfg = load_model_variant( + Path(__file__).parents[1] / "mblt_vision/models/bases/face_recognition.yaml", + "DEFAULT", + ) + path = tmp_path / "synthetic.png" + Image.fromarray( + np.random.default_rng(0).integers(0, 256, (*shape, 3), dtype=np.uint8) + ).save(path) + expected = ops.preprocess(path, cfg["pre_cfg"])[1] + actual = build_preprocess(cfg["pre_cfg"])(path) + np.testing.assert_array_equal( + actual, expected + ) # atol=0, including no-op 112x112 resize + assert np.max(np.abs(actual - expected)) == 0.0 From 77478b4d3eda47524e08319f881e5fe00f6077cc Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 17:02:18 +0900 Subject: [PATCH 5/7] Validate official face pair counts independently for every fold Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/utils/datasets/face_pairs.py | 10 +++++++ tests/test_face_verification.py | 36 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/mblt_vision/utils/datasets/face_pairs.py b/mblt_vision/utils/datasets/face_pairs.py index a72bd66..52e3010 100644 --- a/mblt_vision/utils/datasets/face_pairs.py +++ b/mblt_vision/utils/datasets/face_pairs.py @@ -89,6 +89,16 @@ def __init__( raise ValueError( f"Pair file holds {len(pairs)} pairs, expected {self.config['pairs']}" ) + if validate_counts: + per_fold = self.config["pairs"] // self.config["folds"] + counts = { + fold: sum(pair.fold == fold for pair in pairs) + for fold in range(1, self.config["folds"] + 1) + } + if any(count != per_fold for count in counts.values()): + raise ValueError( + f"Each official fold requires {per_fold} pairs; found {counts}" + ) self.pairs = tuple(pairs) image_dir = self.root / self.config["image_directory"] if not image_dir.is_dir(): diff --git a/tests/test_face_verification.py b/tests/test_face_verification.py index 2d1e60a..62cb9ac 100644 --- a/tests/test_face_verification.py +++ b/tests/test_face_verification.py @@ -216,3 +216,39 @@ def test_ops_preprocessing_byte_parity(tmp_path, shape): actual, expected ) # atol=0, including no-op 112x112 resize assert np.max(np.abs(actual - expected)) == 0.0 + + +@pytest.mark.parametrize( + "name,loader,folder", + [ + ("lfw", LFWDataset, "aligned"), + ("agedb-30", AgeDB30Dataset, "images"), + ("cfp-fp", CFPFPDataset, "images"), + ], +) +def test_rejects_correct_total_with_unbalanced_official_folds( + tmp_path, monkeypatch, name, loader, folder +): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 20 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + rows = [(i, i // 2 + 1, i % 2, "a.png", "b.png") for i in range(20)] + write_pairs(tmp_path, folder, rows) + assert readiness.dataset_ready(tmp_path, "face_recognition", name) + rows[0] = ( + 0, + 2, + 0, + "a.png", + "b.png", + ) # 20 total, all ten folds, but 1/3 instead of 2/2. + write_pairs(tmp_path, folder, rows) + with pytest.raises(ValueError, match="official fold"): + loader(tmp_path) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + assert len(loader(tmp_path, validate_counts=False).pairs) == 20 From 097c810c4d324947ee235d8ec674c787f3213562 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 18:33:36 +0900 Subject: [PATCH 6/7] Reject face-cache symlink roots images and pair manifests Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/utils/datasets/readiness.py | 8 +++++-- tests/test_face_verification.py | 31 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index b1c9d5d..f405ffa 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -1072,14 +1072,18 @@ def _reid_ready(root: Path, dataset: str) -> bool: def _face_ready(root: Path, dataset: str) -> bool: """Validate prepared aligned pairs and image decoding, never realign crops.""" loader = {"lfw": LFWDataset, "agedb-30": AgeDB30Dataset, "cfp-fp": CFPFPDataset}.get(dataset) - if loader is None: + if ( + loader is None + or _path_has_symlink_component(root) + or _path_has_symlink_component(root / "pairs.csv") + ): return False try: data = loader(root) if len({pair.fold for pair in data.pairs}) != 10: return False for sample in data.samples: - if sample.image.is_symlink(): + if _path_has_symlink_component(sample.image): return False with Image.open(sample.image) as image: image.load() diff --git a/tests/test_face_verification.py b/tests/test_face_verification.py index 62cb9ac..bfa0786 100644 --- a/tests/test_face_verification.py +++ b/tests/test_face_verification.py @@ -143,6 +143,37 @@ def tiny(task, dataset): assert not readiness.dataset_ready(tmp_path, "face_recognition", name) +@pytest.mark.parametrize( + "folder,name", [("aligned", "lfw"), ("images", "agedb-30"), ("images", "cfp-fp")] +) +@pytest.mark.parametrize("component", ["root", "images", "manifest"]) +def test_readiness_rejects_symlink_ancestors( + tmp_path, monkeypatch, folder, name, component +): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 10 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + root = tmp_path / "cache" + root.mkdir() + write_pairs(root, folder, [(i, i, 1, "a.png", "b.png") for i in range(1, 11)]) + assert readiness.dataset_ready(root, "face_recognition", name) + if component == "root": + alias = tmp_path / "alias" + alias.symlink_to(root, target_is_directory=True) + root = alias + else: + source = root / (folder if component == "images" else "pairs.csv") + target = tmp_path / "outside" + source.rename(target) + source.symlink_to(target, target_is_directory=component == "images") + assert not readiness.dataset_ready(root, "face_recognition", name) + + @pytest.mark.parametrize("seed", range(5)) def test_ops_metric_exact_parity(seed): ops = reference("verification") From d8475308aca34bd1b0dc4a9ad2609a1fbe7f3bb7 Mon Sep 17 00:00:00 2001 From: jinman Date: Wed, 4 Nov 2026 19:04:47 +0900 Subject: [PATCH 7/7] Validate official face fold labels and malformed CSV readiness Co-Authored-By: Claude Haiku 5.5 --- mblt_vision/utils/datasets/face_pairs.py | 6 +++ mblt_vision/utils/datasets/readiness.py | 3 +- tests/test_face_manifest_review.py | 57 ++++++++++++++++++++++++ tests/test_face_verification.py | 10 +++-- 4 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/test_face_manifest_review.py diff --git a/mblt_vision/utils/datasets/face_pairs.py b/mblt_vision/utils/datasets/face_pairs.py index 52e3010..8622531 100644 --- a/mblt_vision/utils/datasets/face_pairs.py +++ b/mblt_vision/utils/datasets/face_pairs.py @@ -99,6 +99,12 @@ def __init__( raise ValueError( f"Each official fold requires {per_fold} pairs; found {counts}" ) + for fold in counts: + same = sum(pair.fold == fold and pair.is_same for pair in pairs) + if per_fold % 2 or same != per_fold // 2: + raise ValueError( + f"Each official fold requires equal same/different labels; fold {fold} has {same}/{per_fold} same" + ) self.pairs = tuple(pairs) image_dir = self.root / self.config["image_directory"] if not image_dir.is_dir(): diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index f405ffa..3e7d89f 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv import hashlib import json import math @@ -1087,7 +1088,7 @@ def _face_ready(root: Path, dataset: str) -> bool: return False with Image.open(sample.image) as image: image.load() - except (OSError, ValueError): + except (OSError, ValueError, csv.Error): return False return True diff --git a/tests/test_face_manifest_review.py b/tests/test_face_manifest_review.py new file mode 100644 index 0000000..b917299 --- /dev/null +++ b/tests/test_face_manifest_review.py @@ -0,0 +1,57 @@ +"""Official fold label balance and malformed CSV readiness regressions.""" + +import csv + +import pytest +from PIL import Image + +from mblt_vision.utils.datasets import face_pairs, readiness + + +@pytest.mark.parametrize( + "name,folder,loader", + [ + ("lfw", "aligned", face_pairs.LFWDataset), + ("agedb-30", "images", face_pairs.AgeDB30Dataset), + ("cfp-fp", "images", face_pairs.CFPFPDataset), + ], +) +def test_same_different_balance_and_csv_readiness( + tmp_path, monkeypatch, name, folder, loader +): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + config = original(task, dataset) + config["pairs"] = 20 + return config + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + (tmp_path / folder).mkdir() + for image in ["a.png", "b.png"]: + Image.new("RGB", (112, 112)).save(tmp_path / folder / image) + manifest = tmp_path / "pairs.csv" + + def write(labels): + with manifest.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["pair_id", "fold_id", "is_same", "image_a", "image_b"]) + writer.writerows( + (i, i // 2 + 1, label, "a.png", "b.png") + for i, label in enumerate(labels) + ) + + write([i % 2 for i in range(20)]) + assert readiness.dataset_ready(tmp_path, "face_recognition", name) + write([1] * 20) + with pytest.raises(ValueError, match="same/different"): + loader(tmp_path) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + assert len(loader(tmp_path, validate_counts=False).pairs) == 20 + write([i % 2 for i in range(20)]) + old = csv.field_size_limit() + try: + csv.field_size_limit(4) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + finally: + csv.field_size_limit(old) diff --git a/tests/test_face_verification.py b/tests/test_face_verification.py index bfa0786..bd67808 100644 --- a/tests/test_face_verification.py +++ b/tests/test_face_verification.py @@ -130,12 +130,12 @@ def test_readiness(tmp_path, monkeypatch, loader, name, folder): def tiny(task, dataset): cfg = original(task, dataset) - cfg["pairs"] = 10 + cfg["pairs"] = 20 return cfg monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) assert not readiness.dataset_ready(tmp_path, "face_recognition", name) - rows = [(i, i, 1, "a.png", "b.png") for i in range(1, 11)] + rows = [(i, i // 2 + 1, i % 2, "a.png", "b.png") for i in range(20)] write_pairs(tmp_path, folder, rows) assert readiness.dataset_ready(tmp_path, "face_recognition", name) assert not readiness.dataset_ready(tmp_path, "face_recognition", "market1501") @@ -154,13 +154,15 @@ def test_readiness_rejects_symlink_ancestors( def tiny(task, dataset): cfg = original(task, dataset) - cfg["pairs"] = 10 + cfg["pairs"] = 20 return cfg monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) root = tmp_path / "cache" root.mkdir() - write_pairs(root, folder, [(i, i, 1, "a.png", "b.png") for i in range(1, 11)]) + write_pairs( + root, folder, [(i, i // 2 + 1, i % 2, "a.png", "b.png") for i in range(20)] + ) assert readiness.dataset_ready(root, "face_recognition", name) if component == "root": alias = tmp_path / "alias"