diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index e6a9f7a..3fafe11 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -160,6 +160,10 @@ description: >- and prepared query/gallery loaders. Preserve ops stable distance ties, camera exclusions and dataset-specific junk handling; see the ReID API section in `mblt_vision/README.md`. +- Face verification uses prepared 112x112 crops without realignment or implicit flip fusion. + Keep pair/fold order, source threshold search, Results embeddings and readiness connected; + require exact metric and preprocessing parity (face verification API section). + ## Runtime and Packaging - Route NPU runtime access through mblt-npu-python; do not copy backend classes into Vision. diff --git a/AGENTS.md b/AGENTS.md index e7d5d90..baaa824 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -234,6 +234,10 @@ The current ownership boundary is deliberate: stable distance ties, camera exclusions, junk rules, and preprocessing numeric parity tested. Named Normalize styles retain legacy arithmetic; explicit mean/std uses source float32 arithmetic. +- Face verification utilities preserve prepared alignment and official pair/fold order. + Use dataset-bound `FaceRecognitionPost`, `Results.embeddings`, and face readiness checks. + Protocol details and source-parity requirements are in the face verification API section. + ## Python-First Architecture - Keep the public layer independent from a particular backend. Define small internal interfaces diff --git a/benchmark/benchmark_vision_models.py b/benchmark/benchmark_vision_models.py index 3453bba..61aeb17 100644 --- a/benchmark/benchmark_vision_models.py +++ b/benchmark/benchmark_vision_models.py @@ -36,7 +36,7 @@ tuple[CoreMode, ...], core_modes_for_target_device("aries-rb") ) # Utility-only embedding tasks have no benchmark engine/CLI route yet. -TASK_CHOICES = tuple(task for task in VISION_TASKS if task != "person_reid") +TASK_CHOICES = tuple(task for task in VISION_TASKS if task not in {"person_reid", "face_recognition"}) SUPPORTED_TARGET_DEVICES = frozenset({"aries-rb", "regulus-ra", "regulus-rb"}) diff --git a/mblt_vision/README.md b/mblt_vision/README.md index 02a811e..2d4b3bd 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -1022,3 +1022,46 @@ Offline tests vendor the inspected ops functions at distance/CMC/mAP equality; synthetic RGB preprocessing requires byte equality (`atol=0`) against the ops adapter for explicit source mean/std profiles. These are protocol tests, not measured model accuracy claims. + +## Face verification utilities + +`mblt_vision.face_recognition` exports `FaceRecognitionPost(EmbeddingPost)`, +`LFWDataset`, `AgeDB30Dataset`, `CFPFPDataset`, and their `LFWEvaluator`, +`AgeDB30Evaluator`, `CFPEvaluator` subclasses of `VerificationEvaluator`. +The processing-only `bases/face_recognition.yaml` profile is 112x112 PIL bilinear, +explicit float32 0.5 mean/std, 512 dimensions, row-wise L2 normalization. Engine +postprocessing stores dense vectors in `Results.embeddings` and `output`. Model +wrappers/artifact YAMLs are a separate stage; CLI/benchmark routing is unchanged. + +Pair loaders read prepared `pairs.csv` columns `pair_id,fold_id,is_same,image_a,image_b`, +retain pair order and official one-based fold labels, and enumerate unique image +basenames in sorted order. LFW uses `aligned/`; AgeDB-30 and CFP-FP use `images/`. +They require existing 112x112 crops and never detect or realign faces. Source pair +counts are checked by default; caller-owned subsets explicitly opt out with +`validate_counts=False`. Default cache roots end in `/eval`. +`dataset_ready(root, "face_recognition", dataset)` additionally requires all ten +folds and decodable crops; its default dataset is LFW. + +```python +from mblt_vision.face_recognition import LFWDataset, LFWEvaluator + +data = LFWDataset("/path/to/lfw/eval") +# embeddings: one row per data.samples, from your inference callback. +# result = LFWEvaluator().evaluate(embeddings, data.samples, data.pairs) +# print(result.as_dict()) +``` + +Ops at `ec537764e36ca10da963fcb7afab157ea60ca022` owns the protocol: one +normalized embedding per crop, **no flip fusion**, float64 cosine scores, threshold +grid `linspace(-1,1,4001)`, inclusive `score >= threshold`, strictest threshold on +training ties, leave-one-fold-out threshold fitting, mean accuracy and population +standard deviation over folds. `extract_embeddings` shares the ordered bounded +inference helper with retrieval. There is no implicit fusion in scoring. + +Offline tests compare exact cosine scores, fold accuracies, thresholds and aggregate +metrics against a verbatim source snapshot over five synthetic seeds. Three RGB +fixtures (including no-op resize) require preprocessing byte equality (`atol=0`). +Dataset YAMLs record exact source paths/revision. CFP-FP specifically refers to +the prepared cat-claws mirror: ops warns it was not checked against the original +`cfp_fp.bin` and records a training-log discrepancy. This package preserves that +provenance and makes no new accuracy claim. diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index 5a88f1b..177a478 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -9,6 +9,7 @@ from . import depth_estimation as depth_estimation from . import face_detection as face_detection +from . import face_recognition as face_recognition from . import image_classification as image_classification from . import instance_segmentation as instance_segmentation from . import obb as obb @@ -54,6 +55,7 @@ "obb", "pose_estimation", "person_reid", + "face_recognition", "semantic_segmentation", ] + sorted(_LEGACY_MODEL_EXPORTS) # Keep legacy compatibility exports synchronized with their task packages. diff --git a/mblt_vision/_tasks.py b/mblt_vision/_tasks.py index c8a4b6d..310e895 100644 --- a/mblt_vision/_tasks.py +++ b/mblt_vision/_tasks.py @@ -14,6 +14,7 @@ "pose_estimation", "face_detection", "person_reid", + "face_recognition", ) diff --git a/mblt_vision/datasets/agedb-30.yaml b/mblt_vision/datasets/agedb-30.yaml new file mode 100644 index 0000000..f17c8c8 --- /dev/null +++ b/mblt_vision/datasets/agedb-30.yaml @@ -0,0 +1,34 @@ +name: agedb-30 +path: ~/.mblt_model_zoo/datasets/agedb-30 +tasks: +- face_recognition +val: eval +image_directory: images +pairs: 6000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/agedb_30/agedb_30.yaml + name: agedb-30 + dataloader_path: datasets/agedb_30/_private/dataloader.py +eval_protocol: + id: agedb-30-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: agedb-30-ten-fold-verification.v1 + primary_metric: agedb_30_accuracy + secondary_metrics: + - agedb_30_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/agedb_30/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/cfp-fp.yaml b/mblt_vision/datasets/cfp-fp.yaml new file mode 100644 index 0000000..1fb614c --- /dev/null +++ b/mblt_vision/datasets/cfp-fp.yaml @@ -0,0 +1,36 @@ +name: cfp-fp +path: ~/.mblt_model_zoo/datasets/cfp-fp +tasks: +- face_recognition +val: eval +image_directory: images +pairs: 7000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/cfp_fp/cfp_fp.yaml + name: cfp-fp + dataloader_path: datasets/cfp_fp/_private/dataloader.py + caveat: Prepared cat-claws mirror is not checked against original cfp_fp.bin; ops + records lower CFP-FP accuracy than training logs. +eval_protocol: + id: cfp-fp-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: cfp-fp-ten-fold-verification.v1 + primary_metric: cfp_fp_accuracy + secondary_metrics: + - cfp_fp_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/cfp_fp/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/datasets/lfw.yaml b/mblt_vision/datasets/lfw.yaml new file mode 100644 index 0000000..7e83377 --- /dev/null +++ b/mblt_vision/datasets/lfw.yaml @@ -0,0 +1,34 @@ +name: lfw +path: ~/.mblt_model_zoo/datasets/lfw +tasks: +- face_recognition +val: eval +image_directory: aligned +pairs: 6000 +folds: 10 +aligned_size: +- 112 +- 112 +source: + repository: https://git.mobilint.com/algorithm-team/testing/mblt-model-ops + revision: ec537764e36ca10da963fcb7afab157ea60ca022 + config_path: datasets/lfw/lfw.yaml + name: lfw + dataloader_path: datasets/lfw/_private/dataloader.py +eval_protocol: + id: lfw-ten-fold-verification.v1 +evaluation: + split: eval/pairs.csv + protocol: lfw-ten-fold-verification.v1 + primary_metric: lfw_accuracy + secondary_metrics: + - lfw_accuracy_std + embedding_fusion: none; one normalized embedding per crop + implementation_reference: + path: datasets/lfw/_private/evaluator.py + shared_paths: + - datasets/_shared/face_pairs.py + - datasets/_shared/face_verification.py + - models/face_recognition/_shared/evaluate.py + - models/_shared/embeddings.py + revision: ec537764e36ca10da963fcb7afab157ea60ca022 diff --git a/mblt_vision/face_recognition/__init__.py b/mblt_vision/face_recognition/__init__.py new file mode 100644 index 0000000..2d9df56 --- /dev/null +++ b/mblt_vision/face_recognition/__init__.py @@ -0,0 +1,40 @@ +"""Aligned face pair verification utilities; no implicit face detection/alignment.""" + +from ..utils.datasets.face_pairs import ( + AgeDB30Dataset, + CFPFPDataset, + FacePair, + FacePairDataset, + LFWDataset, + PairImage, +) +from ..utils.evaluation.verification import ( + AgeDB30Evaluator, + CFPEvaluator, + LFWEvaluator, + VerificationEvaluator, + VerificationResult, + cosine_scores, + extract_embeddings, + fold_accuracies, +) +from ..utils.postprocess.embedding_post import EmbeddingPost, FaceRecognitionPost + +__all__ = [ + "AgeDB30Dataset", + "AgeDB30Evaluator", + "CFPEvaluator", + "CFPFPDataset", + "EmbeddingPost", + "FacePair", + "FacePairDataset", + "FaceRecognitionPost", + "LFWDataset", + "LFWEvaluator", + "PairImage", + "VerificationEvaluator", + "VerificationResult", + "cosine_scores", + "extract_embeddings", + "fold_accuracies", +] diff --git a/mblt_vision/models/bases/face_recognition.yaml b/mblt_vision/models/bases/face_recognition.yaml new file mode 100644 index 0000000..6545407 --- /dev/null +++ b/mblt_vision/models/bases/face_recognition.yaml @@ -0,0 +1,18 @@ +# Processing only; concrete approved HF artifacts are added separately. +# ops ec537764e36ca10da963fcb7afab157ea60ca022 +# models/face_recognition/ArcFace_IResNet_100/distribution.yaml +DEFAULT: + pre_cfg: + Reader: + style: pil + Resize: + size: [112, 112] + interpolation: bilinear + Normalize: + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + post_cfg: + task: face_recognition + dataset: lfw + embedding_dim: 512 + normalize: true diff --git a/mblt_vision/utils/datasets/face_pairs.py b/mblt_vision/utils/datasets/face_pairs.py new file mode 100644 index 0000000..8622531 --- /dev/null +++ b/mblt_vision/utils/datasets/face_pairs.py @@ -0,0 +1,147 @@ +"""Prepared aligned face pairs, preserving source file order and fold labels.""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +from pathlib import Path + +from PIL import Image + +from ...datasets import get_dataset_config_for_task + + +@dataclass(frozen=True) +class FacePair: + """Source basenames, same/different label and official one-based fold.""" + + image_a: str + image_b: str + is_same: bool + fold: int + + +@dataclass(frozen=True) +class PairImage: + """One prepared 112x112 aligned crop, keyed by its source basename.""" + + image: Path + name: str + + +class FacePairDataset: + """Read prepared pairs.csv and already aligned crops without realignment. + + Root contains pairs.csv and the configured aligned/images directory. + Source pair counts are enforced unless validate_counts=False explicitly + selects a caller-owned subset. Basenames, labels and folds are always checked. + """ + + name: str + + def __init__( + self, root: str | Path | None = None, *, validate_counts: bool = True + ) -> None: + self.config = get_dataset_config_for_task("face_recognition", self.name) + self.root = ( + Path(root).expanduser() + if root is not None + else Path(self.config["path"]) / self.config["val"] + ) + path = self.root / "pairs.csv" + if not path.is_file(): + raise FileNotFoundError(f"Pair file is missing: {path}") + pairs = [] + ids = set() + with path.open(newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + columns = {"pair_id", "fold_id", "is_same", "image_a", "image_b"} + if reader.fieldnames is None or not columns <= set(reader.fieldnames): + raise ValueError(f"{path} lacks the pair columns {sorted(columns)}") + for row in reader: + for key in ("image_a", "image_b"): + name = row[key] + if ( + not name + or name in {".", ".."} + or any(c in name for c in "/\\:") + ): + raise ValueError(f"Invalid pair image basename: {name!r}") + if row["is_same"] not in {"0", "1"}: + raise ValueError("Pair is_same must be 0 or 1") + try: + fold, pair_id = int(row["fold_id"]), int(row["pair_id"]) + except (ValueError, TypeError) as exc: + raise ValueError( + "Pair fold_id and pair_id must be integers" + ) from exc + if not 1 <= fold <= 10 or pair_id in ids: + raise ValueError( + "Pair fold must be 1..10 and pair_id must be unique" + ) + ids.add(pair_id) + pairs.append( + FacePair( + row["image_a"], row["image_b"], row["is_same"] == "1", fold + ) + ) + if not pairs or (validate_counts and len(pairs) != self.config["pairs"]): + raise ValueError( + f"Pair file holds {len(pairs)} pairs, expected {self.config['pairs']}" + ) + if validate_counts: + per_fold = self.config["pairs"] // self.config["folds"] + counts = { + fold: sum(pair.fold == fold for pair in pairs) + for fold in range(1, self.config["folds"] + 1) + } + if any(count != per_fold for count in counts.values()): + raise ValueError( + f"Each official fold requires {per_fold} pairs; found {counts}" + ) + for fold in counts: + same = sum(pair.fold == fold and pair.is_same for pair in pairs) + if per_fold % 2 or same != per_fold // 2: + raise ValueError( + f"Each official fold requires equal same/different labels; fold {fold} has {same}/{per_fold} same" + ) + self.pairs = tuple(pairs) + image_dir = self.root / self.config["image_directory"] + if not image_dir.is_dir(): + raise FileNotFoundError(f"Image directory is missing: {image_dir}") + names = sorted({p.image_a for p in pairs} | {p.image_b for p in pairs}) + images = [] + for name in names: + path = image_dir / name + if not path.is_file(): + raise FileNotFoundError(f"Pair image is missing: {path}") + # Alignment is a dataset preparation contract, never re-estimated here. + with Image.open(path) as image: + if image.size != (112, 112): + raise ValueError(f"Prepared aligned crop must be 112x112: {path}") + images.append(PairImage(path, name)) + self.samples = tuple(images) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> PairImage: + return self.samples[index] + + +class LFWDataset(FacePairDataset): + """LFW official 6000 prepared aligned pairs.""" + + name = "lfw" + + +class AgeDB30Dataset(FacePairDataset): + """Prepared AgeDB-30 mirror pair protocol, without changing its folds.""" + + name = "agedb-30" + + +class CFPFPDataset(FacePairDataset): + """Prepared CFP-FP mirror protocol; source caveats remain in the YAML.""" + + name = "cfp-fp" diff --git a/mblt_vision/utils/datasets/readiness.py b/mblt_vision/utils/datasets/readiness.py index c846aca..3e7d89f 100644 --- a/mblt_vision/utils/datasets/readiness.py +++ b/mblt_vision/utils/datasets/readiness.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv import hashlib import json import math @@ -20,6 +21,7 @@ from ..._tasks import normalize_vision_task from ...datasets import get_dataset_category_ids from .cityscapes import CITYSCAPES_SOURCE_TO_TRAIN_ID +from .face_pairs import LFWDataset, AgeDB30Dataset, CFPFPDataset from .reid import DukeMTMCReIDDataset, Market1501Dataset, OccludedDukeDataset IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} @@ -1068,6 +1070,29 @@ def _reid_ready(root: Path, dataset: str) -> bool: return True +def _face_ready(root: Path, dataset: str) -> bool: + """Validate prepared aligned pairs and image decoding, never realign crops.""" + loader = {"lfw": LFWDataset, "agedb-30": AgeDB30Dataset, "cfp-fp": CFPFPDataset}.get(dataset) + if ( + loader is None + or _path_has_symlink_component(root) + or _path_has_symlink_component(root / "pairs.csv") + ): + return False + try: + data = loader(root) + if len({pair.fold for pair in data.pairs}) != 10: + return False + for sample in data.samples: + if _path_has_symlink_component(sample.image): + return False + with Image.open(sample.image) as image: + image.load() + except (OSError, ValueError, csv.Error): + return False + return True + + def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) -> bool: """Return whether an organized dataset matches its task, taxonomy, and full validation split. @@ -1093,6 +1118,8 @@ def dataset_ready(data_path: str | Path, task: str, dataset: str | None = None) }.get(normalized_task) normalized_dataset = (dataset or expected_dataset or "").lower() + if normalized_task == "face_recognition": + return _face_ready(root, normalized_dataset or "lfw") if normalized_task == "person_reid": return _reid_ready(root, normalized_dataset or "market1501") if normalized_task == "semantic_segmentation": diff --git a/mblt_vision/utils/evaluation/verification.py b/mblt_vision/utils/evaluation/verification.py new file mode 100644 index 0000000..8889c5f --- /dev/null +++ b/mblt_vision/utils/evaluation/verification.py @@ -0,0 +1,160 @@ +"""Fold-based face verification, pinned to ops ec537764 protocol arithmetic.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np + +from ..datasets.face_pairs import FacePair, PairImage +from .retrieval import extract_embeddings + +# Source grid includes both endpoints. Predictions use >= and ties choose highest t. +THRESHOLDS = np.linspace(-1.0, 1.0, 4001) + + +def cosine_scores(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Source float64 cosine scores; the norm-product floor is 1e-12.""" + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + if a.ndim != 2 or b.shape != a.shape or min(a.shape) == 0: + raise ValueError("Cosine inputs require matching nonempty (N,D) matrices") + if not np.isfinite(a).all() or not np.isfinite(b).all(): + raise ValueError("Cosine embeddings must be finite") + norms = np.linalg.norm(a, axis=1) * np.linalg.norm(b, axis=1) + return np.sum(a * b, axis=1) / np.maximum(norms, 1e-12) + + +def fold_accuracies( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Fit threshold on all other folds and score the held-out official fold.""" + scores = np.asarray(scores, dtype=np.float64) + labels = np.asarray(labels) + folds = np.asarray(folds) + if ( + scores.ndim != 1 + or scores.size == 0 + or labels.shape != scores.shape + or folds.shape != scores.shape + ): + raise ValueError("Scores, labels and folds must be matching nonempty vectors") + if not np.isfinite(scores).all() or not np.isin(labels, [0, 1]).all(): + raise ValueError("Verification requires finite scores and binary labels") + if ( + not np.issubdtype(folds.dtype, np.integer) + or not np.isin(folds, np.arange(1, 11)).all() + ): + raise ValueError("Verification fold IDs must be integers in 1..10") + labels = np.asarray(labels, dtype=bool) + fold_ids = np.unique(folds) + if len(fold_ids) < 2: + raise ValueError("Verification protocol needs at least two folds") + accuracies = [] + thresholds = [] + for fold in fold_ids: + test = folds == fold + train_scores, train_labels = scores[~test], labels[~test] + # Keep the source accumulation, threshold grid and strictest-tie policy. + fitted = np.array( + [float(np.mean((train_scores >= t) == train_labels)) for t in THRESHOLDS] + ) + best = float(THRESHOLDS[len(fitted) - 1 - int(np.argmax(fitted[::-1]))]) + thresholds.append(best) + accuracies.append(float(np.mean((scores[test] >= best) == labels[test]))) + return np.asarray(accuracies), np.asarray(thresholds) + + +@dataclass(frozen=True) +class VerificationResult: + """Unrounded fold results, with the source benchmark's rounded score view.""" + + accuracies: np.ndarray + thresholds: np.ndarray + sample_count: int + metric: str + + @property + def primary_score(self) -> float: + return float(np.mean(self.accuracies)) + + @property + def secondary_score(self) -> float: + return float(np.std(self.accuracies)) + + @property + def threshold(self) -> float: + return float(np.mean(self.thresholds)) + + def as_dict(self) -> dict: + return { + "primary": {self.metric: round(self.primary_score, 5)}, + "secondary": {self.metric + "_std": round(self.secondary_score, 5)}, + "sample_count": self.sample_count, + } + + +class VerificationEvaluator: + """One embedding per aligned image, no implicit flip/fusion or alignment. + + Source evaluate_embeddings decodes one L2-normalized vector per crop; + source score_pairs then computes cosine similarity, without flip fusion. + ``extract_embeddings`` is reused unchanged from retrieval evaluation. + """ + + metric = "accuracy" + extract_embeddings = staticmethod(extract_embeddings) + + def evaluate( + self, + embeddings: np.ndarray, + images: Sequence[PairImage], + pairs: Sequence[FacePair], + ) -> VerificationResult: + """Score embeddings in canonical image manifest order against source pairs.""" + values = np.asarray(embeddings) + if ( + values.ndim != 2 + or min(values.shape) == 0 + or values.shape[0] != len(images) + or not np.isfinite(values).all() + ): + raise ValueError("Embeddings require one finite nonempty row per image") + names = [image.name for image in images] + if len(set(names)) != len(names): + raise ValueError("Image manifest names must be unique") + if not pairs: + raise ValueError("No verification pairs supplied") + by_name = dict(zip(names, values)) + try: + a = np.stack([by_name[p.image_a] for p in pairs]) + b = np.stack([by_name[p.image_b] for p in pairs]) + except KeyError as exc: + raise ValueError( + f"Pair image missing from embedding manifest: {exc}" + ) from exc + accuracy, thresholds = fold_accuracies( + cosine_scores(a, b), + np.asarray([p.is_same for p in pairs]), + np.asarray([p.fold for p in pairs]), + ) + return VerificationResult(accuracy, thresholds, len(images), self.metric) + + +class LFWEvaluator(VerificationEvaluator): + """LFW official fold accuracy.""" + + metric = "lfw_accuracy" + + +class AgeDB30Evaluator(VerificationEvaluator): + """AgeDB-30 prepared mirror fold accuracy.""" + + metric = "agedb_30_accuracy" + + +class CFPEvaluator(VerificationEvaluator): + """CFP-FP prepared mirror fold accuracy (see source YAML caveat).""" + + metric = "cfp_fp_accuracy" diff --git a/mblt_vision/utils/postprocess/__init__.py b/mblt_vision/utils/postprocess/__init__.py index 879a06f..1455c4b 100644 --- a/mblt_vision/utils/postprocess/__init__.py +++ b/mblt_vision/utils/postprocess/__init__.py @@ -4,7 +4,14 @@ from .build_post import build_postprocess from .depth_post import DepthPost -from .embedding_post import EmbeddingPost, PersonReIDPost +from .embedding_post import EmbeddingPost, FaceRecognitionPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost -__all__ = ["DepthPost", "EmbeddingPost", "PersonReIDPost", "SemanticSegPost", "build_postprocess"] +__all__ = [ + "DepthPost", + "EmbeddingPost", + "FaceRecognitionPost", + "PersonReIDPost", + "SemanticSegPost", + "build_postprocess", +] diff --git a/mblt_vision/utils/postprocess/build_post.py b/mblt_vision/utils/postprocess/build_post.py index ea0f50f..93645b2 100644 --- a/mblt_vision/utils/postprocess/build_post.py +++ b/mblt_vision/utils/postprocess/build_post.py @@ -9,7 +9,7 @@ from .cls_post import ClsPost from .damoyolo_post import DAMOYOLODetectionPost from .depth_post import DepthPost -from .embedding_post import PersonReIDPost +from .embedding_post import FaceRecognitionPost, PersonReIDPost from .semantic_seg_post import SemanticSegPost from .tiled_obb import YOLOAnchorlessTiledOBBPost, YOLODFLFreeTiledOBBPost from .yolo_anchor_post import ( @@ -72,6 +72,8 @@ def build_postprocess( f"Unsupported post_cfg.head {head!r}; expected one of {sorted(DETECTION_HEADS)}." ) return DETECTION_HEADS[head](pre_cfg, post_cfg, **kwargs) + if task == "face_recognition": + return FaceRecognitionPost(pre_cfg, post_cfg) if task == "person_reid": return PersonReIDPost(pre_cfg, post_cfg) if task == "image_classification": diff --git a/mblt_vision/utils/postprocess/embedding_post.py b/mblt_vision/utils/postprocess/embedding_post.py index 8d55353..4a449a1 100644 --- a/mblt_vision/utils/postprocess/embedding_post.py +++ b/mblt_vision/utils/postprocess/embedding_post.py @@ -83,3 +83,14 @@ def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: if not isinstance(dataset, str) or not dataset: raise ValueError("Person ReID postprocessing requires post_cfg.dataset") self.dataset = get_dataset_config_for_task("person_reid", dataset)["name"] + + +class FaceRecognitionPost(EmbeddingPost): + """One identity embedding per prepared aligned crop; no implicit flip fusion.""" + + def __init__(self, pre_cfg: dict, post_cfg: dict) -> None: + super().__init__(pre_cfg, post_cfg) + dataset = post_cfg.get("dataset") + if not isinstance(dataset, str) or not dataset: + raise ValueError("Face recognition postprocessing requires post_cfg.dataset") + self.dataset = get_dataset_config_for_task("face_recognition", dataset)["name"] diff --git a/mblt_vision/utils/results.py b/mblt_vision/utils/results.py index 7a5b7f6..0b9e3a2 100644 --- a/mblt_vision/utils/results.py +++ b/mblt_vision/utils/results.py @@ -148,7 +148,7 @@ def set_output( self.mask = None self.depth = None self.semantic_mask = None - if self.task == "person_reid": + if self.task in {"person_reid", "face_recognition"}: if not isinstance(output, (np.ndarray, torch.Tensor)): raise TypeError("Person ReID results require an embedding matrix") finite = ( diff --git a/tests/reference/ops_face_preprocess.py b/tests/reference/ops_face_preprocess.py new file mode 100644 index 0000000..62848d1 --- /dev/null +++ b/tests/reference/ops_face_preprocess.py @@ -0,0 +1,57 @@ +# Verbatim models/face_recognition/_shared/preprocess.py at ec537764e36ca10da963fcb7afab157ea60ca022 +"""Default face_recognition preprocess: an aligned face crop, normalized. + +The benchmarks of this task hand the Model already aligned 112x112 crops (five-point +similarity alignment is part of the prepared evaluation layout), so the Model-side +transform is a guard resize to the network input and an explicit ``mean``/``std`` +normalization, in HWC order. Operations run in canonical order regardless of the +sorted-key order of ``pre_cfg`` on disk. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +PIPELINE_ORDER = ("Reader", "Resize", "SetOrder", "Normalize") +REQUIRED = ("Resize", "Normalize") +_RESAMPLING = { + "bilinear": Image.Resampling.BILINEAR, + "bicubic": Image.Resampling.BICUBIC, +} + + +def preprocess(source: Path, config: dict[str, Any]) -> tuple[Path, np.ndarray, None]: + """Return ``(source, HWC float32 tensor, None)``.""" + unknown = set(config) - set(PIPELINE_ORDER) + if unknown: + raise ValueError(f"unsupported preprocess operations: {sorted(unknown)}") + missing = [name for name in REQUIRED if name not in config] + if missing: + raise ValueError(f"pre_cfg is missing required operations: {missing}") + source = Path(source) + value: Any = Image.open(source).convert("RGB") + for operation in PIPELINE_ORDER: + if operation not in config: + continue + options = config[operation] + if operation == "Reader": + continue + if operation == "Resize": + height, width = (int(v) for v in options["size"]) + if value.size != (width, height): + mode = _RESAMPLING[str(options.get("interpolation", "bilinear"))] + value = value.resize((width, height), mode) + elif operation == "SetOrder": + if options.get("shape", "HWC") != "HWC": + raise ValueError( + "face_recognition preprocess produces HWC tensors only" + ) + elif operation == "Normalize": + mean = np.asarray(options["mean"], dtype=np.float32) + std = np.asarray(options["std"], dtype=np.float32) + value = (np.asarray(value, dtype=np.float32) / 255.0 - mean) / std + return source, np.ascontiguousarray(value, dtype=np.float32), None diff --git a/tests/reference/ops_face_verification.py b/tests/reference/ops_face_verification.py new file mode 100644 index 0000000..73f0eab --- /dev/null +++ b/tests/reference/ops_face_verification.py @@ -0,0 +1,75 @@ +# Verbatim datasets/_shared/face_verification.py at ec537764e36ca10da963fcb7afab157ea60ca022 +"""Pair-based face verification protocol shared by LFW-style benchmarks. + +A benchmark lists pairs of images with a same/different label and a fold id. The Model +provides one L2-normalized embedding per image. For every fold, the decision threshold on +cosine similarity is chosen to maximize accuracy on the other folds and then applied to +the held-out fold; the benchmark reports the mean and standard deviation over folds +(Huang et al., "Labeled Faces in the Wild", unrestricted protocol as implemented by +InsightFace / facenet ``evaluate``). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +THRESHOLDS = np.linspace(-1.0, 1.0, 4001) + + +@dataclass(frozen=True) +class Pair: + image_a: str + image_b: str + is_same: bool + fold: int + + +def cosine_scores(a: np.ndarray, b: np.ndarray) -> np.ndarray: + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + norms = np.linalg.norm(a, axis=1) * np.linalg.norm(b, axis=1) + return np.sum(a * b, axis=1) / np.maximum(norms, 1e-12) + + +def _accuracy(scores: np.ndarray, labels: np.ndarray, threshold: float) -> float: + return float(np.mean((scores >= threshold) == labels)) + + +def fold_accuracies( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Held-out accuracy per fold with the threshold fitted on the remaining folds.""" + scores = np.asarray(scores, dtype=np.float64) + labels = np.asarray(labels, dtype=bool) + folds = np.asarray(folds) + fold_ids = np.unique(folds) + if len(fold_ids) < 2: + raise ValueError("Verification protocol needs at least two folds") + accuracies = [] + thresholds = [] + for fold in fold_ids: + test = folds == fold + train_scores, train_labels = scores[~test], labels[~test] + fitted = np.array( + [_accuracy(train_scores, train_labels, t) for t in THRESHOLDS] + ) + # Ties on training accuracy resolve to the strictest threshold (the highest + # similarity), matching the reference implementations that search distance + # thresholds in ascending order and keep the first maximum. + best = float(THRESHOLDS[len(fitted) - 1 - int(np.argmax(fitted[::-1]))]) + thresholds.append(best) + accuracies.append(_accuracy(scores[test], labels[test], best)) + return np.asarray(accuracies), np.asarray(thresholds) + + +def verification_result( + scores: np.ndarray, labels: np.ndarray, folds: np.ndarray +) -> dict[str, float]: + accuracies, thresholds = fold_accuracies(scores, labels, folds) + return { + "accuracy": float(np.mean(accuracies)), + "accuracy_std": float(np.std(accuracies)), + "threshold": float(np.mean(thresholds)), + } diff --git a/tests/test_api.py b/tests/test_api.py index ce184aa..924192f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -22,6 +22,7 @@ def test_public_discovery_exposes_all_supported_tasks() -> None: "pose_estimation", "face_detection", "person_reid", + "face_recognition", ] assert list_models("obb")["obb"] diff --git a/tests/test_face_manifest_review.py b/tests/test_face_manifest_review.py new file mode 100644 index 0000000..b917299 --- /dev/null +++ b/tests/test_face_manifest_review.py @@ -0,0 +1,57 @@ +"""Official fold label balance and malformed CSV readiness regressions.""" + +import csv + +import pytest +from PIL import Image + +from mblt_vision.utils.datasets import face_pairs, readiness + + +@pytest.mark.parametrize( + "name,folder,loader", + [ + ("lfw", "aligned", face_pairs.LFWDataset), + ("agedb-30", "images", face_pairs.AgeDB30Dataset), + ("cfp-fp", "images", face_pairs.CFPFPDataset), + ], +) +def test_same_different_balance_and_csv_readiness( + tmp_path, monkeypatch, name, folder, loader +): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + config = original(task, dataset) + config["pairs"] = 20 + return config + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + (tmp_path / folder).mkdir() + for image in ["a.png", "b.png"]: + Image.new("RGB", (112, 112)).save(tmp_path / folder / image) + manifest = tmp_path / "pairs.csv" + + def write(labels): + with manifest.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["pair_id", "fold_id", "is_same", "image_a", "image_b"]) + writer.writerows( + (i, i // 2 + 1, label, "a.png", "b.png") + for i, label in enumerate(labels) + ) + + write([i % 2 for i in range(20)]) + assert readiness.dataset_ready(tmp_path, "face_recognition", name) + write([1] * 20) + with pytest.raises(ValueError, match="same/different"): + loader(tmp_path) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + assert len(loader(tmp_path, validate_counts=False).pairs) == 20 + write([i % 2 for i in range(20)]) + old = csv.field_size_limit() + try: + csv.field_size_limit(4) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + finally: + csv.field_size_limit(old) diff --git a/tests/test_face_verification.py b/tests/test_face_verification.py new file mode 100644 index 0000000..bd67808 --- /dev/null +++ b/tests/test_face_verification.py @@ -0,0 +1,287 @@ +"""CPU prepared face-pair, engine and source-exact numeric contracts.""" + +import csv +import importlib.util +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch +from mblt_vision._config import load_model_variant +from mblt_vision.face_recognition import ( + AgeDB30Dataset, + AgeDB30Evaluator, + CFPEvaluator, + CFPFPDataset, + EmbeddingPost, + FacePair, + FaceRecognitionPost, + LFWDataset, + LFWEvaluator, + PairImage, + cosine_scores, + fold_accuracies, +) +from mblt_vision.utils.datasets import face_pairs, readiness +from mblt_vision.utils.postprocess import build_postprocess +from mblt_vision.utils.preprocess import build_preprocess +from mblt_vision.wrapper import MBLT_Engine +from PIL import Image + + +def reference(name): + path = Path(__file__).parent / "reference" / f"ops_face_{name}.py" + spec = importlib.util.spec_from_file_location(f"ops_face_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def write_pairs(root, folder, rows): + (root / folder).mkdir(exist_ok=True) + for name in {r[3] for r in rows} | {r[4] for r in rows}: + if "/" not in name: + Image.new("RGB", (112, 112)).save(root / folder / name) + with (root / "pairs.csv").open("w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["pair_id", "fold_id", "is_same", "image_a", "image_b"]) + writer.writerows(rows) + + +@pytest.mark.parametrize( + "loader,folder", + [(LFWDataset, "aligned"), (AgeDB30Dataset, "images"), (CFPFPDataset, "images")], +) +def test_prepared_pairs_preserve_order_folds_and_alignment(tmp_path, loader, folder): + rows = [(9, 3, 0, "b.png", "a.png"), (2, 1, 1, "a.png", "a.png")] + write_pairs(tmp_path, folder, rows) + data = loader(tmp_path, validate_counts=False) + assert [p.fold for p in data.pairs] == [3, 1] + assert [p.is_same for p in data.pairs] == [False, True] + assert [s.name for s in data.samples] == ["a.png", "b.png"] + assert isinstance(data[0], PairImage) and len(data) == 2 + with pytest.raises(ValueError, match="expected"): + loader(tmp_path) + Image.new("RGB", (113, 112)).save(tmp_path / folder / "a.png") + with pytest.raises(ValueError, match="112x112"): + loader(tmp_path, validate_counts=False) + + +@pytest.mark.parametrize( + "bad", + [ + (1, 0, 1, "a.png", "b.png"), + (1, 11, 1, "a.png", "b.png"), + (1, 1, 2, "a.png", "b.png"), + (1, 1, 1, "../a.png", "b.png"), + ], +) +def test_invalid_pair_metadata(tmp_path, bad): + write_pairs(tmp_path, "aligned", [bad]) + with pytest.raises(ValueError): + LFWDataset(tmp_path, validate_counts=False) + + +def test_duplicate_missing_and_empty_pairs(tmp_path): + with pytest.raises(FileNotFoundError): + LFWDataset(tmp_path) + write_pairs(tmp_path, "aligned", [(1, 1, 1, "a.png", "b.png")] * 2) + with pytest.raises(ValueError, match="unique"): + LFWDataset(tmp_path, validate_counts=False) + write_pairs(tmp_path, "aligned", [(1, 1, 1, "a.png", "b.png")]) + (tmp_path / "aligned/b.png").unlink() + with pytest.raises(FileNotFoundError): + LFWDataset(tmp_path, validate_counts=False) + + +def test_engine_embedding_path(): + assert issubclass(FaceRecognitionPost, EmbeddingPost) + engine = MBLT_Engine.__new__(MBLT_Engine) + engine.pre_cfg = {} + engine.post_cfg = { + "task": "face_recognition", + "dataset": "lfw", + "normalize": True, + "embedding_dim": 2, + } + engine.postprocessor = build_postprocess(engine.pre_cfg, engine.post_cfg) + assert isinstance(engine.postprocessor, FaceRecognitionPost) + result = engine.postprocess([np.array([[3.0, 4.0]], dtype=np.float32)]) + torch.testing.assert_close(result.embeddings, torch.tensor([[0.6, 0.8]])) + assert result.output is result.embeddings and result.acc is None + with pytest.raises(ValueError): + FaceRecognitionPost({}, {"dataset": "market1501"}) + with pytest.raises(ValueError): + FaceRecognitionPost({}, {}) + + +@pytest.mark.parametrize( + "loader,name,folder", + [ + (LFWDataset, "lfw", "aligned"), + (AgeDB30Dataset, "agedb-30", "images"), + (CFPFPDataset, "cfp-fp", "images"), + ], +) +def test_readiness(tmp_path, monkeypatch, loader, name, folder): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 20 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + rows = [(i, i // 2 + 1, i % 2, "a.png", "b.png") for i in range(20)] + write_pairs(tmp_path, folder, rows) + assert readiness.dataset_ready(tmp_path, "face_recognition", name) + assert not readiness.dataset_ready(tmp_path, "face_recognition", "market1501") + (tmp_path / folder / "a.png").write_bytes(b"broken") + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + + +@pytest.mark.parametrize( + "folder,name", [("aligned", "lfw"), ("images", "agedb-30"), ("images", "cfp-fp")] +) +@pytest.mark.parametrize("component", ["root", "images", "manifest"]) +def test_readiness_rejects_symlink_ancestors( + tmp_path, monkeypatch, folder, name, component +): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 20 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + root = tmp_path / "cache" + root.mkdir() + write_pairs( + root, folder, [(i, i // 2 + 1, i % 2, "a.png", "b.png") for i in range(20)] + ) + assert readiness.dataset_ready(root, "face_recognition", name) + if component == "root": + alias = tmp_path / "alias" + alias.symlink_to(root, target_is_directory=True) + root = alias + else: + source = root / (folder if component == "images" else "pairs.csv") + target = tmp_path / "outside" + source.rename(target) + source.symlink_to(target, target_is_directory=component == "images") + assert not readiness.dataset_ready(root, "face_recognition", name) + + +@pytest.mark.parametrize("seed", range(5)) +def test_ops_metric_exact_parity(seed): + ops = reference("verification") + rng = np.random.default_rng(seed) + a = rng.integers(-2, 3, (80, 8)).astype(np.float32) + b = rng.integers(-2, 3, (80, 8)).astype(np.float32) + labels = rng.integers(0, 2, 80).astype(bool) + folds = np.tile(np.arange(1, 11), 8) + expected_scores = ops.cosine_scores(a, b) + scores = cosine_scores(a, b) + np.testing.assert_array_equal(scores, expected_scores) + expected_acc, expected_thresholds = ops.fold_accuracies( + expected_scores, labels, folds + ) + acc, thresholds = fold_accuracies(scores, labels, folds) + np.testing.assert_array_equal(acc, expected_acc) + np.testing.assert_array_equal(thresholds, expected_thresholds) + expected = ops.verification_result(scores, labels, folds) + assert float(np.mean(acc)) == expected["accuracy"] + assert float(np.std(acc)) == expected["accuracy_std"] + assert float(np.mean(thresholds)) == expected["threshold"] + + +def test_threshold_ties_and_inclusive_boundary(): + acc, thresholds = fold_accuracies( + np.zeros(4), np.array([0, 1, 0, 1]), np.array([1, 1, 2, 2]) + ) + np.testing.assert_array_equal(thresholds, [1.0, 1.0]) + np.testing.assert_array_equal(acc, [0.5, 0.5]) + # Exactly equal to threshold counts as same. + scores = cosine_scores(np.ones((2, 2)), np.ones((2, 2))) + np.testing.assert_allclose(scores, 1.0) + with pytest.raises(ValueError): + fold_accuracies(np.ones(2), np.ones(2), np.ones(2, dtype=int)) + + +@pytest.mark.parametrize("evaluator", [LFWEvaluator, AgeDB30Evaluator, CFPEvaluator]) +def test_evaluator_mapping_and_no_implicit_fusion(evaluator): + images = [PairImage(Path(n), n) for n in ["a", "b", "c"]] + pairs = [ + FacePair("a", "b", True, 1), + FacePair("a", "c", False, 1), + FacePair("a", "b", True, 2), + FacePair("a", "c", False, 2), + ] + embeddings = np.array([[1.0, 0.0], [1.0, 0.0], [-1.0, 0.0]], dtype=np.float32) + result = evaluator().evaluate(embeddings, images, pairs) + assert result.primary_score == 1.0 and result.secondary_score == 0.0 + assert result.as_dict()["primary"][evaluator.metric] == 1.0 + assert result.sample_count == 3 + with pytest.raises(ValueError): + evaluator().evaluate(embeddings[:2], images, pairs) + with pytest.raises(ValueError): + evaluator().evaluate(embeddings, images, [FacePair("a", "missing", True, 1)]) + + +@pytest.mark.parametrize("shape", [(112, 112), (35, 51), (241, 179)]) +def test_ops_preprocessing_byte_parity(tmp_path, shape): + ops = reference("preprocess") + cfg = load_model_variant( + Path(__file__).parents[1] / "mblt_vision/models/bases/face_recognition.yaml", + "DEFAULT", + ) + path = tmp_path / "synthetic.png" + Image.fromarray( + np.random.default_rng(0).integers(0, 256, (*shape, 3), dtype=np.uint8) + ).save(path) + expected = ops.preprocess(path, cfg["pre_cfg"])[1] + actual = build_preprocess(cfg["pre_cfg"])(path) + np.testing.assert_array_equal( + actual, expected + ) # atol=0, including no-op 112x112 resize + assert np.max(np.abs(actual - expected)) == 0.0 + + +@pytest.mark.parametrize( + "name,loader,folder", + [ + ("lfw", LFWDataset, "aligned"), + ("agedb-30", AgeDB30Dataset, "images"), + ("cfp-fp", CFPFPDataset, "images"), + ], +) +def test_rejects_correct_total_with_unbalanced_official_folds( + tmp_path, monkeypatch, name, loader, folder +): + original = face_pairs.get_dataset_config_for_task + + def tiny(task, dataset): + cfg = original(task, dataset) + cfg["pairs"] = 20 + return cfg + + monkeypatch.setattr(face_pairs, "get_dataset_config_for_task", tiny) + rows = [(i, i // 2 + 1, i % 2, "a.png", "b.png") for i in range(20)] + write_pairs(tmp_path, folder, rows) + assert readiness.dataset_ready(tmp_path, "face_recognition", name) + rows[0] = ( + 0, + 2, + 0, + "a.png", + "b.png", + ) # 20 total, all ten folds, but 1/3 instead of 2/2. + write_pairs(tmp_path, folder, rows) + with pytest.raises(ValueError, match="official fold"): + loader(tmp_path) + assert not readiness.dataset_ready(tmp_path, "face_recognition", name) + assert len(loader(tmp_path, validate_counts=False).pairs) == 20