From bb574888a54dcfa160c1bee3e2c30060c9adf530 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:39:48 +0530 Subject: [PATCH 1/7] fix(models): letterbox YOLO input to preserve aspect ratio Naive 640x640 resize distorted faces and shrank small ones below detectability in large photos. Boxes are unpadded/unscaled back and clipped to image bounds. --- backend/app/models/YOLO.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/backend/app/models/YOLO.py b/backend/app/models/YOLO.py index 879015e7a..e0cced296 100644 --- a/backend/app/models/YOLO.py +++ b/backend/app/models/YOLO.py @@ -120,8 +120,22 @@ def get_output_details(self): def prepare_input(self, image): self.img_height, self.img_width = image.shape[:2] input_img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - input_img = cv2.resize(input_img, (self.input_width, self.input_height)) - input_img = input_img / 255.0 + # Letterbox: resize preserving aspect ratio, pad the rest with gray + self.scale = min( + self.input_width / self.img_width, self.input_height / self.img_height + ) + new_w = round(self.img_width * self.scale) + new_h = round(self.img_height * self.scale) + self.pad_x = (self.input_width - new_w) // 2 + self.pad_y = (self.input_height - new_h) // 2 + resized = cv2.resize(input_img, (new_w, new_h)) + padded = np.full( + (self.input_height, self.input_width, 3), 114, dtype=input_img.dtype + ) + padded[self.pad_y : self.pad_y + new_h, self.pad_x : self.pad_x + new_w] = ( + resized + ) + input_img = padded / 255.0 input_img = input_img.transpose(2, 0, 1) input_tensor = input_img[np.newaxis, :, :, :].astype(np.float32) return input_tensor @@ -145,16 +159,16 @@ def extract_boxes(self, predictions): boxes = predictions[:, :4] boxes = self.rescale_boxes(boxes) boxes = YOLO_util_xywh2xyxy(boxes) + boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, self.img_width) + boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, self.img_height) return boxes def rescale_boxes(self, boxes): - input_shape = np.array( - [self.input_width, self.input_height, self.input_width, self.input_height] - ) - boxes = np.divide(boxes, input_shape, dtype=np.float32) - boxes *= np.array( - [self.img_width, self.img_height, self.img_width, self.img_height] - ) + # Undo the letterbox: remove padding offset, then unscale (boxes are xywh) + boxes = boxes.astype(np.float32).copy() + boxes[:, 0] = (boxes[:, 0] - self.pad_x) / self.scale + boxes[:, 1] = (boxes[:, 1] - self.pad_y) / self.scale + boxes[:, 2:4] /= self.scale return boxes def draw_detections(self, image, draw_scores=True, mask_alpha=0.4): From fcebd73b1548d0094fbf564c8245d68633b8a683 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:40:02 +0530 Subject: [PATCH 2/7] fix(faces): relax quality gate and remove group-photo cap Blur threshold 80 rejected clean studio portraits (Laplacian variance 40-75); face detection was skipped entirely for photos with 7+ people. Lower blur default to 20, min face area to 1000px2, drop the cap. --- backend/app/config/settings.py | 5 +++-- backend/app/utils/images.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/app/config/settings.py b/backend/app/config/settings.py index 0c200e2c1..7ff8994b0 100644 --- a/backend/app/config/settings.py +++ b/backend/app/config/settings.py @@ -180,9 +180,10 @@ def _get_env_int( PICTO_CLUSTERING_CONF_THRESHOLD = _get_env_float( "PICTO_CLUSTERING_CONF_THRESHOLD", 0.45, min_value=0.0, max_value=1.0 ) +# Laplacian variance is low (~40-75) on clean studio portraits, so keep this permissive. PICTO_CLUSTERING_BLUR_THRESHOLD = _get_env_float( - "PICTO_CLUSTERING_BLUR_THRESHOLD", 80.0, min_value=0.0 + "PICTO_CLUSTERING_BLUR_THRESHOLD", 20.0, min_value=0.0 ) PICTO_CLUSTERING_MIN_FACE_SIZE = _get_env_int( - "PICTO_CLUSTERING_MIN_FACE_SIZE", 1600, min_value=1 + "PICTO_CLUSTERING_MIN_FACE_SIZE", 1000, min_value=1 ) diff --git a/backend/app/utils/images.py b/backend/app/utils/images.py index 02ae73f44..dde078fff 100644 --- a/backend/app/utils/images.py +++ b/backend/app/utils/images.py @@ -217,7 +217,7 @@ def image_util_classify_and_face_detect_images( db_insert_image_classes_batch(image_class_pairs) # Step 3: Detect faces if "person" class is present - if classes and 0 in classes and 0 < classes.count(0) < 7: + if classes and 0 in classes: result = face_detector.detect_faces(image_id, image_path) if result: total_faces_skipped += result.get("faces_skipped", 0) From 033095a0a408521f76a06de7267e710594579eee Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:40:07 +0530 Subject: [PATCH 3/7] fix(db): rebuild semantic label tables on stale schema A semantic_labels table created by another schema version (no label_id) breaks the image_semantic_labels FK with 'foreign key mismatch' on any cascading delete, blocking folder deletion. Both tables are derived data, so drop and recreate them. --- backend/app/database/semantic_labels.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/app/database/semantic_labels.py b/backend/app/database/semantic_labels.py index 3c6467dc5..789ddd2d6 100644 --- a/backend/app/database/semantic_labels.py +++ b/backend/app/database/semantic_labels.py @@ -7,6 +7,14 @@ def db_create_semantic_labels_table(): conn = _connect() cursor = conn.cursor() + # A semantic_labels table from another schema version (no label_id column) + # breaks the image_semantic_labels FK with "foreign key mismatch" on any + # cascading delete. Both tables hold derived data, so rebuild them. + cols = [r[1] for r in cursor.execute("PRAGMA table_info(semantic_labels)")] + if cols and "label_id" not in cols: + cursor.execute("DROP TABLE IF EXISTS image_semantic_labels") + cursor.execute("DROP TABLE semantic_labels") + cursor.execute( """ CREATE TABLE IF NOT EXISTS semantic_labels ( From 93559651701b9bc8742bd9302b1141470e2f8677 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:41:29 +0530 Subject: [PATCH 4/7] fix(clusters): keep at most one face per photo per cluster Faces co-occurring in one photo are different people, but degraded embeddings of tiny faces clustered together, making a single image appear multiple times in a cluster. Enforce the cannot-link constraint after clustering (centroid-closest face wins), guard incremental assignment the same way, and dedupe the cluster image listing. --- backend/app/database/face_clusters.py | 8 +++- backend/app/database/faces.py | 27 +++++++++-- backend/app/utils/face_clusters.py | 69 +++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/backend/app/database/face_clusters.py b/backend/app/database/face_clusters.py index 9ba273263..8dd7d16a7 100644 --- a/backend/app/database/face_clusters.py +++ b/backend/app/database/face_clusters.py @@ -311,13 +311,15 @@ def db_get_images_by_cluster_id( FROM images i INNER JOIN faces f ON i.id = f.image_id WHERE f.cluster_id = ? - ORDER BY i.path + ORDER BY i.path, f.confidence DESC """, (cluster_id,), ) rows = cursor.fetchall() + # One row per image: keep only the highest-confidence face + seen_image_ids = set() images = [] for row in rows: ( @@ -330,6 +332,10 @@ def db_get_images_by_cluster_id( bbox_json, ) = row + if image_id in seen_image_ids: + continue + seen_image_ids.add(image_id) + import json metadata_dict = json.loads(metadata) if metadata else None diff --git a/backend/app/database/faces.py b/backend/app/database/faces.py index 4099e7aec..e4301ec30 100644 --- a/backend/app/database/faces.py +++ b/backend/app/database/faces.py @@ -230,16 +230,20 @@ def db_get_faces_unassigned_clusters() -> List[Dict[str, Union[FaceId, FaceEmbed cursor = conn.cursor() try: - cursor.execute("SELECT face_id, embeddings FROM faces WHERE cluster_id IS NULL") + cursor.execute( + "SELECT face_id, image_id, embeddings FROM faces WHERE cluster_id IS NULL" + ) rows = cursor.fetchall() faces = [] for row in rows: - face_id, embeddings_json = row + face_id, image_id, embeddings_json = row # Convert JSON string back to numpy array embeddings = np.array(json.loads(embeddings_json)) - faces.append({"face_id": face_id, "embeddings": embeddings}) + faces.append( + {"face_id": face_id, "image_id": image_id, "embeddings": embeddings} + ) return faces finally: @@ -261,7 +265,7 @@ def db_get_all_faces_with_cluster_names() -> ( try: cursor.execute( """ - SELECT f.face_id, f.embeddings, fc.cluster_name + SELECT f.face_id, f.image_id, f.embeddings, fc.cluster_name FROM faces f LEFT JOIN face_clusters fc ON f.cluster_id = fc.cluster_id ORDER BY f.face_id @@ -272,12 +276,13 @@ def db_get_all_faces_with_cluster_names() -> ( faces = [] for row in rows: - face_id, embeddings_json, cluster_name = row + face_id, image_id, embeddings_json, cluster_name = row # Convert JSON string back to numpy array embeddings = np.array(json.loads(embeddings_json)) faces.append( { "face_id": face_id, + "image_id": image_id, "embeddings": embeddings, "cluster_name": cluster_name, } @@ -352,6 +357,18 @@ def db_update_face_cluster_ids_batch( conn.close() +def db_get_cluster_image_pairs() -> set: + """Distinct (cluster_id, image_id) pairs for all cluster-assigned faces.""" + conn = sqlite3.connect(DATABASE_PATH) + try: + rows = conn.execute( + "SELECT DISTINCT cluster_id, image_id FROM faces WHERE cluster_id IS NOT NULL" + ).fetchall() + return set(rows) + finally: + conn.close() + + def db_get_cluster_mean_embeddings() -> List[Dict[str, Union[str, FaceEmbedding]]]: """ Get cluster IDs and their corresponding mean face embeddings. diff --git a/backend/app/utils/face_clusters.py b/backend/app/utils/face_clusters.py index fc5ea3c2e..632c9678f 100644 --- a/backend/app/utils/face_clusters.py +++ b/backend/app/utils/face_clusters.py @@ -21,6 +21,7 @@ db_update_face_cluster_ids_batch, db_get_faces_unassigned_clusters, db_get_cluster_mean_embeddings, + db_get_cluster_image_pairs, ) from app.database.face_clusters import db_delete_all_clusters, db_insert_clusters_batch from app.database.metadata import ( @@ -235,6 +236,7 @@ def cluster_util_cluster_all_face_embeddings( # Extract embeddings and face IDs with validation embeddings = [] face_ids = [] + image_ids = [] existing_cluster_names = [] invalid_count = 0 @@ -244,6 +246,7 @@ def cluster_util_cluster_all_face_embeddings( # Validate embedding before adding if _validate_embedding(embedding): face_ids.append(face["face_id"]) + image_ids.append(face.get("image_id")) embeddings.append(embedding) existing_cluster_names.append(face["cluster_name"]) else: @@ -360,6 +363,10 @@ def cluster_util_cluster_all_face_embeddings( results, merge_threshold=effective_merge_threshold ) + # Cannot-link constraint: faces co-occurring in one photo are different people + face_to_image = dict(zip(face_ids, image_ids)) + results = _enforce_one_face_per_image(results, face_to_image) + return results, total_faces_skipped @@ -422,6 +429,9 @@ def cluster_util_assign_cluster_to_faces_without_clusterId( mean_embeddings_array = np.array(mean_embeddings) + # (cluster_id, image_id) pairs already taken; a photo's faces are distinct people + occupied_pairs = db_get_cluster_image_pairs() + # Prepare batch update data face_cluster_mappings = [] skipped_invalid = 0 @@ -454,9 +464,18 @@ def cluster_util_assign_cluster_to_faces_without_clusterId( nearest_cluster_idx = np.argmin(distances) nearest_cluster_id = cluster_ids[nearest_cluster_idx] + image_id = face.get("image_id") + if ( + image_id is not None + and (nearest_cluster_id, image_id) in occupied_pairs + ): + continue + face_cluster_mappings.append( {"face_id": face_id, "cluster_id": nearest_cluster_id} ) + if image_id is not None: + occupied_pairs.add((nearest_cluster_id, image_id)) if skipped_invalid > 0: logger.warning( @@ -468,6 +487,56 @@ def cluster_util_assign_cluster_to_faces_without_clusterId( return face_cluster_mappings, total_faces_skipped +def _enforce_one_face_per_image( + results: List[ClusterResult], face_to_image: Dict[int, Optional[str]] +) -> List[ClusterResult]: + """ + Keep at most one face per image in each cluster: co-occurring faces belong + to different people. The face closest to the cluster centroid wins; the + rest are dropped from results (left unclustered). + """ + by_cluster = defaultdict(list) + for result in results: + by_cluster[result.cluster_uuid].append(result) + + kept = [] + dropped = 0 + for cluster_results in by_cluster.values(): + centroid = np.mean([r.embedding for r in cluster_results], axis=0) + centroid_norm = np.linalg.norm(centroid) + + best_per_image: Dict[str, Tuple[float, ClusterResult]] = {} + for result in cluster_results: + image_id = face_to_image.get(result.face_id) + if image_id is None: + kept.append(result) + continue + + emb_norm = np.linalg.norm(result.embedding) + if centroid_norm < 1e-6 or emb_norm < 1e-6: + similarity = 0.0 + else: + similarity = float( + np.dot(result.embedding, centroid) / (emb_norm * centroid_norm) + ) + + prev = best_per_image.get(image_id) + if prev is None: + best_per_image[image_id] = (similarity, result) + else: + dropped += 1 + if similarity > prev[0]: + best_per_image[image_id] = (similarity, result) + + kept.extend(result for _, result in best_per_image.values()) + + if dropped: + logger.info( + f"Cannot-link constraint: dropped {dropped} same-image duplicate face(s) from clusters" + ) + return kept + + def _merge_similar_clusters( results: List[ClusterResult], merge_threshold: float = 0.85 ) -> List[ClusterResult]: From 3f75c5f09b8c96c163b77626616b6d8c2e38a3ab Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:41:48 +0530 Subject: [PATCH 5/7] feat(clusters): rank clusters by confidence-weighted prominence Order the people view by avg detection confidence x log2(1+face_count) instead of random UUID order, so frequently photographed, clearly detected people appear first. --- backend/app/database/face_clusters.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/backend/app/database/face_clusters.py b/backend/app/database/face_clusters.py index 8dd7d16a7..3c4ed5ded 100644 --- a/backend/app/database/face_clusters.py +++ b/backend/app/database/face_clusters.py @@ -1,3 +1,4 @@ +import math import sqlite3 from typing import Optional, List, Dict, TypedDict, Union from app.config.settings import DATABASE_PATH @@ -241,6 +242,9 @@ def db_get_all_clusters_with_face_counts() -> ( ): """ Retrieve all clusters with their face counts and stored face images. + Ordered by prominence: avg detection confidence weighted by log of face count, + so frequently-photographed people (likely the device owner) rank first + without letting large low-quality clusters outrank clean ones. Returns: List of dictionaries containing cluster_id, cluster_name, face_count, and face_image_base64 @@ -251,15 +255,15 @@ def db_get_all_clusters_with_face_counts() -> ( try: cursor.execute( """ - SELECT - fc.cluster_id, - fc.cluster_name, + SELECT + fc.cluster_id, + fc.cluster_name, COUNT(f.face_id) as face_count, - fc.face_image_base64 + fc.face_image_base64, + COALESCE(AVG(f.confidence), 0) as avg_confidence FROM face_clusters fc LEFT JOIN faces f ON fc.cluster_id = f.cluster_id GROUP BY fc.cluster_id, fc.cluster_name, fc.face_image_base64 - ORDER BY fc.cluster_id """ ) @@ -267,16 +271,20 @@ def db_get_all_clusters_with_face_counts() -> ( clusters = [] for row in rows: - cluster_id, cluster_name, face_count, face_image_base64 = row + cluster_id, cluster_name, face_count, face_image_base64, avg_conf = row clusters.append( { "cluster_id": cluster_id, "cluster_name": cluster_name, "face_count": face_count, "face_image_base64": face_image_base64, + "_score": avg_conf * math.log2(1 + face_count), } ) + clusters.sort(key=lambda c: c["_score"], reverse=True) + for cluster in clusters: + del cluster["_score"] return clusters finally: conn.close() From 9dae7222f221b4f2c0ca2e734450d9c7e6165312 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:41:54 +0530 Subject: [PATCH 6/7] fix(clusters): return consistent tuple when no faces exist cluster_util_cluster_all_face_embeddings returned a bare list on an empty library, crashing the caller's tuple unpack. --- backend/app/utils/face_clusters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/utils/face_clusters.py b/backend/app/utils/face_clusters.py index 632c9678f..4dc608ee5 100644 --- a/backend/app/utils/face_clusters.py +++ b/backend/app/utils/face_clusters.py @@ -231,7 +231,7 @@ def cluster_util_cluster_all_face_embeddings( faces_data = db_get_all_faces_with_cluster_names() if not faces_data: - return [] + return [], 0 # Extract embeddings and face IDs with validation embeddings = [] From 35a922aa4e1ed2f944244cf898e5ccdb1311eaab Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:42:00 +0530 Subject: [PATCH 7/7] chore: ignore semantic vocabulary source caches --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index b5f42f6ca..530e3da08 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ videos_cache.txt venv/ frontend/dist env/ + +# semantic vocabulary source downloads (cache for scripts/build_semantic_vocabulary.py) +backend/scripts/vocabulary/sources/ +backend/scripts/vocabulary/eval_images/