feat: replace face pipeline with InsightFace, add content classifier
Face detection/recognition: - Replace YuNet + SFace with InsightFace buffalo_l (RetinaFace + ArcFace) - 512-d ArcFace embeddings (was 128-d SFace), migration 0006 resizes column - Remove YOLO person-bbox workaround — RetinaFace is accurate enough - Detection threshold 0.65 cleanly separates real faces (0.72+) from false positives on dogs/paintings (0.56-0.61) Content-type classification: - CLIP zero-shot classifier using native PyTorch text encoder + ONNX image encoder for high-quality text-image similarity - Categories: photograph, screenshot, document, receipt, meme, artwork - Writes Tag(kind=content_type) per photo via photo_tags - Margin-based confidence: top-1 vs top-2 score difference - New ClassifierSettings in config (enabled, min_confidence) - Wired into vision_fanout pipeline Tested: 6 real faces from 4 photos (zero false positives), 11/13 photos classified (8 photograph, 2 artwork, 1 meme). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
70
backend/app/services/vision/insightface_processor.py
Normal file
70
backend/app/services/vision/insightface_processor.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Face detection + recognition using InsightFace (RetinaFace + ArcFace).
|
||||
|
||||
Uses the buffalo_l model pack which auto-downloads on first use (~300MB).
|
||||
Produces 512-d ArcFace embeddings. Non-commercial research license —
|
||||
fine for homelab self-hosting.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import FaceProcessor, FaceDetection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InsightFaceProcessor(FaceProcessor):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
from insightface.app import FaceAnalysis
|
||||
|
||||
model_root = str(Path(settings.models_dir) / "face" / "insightface")
|
||||
logger.info("Loading InsightFace buffalo_l from %s", model_root)
|
||||
|
||||
self._app = FaceAnalysis(
|
||||
name="buffalo_l",
|
||||
root=model_root,
|
||||
providers=["CPUExecutionProvider"],
|
||||
)
|
||||
self._app.prepare(ctx_id=-1, det_size=(640, 640))
|
||||
self._min_det_score = settings.faces.recognition_threshold
|
||||
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
|
||||
# InsightFace expects BGR
|
||||
bgr = image[:, :, ::-1].copy()
|
||||
|
||||
faces = self._app.get(bgr)
|
||||
|
||||
if not faces:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for face in faces:
|
||||
if face.det_score < self._min_det_score:
|
||||
continue
|
||||
|
||||
# face.bbox is [x1, y1, x2, y2] in pixel coords
|
||||
x1, y1, x2, y2 = face.bbox
|
||||
bbox = [
|
||||
max(0, float(x1) / orig_w),
|
||||
max(0, float(y1) / orig_h),
|
||||
min(1, float(x2) / orig_w),
|
||||
min(1, float(y2) / orig_h),
|
||||
]
|
||||
|
||||
embedding = face.normed_embedding # already L2-normalized, 512-d
|
||||
results.append(FaceDetection(
|
||||
bbox=bbox,
|
||||
embedding=embedding.astype(np.float32),
|
||||
quality=float(face.det_score),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return 512
|
||||
Reference in New Issue
Block a user