dtoro
fa9b21856f
Face detection/recognition: - Replace YuNet + SFace with InsightFace buffalo_l (RetinaFace + ArcFace) - 512-d ArcFace embeddings (was 128-d SFace), migration 0006 resizes column - Remove YOLO person-bbox workaround — RetinaFace is accurate enough - Detection threshold 0.65 cleanly separates real faces (0.72+) from false positives on dogs/paintings (0.56-0.61) Content-type classification: - CLIP zero-shot classifier using native PyTorch text encoder + ONNX image encoder for high-quality text-image similarity - Categories: photograph, screenshot, document, receipt, meme, artwork - Writes Tag(kind=content_type) per photo via photo_tags - Margin-based confidence: top-1 vs top-2 score difference - New ClassifierSettings in config (enabled, min_confidence) - Wired into vision_fanout pipeline Tested: 6 real faces from 4 photos (zero false positives), 11/13 photos classified (8 photograph, 2 artwork, 1 meme). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
106 lines
2.7 KiB
Python
106 lines
2.7 KiB
Python
"""
|
|
Abstract base classes for vision backends.
|
|
|
|
Each ABC defines the contract a backend must satisfy. The default
|
|
implementation is ONNXBackend (onnx_backend.py). A ROCm backend can be
|
|
added later by subclassing these ABCs and registering via
|
|
settings.vision.backend.
|
|
"""
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
import numpy as np
|
|
|
|
|
|
@dataclass
|
|
class DetectionBox:
|
|
"""A single object detection result."""
|
|
label: str
|
|
confidence: float
|
|
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
|
|
|
|
|
@dataclass
|
|
class OCRResult:
|
|
"""A single OCR text region."""
|
|
text: str
|
|
confidence: float
|
|
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
|
language: str = ""
|
|
|
|
|
|
@dataclass
|
|
class FaceDetection:
|
|
"""A detected face with its recognition embedding."""
|
|
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
|
embedding: np.ndarray # float32 vector (128-d for SFace)
|
|
quality: float
|
|
|
|
|
|
@dataclass
|
|
class ClassificationResult:
|
|
"""A content-type classification."""
|
|
label: str
|
|
confidence: float
|
|
|
|
|
|
class Embedder(ABC):
|
|
"""Generates image and text embeddings (e.g. OpenCLIP ViT-B/32)."""
|
|
|
|
@abstractmethod
|
|
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
|
"""Return a normalized float32 embedding vector for an RGB image."""
|
|
...
|
|
|
|
@abstractmethod
|
|
def embed_text(self, text: str) -> np.ndarray:
|
|
"""Return a normalized float32 embedding vector for a text query."""
|
|
...
|
|
|
|
@property
|
|
@abstractmethod
|
|
def dim(self) -> int:
|
|
"""Dimensionality of the output embedding."""
|
|
...
|
|
|
|
|
|
class OCREngine(ABC):
|
|
"""Extracts text from images (e.g. rapidocr-onnxruntime)."""
|
|
|
|
@abstractmethod
|
|
def run(self, image: np.ndarray) -> list[OCRResult]:
|
|
"""Return OCR results for an RGB image."""
|
|
...
|
|
|
|
|
|
class ObjectDetector(ABC):
|
|
"""Detects objects in images (e.g. YOLOv8n)."""
|
|
|
|
@abstractmethod
|
|
def detect(self, image: np.ndarray) -> list[DetectionBox]:
|
|
"""Return detections for an RGB image."""
|
|
...
|
|
|
|
|
|
class ContentClassifier(ABC):
|
|
"""Classifies images into content types (screenshot, document, etc.)."""
|
|
|
|
@abstractmethod
|
|
def classify(self, image: np.ndarray) -> list[ClassificationResult]:
|
|
"""Return content type classifications for an RGB image."""
|
|
...
|
|
|
|
|
|
class FaceProcessor(ABC):
|
|
"""Detects faces and extracts recognition embeddings (e.g. YuNet + SFace)."""
|
|
|
|
@abstractmethod
|
|
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
|
"""Return face detections with embeddings for an RGB image."""
|
|
...
|
|
|
|
@property
|
|
@abstractmethod
|
|
def embedding_dim(self) -> int:
|
|
"""Dimensionality of face embedding vectors."""
|
|
...
|