dtoro
fa9b21856f
Face detection/recognition: - Replace YuNet + SFace with InsightFace buffalo_l (RetinaFace + ArcFace) - 512-d ArcFace embeddings (was 128-d SFace), migration 0006 resizes column - Remove YOLO person-bbox workaround — RetinaFace is accurate enough - Detection threshold 0.65 cleanly separates real faces (0.72+) from false positives on dogs/paintings (0.56-0.61) Content-type classification: - CLIP zero-shot classifier using native PyTorch text encoder + ONNX image encoder for high-quality text-image similarity - Categories: photograph, screenshot, document, receipt, meme, artwork - Writes Tag(kind=content_type) per photo via photo_tags - Margin-based confidence: top-1 vs top-2 score difference - New ClassifierSettings in config (enabled, min_confidence) - Wired into vision_fanout pipeline Tested: 6 real faces from 4 photos (zero false positives), 11/13 photos classified (8 photograph, 2 artwork, 1 meme). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
107 lines
3.5 KiB
Python
107 lines
3.5 KiB
Python
"""
|
|
CLIP zero-shot content-type classifier.
|
|
|
|
Uses the native OpenCLIP PyTorch text encoder for high-quality text
|
|
embeddings (the ONNX text encoder has degraded quality due to the
|
|
eot_indices workaround). Image embeddings use the ONNX visual encoder
|
|
which works well.
|
|
"""
|
|
import logging
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from app.config import VisionSettings
|
|
from app.services.vision.base import ContentClassifier, ClassificationResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
CATEGORY_PROMPTS = {
|
|
"screenshot": [
|
|
"a screenshot of a computer screen",
|
|
"a screenshot of a phone screen",
|
|
"a screen capture of a user interface",
|
|
],
|
|
"document": [
|
|
"a scanned document",
|
|
"a photo of a document with printed text",
|
|
"a photo of a page of text on paper",
|
|
],
|
|
"receipt": [
|
|
"a photo of a receipt",
|
|
"a photo of a bill or invoice",
|
|
],
|
|
"meme": [
|
|
"an internet meme with text overlay",
|
|
"a funny image with caption text",
|
|
],
|
|
"artwork": [
|
|
"a painting or drawing",
|
|
"a sketch or illustration",
|
|
"digital art or graphic design",
|
|
],
|
|
"photograph": [
|
|
"a photograph taken with a camera",
|
|
"a real photo of a real scene or person",
|
|
"a candid photograph",
|
|
],
|
|
}
|
|
|
|
|
|
class CLIPContentClassifier(ContentClassifier):
|
|
"""Zero-shot content classifier using CLIP text-image similarity.
|
|
Uses native PyTorch for text encoding, ONNX for image encoding."""
|
|
|
|
def __init__(self, settings: VisionSettings):
|
|
import open_clip
|
|
|
|
self._min_confidence = settings.classifier.min_confidence
|
|
|
|
# Load native model for text encoding only
|
|
logger.info("Loading OpenCLIP text encoder for content classification")
|
|
model, _, _ = open_clip.create_model_and_transforms(
|
|
"ViT-B-32", pretrained="laion2b_s34b_b79k"
|
|
)
|
|
model.eval()
|
|
self._model = model
|
|
self._tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
|
|
|
# Get the ONNX image embedder from the registry
|
|
from app.services.vision.registry import registry
|
|
self._embedder = registry.get_embedder()
|
|
|
|
# Pre-compute text embeddings for each category
|
|
self._category_embeddings: dict[str, np.ndarray] = {}
|
|
for category, prompts in CATEGORY_PROMPTS.items():
|
|
tokens = self._tokenizer(prompts)
|
|
with torch.no_grad():
|
|
text_features = model.encode_text(tokens)
|
|
text_features /= text_features.norm(dim=-1, keepdim=True)
|
|
avg = text_features.mean(dim=0)
|
|
avg /= avg.norm()
|
|
self._category_embeddings[category] = avg.numpy().astype(np.float32)
|
|
|
|
logger.info("Content classifier ready with %d categories", len(self._category_embeddings))
|
|
|
|
def classify(self, image: np.ndarray) -> list[ClassificationResult]:
|
|
img_vec = self._embedder.embed_image(image)
|
|
|
|
# Cosine similarity against each category
|
|
scores = {}
|
|
for category, cat_vec in self._category_embeddings.items():
|
|
scores[category] = float(np.dot(img_vec, cat_vec))
|
|
|
|
# Sort by score descending
|
|
ranked = sorted(scores.items(), key=lambda x: -x[1])
|
|
best_cat, best_score = ranked[0]
|
|
second_score = ranked[1][1]
|
|
|
|
margin = best_score - second_score
|
|
# Normalize: 0.01 margin → ~0.5 confidence, 0.03+ → ~1.0
|
|
confidence = min(1.0, margin * 30)
|
|
|
|
if confidence >= self._min_confidence:
|
|
return [ClassificationResult(label=best_cat, confidence=confidence)]
|
|
|
|
return []
|