dtoro
9282a5c734
Introduce the app/services/vision/ module with ABC interfaces, ONNX Runtime backend, model registry, and per-task implementations: - OpenCLIP ViT-B/32 embedder (image + text, 512-d) - RapidOCR engine (PP-OCRv4 via ONNX, no PaddlePaddle) - YOLOv8n object detector (raw ONNX, no ultralytics runtime) - YuNet + SFace face processor (Apache 2.0, opencv_zoo, 128-d) - DBSCAN face clustering helper Add VisionSettings to config (mulita.yml + Pydantic), bootstrap_models.py for first-boot weight downloads, models_data Docker volume, and ROCm backend stub for future GPU acceleration. No Celery tasks wired yet — models load but nothing invokes them. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
90 lines
2.3 KiB
Python
90 lines
2.3 KiB
Python
"""
|
|
Abstract base classes for vision backends.
|
|
|
|
Each ABC defines the contract a backend must satisfy. The default
|
|
implementation is ONNXBackend (onnx_backend.py). A ROCm backend can be
|
|
added later by subclassing these ABCs and registering via
|
|
settings.vision.backend.
|
|
"""
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
import numpy as np
|
|
|
|
|
|
@dataclass
|
|
class DetectionBox:
|
|
"""A single object detection result."""
|
|
label: str
|
|
confidence: float
|
|
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
|
|
|
|
|
@dataclass
|
|
class OCRResult:
|
|
"""A single OCR text region."""
|
|
text: str
|
|
confidence: float
|
|
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
|
language: str = ""
|
|
|
|
|
|
@dataclass
|
|
class FaceDetection:
|
|
"""A detected face with its recognition embedding."""
|
|
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
|
embedding: np.ndarray # float32 vector (128-d for SFace)
|
|
quality: float
|
|
|
|
|
|
class Embedder(ABC):
|
|
"""Generates image and text embeddings (e.g. OpenCLIP ViT-B/32)."""
|
|
|
|
@abstractmethod
|
|
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
|
"""Return a normalized float32 embedding vector for an RGB image."""
|
|
...
|
|
|
|
@abstractmethod
|
|
def embed_text(self, text: str) -> np.ndarray:
|
|
"""Return a normalized float32 embedding vector for a text query."""
|
|
...
|
|
|
|
@property
|
|
@abstractmethod
|
|
def dim(self) -> int:
|
|
"""Dimensionality of the output embedding."""
|
|
...
|
|
|
|
|
|
class OCREngine(ABC):
|
|
"""Extracts text from images (e.g. rapidocr-onnxruntime)."""
|
|
|
|
@abstractmethod
|
|
def run(self, image: np.ndarray) -> list[OCRResult]:
|
|
"""Return OCR results for an RGB image."""
|
|
...
|
|
|
|
|
|
class ObjectDetector(ABC):
|
|
"""Detects objects in images (e.g. YOLOv8n)."""
|
|
|
|
@abstractmethod
|
|
def detect(self, image: np.ndarray) -> list[DetectionBox]:
|
|
"""Return detections for an RGB image."""
|
|
...
|
|
|
|
|
|
class FaceProcessor(ABC):
|
|
"""Detects faces and extracts recognition embeddings (e.g. YuNet + SFace)."""
|
|
|
|
@abstractmethod
|
|
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
|
"""Return face detections with embeddings for an RGB image."""
|
|
...
|
|
|
|
@property
|
|
@abstractmethod
|
|
def embedding_dim(self) -> int:
|
|
"""Dimensionality of face embedding vectors."""
|
|
...
|