feat: add vision pipeline scaffolding with ONNX backend
Introduce the app/services/vision/ module with ABC interfaces, ONNX Runtime backend, model registry, and per-task implementations: - OpenCLIP ViT-B/32 embedder (image + text, 512-d) - RapidOCR engine (PP-OCRv4 via ONNX, no PaddlePaddle) - YOLOv8n object detector (raw ONNX, no ultralytics runtime) - YuNet + SFace face processor (Apache 2.0, opencv_zoo, 128-d) - DBSCAN face clustering helper Add VisionSettings to config (mulita.yml + Pydantic), bootstrap_models.py for first-boot weight downloads, models_data Docker volume, and ROCm backend stub for future GPU acceleration. No Celery tasks wired yet — models load but nothing invokes them. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
77
backend/app/services/vision/registry.py
Normal file
77
backend/app/services/vision/registry.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""
|
||||
ModelRegistry — singleton that lazy-loads vision models per worker process.
|
||||
|
||||
Usage from Celery tasks:
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
vec = embedder.embed_image(img)
|
||||
|
||||
Models are created on first access and cached for the worker's lifetime.
|
||||
The registry reads settings.vision to decide which backend to use and
|
||||
where model weights live.
|
||||
"""
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
|
||||
from app.config import settings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ModelRegistry:
|
||||
"""Central access point for all vision models."""
|
||||
|
||||
def __init__(self):
|
||||
self._vision = settings.vision
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_embedder(self) -> Embedder:
|
||||
logger.info("Loading embedder: %s (backend=%s)", self._vision.embedder.name, self._vision.backend)
|
||||
return self._load_backend().create_embedder()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_ocr(self) -> OCREngine:
|
||||
logger.info("Loading OCR engine (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_ocr()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_detector(self) -> ObjectDetector:
|
||||
logger.info("Loading object detector (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_detector()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_face_processor(self) -> FaceProcessor:
|
||||
logger.info("Loading face processor (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_face_processor()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_backend(self):
|
||||
"""Import and instantiate the configured backend."""
|
||||
backend_name = self._vision.backend
|
||||
if backend_name == "onnx":
|
||||
from app.services.vision.onnx_backend import ONNXBackend
|
||||
return ONNXBackend(self._vision)
|
||||
elif backend_name == "rocm":
|
||||
from app.services.vision.rocm_backend import ROCmBackend
|
||||
return ROCmBackend(self._vision)
|
||||
else:
|
||||
raise ValueError(f"Unknown vision backend: {backend_name}")
|
||||
|
||||
def warmup(self):
|
||||
"""Pre-load all enabled models. Called from Celery worker_process_init
|
||||
on the vision queue to avoid cold-start latency on the first task."""
|
||||
logger.info("Warming up vision models...")
|
||||
self.get_embedder()
|
||||
if self._vision.ocr.enabled:
|
||||
self.get_ocr()
|
||||
if self._vision.detector.enabled:
|
||||
self.get_detector()
|
||||
if self._vision.faces.enabled:
|
||||
self.get_face_processor()
|
||||
logger.info("Vision model warmup complete")
|
||||
|
||||
|
||||
# Module-level singleton. Import this from tasks.
|
||||
registry = ModelRegistry()
|
||||
Reference in New Issue
Block a user