feat: add vision pipeline scaffolding with ONNX backend
Introduce the app/services/vision/ module with ABC interfaces, ONNX Runtime backend, model registry, and per-task implementations: - OpenCLIP ViT-B/32 embedder (image + text, 512-d) - RapidOCR engine (PP-OCRv4 via ONNX, no PaddlePaddle) - YOLOv8n object detector (raw ONNX, no ultralytics runtime) - YuNet + SFace face processor (Apache 2.0, opencv_zoo, 128-d) - DBSCAN face clustering helper Add VisionSettings to config (mulita.yml + Pydantic), bootstrap_models.py for first-boot weight downloads, models_data Docker volume, and ROCm backend stub for future GPU acceleration. No Celery tasks wired yet — models load but nothing invokes them. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
45
backend/app/services/vision/ocr.py
Normal file
45
backend/app/services/vision/ocr.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
OCR engine using rapidocr-onnxruntime (PP-OCRv4 weights).
|
||||
|
||||
No PaddlePaddle dependency — pure ONNX Runtime. Language packs are
|
||||
downloaded automatically by rapidocr on first use.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import OCREngine, OCRResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RapidOCREngine(OCREngine):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
self._min_confidence = settings.ocr.min_confidence
|
||||
self._engine = RapidOCR()
|
||||
logger.info("RapidOCR engine initialized")
|
||||
|
||||
def run(self, image: np.ndarray) -> list[OCRResult]:
|
||||
result, _ = self._engine(image)
|
||||
if not result:
|
||||
return []
|
||||
|
||||
out = []
|
||||
for box, text, score in result:
|
||||
if score < self._min_confidence:
|
||||
continue
|
||||
# box is [[x1,y1],[x2,y2],[x3,y3],[x4,y4]] — take bounding rect
|
||||
xs = [p[0] for p in box]
|
||||
ys = [p[1] for p in box]
|
||||
h, w = image.shape[:2]
|
||||
bbox = [
|
||||
min(xs) / w,
|
||||
min(ys) / h,
|
||||
max(xs) / w,
|
||||
max(ys) / h,
|
||||
]
|
||||
out.append(OCRResult(text=text, confidence=float(score), bbox=bbox))
|
||||
return out
|
||||
Reference in New Issue
Block a user