refactor: strip AI pipeline to binary photo/other classifier
Drops face recognition, OCR, object detection, and semantic embeddings. The sole remaining vision task is a CLIP-based binary classifier (photography vs other); photos in "other" get needs_review=true so screenshots, documents, memes and scans can be triaged from a new filter pill in the UI. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -38,7 +38,6 @@ from app.models import ( # noqa: E402, F401
|
||||
Tag,
|
||||
Heap,
|
||||
HeapPhoto,
|
||||
Embedding,
|
||||
)
|
||||
|
||||
config = context.config
|
||||
|
||||
65
backend/alembic/versions/0012_strip_ai_pipeline.py
Normal file
65
backend/alembic/versions/0012_strip_ai_pipeline.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""Strip AI pipeline to binary classifier only
|
||||
|
||||
Revision ID: 0012_strip_ai
|
||||
Revises: 0011_sharing
|
||||
Create Date: 2026-04-14
|
||||
|
||||
Removes face recognition, OCR, object detection, and semantic embeddings.
|
||||
The remaining AI is a single binary 'photography' vs 'other' classifier
|
||||
whose output feeds Tag(kind='content_type') and a new Photo.needs_review
|
||||
flag.
|
||||
|
||||
Drops: embeddings, face_embeddings, ocr_text tables.
|
||||
Drops: photo_tags rows produced by 'vision:yolov8n' and 'vision:sface'.
|
||||
Drops: tags with kind IN ('object','scene','face_cluster').
|
||||
Drops: tags.representative_photo_id column.
|
||||
Adds: photos.needs_review (bool, default false) + partial index.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0012_strip_ai"
|
||||
down_revision: Union[str, None] = "0011_sharing"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Drop dropped-AI tables. CASCADE clears any lingering FKs/indices.
|
||||
op.execute("DROP TABLE IF EXISTS embeddings CASCADE")
|
||||
op.execute("DROP TABLE IF EXISTS face_embeddings CASCADE")
|
||||
op.execute("DROP TABLE IF EXISTS ocr_text CASCADE")
|
||||
|
||||
# Clear ML-produced photo_tags rows and their parent tags.
|
||||
op.execute(
|
||||
"DELETE FROM photo_tags WHERE source IN ('vision:yolov8n','vision:sface')"
|
||||
)
|
||||
op.execute(
|
||||
"DELETE FROM tags WHERE kind IN ('object','scene','face_cluster')"
|
||||
)
|
||||
|
||||
# Drop the face-cluster representative column.
|
||||
op.execute("ALTER TABLE tags DROP COLUMN IF EXISTS representative_photo_id")
|
||||
|
||||
# Add the needs_review flag.
|
||||
op.execute(
|
||||
"ALTER TABLE photos ADD COLUMN IF NOT EXISTS needs_review "
|
||||
"BOOLEAN NOT NULL DEFAULT false"
|
||||
)
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS ix_photos_needs_review "
|
||||
"ON photos(needs_review) WHERE needs_review"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Data is not recoverable on downgrade — only the schema stubs are
|
||||
# put back so a future reinstall of the old pipeline can re-populate.
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_needs_review")
|
||||
op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS needs_review")
|
||||
|
||||
op.execute(
|
||||
"ALTER TABLE tags ADD COLUMN IF NOT EXISTS representative_photo_id "
|
||||
"VARCHAR REFERENCES photos(id) ON DELETE SET NULL"
|
||||
)
|
||||
35
backend/alembic/versions/0013_drop_old_content_types.py
Normal file
35
backend/alembic/versions/0013_drop_old_content_types.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""Drop legacy content_type tags from the 6-category classifier
|
||||
|
||||
Revision ID: 0013_drop_old_ct
|
||||
Revises: 0012_strip_ai
|
||||
Create Date: 2026-04-14
|
||||
|
||||
The previous classifier wrote Tag(kind='content_type', name IN
|
||||
('photograph','screenshot','document','receipt','meme','artwork')).
|
||||
The new binary classifier writes names ('photography','other'). Both
|
||||
coexisted after the cutover so users saw duplicate groupings like
|
||||
'photography' alongside 'photograph'. Drop the old names — photo_tags
|
||||
rows cascade-delete via the FK.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0013_drop_old_ct"
|
||||
down_revision: Union[str, None] = "0012_strip_ai"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
LEGACY_NAMES = ('photograph', 'screenshot', 'document', 'receipt', 'meme', 'artwork')
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"DELETE FROM tags WHERE kind = 'content_type' "
|
||||
f"AND name IN {LEGACY_NAMES}"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -30,51 +30,16 @@ class PerformanceSettings(BaseModel):
|
||||
db_pool_max_overflow: int = 10
|
||||
db_pool_recycle: int = 3600
|
||||
|
||||
class EmbedderSettings(BaseModel):
|
||||
"""CLIP / SigLIP embedding model settings.
|
||||
Supported: "openclip_vitb32" (512-d), "siglip2_vitb16" (768-d, default)."""
|
||||
name: str = "openclip_vitb32"
|
||||
batch_size: int = 8
|
||||
|
||||
class OCRSettings(BaseModel):
|
||||
"""PaddleOCR / rapidocr settings"""
|
||||
enabled: bool = True
|
||||
languages: list[str] = ["en"]
|
||||
min_confidence: float = 0.5
|
||||
|
||||
class DetectorSettings(BaseModel):
|
||||
"""YOLOv8n object detection settings"""
|
||||
enabled: bool = True
|
||||
min_confidence: float = 0.35
|
||||
max_detections: int = 50
|
||||
|
||||
class FacesSettings(BaseModel):
|
||||
"""YuNet + SFace face detection/recognition settings"""
|
||||
enabled: bool = True
|
||||
min_face_size: int = 40
|
||||
recognition_threshold: float = 0.65
|
||||
cluster_eps: float = 0.5
|
||||
|
||||
class ClassifierSettings(BaseModel):
|
||||
"""CLIP zero-shot content classification settings"""
|
||||
enabled: bool = True
|
||||
"""Binary content classifier (photography vs other)."""
|
||||
min_confidence: float = 0.3
|
||||
|
||||
class VisionSettings(BaseModel):
|
||||
"""AI vision pipeline settings. Disabled when running on SQLite
|
||||
(pgvector is required for embedding storage)."""
|
||||
"""Vision pipeline — one binary classifier (photography vs other)."""
|
||||
enabled: bool = True
|
||||
backend: str = "onnx" # "onnx" | "rocm" (future)
|
||||
backend: str = "onnx"
|
||||
models_dir: str = "/data/models"
|
||||
# ONNX Runtime execution providers in priority order.
|
||||
# Auto-detected at startup; falls back to CPU if GPU is unavailable.
|
||||
# Options: "CUDAExecutionProvider", "ROCMExecutionProvider",
|
||||
# "OpenVINOExecutionProvider", "CPUExecutionProvider"
|
||||
execution_providers: list[str] = ["CPUExecutionProvider"]
|
||||
embedder: EmbedderSettings = EmbedderSettings()
|
||||
ocr: OCRSettings = OCRSettings()
|
||||
detector: DetectorSettings = DetectorSettings()
|
||||
faces: FacesSettings = FacesSettings()
|
||||
classifier: ClassifierSettings = ClassifierSettings()
|
||||
worker_concurrency: int = 2
|
||||
|
||||
|
||||
@@ -111,13 +111,7 @@ async def init_db():
|
||||
"""Initialize database, create tables if they don't exist"""
|
||||
async with engine.begin() as conn:
|
||||
# Import all models to register them with Base
|
||||
from app.models import User, Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto, Embedding
|
||||
|
||||
# Postgres: ensure pgvector is available before create_all touches
|
||||
# any Vector columns (added in later PRs but the extension is cheap
|
||||
# and idempotent to create now).
|
||||
if _is_postgres:
|
||||
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
|
||||
from app.models import User, Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto # noqa: F401
|
||||
|
||||
# Create all tables. Note: create_all only creates *missing* tables —
|
||||
# it does NOT add new columns to existing tables when the model gains
|
||||
|
||||
@@ -6,9 +6,6 @@ from app.models.photos import Photo
|
||||
from app.models.folders import Folder, SourceRoot
|
||||
from app.models.tags import Tag, PhotoTag
|
||||
from app.models.heaps import Heap, HeapPhoto
|
||||
from app.models.embeddings import Embedding
|
||||
from app.models.ocr_text import OCRText
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
from app.models.sharing import HeapShare, FolderShare
|
||||
|
||||
__all__ = [
|
||||
@@ -20,9 +17,6 @@ __all__ = [
|
||||
'PhotoTag',
|
||||
'Heap',
|
||||
'HeapPhoto',
|
||||
'Embedding',
|
||||
'OCRText',
|
||||
'FaceEmbedding',
|
||||
'HeapShare',
|
||||
'FolderShare',
|
||||
]
|
||||
]
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
"""
|
||||
Embedding model — stores CLIP/SigLIP image embeddings via pgvector.
|
||||
|
||||
Composite PK (photo_id, model) allows re-embedding with newer models
|
||||
without clobbering old vectors.
|
||||
|
||||
Vector dimension is 768 to support SigLIP2 ViT-B/16 (the default).
|
||||
OpenCLIP ViT-B/32 (512-d) embeddings are zero-padded on insert so
|
||||
both models coexist in the same column. The padding is invisible to
|
||||
cosine similarity (zeros don't affect the angle).
|
||||
"""
|
||||
from sqlalchemy import Column, String, ForeignKey, DateTime, func
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Embedding(Base):
|
||||
__tablename__ = 'embeddings'
|
||||
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True)
|
||||
model = Column(String(64), primary_key=True) # e.g. 'siglip2_vitb16'
|
||||
vector = Column(Vector(512)) # OpenCLIP ViT-B/32 → 512-d
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -1,24 +0,0 @@
|
||||
"""
|
||||
Face embedding model — stores per-face detection + recognition vectors.
|
||||
|
||||
cluster_id FKs to tags.id where kind='face_cluster'. Null means
|
||||
unclustered (will be assigned by recluster_faces).
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, DateTime, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from pgvector.sqlalchemy import Vector
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class FaceEmbedding(Base):
|
||||
__tablename__ = 'face_embeddings'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
bbox = Column(JSONB) # [x1, y1, x2, y2] normalized 0-1
|
||||
vector = Column(Vector(512)) # ArcFace → 512-d
|
||||
cluster_id = Column(String, ForeignKey('tags.id', ondelete='SET NULL'), nullable=True, index=True)
|
||||
quality = Column(Float)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -1,20 +0,0 @@
|
||||
"""
|
||||
OCR text model — stores text regions extracted from photos via rapidocr.
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, Text, DateTime, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class OCRText(Base):
|
||||
__tablename__ = 'ocr_text'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
text = Column(Text, nullable=False)
|
||||
language = Column(String(8), default='')
|
||||
confidence = Column(Float)
|
||||
bbox = Column(JSONB) # [x1, y1, x2, y2] normalized 0-1
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -50,6 +50,11 @@ class Photo(Base):
|
||||
# rows, and POST /folders/{id}/hide recomputes it on toggle.
|
||||
is_hidden = Column(Boolean, nullable=False, default=False, server_default='false', index=True)
|
||||
|
||||
# "Needs review" — set by the content classifier when a photo is
|
||||
# classified as 'other' (screenshot, document, meme, scan, etc.) so
|
||||
# the user can page through non-photographs in the UI and triage them.
|
||||
needs_review = Column(Boolean, nullable=False, default=False, server_default='false', index=True)
|
||||
|
||||
# "Capture date is probably wrong" — denormalized from the folder/filename
|
||||
# date-guesser. Set at scan time and recomputed on every taken_at edit so
|
||||
# the filter bar can query it directly. See services/date_guess.py for
|
||||
@@ -118,4 +123,5 @@ class Photo(Base):
|
||||
Index('ix_photos_media_type', 'media_type'),
|
||||
Index('ix_photos_processing_status', 'processing_status'),
|
||||
Index('ix_photos_lat_lon', 'latitude', 'longitude'),
|
||||
Index('ix_photos_needs_review', 'needs_review'),
|
||||
)
|
||||
@@ -6,7 +6,7 @@ labels, and face clusters via the `kind` column. The `photo_tags`
|
||||
association carries per-photo ML metadata (confidence, bounding box,
|
||||
source model).
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, Table, Index, UniqueConstraint
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, Table, Index, UniqueConstraint # noqa: F401
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
import uuid
|
||||
@@ -22,7 +22,7 @@ photo_tags = Table(
|
||||
# ML metadata — null for user-applied tags
|
||||
Column('confidence', Float, nullable=True),
|
||||
Column('bbox', JSONB, nullable=True), # [x1, y1, x2, y2] normalized 0-1
|
||||
Column('source', String, nullable=True), # e.g. "vision:yolov8n", "vision:sface"
|
||||
Column('source', String, nullable=True), # e.g. "vision:clip_classifier"
|
||||
Index('ix_photo_tags_photo_id', 'photo_id'),
|
||||
Index('ix_photo_tags_tag_id', 'tag_id'),
|
||||
)
|
||||
@@ -42,16 +42,11 @@ class Tag(Base):
|
||||
|
||||
# Tag classification
|
||||
kind = Column(String, nullable=False, default='user', index=True)
|
||||
# kind values: 'user' | 'object' | 'scene' | 'face_cluster'
|
||||
# kind values: 'user' | 'content_type'
|
||||
|
||||
# Which model produced this tag (null for user-created)
|
||||
source = Column(String, nullable=True)
|
||||
# e.g. "vision:yolov8n", "vision:sface", null
|
||||
|
||||
# For face clusters: the photo used as the cluster representative thumbnail
|
||||
representative_photo_id = Column(
|
||||
String, ForeignKey('photos.id', ondelete='SET NULL'), nullable=True
|
||||
)
|
||||
# e.g. "vision:clip_classifier", null
|
||||
|
||||
# Relationships
|
||||
photos = relationship("Photo", secondary=photo_tags, backref="tags")
|
||||
|
||||
@@ -320,13 +320,8 @@ async def update_feature_flag(
|
||||
|
||||
|
||||
class BackfillVisionBody(BaseModel):
|
||||
"""POST body for triggering a vision backfill. ``task`` picks a
|
||||
specific stage (``embed`` / ``ocr`` / ``detect`` / ``faces`` /
|
||||
``classify``); leaving it null runs every enabled stage. ``limit``
|
||||
caps how many photos per stage are queued — useful for smoke-
|
||||
testing a newly-enabled feature before committing a full run.
|
||||
"""
|
||||
task: Optional[str] = None
|
||||
"""POST body for triggering a classifier backfill. ``limit`` caps how
|
||||
many photos are queued."""
|
||||
limit: Optional[int] = None
|
||||
|
||||
|
||||
@@ -335,61 +330,29 @@ async def trigger_ai_backfill(
|
||||
body: BackfillVisionBody,
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Queue a vision backfill pass. Identical code path as the automatic
|
||||
post-scan backfill — just triggered manually from the UI."""
|
||||
"""Queue a classifier backfill pass."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Vision is currently disabled; enable it before running a backfill.",
|
||||
)
|
||||
valid_tasks = {'embed', 'ocr', 'detect', 'faces', 'classify'}
|
||||
if body.task is not None and body.task not in valid_tasks:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"task must be one of {sorted(valid_tasks)} or null",
|
||||
)
|
||||
if body.limit is not None and body.limit <= 0:
|
||||
raise HTTPException(status_code=400, detail="limit must be positive")
|
||||
|
||||
# Import lazily so importing admin.py doesn't pull in the whole
|
||||
# vision stack on startup (Celery task module loads numpy etc.).
|
||||
from app.tasks.vision import backfill_vision
|
||||
|
||||
result = backfill_vision.apply_async(
|
||||
kwargs={'task': body.task, 'limit': body.limit}
|
||||
)
|
||||
result = backfill_vision.apply_async(kwargs={'limit': body.limit})
|
||||
logger.info(
|
||||
f"Admin '{admin.username}' queued vision backfill "
|
||||
f"(task={body.task}, limit={body.limit}, celery_id={result.id})"
|
||||
f"(limit={body.limit}, celery_id={result.id})"
|
||||
)
|
||||
return {
|
||||
"status": "queued",
|
||||
"task_id": result.id,
|
||||
"task": body.task,
|
||||
"limit": body.limit,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/ai/recluster-faces")
|
||||
async def trigger_face_recluster(admin: User = Depends(require_admin)):
|
||||
"""Kick off face recluster. Normally auto-fires after a scan via a
|
||||
debounced scheduler; this endpoint is for admins who want to force
|
||||
a fresh clustering pass (e.g. after tweaking ``cluster_eps`` in
|
||||
the YAML config)."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Vision is currently disabled; enable it before reclustering.",
|
||||
)
|
||||
from app.tasks.vision import recluster_faces
|
||||
|
||||
result = recluster_faces.apply_async()
|
||||
logger.info(
|
||||
f"Admin '{admin.username}' queued face recluster (celery_id={result.id})"
|
||||
)
|
||||
return {"status": "queued", "task_id": result.id}
|
||||
|
||||
|
||||
@router.post("/ai/rescan")
|
||||
async def trigger_full_rescan(admin: User = Depends(require_admin)):
|
||||
"""Dispatch the same scan_all_source_roots job the backend runs at
|
||||
|
||||
@@ -89,6 +89,12 @@ async def get_library_stats(
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
needs_review_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(visible, Photo.needs_review.is_(True))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
# Legacy split (kept for the existing /stats consumers).
|
||||
photo_count = (
|
||||
await db.execute(
|
||||
@@ -120,6 +126,7 @@ async def get_library_stats(
|
||||
"with_gps": with_gps_count,
|
||||
"duplicates": duplicates_count,
|
||||
"discarded": discarded_count,
|
||||
"needs_review": needs_review_count,
|
||||
"total_photos": photo_count,
|
||||
"total_videos": video_count,
|
||||
"total_size": size,
|
||||
@@ -437,10 +444,7 @@ async def get_worker_status(
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
r.ping()
|
||||
broker_ok = True
|
||||
# `vision` is the big one — embed / classify / detect / ocr /
|
||||
# extract_faces all land here, so it's where backlogs actually
|
||||
# pile up. Leaving it off the dashboard made it look like the
|
||||
# queue was always empty while the worker was clearly busy.
|
||||
# `vision` runs the content classifier — the only heavy queue.
|
||||
for q in ('default', 'high', 'low', 'vision'):
|
||||
try:
|
||||
queue_depths[q] = int(r.llen(q) or 0)
|
||||
@@ -525,8 +529,6 @@ async def get_pipeline_stats(
|
||||
Keep the shape flat + serialisable; the frontend turns it straight
|
||||
into a list of rows without needing to know about the models.
|
||||
"""
|
||||
from app.config import settings as _settings
|
||||
from app.models import Embedding, FaceEmbedding, OCRText
|
||||
from app.models.tags import photo_tags # association Table, not a model
|
||||
|
||||
owner = _owner_filter(current_user, scope)
|
||||
@@ -572,45 +574,15 @@ async def get_pipeline_stats(
|
||||
)
|
||||
)
|
||||
|
||||
# Embeddings: count distinct photos that have a row for the currently
|
||||
# configured embedder model. A photo can have multiple model rows
|
||||
# (historical re-embeds) so COUNT(DISTINCT) is the right thing here.
|
||||
embedder_model = _settings.vision.embedder.name
|
||||
embeddings_done = await scalar_count(
|
||||
select(func.count(func.distinct(Embedding.photo_id)))
|
||||
.select_from(Embedding)
|
||||
.join(Photo, Photo.id == Embedding.photo_id)
|
||||
.where(not_discarded, Embedding.model == embedder_model)
|
||||
)
|
||||
|
||||
tagged_photos = await scalar_count(
|
||||
# Classified: distinct photos with a content_type tag.
|
||||
classified_done = await scalar_count(
|
||||
select(func.count(func.distinct(photo_tags.c.photo_id)))
|
||||
.select_from(photo_tags)
|
||||
.join(Photo, Photo.id == photo_tags.c.photo_id)
|
||||
.where(not_discarded)
|
||||
.where(not_discarded, photo_tags.c.source == 'vision:clip_classifier')
|
||||
)
|
||||
ocr_done = await scalar_count(
|
||||
select(func.count(func.distinct(OCRText.photo_id)))
|
||||
.select_from(OCRText)
|
||||
.join(Photo, Photo.id == OCRText.photo_id)
|
||||
.where(not_discarded)
|
||||
)
|
||||
|
||||
# Faces: photos that have at least one face_embeddings row. A photo
|
||||
# with no faces legitimately finishes face extraction with zero rows,
|
||||
# so this undercounts by exactly "images with no visible people". We
|
||||
# surface the photo-with-faces count rather than "images scanned for
|
||||
# faces" because the latter isn't tracked anywhere.
|
||||
faces_photos = await scalar_count(
|
||||
select(func.count(func.distinct(FaceEmbedding.photo_id)))
|
||||
.select_from(FaceEmbedding)
|
||||
.join(Photo, Photo.id == FaceEmbedding.photo_id)
|
||||
.where(not_discarded)
|
||||
)
|
||||
face_rows = await scalar_count(select(func.count(FaceEmbedding.id)))
|
||||
face_clusters = await scalar_count(
|
||||
select(func.count(func.distinct(FaceEmbedding.cluster_id)))
|
||||
.where(FaceEmbedding.cluster_id.is_not(None))
|
||||
needs_review_count = await scalar_count(
|
||||
select(func.count(Photo.id)).where(not_discarded, Photo.needs_review.is_(True))
|
||||
)
|
||||
|
||||
duplicate_groups = await scalar_count(
|
||||
@@ -657,43 +629,11 @@ async def get_pipeline_stats(
|
||||
"hint": "Feeds duplicate detection.",
|
||||
},
|
||||
{
|
||||
"key": "embeddings",
|
||||
"label": f"Embeddings ({embedder_model})",
|
||||
"done": embeddings_done,
|
||||
"key": "classification",
|
||||
"label": "Content classification (photo vs other)",
|
||||
"done": classified_done,
|
||||
"total": total_images,
|
||||
"hint": "Semantic search + content classification.",
|
||||
},
|
||||
{
|
||||
"key": "tags",
|
||||
"label": "Object tags (YOLO)",
|
||||
"done": tagged_photos,
|
||||
"total": total_images,
|
||||
"hint": "Auto-generated object labels. Not every photo has a detectable object.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "ocr",
|
||||
"label": "OCR text",
|
||||
"done": ocr_done,
|
||||
"total": total_images,
|
||||
"hint": "Extracted text from screenshots / documents. Many photos have none.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "faces",
|
||||
"label": "Face detection",
|
||||
"done": faces_photos,
|
||||
"total": total_images,
|
||||
"hint": f"{face_rows} face rows detected across {faces_photos} photos.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "face_clusters",
|
||||
"label": "Face clusters",
|
||||
"done": face_clusters,
|
||||
"total": face_clusters, # no meaningful "total" — it's just the current count
|
||||
"hint": "Built by recluster_faces. Run it after backfill to populate the People view.",
|
||||
"standalone": True,
|
||||
"hint": f"{needs_review_count} photos flagged for review.",
|
||||
},
|
||||
{
|
||||
"key": "duplicates",
|
||||
@@ -708,7 +648,6 @@ async def get_pipeline_stats(
|
||||
return {
|
||||
"total_photos": total_photos,
|
||||
"total_images": total_images,
|
||||
"embedder_model": embedder_model,
|
||||
"stages": stages,
|
||||
}
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ async def list_photos(
|
||||
color_label: Optional[str] = None,
|
||||
is_discarded: Optional[bool] = False,
|
||||
is_duplicate: Optional[bool] = None,
|
||||
needs_review: Optional[bool] = None,
|
||||
has_date_warning: Optional[bool] = None,
|
||||
heap_id: Optional[str] = None,
|
||||
sort: str = "taken_at",
|
||||
@@ -196,6 +197,8 @@ async def list_photos(
|
||||
# view shows everything regardless of duplicate status.
|
||||
if is_duplicate is not None:
|
||||
filters.append(Photo.is_duplicate == is_duplicate)
|
||||
if needs_review is not None:
|
||||
filters.append(Photo.needs_review == needs_review)
|
||||
if has_date_warning is not None:
|
||||
filters.append(Photo.has_date_warning == has_date_warning)
|
||||
|
||||
|
||||
@@ -25,13 +25,7 @@ class SearchRequest(BaseModel):
|
||||
|
||||
@router.post("")
|
||||
async def search_photos(body: SearchRequest, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Unified search endpoint. Every query runs hybrid (FTS + semantic)
|
||||
by default — the user never picks a mode.
|
||||
|
||||
Filters:
|
||||
- tag_ids: list of tag IDs (any kind: user, object, face_cluster)
|
||||
- date_from / date_to: ISO date strings
|
||||
"""
|
||||
"""FTS search over photo metadata with optional tag and date filters."""
|
||||
filters = body.filters or {}
|
||||
|
||||
results = await hybrid_search(
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
"""
|
||||
Tags API router.
|
||||
|
||||
Unified across user tags, ML-detected objects, and face clusters via
|
||||
the `kind` query parameter. Default behaviour (no kind filter) returns
|
||||
all tags — the frontend's "Hide auto-generated tags" toggle filters
|
||||
client-side or passes `kind=user`.
|
||||
Unified across user tags and the binary content-type classifier
|
||||
('photography' | 'other') via the `kind` column.
|
||||
"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
@@ -34,15 +32,11 @@ class TagUpdate(BaseModel):
|
||||
color: Optional[str] = None
|
||||
|
||||
|
||||
class TagMerge(BaseModel):
|
||||
target_id: str # tag to merge INTO
|
||||
|
||||
|
||||
# ── Endpoints ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("")
|
||||
async def list_tags(
|
||||
kind: Optional[str] = Query(None, description="Filter by kind: user, object, scene, face_cluster"),
|
||||
kind: Optional[str] = Query(None, description="Filter by kind: user, content_type"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
@@ -90,7 +84,7 @@ async def list_tags(
|
||||
"color": tag.color,
|
||||
"kind": tag.kind,
|
||||
"source": tag.source,
|
||||
"representative_photo_id": tag.representative_photo_id or first_photo_id,
|
||||
"representative_photo_id": first_photo_id,
|
||||
"photo_count": int(count or 0),
|
||||
}
|
||||
for tag, count, first_photo_id in rows
|
||||
@@ -130,7 +124,7 @@ async def update_tag(
|
||||
tag_id: str, body: TagUpdate, db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Rename or recolor a tag (works for any kind — user, object, face_cluster)."""
|
||||
"""Rename or recolor a tag."""
|
||||
result = await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id))
|
||||
tag = result.scalar_one_or_none()
|
||||
if not tag:
|
||||
@@ -149,52 +143,6 @@ async def update_tag(
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "kind": tag.kind}
|
||||
|
||||
|
||||
@router.post("/{tag_id}/merge")
|
||||
async def merge_tag(
|
||||
tag_id: str, body: TagMerge, db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Merge tag_id INTO target_id. All photo associations from the source
|
||||
tag are moved to the target, then the source tag is deleted.
|
||||
|
||||
Useful for merging auto-detected face clusters (e.g. "Person 3" → "Alice")
|
||||
or merging duplicate object labels."""
|
||||
if tag_id == body.target_id:
|
||||
raise HTTPException(status_code=400, detail="Cannot merge a tag into itself")
|
||||
|
||||
source = (await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id))).scalar_one_or_none()
|
||||
target = (await db.execute(select(Tag).where(Tag.id == body.target_id, Tag.user_id == current_user.id))).scalar_one_or_none()
|
||||
if not source:
|
||||
raise HTTPException(status_code=404, detail="Source tag not found")
|
||||
if not target:
|
||||
raise HTTPException(status_code=404, detail="Target tag not found")
|
||||
|
||||
# Move photo associations: update tag_id from source → target.
|
||||
# Skip any that would violate the PK (photo already tagged with target).
|
||||
existing_target_photos = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.tag_id == body.target_id
|
||||
)
|
||||
await db.execute(
|
||||
update(photo_tags)
|
||||
.where(
|
||||
photo_tags.c.tag_id == tag_id,
|
||||
photo_tags.c.photo_id.notin_(existing_target_photos),
|
||||
)
|
||||
.values(tag_id=body.target_id)
|
||||
)
|
||||
# Delete remaining source associations (duplicates that couldn't move)
|
||||
from sqlalchemy import delete as sa_delete
|
||||
await db.execute(
|
||||
sa_delete(photo_tags).where(photo_tags.c.tag_id == tag_id)
|
||||
)
|
||||
|
||||
# Delete source tag
|
||||
await db.delete(source)
|
||||
await db.commit()
|
||||
|
||||
return {"merged_into": target.id, "target_name": target.name}
|
||||
|
||||
|
||||
@router.delete("/{tag_id}", status_code=204)
|
||||
async def delete_tag(tag_id: str, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Delete a tag. Photo associations cascade-delete via the FK."""
|
||||
|
||||
@@ -39,6 +39,7 @@ class PhotoResponse(PhotoBase):
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
is_duplicate: bool = False
|
||||
needs_review: bool = False
|
||||
has_date_warning: bool = False
|
||||
live_photo_video_id: Optional[str] = None
|
||||
owner_username: Optional[str] = None
|
||||
|
||||
@@ -41,12 +41,10 @@ import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select, update, text
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.photos import Photo
|
||||
from app.models.embeddings import Embedding
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -54,11 +52,6 @@ logger = logging.getLogger(__name__)
|
||||
# pHash Hamming distance threshold (6 out of 64 bits).
|
||||
DEFAULT_PHASH_THRESHOLD = 6
|
||||
|
||||
# CLIP cosine distance threshold. CLIP embeddings are L2-normalized,
|
||||
# so cosine distance = 1 - dot(a, b). A threshold of 0.08 catches
|
||||
# visually near-identical shots; 0.15 catches similar compositions.
|
||||
DEFAULT_CLIP_THRESHOLD = 0.10
|
||||
|
||||
|
||||
def _hex_to_int(h: str) -> int:
|
||||
"""Parse a 16-char hex pHash to a Python int. Returns -1 on bad input
|
||||
@@ -119,14 +112,12 @@ class _UnionFind:
|
||||
|
||||
async def regroup_duplicates(
|
||||
phash_threshold: int = DEFAULT_PHASH_THRESHOLD,
|
||||
clip_threshold: float = DEFAULT_CLIP_THRESHOLD,
|
||||
**_ignored,
|
||||
) -> dict:
|
||||
"""Full recompute of duplicate groups using pHash + CLIP similarity.
|
||||
"""Full recompute of duplicate groups using pHash similarity.
|
||||
|
||||
Idempotent — safe to call as often as you like. Returns a summary dict.
|
||||
"""
|
||||
embedder_model = settings.vision.embedder.name
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Pull all visible photos with a phash or embedding.
|
||||
rows = (
|
||||
@@ -162,15 +153,6 @@ async def regroup_duplicates(
|
||||
if _hamming(hi, hj) <= phash_threshold:
|
||||
uf.union_by_key(phash_ids[i], phash_ids[j])
|
||||
|
||||
# ── Phase 2: CLIP similarity via pgvector ──
|
||||
# For each photo with an embedding, find its nearest neighbors
|
||||
# within the cosine distance threshold using the HNSW index.
|
||||
clip_matches = await _clip_neighbor_scan(
|
||||
session, ids, embedder_model, clip_threshold
|
||||
)
|
||||
for photo_id, neighbor_id in clip_matches:
|
||||
uf.union_by_key(photo_id, neighbor_id)
|
||||
|
||||
# ── Write results ──
|
||||
await _clear_all_groups(session)
|
||||
|
||||
@@ -202,16 +184,9 @@ async def regroup_duplicates(
|
||||
async def incremental_regroup(
|
||||
since: Optional[datetime] = None,
|
||||
phash_threshold: int = DEFAULT_PHASH_THRESHOLD,
|
||||
clip_threshold: float = DEFAULT_CLIP_THRESHOLD,
|
||||
**_ignored,
|
||||
) -> dict:
|
||||
"""Incremental duplicate detection for newly added photos.
|
||||
|
||||
Only photos added after `since` are compared against the full library.
|
||||
Much faster than a full regroup for post-scan updates:
|
||||
O(new × log N) via HNSW instead of O(N²).
|
||||
"""
|
||||
embedder_model = settings.vision.embedder.name
|
||||
|
||||
"""Incremental duplicate detection for newly added photos using pHash."""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# If no watermark, fall back to full regroup.
|
||||
if since is None:
|
||||
@@ -282,13 +257,6 @@ async def incremental_regroup(
|
||||
if _hamming(nh, eh) <= phash_threshold:
|
||||
uf.union_by_key(new_id, existing_id)
|
||||
|
||||
# ── Phase 2: CLIP — vector similarity for new photos only ──
|
||||
clip_matches = await _clip_neighbor_scan(
|
||||
session, new_ids, embedder_model, clip_threshold
|
||||
)
|
||||
for photo_id, neighbor_id in clip_matches:
|
||||
uf.union_by_key(photo_id, neighbor_id)
|
||||
|
||||
# ── Write results ──
|
||||
# Only update groups that contain at least one new photo.
|
||||
# Clear all groups first, then rewrite.
|
||||
@@ -323,73 +291,6 @@ async def incremental_regroup(
|
||||
}
|
||||
|
||||
|
||||
async def _clip_neighbor_scan(
|
||||
session,
|
||||
photo_ids: list[str],
|
||||
embedder_model: str,
|
||||
threshold: float,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""For each photo in `photo_ids` that has a CLIP embedding, find
|
||||
neighbors within cosine distance `threshold` using pgvector HNSW.
|
||||
|
||||
Returns a list of (photo_id, neighbor_id) pairs.
|
||||
"""
|
||||
matches: list[tuple[str, str]] = []
|
||||
|
||||
if not photo_ids:
|
||||
return matches
|
||||
|
||||
# Batch: get all embeddings for the target photos.
|
||||
target_embeddings = (
|
||||
await session.execute(
|
||||
select(Embedding.photo_id, Embedding.vector)
|
||||
.where(Embedding.photo_id.in_(photo_ids))
|
||||
.where(Embedding.model == embedder_model)
|
||||
)
|
||||
).all()
|
||||
|
||||
if not target_embeddings:
|
||||
return matches
|
||||
|
||||
# For each target, query nearest neighbors via pgvector.
|
||||
# We use raw SQL for the <=> cosine distance operator.
|
||||
for photo_id, vector in target_embeddings:
|
||||
# pgvector cosine distance: <=> operator
|
||||
# Find top 20 nearest neighbors within threshold.
|
||||
# Serialize the vector as "[a,b,c,...]" — pgvector's text
|
||||
# format uses commas; numpy's default str() joins with spaces
|
||||
# which Postgres rejects with "invalid input syntax for vector".
|
||||
if hasattr(vector, 'tolist'):
|
||||
vec_seq = vector.tolist()
|
||||
else:
|
||||
vec_seq = list(vector)
|
||||
vec_text = '[' + ','.join(f'{float(x):.8f}' for x in vec_seq) + ']'
|
||||
result = await session.execute(
|
||||
text("""
|
||||
SELECT e.photo_id, (e.vector <=> :vec) AS distance
|
||||
FROM embeddings e
|
||||
JOIN photos p ON p.id = e.photo_id
|
||||
WHERE e.model = :model
|
||||
AND e.photo_id != :pid
|
||||
AND p.is_trashed = false
|
||||
AND p.is_hidden = false
|
||||
AND (e.vector <=> :vec) < :threshold
|
||||
ORDER BY e.vector <=> :vec
|
||||
LIMIT 20
|
||||
"""),
|
||||
{
|
||||
'vec': vec_text,
|
||||
'pid': photo_id,
|
||||
'model': embedder_model,
|
||||
'threshold': threshold,
|
||||
}
|
||||
)
|
||||
for row in result.all():
|
||||
matches.append((photo_id, row[0]))
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
async def _clear_all_groups(session) -> None:
|
||||
"""Reset duplicate_group_id / is_duplicate on every photo."""
|
||||
await session.execute(
|
||||
|
||||
@@ -1,23 +1,9 @@
|
||||
"""
|
||||
Runtime feature flags for expensive pipeline stages.
|
||||
Runtime feature flags for the vision pipeline.
|
||||
|
||||
The YAML config (``mulita.yml``) ships reasonable defaults. Admins can
|
||||
toggle these at runtime from the Settings → AI Features tab without
|
||||
rebuilding the image or editing the bind-mounted YAML; the overrides
|
||||
live in Redis so both the FastAPI backend and the Celery workers see
|
||||
the same value within ~1s of the write.
|
||||
|
||||
The key namespace is:
|
||||
|
||||
mulita:flags:<name> → "true" | "false"
|
||||
|
||||
An unset key means "fall back to the YAML default" — so an admin who
|
||||
has never touched the tab sees exactly the config-file behaviour.
|
||||
|
||||
Only bool flags live here. Thresholds, batch sizes, model names etc.
|
||||
stay in the YAML file because flipping them safely requires restarting
|
||||
the vision workers (model reload, ONNX session re-init); that's not
|
||||
something a single admin click should do.
|
||||
Only one flag now — the master vision switch. Runtime overrides live in
|
||||
Redis under ``mulita:flags:<name>``; an unset key falls back to the
|
||||
YAML default.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -31,30 +17,16 @@ from app.config import settings
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Feature identifiers. The public name is what the admin UI sends; the
|
||||
# ``yaml_default`` getter returns the value the YAML would have set.
|
||||
# Keep these in sync with the VisionSettings fields in ``config.py``.
|
||||
FLAG_VISION_ENABLED = 'vision.enabled'
|
||||
FLAG_OCR_ENABLED = 'vision.ocr.enabled'
|
||||
FLAG_DETECTOR_ENABLED = 'vision.detector.enabled'
|
||||
FLAG_FACES_ENABLED = 'vision.faces.enabled'
|
||||
FLAG_CLASSIFIER_ENABLED = 'vision.classifier.enabled'
|
||||
|
||||
ALL_FLAGS = (
|
||||
FLAG_VISION_ENABLED,
|
||||
FLAG_OCR_ENABLED,
|
||||
FLAG_DETECTOR_ENABLED,
|
||||
FLAG_FACES_ENABLED,
|
||||
FLAG_CLASSIFIER_ENABLED,
|
||||
)
|
||||
ALL_FLAGS = (FLAG_VISION_ENABLED,)
|
||||
|
||||
_VISION_QUEUE = 'vision'
|
||||
|
||||
_REDIS: Optional[redis.Redis] = None
|
||||
|
||||
|
||||
def _redis() -> Optional[redis.Redis]:
|
||||
"""Lazy Redis client. Returns None if the broker is unreachable so
|
||||
callers can fall back to YAML defaults instead of crashing."""
|
||||
global _REDIS
|
||||
if _REDIS is None:
|
||||
try:
|
||||
@@ -69,19 +41,8 @@ def _redis() -> Optional[redis.Redis]:
|
||||
|
||||
|
||||
def _yaml_default(name: str) -> bool:
|
||||
"""Return the YAML-configured default for a flag. Used when Redis
|
||||
has no value set (fresh install or admin never touched the tab)."""
|
||||
v = settings.vision
|
||||
if name == FLAG_VISION_ENABLED:
|
||||
return bool(v.enabled)
|
||||
if name == FLAG_OCR_ENABLED:
|
||||
return bool(v.ocr.enabled)
|
||||
if name == FLAG_DETECTOR_ENABLED:
|
||||
return bool(v.detector.enabled)
|
||||
if name == FLAG_FACES_ENABLED:
|
||||
return bool(v.faces.enabled)
|
||||
if name == FLAG_CLASSIFIER_ENABLED:
|
||||
return bool(v.classifier.enabled)
|
||||
return bool(settings.vision.enabled)
|
||||
raise ValueError(f"Unknown feature flag: {name!r}")
|
||||
|
||||
|
||||
@@ -90,16 +51,6 @@ def _redis_key(name: str) -> str:
|
||||
|
||||
|
||||
def is_enabled(name: str) -> bool:
|
||||
"""Return True if feature ``name`` is currently enabled.
|
||||
|
||||
Order of precedence:
|
||||
1. Redis override (set by PATCH /admin/feature-flags)
|
||||
2. YAML default
|
||||
|
||||
Reads are cheap (~ms) and we intentionally do NOT add a local
|
||||
process cache — the whole point of runtime flags is that a toggle
|
||||
takes effect on the next task without a worker restart.
|
||||
"""
|
||||
r = _redis()
|
||||
if r is not None:
|
||||
try:
|
||||
@@ -112,9 +63,6 @@ def is_enabled(name: str) -> bool:
|
||||
|
||||
|
||||
def set_flag(name: str, value: bool) -> None:
|
||||
"""Persist a flag override to Redis. No-op if Redis is unreachable
|
||||
(we don't silently pretend to have written; raise so the admin
|
||||
request returns a 500 instead of misleading success)."""
|
||||
if name not in ALL_FLAGS:
|
||||
raise ValueError(f"Unknown feature flag: {name!r}")
|
||||
r = _redis()
|
||||
@@ -125,9 +73,6 @@ def set_flag(name: str, value: bool) -> None:
|
||||
|
||||
|
||||
def reset_flag(name: str) -> None:
|
||||
"""Delete the Redis override so the flag falls back to its YAML
|
||||
default. Useful if an admin wants a clean slate without guessing
|
||||
what the config defaults are."""
|
||||
if name not in ALL_FLAGS:
|
||||
raise ValueError(f"Unknown feature flag: {name!r}")
|
||||
r = _redis()
|
||||
@@ -137,150 +82,41 @@ def reset_flag(name: str) -> None:
|
||||
_apply_worker_side_effects(name)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker-level side effects: when the admin flips a flag we don't just want
|
||||
# gating at task-start (which still executes the message, it just returns
|
||||
# 'skipped'). We also want queued work gone and the vision worker genuinely
|
||||
# idle when the master switch is off.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VISION_QUEUE = 'vision'
|
||||
|
||||
# Flag → celery task name(s) whose queued messages should be dropped when
|
||||
# the flag goes off. Keeps the queue from replaying yesterday's work the
|
||||
# moment someone re-enables the stage.
|
||||
_TASKS_BY_FLAG: dict[str, tuple[str, ...]] = {
|
||||
FLAG_VISION_ENABLED: (
|
||||
'embed_photo', 'ocr_photo', 'detect_objects', 'extract_faces',
|
||||
'classify_content', 'vision_fanout', 'recluster_faces',
|
||||
),
|
||||
FLAG_OCR_ENABLED: ('ocr_photo',),
|
||||
FLAG_DETECTOR_ENABLED: ('detect_objects',),
|
||||
FLAG_FACES_ENABLED: ('extract_faces', 'recluster_faces'),
|
||||
FLAG_CLASSIFIER_ENABLED: ('classify_content',),
|
||||
}
|
||||
|
||||
|
||||
def _apply_worker_side_effects(name: str) -> None:
|
||||
"""Bring the live workers in line with the new flag value.
|
||||
|
||||
For the master ``vision.enabled`` flag we go beyond task gating and
|
||||
actually stop consumption from the ``vision`` queue — flipping it
|
||||
off puts the vision worker to sleep (no CPU, no model memory
|
||||
churn) until it's flipped back on. For per-feature flags, the
|
||||
running tasks already skip via ``is_enabled``; we just purge any
|
||||
messages already sitting in the queue so the admin doesn't pay for
|
||||
a backlog on re-enable.
|
||||
|
||||
All operations are best-effort — if control messaging or a Redis
|
||||
op fails, we log and return; the flag state itself is already
|
||||
persisted so the gating path continues to work.
|
||||
"""
|
||||
"""Attach or detach the vision consumer and purge queued work when
|
||||
the master flag flips. Best-effort — state is already persisted."""
|
||||
if name != FLAG_VISION_ENABLED:
|
||||
return
|
||||
try:
|
||||
# Lazy import: avoids a circular dependency between the services
|
||||
# module (imported from tasks.vision) and the celery app config.
|
||||
from app.tasks.celery import celery_app
|
||||
except Exception as e:
|
||||
logger.warning(f"feature_flags: celery app unavailable for side effects ({e})")
|
||||
return
|
||||
|
||||
try:
|
||||
if name == FLAG_VISION_ENABLED:
|
||||
if is_enabled(FLAG_VISION_ENABLED):
|
||||
# Re-attach the vision consumer so workers pick up tasks
|
||||
# again. broadcast=True ensures every running worker
|
||||
# receives the command.
|
||||
celery_app.control.add_consumer(_VISION_QUEUE, reply=False)
|
||||
logger.info("feature_flags: vision re-enabled; consumer added")
|
||||
else:
|
||||
celery_app.control.cancel_consumer(_VISION_QUEUE, reply=False)
|
||||
_purge_queue(_VISION_QUEUE)
|
||||
logger.info(
|
||||
"feature_flags: vision disabled; consumer cancelled "
|
||||
"and queue purged"
|
||||
)
|
||||
return
|
||||
|
||||
# Per-feature flag going off → drop pending tasks of its types.
|
||||
if not is_enabled(name):
|
||||
targets = _TASKS_BY_FLAG.get(name, ())
|
||||
if targets:
|
||||
removed = _purge_queue_by_task_names(_VISION_QUEUE, targets)
|
||||
logger.info(
|
||||
f"feature_flags: {name} disabled; removed {removed} "
|
||||
f"pending messages from {_VISION_QUEUE}"
|
||||
)
|
||||
if is_enabled(FLAG_VISION_ENABLED):
|
||||
celery_app.control.add_consumer(_VISION_QUEUE, reply=False)
|
||||
logger.info("feature_flags: vision re-enabled; consumer added")
|
||||
else:
|
||||
celery_app.control.cancel_consumer(_VISION_QUEUE, reply=False)
|
||||
_purge_queue(_VISION_QUEUE)
|
||||
logger.info("feature_flags: vision disabled; consumer cancelled and queue purged")
|
||||
except Exception as e:
|
||||
logger.warning(f"feature_flags: worker side effects failed for {name}: {e}")
|
||||
logger.warning(f"feature_flags: worker side effects failed: {e}")
|
||||
|
||||
|
||||
def _purge_queue(queue: str) -> int:
|
||||
"""Drop every pending message from ``queue``. Returns the count
|
||||
deleted. Celery's control.purge() purges the default queue only,
|
||||
so we delete the Redis key directly (the broker's queue list)."""
|
||||
r = _redis()
|
||||
if r is None:
|
||||
return 0
|
||||
try:
|
||||
removed = r.delete(queue)
|
||||
return int(removed or 0)
|
||||
return int(r.delete(queue) or 0)
|
||||
except Exception as e:
|
||||
logger.warning(f"feature_flags: purge {queue} failed: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
def _purge_queue_by_task_names(queue: str, task_names: tuple[str, ...]) -> int:
|
||||
"""Walk ``queue`` and drop any message whose Celery task name is in
|
||||
``task_names``. Other messages are preserved (pushed back in order)
|
||||
so we don't flush embed tasks when the admin disabled only OCR.
|
||||
|
||||
Celery stores each message as a JSON blob in a Redis list; the
|
||||
task name lives at ``headers.task``.
|
||||
"""
|
||||
import json
|
||||
r = _redis()
|
||||
if r is None:
|
||||
return 0
|
||||
try:
|
||||
# Snapshot the queue, then rebuild it without the filtered names.
|
||||
# Done inside a Redis transaction so a concurrent enqueue doesn't
|
||||
# race with us (worst case it gets re-delivered after we release,
|
||||
# which is the normal enqueue path anyway).
|
||||
pipe = r.pipeline()
|
||||
pipe.lrange(queue, 0, -1)
|
||||
pipe.delete(queue)
|
||||
raw_items, _ = pipe.execute()
|
||||
kept: list[bytes | str] = []
|
||||
removed = 0
|
||||
for raw in raw_items or []:
|
||||
try:
|
||||
# Messages can be bytes or str depending on decode_responses.
|
||||
payload = raw.decode() if isinstance(raw, bytes) else raw
|
||||
msg = json.loads(payload)
|
||||
task = (
|
||||
msg.get('headers', {}).get('task')
|
||||
or msg.get('task')
|
||||
)
|
||||
if task in task_names:
|
||||
removed += 1
|
||||
continue
|
||||
except Exception:
|
||||
# Unparseable message — keep it, better to leak than
|
||||
# to silently drop a message we can't identify.
|
||||
pass
|
||||
kept.append(raw)
|
||||
if kept:
|
||||
r.rpush(queue, *kept)
|
||||
return removed
|
||||
except Exception as e:
|
||||
logger.warning(f"feature_flags: selective purge failed on {queue}: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
def snapshot() -> dict[str, dict[str, object]]:
|
||||
"""Return every flag's current effective value, YAML default, and
|
||||
whether it's overridden. Powers the admin UI tab.
|
||||
"""
|
||||
r = _redis()
|
||||
out: dict[str, dict[str, object]] = {}
|
||||
for name in ALL_FLAGS:
|
||||
|
||||
@@ -1,19 +1,13 @@
|
||||
"""
|
||||
Unified search service — hybrid FTS + semantic (RRF) search.
|
||||
|
||||
Phase 1 (PR4): semantic-only via pgvector cosine similarity.
|
||||
Phase 2 (PR5): adds FTS via tsvector, enables RRF fusion.
|
||||
FTS search over photos.search_vector with optional tag/date filters.
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
from sqlalchemy import select, text, func
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Photo
|
||||
from app.models.embeddings import Embedding
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,135 +21,57 @@ async def hybrid_search(
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
"""Run hybrid search (FTS + semantic) with RRF fusion.
|
||||
|
||||
Currently semantic-only; FTS leg added in PR5.
|
||||
"""
|
||||
model_name = settings.vision.embedder.name
|
||||
results = {}
|
||||
|
||||
# ── Semantic search (CLIP text → pgvector cosine) ─────────────────
|
||||
"""Full-text search using photos.search_vector. No embeddings, no OCR."""
|
||||
if q:
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
query_vec = embedder.embed_text(q)
|
||||
|
||||
# pgvector cosine distance: <=> returns distance (lower = closer).
|
||||
# Join photos so we can filter out discarded / hidden rows
|
||||
# inside the same query — otherwise a hidden-folder photo
|
||||
# can take a top-N rank and starve the visible results.
|
||||
vec_str = "[" + ",".join(str(float(v)) for v in query_vec) + "]"
|
||||
stmt = text("""
|
||||
SELECT e.photo_id,
|
||||
(e.vector <=> :qvec::vector) AS distance
|
||||
FROM embeddings e
|
||||
JOIN photos p ON p.id = e.photo_id
|
||||
WHERE e.model = :model
|
||||
AND p.is_trashed = false
|
||||
AND p.is_hidden = false
|
||||
ORDER BY e.vector <=> :qvec::vector
|
||||
LIMIT 200
|
||||
""")
|
||||
rows = (await db.execute(stmt, {"qvec": vec_str, "model": model_name})).fetchall()
|
||||
|
||||
for rank, (photo_id, distance) in enumerate(rows):
|
||||
if photo_id not in results:
|
||||
results[photo_id] = {"semantic_rank": rank, "fts_rank": None}
|
||||
else:
|
||||
results[photo_id]["semantic_rank"] = rank
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Semantic search failed (models may not be loaded): %s", e)
|
||||
|
||||
# ── FTS search (photos.search_vector + ocr_text) ────────────────
|
||||
if q:
|
||||
try:
|
||||
# Same discarded/hidden filter as the semantic leg.
|
||||
# The OCR branch joins photos (through photo_id) so we can
|
||||
# filter there too; otherwise OCR hits in hidden folders
|
||||
# would leak into results.
|
||||
fts_stmt = text("""
|
||||
SELECT id, ts_rank(search_vector, plainto_tsquery('english', :q)) AS rank
|
||||
FROM photos
|
||||
WHERE search_vector @@ plainto_tsquery('english', :q)
|
||||
AND is_trashed = false
|
||||
AND is_hidden = false
|
||||
UNION
|
||||
SELECT o.photo_id AS id,
|
||||
MAX(o.confidence) AS rank
|
||||
FROM ocr_text o
|
||||
JOIN photos p ON p.id = o.photo_id
|
||||
WHERE to_tsvector('english', o.text) @@ plainto_tsquery('english', :q)
|
||||
AND p.is_trashed = false
|
||||
AND p.is_hidden = false
|
||||
GROUP BY o.photo_id
|
||||
ORDER BY rank DESC
|
||||
LIMIT 200
|
||||
LIMIT 500
|
||||
""")
|
||||
fts_rows = (await db.execute(fts_stmt, {"q": q})).fetchall()
|
||||
for rank, (photo_id, score) in enumerate(fts_rows):
|
||||
if photo_id not in results:
|
||||
results[photo_id] = {"semantic_rank": None, "fts_rank": rank}
|
||||
else:
|
||||
results[photo_id]["fts_rank"] = rank
|
||||
rows = (await db.execute(fts_stmt, {"q": q})).fetchall()
|
||||
except Exception as e:
|
||||
logger.warning("FTS search failed: %s", e)
|
||||
rows = []
|
||||
|
||||
# ── RRF fusion ────────────────────────────────────────────────────
|
||||
k = 60
|
||||
scored = []
|
||||
for photo_id, ranks in results.items():
|
||||
score = 0.0
|
||||
if ranks["semantic_rank"] is not None:
|
||||
score += 1.0 / (k + ranks["semantic_rank"])
|
||||
if ranks.get("fts_rank") is not None:
|
||||
score += 1.0 / (k + ranks["fts_rank"])
|
||||
scored.append((photo_id, score))
|
||||
scored = [(pid, float(rank)) for pid, rank in rows]
|
||||
|
||||
scored.sort(key=lambda x: -x[1])
|
||||
|
||||
# If no text query, fall back to recent photos. Always filter out
|
||||
# discarded + hidden here — this path backs the "Tags" and "People"
|
||||
# browse views, which should honor the folder hide flag.
|
||||
if not q:
|
||||
if tag_ids:
|
||||
from app.models.tags import photo_tags
|
||||
# Subquery to get distinct photo_ids matching the tag filter
|
||||
sub = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.tag_id.in_(tag_ids)
|
||||
).distinct().subquery()
|
||||
stmt = select(Photo.id).join(sub, Photo.id == sub.c.photo_id)
|
||||
else:
|
||||
stmt = select(Photo.id)
|
||||
stmt = stmt.where(
|
||||
Photo.is_discarded.is_(False),
|
||||
Photo.is_hidden.is_(False),
|
||||
)
|
||||
stmt = stmt.order_by(Photo.added_at.desc())
|
||||
if date_from:
|
||||
stmt = stmt.where(Photo.taken_at >= date_from)
|
||||
if date_to:
|
||||
stmt = stmt.where(Photo.taken_at <= date_to)
|
||||
stmt = stmt.offset(offset).limit(limit)
|
||||
rows = (await db.execute(stmt)).fetchall()
|
||||
return [{"photo_id": row[0], "score": 0.0} for row in rows]
|
||||
photo_ids = [pid for pid, _ in scored]
|
||||
if not photo_ids:
|
||||
return []
|
||||
stmt = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.photo_id.in_(photo_ids),
|
||||
photo_tags.c.tag_id.in_(tag_ids),
|
||||
).distinct()
|
||||
valid = {row[0] for row in (await db.execute(stmt)).fetchall()}
|
||||
scored = [(pid, s) for pid, s in scored if pid in valid]
|
||||
|
||||
# Apply filters to scored results
|
||||
photo_ids = [pid for pid, _ in scored]
|
||||
if not photo_ids:
|
||||
return []
|
||||
page = scored[offset : offset + limit]
|
||||
return [{"photo_id": pid, "score": s} for pid, s in page]
|
||||
|
||||
# Filter by tags if requested
|
||||
# No text query — recent photos with tag/date filters.
|
||||
if tag_ids:
|
||||
from app.models.tags import photo_tags
|
||||
stmt = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.photo_id.in_(photo_ids),
|
||||
photo_tags.c.tag_id.in_(tag_ids),
|
||||
).distinct()
|
||||
valid_ids = {row[0] for row in (await db.execute(stmt)).fetchall()}
|
||||
scored = [(pid, s) for pid, s in scored if pid in valid_ids]
|
||||
|
||||
# Paginate
|
||||
page = scored[offset : offset + limit]
|
||||
return [{"photo_id": pid, "score": score} for pid, score in page]
|
||||
sub = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.tag_id.in_(tag_ids)
|
||||
).distinct().subquery()
|
||||
stmt = select(Photo.id).join(sub, Photo.id == sub.c.photo_id)
|
||||
else:
|
||||
stmt = select(Photo.id)
|
||||
stmt = stmt.where(
|
||||
Photo.is_discarded.is_(False),
|
||||
Photo.is_hidden.is_(False),
|
||||
)
|
||||
if date_from:
|
||||
stmt = stmt.where(Photo.taken_at >= date_from)
|
||||
if date_to:
|
||||
stmt = stmt.where(Photo.taken_at <= date_to)
|
||||
stmt = stmt.order_by(Photo.added_at.desc()).offset(offset).limit(limit)
|
||||
rows = (await db.execute(stmt)).fetchall()
|
||||
return [{"photo_id": row[0], "score": 0.0} for row in rows]
|
||||
|
||||
@@ -1,105 +1,25 @@
|
||||
"""
|
||||
Abstract base classes for vision backends.
|
||||
Abstract base classes for the vision backend.
|
||||
|
||||
Each ABC defines the contract a backend must satisfy. The default
|
||||
implementation is ONNXBackend (onnx_backend.py). A ROCm backend can be
|
||||
added later by subclassing these ABCs and registering via
|
||||
settings.vision.backend.
|
||||
The pipeline is now a single binary classifier: photography vs other.
|
||||
Feature extraction is an internal detail of the classifier and is not
|
||||
exposed as a separate service.
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectionBox:
|
||||
"""A single object detection result."""
|
||||
label: str
|
||||
confidence: float
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
|
||||
|
||||
@dataclass
|
||||
class OCRResult:
|
||||
"""A single OCR text region."""
|
||||
text: str
|
||||
confidence: float
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
language: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FaceDetection:
|
||||
"""A detected face with its recognition embedding."""
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
embedding: np.ndarray # float32 vector (128-d for SFace)
|
||||
quality: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassificationResult:
|
||||
"""A content-type classification."""
|
||||
label: str
|
||||
confidence: float
|
||||
|
||||
|
||||
class Embedder(ABC):
|
||||
"""Generates image and text embeddings (e.g. OpenCLIP ViT-B/32)."""
|
||||
|
||||
@abstractmethod
|
||||
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
||||
"""Return a normalized float32 embedding vector for an RGB image."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def embed_text(self, text: str) -> np.ndarray:
|
||||
"""Return a normalized float32 embedding vector for a text query."""
|
||||
...
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def dim(self) -> int:
|
||||
"""Dimensionality of the output embedding."""
|
||||
...
|
||||
|
||||
|
||||
class OCREngine(ABC):
|
||||
"""Extracts text from images (e.g. rapidocr-onnxruntime)."""
|
||||
|
||||
@abstractmethod
|
||||
def run(self, image: np.ndarray) -> list[OCRResult]:
|
||||
"""Return OCR results for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class ObjectDetector(ABC):
|
||||
"""Detects objects in images (e.g. YOLOv8n)."""
|
||||
|
||||
@abstractmethod
|
||||
def detect(self, image: np.ndarray) -> list[DetectionBox]:
|
||||
"""Return detections for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class ContentClassifier(ABC):
|
||||
"""Classifies images into content types (screenshot, document, etc.)."""
|
||||
"""Classifies an image into 'photography' or 'other'."""
|
||||
|
||||
@abstractmethod
|
||||
def classify(self, image: np.ndarray) -> list[ClassificationResult]:
|
||||
"""Return content type classifications for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class FaceProcessor(ABC):
|
||||
"""Detects faces and extracts recognition embeddings (e.g. YuNet + SFace)."""
|
||||
|
||||
@abstractmethod
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
"""Return face detections with embeddings for an RGB image."""
|
||||
...
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def embedding_dim(self) -> int:
|
||||
"""Dimensionality of face embedding vectors."""
|
||||
def classify(self, image: np.ndarray) -> ClassificationResult:
|
||||
...
|
||||
|
||||
@@ -1,124 +1,46 @@
|
||||
"""
|
||||
Download vision model weights on first worker boot.
|
||||
|
||||
Run as: python -m app.services.vision.bootstrap_models
|
||||
|
||||
Or called from the vision worker entrypoint before Celery starts.
|
||||
Downloads are idempotent — existing files are skipped.
|
||||
|
||||
For models that require export (OpenCLIP, YOLOv8n), see export_models.py.
|
||||
Those must be exported once on any machine with pip, then placed in
|
||||
the models volume before the worker starts.
|
||||
Ensure the OpenCLIP ViT-B/32 visual encoder is present on worker boot.
|
||||
Exported via export_models.py if missing.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# (relative_path, url, description)
|
||||
# Models with url=None must be pre-exported via export_models.py.
|
||||
# InsightFace (RetinaFace + ArcFace) auto-downloads via the insightface
|
||||
# package on first use — no manual download entries needed.
|
||||
DOWNLOADS = []
|
||||
|
||||
# Models that need manual export via export_models.py
|
||||
EXPORTS = [
|
||||
REQUIRED = [
|
||||
("embed/visual.onnx", "OpenCLIP ViT-B/32 visual encoder"),
|
||||
("embed/textual.onnx", "OpenCLIP ViT-B/32 textual encoder"),
|
||||
("detect/yolov8n.onnx", "YOLOv8n object detector"),
|
||||
]
|
||||
|
||||
|
||||
def bootstrap(models_dir: str | None = None):
|
||||
"""Ensure all model files are present. Download what we can, warn about
|
||||
files that need manual export."""
|
||||
base = Path(models_dir or settings.vision.models_dir)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Download auto-downloadable models
|
||||
for rel_path, url, desc in DOWNLOADS:
|
||||
dest = base / rel_path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if dest.exists():
|
||||
logger.debug("Already exists: %s (%s)", dest, desc)
|
||||
continue
|
||||
|
||||
logger.info("Downloading %s → %s", desc, dest)
|
||||
try:
|
||||
urlretrieve(url, str(dest))
|
||||
size_kb = dest.stat().st_size / 1024
|
||||
logger.info("Downloaded %s (%.0f KB)", desc, size_kb)
|
||||
except Exception as e:
|
||||
logger.error("Failed to download %s: %s", desc, e)
|
||||
if dest.exists():
|
||||
dest.unlink()
|
||||
|
||||
# Check for manually-exported models
|
||||
missing = []
|
||||
for rel_path, desc in EXPORTS:
|
||||
dest = base / rel_path
|
||||
if not dest.exists():
|
||||
missing.append((rel_path, desc))
|
||||
|
||||
missing = [(rel, desc) for rel, desc in REQUIRED if not (base / rel).exists()]
|
||||
if missing:
|
||||
logger.warning(
|
||||
"Missing %d model file(s); attempting automatic export:",
|
||||
len(missing),
|
||||
)
|
||||
for rel_path, desc in missing:
|
||||
logger.warning(" %s — %s", base / rel_path, desc)
|
||||
logger.warning("Missing %d model file(s); attempting automatic export", len(missing))
|
||||
try:
|
||||
from app.services.vision import export_models
|
||||
export_models.export_openclip_visual(base)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Export failed: %s. Run `python -m app.services.vision.export_models "
|
||||
"--models-dir %s` manually to retry.",
|
||||
e, base,
|
||||
)
|
||||
|
||||
from app.services.vision import export_models
|
||||
|
||||
missing_paths = {rel for rel, _ in missing}
|
||||
|
||||
# Only run the export functions whose outputs are actually missing.
|
||||
# Each function is mapped to the file(s) it produces.
|
||||
export_map = [
|
||||
(export_models.export_openclip, ["embed/visual.onnx", "embed/textual.onnx"]),
|
||||
(export_models.export_siglip2, ["embed_siglip2/visual.onnx", "embed_siglip2/textual.onnx"]),
|
||||
(export_models.export_yolov8n, ["detect/yolov8n.onnx"]),
|
||||
]
|
||||
for export_fn, outputs in export_map:
|
||||
if not any(o in missing_paths for o in outputs):
|
||||
continue
|
||||
try:
|
||||
export_fn(base)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Export step %s failed: %s. "
|
||||
"Run `python -m app.services.vision.export_models "
|
||||
"--models-dir %s` manually to retry.",
|
||||
export_fn.__name__,
|
||||
e,
|
||||
base,
|
||||
)
|
||||
|
||||
# Re-check what's still missing after the export pass.
|
||||
still_missing = [
|
||||
(rel_path, desc)
|
||||
for rel_path, desc in EXPORTS
|
||||
if not (base / rel_path).exists()
|
||||
]
|
||||
if still_missing:
|
||||
for rel_path, desc in still_missing:
|
||||
logger.error(" still missing: %s — %s", base / rel_path, desc)
|
||||
else:
|
||||
logger.info("All model files present in %s", base)
|
||||
still_missing = [(r, d) for r, d in REQUIRED if not (base / r).exists()]
|
||||
if still_missing:
|
||||
for rel, desc in still_missing:
|
||||
logger.error(" still missing: %s — %s", base / rel, desc)
|
||||
else:
|
||||
logger.info("All model files present in %s", base)
|
||||
|
||||
# Signal readiness via Redis so the scan pipeline knows the vision
|
||||
# worker can accept tasks.
|
||||
try:
|
||||
import redis as _redis
|
||||
r = _redis.from_url(settings.redis_url)
|
||||
r.set("mulita:vision:ready", "1")
|
||||
_redis.from_url(settings.redis_url).set("mulita:vision:ready", "1")
|
||||
logger.info("Set mulita:vision:ready in Redis")
|
||||
except Exception as e:
|
||||
logger.warning("Could not set vision readiness flag in Redis: %s", e)
|
||||
|
||||
@@ -1,117 +1,106 @@
|
||||
"""
|
||||
CLIP zero-shot content-type classifier.
|
||||
Binary content classifier: 'photography' vs 'other'.
|
||||
|
||||
Uses the native OpenCLIP PyTorch text encoder for high-quality text
|
||||
embeddings (the ONNX text encoder has degraded quality due to the
|
||||
eot_indices workaround). Image embeddings use the ONNX visual encoder
|
||||
which works well.
|
||||
Uses OpenCLIP ViT-B/32 image features (ONNX) and two pre-computed text
|
||||
prompt centroids. Text centroids are computed once with the native
|
||||
open_clip text encoder and cached to {models_dir}/classifier/vectors.npz
|
||||
so steady-state worker startup doesn't pay the PyTorch cost.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import ContentClassifier, ClassificationResult
|
||||
from app.services.vision.base import ClassificationResult, ContentClassifier
|
||||
from app.services.vision.embed import CLIPVisualEncoder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CATEGORY_PROMPTS = {
|
||||
"screenshot": [
|
||||
"a screenshot of a computer screen",
|
||||
"a screenshot of a phone screen",
|
||||
"a screen capture of a user interface",
|
||||
],
|
||||
"document": [
|
||||
"a scanned document",
|
||||
"a photo of a document with printed text",
|
||||
"a photo of a page of text on paper",
|
||||
],
|
||||
"receipt": [
|
||||
"a photo of a receipt",
|
||||
"a photo of a bill or invoice",
|
||||
],
|
||||
"meme": [
|
||||
"an internet meme with text overlay",
|
||||
"a funny image with caption text",
|
||||
],
|
||||
"artwork": [
|
||||
"a painting or drawing",
|
||||
"a sketch or illustration",
|
||||
"digital art or graphic design",
|
||||
],
|
||||
"photograph": [
|
||||
|
||||
PROMPTS = {
|
||||
"photography": [
|
||||
"a photograph taken with a camera",
|
||||
"a real photo of a real scene or person",
|
||||
"a candid photograph",
|
||||
"a portrait photograph",
|
||||
"a landscape photograph",
|
||||
],
|
||||
"other": [
|
||||
"a screenshot of a computer screen",
|
||||
"a screenshot of a phone screen",
|
||||
"a screen capture of a user interface",
|
||||
"a scanned document",
|
||||
"a photo of a document with printed text",
|
||||
"a photo of a receipt",
|
||||
"a photo of a bill or invoice",
|
||||
"an internet meme with text overlay",
|
||||
"a funny image with caption text",
|
||||
"a digital illustration or graphic design",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _compute_text_centroids() -> dict[str, np.ndarray]:
|
||||
"""Compute the 'photography' and 'other' centroid vectors using the
|
||||
open_clip text encoder. Only called on the cache-miss path."""
|
||||
import open_clip
|
||||
import torch
|
||||
|
||||
logger.info("Computing CLIP text centroids for binary classifier")
|
||||
model, _, _ = open_clip.create_model_and_transforms(
|
||||
"ViT-B-32", pretrained="laion2b_s34b_b79k"
|
||||
)
|
||||
model.eval()
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
||||
|
||||
centroids: dict[str, np.ndarray] = {}
|
||||
for label, prompts in PROMPTS.items():
|
||||
tokens = tokenizer(prompts)
|
||||
with torch.no_grad():
|
||||
feats = model.encode_text(tokens)
|
||||
feats = feats / feats.norm(dim=-1, keepdim=True)
|
||||
avg = feats.mean(dim=0)
|
||||
avg = avg / avg.norm()
|
||||
centroids[label] = avg.numpy().astype(np.float32)
|
||||
return centroids
|
||||
|
||||
|
||||
class CLIPContentClassifier(ContentClassifier):
|
||||
"""Zero-shot content classifier using CLIP text-image similarity.
|
||||
Uses native PyTorch for text encoding, ONNX for image encoding."""
|
||||
|
||||
def __init__(self, settings: VisionSettings):
|
||||
import open_clip
|
||||
|
||||
self._min_confidence = settings.classifier.min_confidence
|
||||
self._encoder = CLIPVisualEncoder(settings)
|
||||
|
||||
# Load native model for text encoding only.
|
||||
# Use whichever model family the embedder is configured for so
|
||||
# the classification text vectors live in the same space as the
|
||||
# image embeddings.
|
||||
embedder_name = settings.embedder.name
|
||||
if embedder_name.startswith("siglip2"):
|
||||
model_arch = "ViT-B-16-SigLIP-384"
|
||||
pretrained = "webli"
|
||||
cache_dir = Path(settings.models_dir) / "classifier"
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
cache_path = cache_dir / "vectors.npz"
|
||||
|
||||
if cache_path.exists():
|
||||
logger.info("Loading cached text centroids from %s", cache_path)
|
||||
data = np.load(cache_path)
|
||||
self._photo = data["photography"].astype(np.float32)
|
||||
self._other = data["other"].astype(np.float32)
|
||||
else:
|
||||
model_arch = "ViT-B-32"
|
||||
pretrained = "laion2b_s34b_b79k"
|
||||
centroids = _compute_text_centroids()
|
||||
self._photo = centroids["photography"]
|
||||
self._other = centroids["other"]
|
||||
np.savez(cache_path, photography=self._photo, other=self._other)
|
||||
logger.info("Cached text centroids to %s", cache_path)
|
||||
|
||||
logger.info("Loading %s text encoder for content classification", model_arch)
|
||||
model, _, _ = open_clip.create_model_and_transforms(
|
||||
model_arch, pretrained=pretrained
|
||||
)
|
||||
model.eval()
|
||||
self._model = model
|
||||
self._tokenizer = open_clip.get_tokenizer(model_arch)
|
||||
def classify(self, image: np.ndarray) -> ClassificationResult:
|
||||
vec = self._encoder.encode(image)
|
||||
s_photo = float(np.dot(vec, self._photo))
|
||||
s_other = float(np.dot(vec, self._other))
|
||||
|
||||
# Get the ONNX image embedder from the registry
|
||||
from app.services.vision.registry import registry
|
||||
self._embedder = registry.get_embedder()
|
||||
if s_photo >= s_other:
|
||||
label = "photography"
|
||||
margin = s_photo - s_other
|
||||
else:
|
||||
label = "other"
|
||||
margin = s_other - s_photo
|
||||
|
||||
# Pre-compute text embeddings for each category
|
||||
self._category_embeddings: dict[str, np.ndarray] = {}
|
||||
for category, prompts in CATEGORY_PROMPTS.items():
|
||||
tokens = self._tokenizer(prompts)
|
||||
with torch.no_grad():
|
||||
text_features = model.encode_text(tokens)
|
||||
text_features /= text_features.norm(dim=-1, keepdim=True)
|
||||
avg = text_features.mean(dim=0)
|
||||
avg /= avg.norm()
|
||||
self._category_embeddings[category] = avg.numpy().astype(np.float32)
|
||||
|
||||
logger.info("Content classifier ready with %d categories", len(self._category_embeddings))
|
||||
|
||||
def classify(self, image: np.ndarray) -> list[ClassificationResult]:
|
||||
img_vec = self._embedder.embed_image(image)
|
||||
|
||||
# Cosine similarity against each category
|
||||
scores = {}
|
||||
for category, cat_vec in self._category_embeddings.items():
|
||||
scores[category] = float(np.dot(img_vec, cat_vec))
|
||||
|
||||
# Sort by score descending
|
||||
ranked = sorted(scores.items(), key=lambda x: -x[1])
|
||||
best_cat, best_score = ranked[0]
|
||||
second_score = ranked[1][1]
|
||||
|
||||
margin = best_score - second_score
|
||||
# Normalize: 0.01 margin → ~0.5 confidence, 0.03+ → ~1.0
|
||||
# 0.01 margin → ~0.3 conf, 0.03+ → ~1.0
|
||||
confidence = min(1.0, margin * 30)
|
||||
|
||||
if confidence >= self._min_confidence:
|
||||
return [ClassificationResult(label=best_cat, confidence=confidence)]
|
||||
|
||||
return []
|
||||
return ClassificationResult(label=label, confidence=confidence)
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
"""
|
||||
Face embedding clustering using DBSCAN with cosine distance.
|
||||
|
||||
Called by the periodic `recluster_faces` Celery task (PR7).
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
from sklearn.cluster import DBSCAN
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cluster_faces(
|
||||
embeddings: np.ndarray,
|
||||
eps: float = 0.35,
|
||||
min_samples: int = 2,
|
||||
) -> np.ndarray:
|
||||
"""Cluster face embeddings using DBSCAN with cosine metric.
|
||||
|
||||
Args:
|
||||
embeddings: (N, D) float32 array of L2-normalized face embeddings.
|
||||
eps: Maximum cosine distance between two samples to be in the
|
||||
same neighborhood. Lower = tighter clusters.
|
||||
min_samples: Minimum cluster size.
|
||||
|
||||
Returns:
|
||||
(N,) int array of cluster labels. -1 = noise / unclustered.
|
||||
"""
|
||||
if len(embeddings) < min_samples:
|
||||
return np.full(len(embeddings), -1, dtype=int)
|
||||
|
||||
db = DBSCAN(eps=eps, min_samples=min_samples, metric="cosine")
|
||||
labels = db.fit_predict(embeddings)
|
||||
|
||||
n_clusters = len(set(labels) - {-1})
|
||||
n_noise = (labels == -1).sum()
|
||||
logger.info(
|
||||
"Face clustering: %d embeddings → %d clusters, %d noise",
|
||||
len(embeddings), n_clusters, n_noise,
|
||||
)
|
||||
return labels
|
||||
@@ -1,138 +0,0 @@
|
||||
"""
|
||||
YOLOv8n object detector using raw ONNX Runtime.
|
||||
|
||||
Expects {models_dir}/detect/yolov8n.onnx, exported from ultralytics
|
||||
via bootstrap_models.py. We do NOT ship ultralytics at runtime to
|
||||
avoid dragging in torch.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import ObjectDetector, DetectionBox
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_INPUT_SIZE = 640
|
||||
|
||||
# COCO class names (80 classes)
|
||||
COCO_LABELS = [
|
||||
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
|
||||
"truck", "boat", "traffic light", "fire hydrant", "stop sign",
|
||||
"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep",
|
||||
"cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella",
|
||||
"handbag", "tie", "suitcase", "frisbee", "skis", "snowboard",
|
||||
"sports ball", "kite", "baseball bat", "baseball glove", "skateboard",
|
||||
"surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork",
|
||||
"knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
|
||||
"broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
|
||||
"couch", "potted plant", "bed", "dining table", "toilet", "tv",
|
||||
"laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
|
||||
"oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
|
||||
"scissors", "teddy bear", "hair drier", "toothbrush",
|
||||
]
|
||||
|
||||
|
||||
def _preprocess(image: np.ndarray) -> tuple[np.ndarray, float, float]:
|
||||
"""Letterbox-resize + normalize to NCHW float32. Returns input tensor
|
||||
and scale factors for mapping boxes back to original coords."""
|
||||
from PIL import Image
|
||||
|
||||
img = Image.fromarray(image).convert("RGB")
|
||||
orig_w, orig_h = img.size
|
||||
|
||||
scale = min(_INPUT_SIZE / orig_w, _INPUT_SIZE / orig_h)
|
||||
new_w = int(orig_w * scale)
|
||||
new_h = int(orig_h * scale)
|
||||
img = img.resize((new_w, new_h), Image.BICUBIC)
|
||||
|
||||
# Paste onto gray canvas
|
||||
canvas = np.full((_INPUT_SIZE, _INPUT_SIZE, 3), 114, dtype=np.uint8)
|
||||
pad_x = (_INPUT_SIZE - new_w) // 2
|
||||
pad_y = (_INPUT_SIZE - new_h) // 2
|
||||
canvas[pad_y : pad_y + new_h, pad_x : pad_x + new_w] = np.array(img)
|
||||
|
||||
blob = canvas.astype(np.float32) / 255.0
|
||||
blob = blob.transpose(2, 0, 1)[np.newaxis] # NCHW
|
||||
return blob, scale, pad_x, pad_y
|
||||
|
||||
|
||||
def _postprocess(
|
||||
outputs: np.ndarray,
|
||||
scale: float,
|
||||
pad_x: int,
|
||||
pad_y: int,
|
||||
orig_w: int,
|
||||
orig_h: int,
|
||||
conf_threshold: float,
|
||||
max_detections: int,
|
||||
) -> list[DetectionBox]:
|
||||
"""Parse YOLOv8 output (1, 84, N) → list of DetectionBox."""
|
||||
# outputs shape: (1, 84, N) where 84 = 4 box coords + 80 class scores
|
||||
preds = outputs[0] # (84, N)
|
||||
preds = preds.T # (N, 84)
|
||||
|
||||
boxes_xywh = preds[:, :4]
|
||||
scores = preds[:, 4:]
|
||||
|
||||
class_ids = np.argmax(scores, axis=1)
|
||||
confidences = scores[np.arange(len(scores)), class_ids]
|
||||
|
||||
mask = confidences >= conf_threshold
|
||||
boxes_xywh = boxes_xywh[mask]
|
||||
class_ids = class_ids[mask]
|
||||
confidences = confidences[mask]
|
||||
|
||||
if len(confidences) == 0:
|
||||
return []
|
||||
|
||||
# Sort by confidence, take top N
|
||||
order = np.argsort(-confidences)[:max_detections]
|
||||
boxes_xywh = boxes_xywh[order]
|
||||
class_ids = class_ids[order]
|
||||
confidences = confidences[order]
|
||||
|
||||
results = []
|
||||
for i in range(len(confidences)):
|
||||
cx, cy, w, h = boxes_xywh[i]
|
||||
# Remove letterbox padding and rescale to original image
|
||||
x1 = (cx - w / 2 - pad_x) / scale
|
||||
y1 = (cy - h / 2 - pad_y) / scale
|
||||
x2 = (cx + w / 2 - pad_x) / scale
|
||||
y2 = (cy + h / 2 - pad_y) / scale
|
||||
# Normalize to 0-1
|
||||
bbox = [
|
||||
max(0, x1 / orig_w),
|
||||
max(0, y1 / orig_h),
|
||||
min(1, x2 / orig_w),
|
||||
min(1, y2 / orig_h),
|
||||
]
|
||||
label = COCO_LABELS[class_ids[i]] if class_ids[i] < len(COCO_LABELS) else f"class_{class_ids[i]}"
|
||||
results.append(DetectionBox(label=label, confidence=float(confidences[i]), bbox=bbox))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class YOLOv8Detector(ObjectDetector):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
model_path = Path(settings.models_dir) / "detect" / "yolov8n.onnx"
|
||||
|
||||
from app.services.vision.providers import create_session
|
||||
|
||||
logger.info("Loading YOLOv8n from %s", model_path)
|
||||
self._session = create_session(str(model_path), configured_providers=settings.execution_providers)
|
||||
self._conf_threshold = settings.detector.min_confidence
|
||||
self._max_detections = settings.detector.max_detections
|
||||
|
||||
def detect(self, image: np.ndarray) -> list[DetectionBox]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
blob, scale, pad_x, pad_y = _preprocess(image)
|
||||
input_name = self._session.get_inputs()[0].name
|
||||
outputs = self._session.run(None, {input_name: blob})[0]
|
||||
return _postprocess(
|
||||
outputs, scale, pad_x, pad_y, orig_w, orig_h,
|
||||
self._conf_threshold, self._max_detections,
|
||||
)
|
||||
@@ -1,146 +1,58 @@
|
||||
"""
|
||||
CLIP / SigLIP2 embedder using ONNX Runtime.
|
||||
|
||||
Supports two model families:
|
||||
- OpenCLIP ViT-B/32 (512-d) — legacy, config name "openclip_vitb32"
|
||||
- SigLIP2 ViT-B/16 (768-d) — default, config name "siglip2_vitb16"
|
||||
|
||||
Expects two ONNX files under {models_dir}/embed/:
|
||||
- visual.onnx (image encoder)
|
||||
- textual.onnx (text encoder)
|
||||
|
||||
These are exported from open_clip via export_models.py / bootstrap_models.py.
|
||||
OpenCLIP ViT-B/32 visual encoder (ONNX). Produces 512-d image features
|
||||
consumed by the content classifier. Not exposed as a standalone service;
|
||||
the classifier owns the lifecycle.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
import onnxruntime as ort # noqa: F401 (provider plumbing relies on this)
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Model-specific constants ──────────────────────────────────────────
|
||||
|
||||
# OpenCLIP ViT-B/32 (ImageNet norm, 224px)
|
||||
_OPENCLIP_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
|
||||
_OPENCLIP_STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32)
|
||||
_OPENCLIP_SIZE = 224
|
||||
|
||||
# SigLIP2 ViT-B/16 (SigLIP norm, 384px)
|
||||
_SIGLIP2_MEAN = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
||||
_SIGLIP2_STD = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
||||
_SIGLIP2_SIZE = 384
|
||||
_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
|
||||
_STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32)
|
||||
_SIZE = 224
|
||||
|
||||
|
||||
def _preprocess_image(
|
||||
image: np.ndarray,
|
||||
input_size: int,
|
||||
mean: np.ndarray,
|
||||
std: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""Resize, center-crop, normalize an RGB uint8 image to NCHW float32."""
|
||||
def _preprocess(image: np.ndarray) -> np.ndarray:
|
||||
from PIL import Image
|
||||
|
||||
img = Image.fromarray(image).convert("RGB")
|
||||
w, h = img.size
|
||||
scale = input_size / min(w, h)
|
||||
scale = _SIZE / min(w, h)
|
||||
img = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC)
|
||||
w, h = img.size
|
||||
left = (w - input_size) // 2
|
||||
top = (h - input_size) // 2
|
||||
img = img.crop((left, top, left + input_size, top + input_size))
|
||||
left = (w - _SIZE) // 2
|
||||
top = (h - _SIZE) // 2
|
||||
img = img.crop((left, top, left + _SIZE, top + _SIZE))
|
||||
|
||||
arr = np.array(img, dtype=np.float32) / 255.0
|
||||
arr = (arr - mean) / std
|
||||
arr = arr.transpose(2, 0, 1) # HWC → CHW
|
||||
return arr[np.newaxis] # NCHW
|
||||
arr = (arr - _MEAN) / _STD
|
||||
arr = arr.transpose(2, 0, 1)
|
||||
return arr[np.newaxis]
|
||||
|
||||
|
||||
class OpenCLIPEmbedder(Embedder):
|
||||
"""Legacy OpenCLIP ViT-B/32 embedder (512-d)."""
|
||||
class CLIPVisualEncoder:
|
||||
"""OpenCLIP ViT-B/32 image encoder, 512-d normalized output."""
|
||||
|
||||
def __init__(self, settings: VisionSettings):
|
||||
model_dir = Path(settings.models_dir) / "embed"
|
||||
visual_path = model_dir / "visual.onnx"
|
||||
textual_path = model_dir / "textual.onnx"
|
||||
|
||||
model_path = Path(settings.models_dir) / "embed" / "visual.onnx"
|
||||
from app.services.vision.providers import create_session
|
||||
from app.config import settings as app_settings
|
||||
providers = app_settings.vision.execution_providers
|
||||
|
||||
logger.info("Loading OpenCLIP visual encoder from %s", visual_path)
|
||||
self._visual = create_session(str(visual_path), configured_providers=providers)
|
||||
logger.info("Loading CLIP visual encoder from %s", model_path)
|
||||
self._session = create_session(
|
||||
str(model_path),
|
||||
configured_providers=app_settings.vision.execution_providers,
|
||||
)
|
||||
|
||||
logger.info("Loading OpenCLIP textual encoder from %s", textual_path)
|
||||
self._textual = create_session(str(textual_path), configured_providers=providers)
|
||||
|
||||
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
||||
inp = _preprocess_image(image, _OPENCLIP_SIZE, _OPENCLIP_MEAN, _OPENCLIP_STD)
|
||||
input_name = self._visual.get_inputs()[0].name
|
||||
out = self._visual.run(None, {input_name: inp})[0][0]
|
||||
def encode(self, image: np.ndarray) -> np.ndarray:
|
||||
inp = _preprocess(image)
|
||||
name = self._session.get_inputs()[0].name
|
||||
out = self._session.run(None, {name: inp})[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
def embed_text(self, text: str) -> np.ndarray:
|
||||
import open_clip
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
||||
tokens = tokenizer([text]).numpy().astype(np.int64)
|
||||
eot_indices = tokens.argmax(axis=-1).astype(np.int64)
|
||||
inputs = self._textual.get_inputs()
|
||||
out = self._textual.run(None, {
|
||||
inputs[0].name: tokens,
|
||||
inputs[1].name: eot_indices,
|
||||
})[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
@property
|
||||
def dim(self) -> int:
|
||||
return 512
|
||||
|
||||
|
||||
class SigLIP2Embedder(Embedder):
|
||||
"""SigLIP2 ViT-B/16 embedder (768-d) — higher recall than OpenCLIP."""
|
||||
|
||||
def __init__(self, settings: VisionSettings):
|
||||
model_dir = Path(settings.models_dir) / "embed_siglip2"
|
||||
visual_path = model_dir / "visual.onnx"
|
||||
textual_path = model_dir / "textual.onnx"
|
||||
|
||||
from app.services.vision.providers import create_session
|
||||
from app.config import settings as app_settings
|
||||
providers = app_settings.vision.execution_providers
|
||||
|
||||
logger.info("Loading SigLIP2 visual encoder from %s", visual_path)
|
||||
self._visual = create_session(str(visual_path), configured_providers=providers)
|
||||
|
||||
logger.info("Loading SigLIP2 textual encoder from %s", textual_path)
|
||||
self._textual = create_session(str(textual_path), configured_providers=providers)
|
||||
|
||||
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
||||
inp = _preprocess_image(image, _SIGLIP2_SIZE, _SIGLIP2_MEAN, _SIGLIP2_STD)
|
||||
input_name = self._visual.get_inputs()[0].name
|
||||
out = self._visual.run(None, {input_name: inp})[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
def embed_text(self, text: str) -> np.ndarray:
|
||||
import open_clip
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-16-SigLIP-384")
|
||||
tokens = tokenizer([text]).numpy().astype(np.int64)
|
||||
inputs = self._textual.get_inputs()
|
||||
feed = {inputs[0].name: tokens}
|
||||
# SigLIP2 text encoder may need attention mask
|
||||
if len(inputs) > 1:
|
||||
attention_mask = (tokens != 0).astype(np.int64)
|
||||
feed[inputs[1].name] = attention_mask
|
||||
out = self._textual.run(None, feed)[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
@property
|
||||
def dim(self) -> int:
|
||||
return 768
|
||||
|
||||
@@ -1,253 +1,61 @@
|
||||
"""
|
||||
Export / download all vision model weights to ONNX format.
|
||||
Export the OpenCLIP ViT-B/32 visual encoder to ONNX.
|
||||
|
||||
Run ONCE on any machine with Python + pip (doesn't need GPU):
|
||||
Run once on any machine with Python + pip (no GPU needed):
|
||||
|
||||
pip install open-clip-torch ultralytics onnx
|
||||
pip install open-clip-torch onnx
|
||||
python -m app.services.vision.export_models [--models-dir /data/models]
|
||||
|
||||
This produces:
|
||||
embed/visual.onnx (~350 MB)
|
||||
embed/textual.onnx (~250 MB)
|
||||
detect/yolov8n.onnx (~12 MB)
|
||||
|
||||
YuNet and SFace are downloaded by bootstrap_models.py at worker boot
|
||||
(Apache 2.0, lightweight, no export step needed).
|
||||
|
||||
After export, copy the /data/models directory into your Docker volume:
|
||||
docker cp /data/models mulita-worker:/data/models
|
||||
Or mount a host path in docker-compose.yml.
|
||||
Produces:
|
||||
embed/visual.onnx (~350 MB)
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def export_openclip(models_dir: Path):
|
||||
"""Export OpenCLIP ViT-B/32 to two ONNX files (visual + textual)."""
|
||||
def export_openclip_visual(models_dir: Path):
|
||||
import torch
|
||||
import open_clip
|
||||
|
||||
out_dir = models_dir / "embed"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
visual_path = out_dir / "visual.onnx"
|
||||
textual_path = out_dir / "textual.onnx"
|
||||
|
||||
if visual_path.exists() and textual_path.exists():
|
||||
logger.info("OpenCLIP ONNX files already exist, skipping export")
|
||||
if visual_path.exists():
|
||||
logger.info("OpenCLIP visual.onnx already exists, skipping export")
|
||||
return
|
||||
|
||||
logger.info("Loading OpenCLIP ViT-B-32 laion2b_s34b_b79k...")
|
||||
model, _, preprocess = open_clip.create_model_and_transforms(
|
||||
model, _, _ = open_clip.create_model_and_transforms(
|
||||
"ViT-B-32", pretrained="laion2b_s34b_b79k"
|
||||
)
|
||||
model.eval()
|
||||
|
||||
# Use dynamo=False to get the legacy TorchScript exporter which
|
||||
# produces IR version 9 (compatible with onnxruntime 1.17.x).
|
||||
# The new torch.onnx.export default (dynamo=True) emits IR 10.
|
||||
export_kwargs = dict(opset_version=14, dynamo=False)
|
||||
|
||||
# ── Visual encoder ────────────────────────────────────────────────
|
||||
if not visual_path.exists():
|
||||
logger.info("Exporting visual encoder → %s", visual_path)
|
||||
dummy_image = torch.randn(1, 3, 224, 224)
|
||||
torch.onnx.export(
|
||||
model.visual,
|
||||
dummy_image,
|
||||
str(visual_path),
|
||||
input_names=["image"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"image": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = visual_path.stat().st_size / 1e6
|
||||
logger.info("Visual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
# ── Textual encoder ───────────────────────────────────────────────
|
||||
if not textual_path.exists():
|
||||
logger.info("Exporting textual encoder → %s", textual_path)
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
||||
dummy_text = tokenizer(["a photo"]).to(torch.int64)
|
||||
|
||||
class TextEncoder(torch.nn.Module):
|
||||
"""Wrap the CLIP text encoder to avoid argmax in the ONNX graph.
|
||||
OpenCLIP uses argmax to find the EOT token position, but ORT
|
||||
ARM64 doesn't support ArgMax(13). We pre-compute the EOT index
|
||||
from the token sequence and pass it directly."""
|
||||
def __init__(self, clip_model):
|
||||
super().__init__()
|
||||
self.transformer = clip_model.transformer
|
||||
self.token_embedding = clip_model.token_embedding
|
||||
self.positional_embedding = clip_model.positional_embedding
|
||||
self.ln_final = clip_model.ln_final
|
||||
self.text_projection = clip_model.text_projection
|
||||
|
||||
def forward(self, text, eot_indices):
|
||||
x = self.token_embedding(text)
|
||||
x = x + self.positional_embedding
|
||||
x = x.permute(1, 0, 2) # NLD -> LND
|
||||
x = self.transformer(x)
|
||||
x = x.permute(1, 0, 2) # LND -> NLD
|
||||
x = self.ln_final(x)
|
||||
# Take the feature at the EOT token. The EOT index is
|
||||
# passed in as a separate input (computed outside ONNX)
|
||||
# to avoid ArgMax(13) which ORT ARM64 doesn't support.
|
||||
x = x[torch.arange(x.shape[0]), eot_indices]
|
||||
x = x @ self.text_projection
|
||||
return x
|
||||
|
||||
text_enc = TextEncoder(model)
|
||||
text_enc.eval()
|
||||
|
||||
# Compute EOT indices from dummy tokens (argmax of token ids)
|
||||
dummy_eot = dummy_text.argmax(dim=-1)
|
||||
|
||||
torch.onnx.export(
|
||||
text_enc,
|
||||
(dummy_text, dummy_eot),
|
||||
str(textual_path),
|
||||
input_names=["text", "eot_indices"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"text": {0: "batch"}, "eot_indices": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = textual_path.stat().st_size / 1e6
|
||||
logger.info("Textual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
|
||||
def export_siglip2(models_dir: Path):
|
||||
"""Export SigLIP2 ViT-B/16 to two ONNX files (visual + textual)."""
|
||||
import torch
|
||||
import open_clip
|
||||
|
||||
out_dir = models_dir / "embed_siglip2"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
visual_path = out_dir / "visual.onnx"
|
||||
textual_path = out_dir / "textual.onnx"
|
||||
|
||||
if visual_path.exists() and textual_path.exists():
|
||||
logger.info("SigLIP2 ONNX files already exist, skipping export")
|
||||
return
|
||||
|
||||
logger.info("Loading SigLIP2 ViT-B-16-SigLIP-384 webli...")
|
||||
model, _, preprocess = open_clip.create_model_and_transforms(
|
||||
"ViT-B-16-SigLIP-384", pretrained="webli"
|
||||
logger.info("Exporting visual encoder → %s", visual_path)
|
||||
dummy = torch.randn(1, 3, 224, 224)
|
||||
torch.onnx.export(
|
||||
model.visual,
|
||||
dummy,
|
||||
str(visual_path),
|
||||
input_names=["image"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"image": {0: "batch"}},
|
||||
opset_version=14,
|
||||
dynamo=False,
|
||||
)
|
||||
model.eval()
|
||||
|
||||
export_kwargs = dict(opset_version=14, dynamo=False)
|
||||
|
||||
# ── Visual encoder ────────────────────────────────────────────────
|
||||
if not visual_path.exists():
|
||||
logger.info("Exporting SigLIP2 visual encoder → %s", visual_path)
|
||||
dummy_image = torch.randn(1, 3, 384, 384)
|
||||
torch.onnx.export(
|
||||
model.visual,
|
||||
dummy_image,
|
||||
str(visual_path),
|
||||
input_names=["image"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"image": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = visual_path.stat().st_size / 1e6
|
||||
logger.info("SigLIP2 visual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
# ── Textual encoder ───────────────────────────────────────────────
|
||||
if not textual_path.exists():
|
||||
logger.info("Exporting SigLIP2 textual encoder → %s", textual_path)
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-16-SigLIP-384")
|
||||
dummy_text = tokenizer(["a photo"]).to(torch.int64)
|
||||
|
||||
class SigLIP2TextEncoder(torch.nn.Module):
|
||||
"""Wrap the SigLIP2 text transformer for ONNX export."""
|
||||
def __init__(self, clip_model):
|
||||
super().__init__()
|
||||
self.text = clip_model.text
|
||||
|
||||
def forward(self, text):
|
||||
return self.text(text)
|
||||
|
||||
text_enc = SigLIP2TextEncoder(model)
|
||||
text_enc.eval()
|
||||
|
||||
torch.onnx.export(
|
||||
text_enc,
|
||||
dummy_text,
|
||||
str(textual_path),
|
||||
input_names=["text"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"text": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = textual_path.stat().st_size / 1e6
|
||||
logger.info("SigLIP2 textual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
|
||||
def export_yolov8n(models_dir: Path):
|
||||
"""Export YOLOv8n to ONNX."""
|
||||
out_dir = models_dir / "detect"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
onnx_path = out_dir / "yolov8n.onnx"
|
||||
|
||||
if onnx_path.exists():
|
||||
logger.info("YOLOv8n ONNX already exists, skipping export")
|
||||
return
|
||||
|
||||
logger.info("Exporting YOLOv8n → %s", onnx_path)
|
||||
|
||||
from ultralytics import YOLO
|
||||
|
||||
model = YOLO("yolov8n.pt")
|
||||
model.export(format="onnx", imgsz=640, simplify=True)
|
||||
|
||||
# ultralytics exports to cwd as yolov8n.onnx — move to target. Use
|
||||
# shutil.move rather than Path.rename so it works across filesystems
|
||||
# (the cwd is typically /app inside the container, while the target
|
||||
# /data/models is a separately-mounted volume — Path.rename raises
|
||||
# "Invalid cross-device link" in that case).
|
||||
import shutil
|
||||
|
||||
exported = Path("yolov8n.onnx")
|
||||
if exported.exists():
|
||||
shutil.move(str(exported), str(onnx_path))
|
||||
|
||||
size_mb = onnx_path.stat().st_size / 1e6
|
||||
logger.info("YOLOv8n exported (%.1f MB)", size_mb)
|
||||
logger.info("Visual encoder exported (%.1f MB)", visual_path.stat().st_size / 1e6)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Export vision model weights to ONNX")
|
||||
parser.add_argument(
|
||||
"--models-dir",
|
||||
type=Path,
|
||||
default=Path("/data/models"),
|
||||
help="Directory to write model files (default: /data/models)",
|
||||
)
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--models-dir", type=Path, default=Path("/data/models"))
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
|
||||
models_dir = args.models_dir
|
||||
models_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info("Exporting models to %s", models_dir)
|
||||
|
||||
export_openclip(models_dir)
|
||||
export_siglip2(models_dir)
|
||||
export_yolov8n(models_dir)
|
||||
|
||||
logger.info("Done. Run bootstrap_models.py next to download YuNet + SFace.")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
args.models_dir.mkdir(parents=True, exist_ok=True)
|
||||
export_openclip_visual(args.models_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
"""
|
||||
Face detection (YuNet) + recognition (SFace) using OpenCV DNN.
|
||||
|
||||
YuNet is loaded via cv2.FaceDetectorYN which handles the multi-scale
|
||||
anchor decoding and NMS internally. SFace recognition uses raw ONNX
|
||||
Runtime for the 128-d embedding.
|
||||
|
||||
Both models are from opencv_zoo (Apache 2.0 license).
|
||||
Expects {models_dir}/face/:
|
||||
- yunet.onnx (~233 KB)
|
||||
- sface.onnx (~37 MB, 128-d embeddings)
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import cv2
|
||||
import onnxruntime as ort
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import FaceProcessor, FaceDetection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _align_face(image: np.ndarray, landmarks: np.ndarray) -> np.ndarray:
|
||||
"""Align and crop a 112x112 face patch using 5-point landmarks."""
|
||||
left_eye = landmarks[0]
|
||||
right_eye = landmarks[1]
|
||||
|
||||
dx = right_eye[0] - left_eye[0]
|
||||
dy = right_eye[1] - left_eye[1]
|
||||
angle = np.degrees(np.arctan2(dy, dx))
|
||||
eye_center = ((left_eye[0] + right_eye[0]) / 2, (left_eye[1] + right_eye[1]) / 2)
|
||||
eye_dist = np.sqrt(dx * dx + dy * dy)
|
||||
|
||||
M = cv2.getRotationMatrix2D(eye_center, angle, 1.0)
|
||||
rotated = cv2.warpAffine(image, M, (image.shape[1], image.shape[0]))
|
||||
|
||||
# Crop around face center
|
||||
scale = 64.0 / max(eye_dist, 1e-6)
|
||||
cx, cy = eye_center
|
||||
half = 56.0 / scale
|
||||
x1 = max(0, int(cx - half))
|
||||
y1 = max(0, int(cy - half * 0.8))
|
||||
x2 = min(rotated.shape[1], int(cx + half))
|
||||
y2 = min(rotated.shape[0], int(cy + half * 1.2))
|
||||
crop = rotated[y1:y2, x1:x2]
|
||||
|
||||
if crop.size == 0:
|
||||
return np.zeros((112, 112, 3), dtype=np.float32)
|
||||
|
||||
return cv2.resize(crop, (112, 112)).astype(np.float32)
|
||||
|
||||
|
||||
class YuNetSFaceProcessor(FaceProcessor):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
face_dir = Path(settings.models_dir) / "face"
|
||||
yunet_path = str(face_dir / "yunet.onnx")
|
||||
sface_path = str(face_dir / "sface.onnx")
|
||||
|
||||
# YuNet via OpenCV's FaceDetectorYN — handles anchor decoding + NMS
|
||||
self._detector = cv2.FaceDetectorYN.create(
|
||||
yunet_path,
|
||||
"",
|
||||
(640, 640),
|
||||
settings.faces.recognition_threshold,
|
||||
0.3, # NMS threshold
|
||||
5000, # top_k
|
||||
)
|
||||
logger.info("YuNet face detector loaded via OpenCV")
|
||||
|
||||
# SFace via ONNX Runtime
|
||||
from app.services.vision.providers import create_session
|
||||
ort.set_default_logger_severity(3)
|
||||
self._recognizer = create_session(sface_path, configured_providers=settings.execution_providers)
|
||||
logger.info("SFace recognizer loaded via ONNX Runtime")
|
||||
|
||||
self._min_face_size = settings.faces.min_face_size
|
||||
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
|
||||
# Convert RGB → BGR for OpenCV
|
||||
bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
|
||||
# Set input size to actual image dimensions
|
||||
self._detector.setInputSize((orig_w, orig_h))
|
||||
|
||||
# Detect faces
|
||||
_, faces_raw = self._detector.detect(bgr)
|
||||
|
||||
if faces_raw is None or len(faces_raw) == 0:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for face in faces_raw:
|
||||
# face: [x, y, w, h, right_eye_x, right_eye_y, left_eye_x, left_eye_y,
|
||||
# nose_x, nose_y, right_mouth_x, right_mouth_y, left_mouth_x, left_mouth_y, score]
|
||||
x, y, w, h = int(face[0]), int(face[1]), int(face[2]), int(face[3])
|
||||
score = float(face[14])
|
||||
|
||||
# Filter small faces
|
||||
face_size = max(w, h)
|
||||
if face_size < self._min_face_size:
|
||||
continue
|
||||
|
||||
# Normalized bbox
|
||||
bbox = [
|
||||
max(0, x / orig_w),
|
||||
max(0, y / orig_h),
|
||||
min(1, (x + w) / orig_w),
|
||||
min(1, (y + h) / orig_h),
|
||||
]
|
||||
|
||||
# Extract 5-point landmarks for alignment
|
||||
landmarks = np.array([
|
||||
[face[4], face[5]], # right eye
|
||||
[face[6], face[7]], # left eye
|
||||
[face[8], face[9]], # nose
|
||||
[face[10], face[11]], # right mouth
|
||||
[face[12], face[13]], # left mouth
|
||||
], dtype=np.float32)
|
||||
|
||||
# Align face for recognition
|
||||
face_crop = _align_face(image, landmarks)
|
||||
|
||||
# SFace expects (1, 3, 112, 112) float32, BGR
|
||||
face_bgr = cv2.cvtColor(face_crop.astype(np.uint8), cv2.COLOR_RGB2BGR)
|
||||
face_blob = (face_bgr.astype(np.float32) / 255.0).transpose(2, 0, 1)[np.newaxis]
|
||||
|
||||
rec_input = self._recognizer.get_inputs()[0].name
|
||||
embedding = self._recognizer.run(None, {rec_input: face_blob})[0][0]
|
||||
embedding = embedding / np.linalg.norm(embedding)
|
||||
|
||||
results.append(FaceDetection(
|
||||
bbox=bbox,
|
||||
embedding=embedding.astype(np.float32),
|
||||
quality=score,
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return 128
|
||||
@@ -1,73 +0,0 @@
|
||||
"""
|
||||
Face detection + recognition using InsightFace (RetinaFace + ArcFace).
|
||||
|
||||
Uses the buffalo_l model pack which auto-downloads on first use (~300MB).
|
||||
Produces 512-d ArcFace embeddings. Non-commercial research license —
|
||||
fine for homelab self-hosting.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import FaceProcessor, FaceDetection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InsightFaceProcessor(FaceProcessor):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
from insightface.app import FaceAnalysis
|
||||
|
||||
model_root = str(Path(settings.models_dir) / "face" / "insightface")
|
||||
logger.info("Loading InsightFace buffalo_l from %s", model_root)
|
||||
|
||||
from app.services.vision.providers import get_providers
|
||||
providers = get_providers(settings.execution_providers)
|
||||
|
||||
self._app = FaceAnalysis(
|
||||
name="buffalo_l",
|
||||
root=model_root,
|
||||
providers=providers,
|
||||
)
|
||||
self._app.prepare(ctx_id=-1, det_size=(640, 640))
|
||||
self._min_det_score = settings.faces.recognition_threshold
|
||||
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
|
||||
# InsightFace expects BGR
|
||||
bgr = image[:, :, ::-1].copy()
|
||||
|
||||
faces = self._app.get(bgr)
|
||||
|
||||
if not faces:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for face in faces:
|
||||
if face.det_score < self._min_det_score:
|
||||
continue
|
||||
|
||||
# face.bbox is [x1, y1, x2, y2] in pixel coords
|
||||
x1, y1, x2, y2 = face.bbox
|
||||
bbox = [
|
||||
max(0, float(x1) / orig_w),
|
||||
max(0, float(y1) / orig_h),
|
||||
min(1, float(x2) / orig_w),
|
||||
min(1, float(y2) / orig_h),
|
||||
]
|
||||
|
||||
embedding = face.normed_embedding # already L2-normalized, 512-d
|
||||
results.append(FaceDetection(
|
||||
bbox=bbox,
|
||||
embedding=embedding.astype(np.float32),
|
||||
quality=float(face.det_score),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return 512
|
||||
@@ -1,45 +0,0 @@
|
||||
"""
|
||||
OCR engine using rapidocr-onnxruntime (PP-OCRv4 weights).
|
||||
|
||||
No PaddlePaddle dependency — pure ONNX Runtime. Language packs are
|
||||
downloaded automatically by rapidocr on first use.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import OCREngine, OCRResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RapidOCREngine(OCREngine):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
self._min_confidence = settings.ocr.min_confidence
|
||||
self._engine = RapidOCR()
|
||||
logger.info("RapidOCR engine initialized")
|
||||
|
||||
def run(self, image: np.ndarray) -> list[OCRResult]:
|
||||
result, _ = self._engine(image)
|
||||
if not result:
|
||||
return []
|
||||
|
||||
out = []
|
||||
for box, text, score in result:
|
||||
if score < self._min_confidence:
|
||||
continue
|
||||
# box is [[x1,y1],[x2,y2],[x3,y3],[x4,y4]] — take bounding rect
|
||||
xs = [p[0] for p in box]
|
||||
ys = [p[1] for p in box]
|
||||
h, w = image.shape[:2]
|
||||
bbox = [
|
||||
min(xs) / w,
|
||||
min(ys) / h,
|
||||
max(xs) / w,
|
||||
max(ys) / h,
|
||||
]
|
||||
out.append(OCRResult(text=text, confidence=float(score), bbox=bbox))
|
||||
return out
|
||||
@@ -1,46 +0,0 @@
|
||||
"""
|
||||
ONNX Runtime backend — default CPU inference for all vision models.
|
||||
|
||||
Each create_* method returns a concrete implementation of the
|
||||
corresponding ABC from base.py. Models are loaded from ONNX files
|
||||
under settings.vision.models_dir, downloaded on first boot by
|
||||
bootstrap_models.py.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor, ContentClassifier
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ONNXBackend:
|
||||
"""Factory for ONNX Runtime-based vision model instances."""
|
||||
|
||||
def __init__(self, vision_settings: VisionSettings):
|
||||
self._settings = vision_settings
|
||||
|
||||
def create_embedder(self) -> Embedder:
|
||||
model_name = self._settings.embedder.name
|
||||
if model_name.startswith("siglip2"):
|
||||
from app.services.vision.embed import SigLIP2Embedder
|
||||
return SigLIP2Embedder(self._settings)
|
||||
else:
|
||||
from app.services.vision.embed import OpenCLIPEmbedder
|
||||
return OpenCLIPEmbedder(self._settings)
|
||||
|
||||
def create_ocr(self) -> OCREngine:
|
||||
from app.services.vision.ocr import RapidOCREngine
|
||||
return RapidOCREngine(self._settings)
|
||||
|
||||
def create_detector(self) -> ObjectDetector:
|
||||
from app.services.vision.detect import YOLOv8Detector
|
||||
return YOLOv8Detector(self._settings)
|
||||
|
||||
def create_face_processor(self) -> FaceProcessor:
|
||||
from app.services.vision.insightface_processor import InsightFaceProcessor
|
||||
return InsightFaceProcessor(self._settings)
|
||||
|
||||
def create_classifier(self) -> ContentClassifier:
|
||||
from app.services.vision.classify import CLIPContentClassifier
|
||||
return CLIPContentClassifier(self._settings)
|
||||
@@ -1,84 +1,29 @@
|
||||
"""
|
||||
ModelRegistry — singleton that lazy-loads vision models per worker process.
|
||||
|
||||
Usage from Celery tasks:
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
vec = embedder.embed_image(img)
|
||||
|
||||
Models are created on first access and cached for the worker's lifetime.
|
||||
The registry reads settings.vision to decide which backend to use and
|
||||
where model weights live.
|
||||
ModelRegistry — lazy-loads the single content classifier per worker.
|
||||
"""
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
|
||||
from app.config import settings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor, ContentClassifier
|
||||
from app.services.vision.base import ContentClassifier
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ModelRegistry:
|
||||
"""Central access point for all vision models."""
|
||||
|
||||
def __init__(self):
|
||||
self._vision = settings.vision
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_embedder(self) -> Embedder:
|
||||
logger.info("Loading embedder: %s (backend=%s)", self._vision.embedder.name, self._vision.backend)
|
||||
return self._load_backend().create_embedder()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_ocr(self) -> OCREngine:
|
||||
logger.info("Loading OCR engine (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_ocr()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_detector(self) -> ObjectDetector:
|
||||
logger.info("Loading object detector (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_detector()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_face_processor(self) -> FaceProcessor:
|
||||
logger.info("Loading face processor (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_face_processor()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_classifier(self) -> ContentClassifier:
|
||||
logger.info("Loading content classifier (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_classifier()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_backend(self):
|
||||
"""Import and instantiate the configured backend."""
|
||||
backend_name = self._vision.backend
|
||||
if backend_name == "onnx":
|
||||
from app.services.vision.onnx_backend import ONNXBackend
|
||||
return ONNXBackend(self._vision)
|
||||
elif backend_name == "rocm":
|
||||
from app.services.vision.rocm_backend import ROCmBackend
|
||||
return ROCmBackend(self._vision)
|
||||
else:
|
||||
raise ValueError(f"Unknown vision backend: {backend_name}")
|
||||
from app.services.vision.classify import CLIPContentClassifier
|
||||
return CLIPContentClassifier(self._vision)
|
||||
|
||||
def warmup(self):
|
||||
"""Pre-load all enabled models. Called from Celery worker_process_init
|
||||
on the vision queue to avoid cold-start latency on the first task."""
|
||||
logger.info("Warming up vision models...")
|
||||
self.get_embedder()
|
||||
if self._vision.ocr.enabled:
|
||||
self.get_ocr()
|
||||
if self._vision.detector.enabled:
|
||||
self.get_detector()
|
||||
if self._vision.faces.enabled:
|
||||
self.get_face_processor()
|
||||
if self._vision.classifier.enabled:
|
||||
self.get_classifier()
|
||||
logger.info("Vision model warmup complete")
|
||||
logger.info("Warming up vision classifier...")
|
||||
self.get_classifier()
|
||||
logger.info("Vision warmup complete")
|
||||
|
||||
|
||||
# Module-level singleton. Import this from tasks.
|
||||
registry = ModelRegistry()
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
"""
|
||||
ROCm backend — GPU-accelerated inference for Radeon 760M-class hardware.
|
||||
|
||||
Stub: raises NotImplementedError on all factory methods. To enable,
|
||||
set `vision.backend: rocm` in mulita.yml once ROCm support is implemented.
|
||||
"""
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor
|
||||
|
||||
|
||||
class ROCmBackend:
|
||||
def __init__(self, vision_settings: VisionSettings):
|
||||
self._settings = vision_settings
|
||||
|
||||
def create_embedder(self) -> Embedder:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_ocr(self) -> OCREngine:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_detector(self) -> ObjectDetector:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_face_processor(self) -> FaceProcessor:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
@@ -39,14 +39,9 @@ celery_app.conf.update(
|
||||
# Explicit routes for every task name. Wildcard patterns don't match
|
||||
# short names produced by @shared_task(name='...').
|
||||
task_routes={
|
||||
# Vision queue — GPU/CPU-bound inference
|
||||
'embed_photo': {'queue': 'vision'},
|
||||
'ocr_photo': {'queue': 'vision'},
|
||||
'detect_objects': {'queue': 'vision'},
|
||||
'extract_faces': {'queue': 'vision'},
|
||||
# Vision queue — CPU-bound binary classification
|
||||
'classify_content': {'queue': 'vision'},
|
||||
'vision_fanout': {'queue': 'vision'},
|
||||
'recluster_faces': {'queue': 'vision'},
|
||||
# High-priority queue — thumbnails & duplicates
|
||||
'generate_thumbnails': {'queue': 'high'},
|
||||
'regenerate_all_thumbnails': {'queue': 'high'},
|
||||
|
||||
@@ -422,7 +422,7 @@ async def _scan_all_source_roots_async():
|
||||
still hit Settings → Re-detect duplicates to force a fresh pass.
|
||||
"""
|
||||
from app.tasks.thumbs import incremental_regroup_duplicates_task
|
||||
from app.tasks.vision import backfill_vision, recluster_faces
|
||||
from app.tasks.vision import backfill_vision
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
@@ -461,14 +461,6 @@ async def _scan_all_source_roots_async():
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue post-scan vision backfill: {e}")
|
||||
|
||||
# 300s gives face extraction time to run before reclustering.
|
||||
# Fires even if some faces are still in-flight — the task is
|
||||
# idempotent and the user can re-trigger from Settings.
|
||||
try:
|
||||
recluster_faces.apply_async(countdown=300)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue post-scan face recluster: {e}")
|
||||
|
||||
# Re-extract metadata for photos missing GPS coordinates.
|
||||
# Runs on every startup so photos scanned before the GPS fix
|
||||
# eventually get their coordinates populated.
|
||||
|
||||
@@ -1,33 +1,21 @@
|
||||
"""
|
||||
Celery tasks for the vision pipeline — embedding, OCR, object detection,
|
||||
face recognition.
|
||||
Celery tasks for the vision pipeline.
|
||||
|
||||
All tasks run on the dedicated `vision` queue with limited concurrency
|
||||
(memory-bound CPU inference). They read thumbnails generated by
|
||||
generate_thumbnails, so they MUST run after thumbs complete.
|
||||
|
||||
DB access uses sync psycopg2 sessions (not asyncpg) because Celery
|
||||
forks workers and asyncpg connections can't be shared across forks.
|
||||
A single binary classifier decides whether a photo is 'photography' or
|
||||
'other'. Photos classified as 'other' get needs_review=true so the user
|
||||
can triage screenshots / documents / memes in the UI.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from celery import shared_task
|
||||
from sqlalchemy import create_engine, text as sa_text, select, delete
|
||||
from sqlalchemy import create_engine, text as sa_text, select, delete, update
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from PIL import Image
|
||||
|
||||
from app.models.embeddings import Embedding
|
||||
from app.config import settings
|
||||
from app.services.feature_flags import (
|
||||
is_enabled,
|
||||
FLAG_VISION_ENABLED,
|
||||
FLAG_OCR_ENABLED,
|
||||
FLAG_DETECTOR_ENABLED,
|
||||
FLAG_FACES_ENABLED,
|
||||
FLAG_CLASSIFIER_ENABLED,
|
||||
)
|
||||
from app.services.feature_flags import is_enabled, FLAG_VISION_ENABLED
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -35,7 +23,6 @@ VISION_READY_KEY = "mulita:vision:ready"
|
||||
|
||||
|
||||
def _vision_worker_ready() -> bool:
|
||||
"""Check whether the vision worker has finished model bootstrap."""
|
||||
try:
|
||||
import redis as _redis
|
||||
return bool(_redis.from_url(settings.redis_url).exists(VISION_READY_KEY))
|
||||
@@ -47,7 +34,6 @@ _sync_engine = None
|
||||
|
||||
|
||||
def _get_sync_engine():
|
||||
"""Return a module-level singleton engine (one per worker process)."""
|
||||
global _sync_engine
|
||||
if _sync_engine is None:
|
||||
sync_url = settings.database_url.replace("+asyncpg", "+psycopg2").replace("+aiosqlite", "")
|
||||
@@ -56,21 +42,13 @@ def _get_sync_engine():
|
||||
|
||||
|
||||
def _get_sync_session() -> Session:
|
||||
"""Create a sync DB session backed by the shared engine."""
|
||||
return sessionmaker(bind=_get_sync_engine())()
|
||||
|
||||
|
||||
def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None:
|
||||
"""Load a thumbnail as an RGB numpy array.
|
||||
|
||||
Thumbnails may live at ``/data/thumbs/{photo_id}/`` (legacy) or
|
||||
``/data/thumbs/{user_id}/{photo_id}/`` (multi-user). Try both.
|
||||
"""
|
||||
thumb_base = Path("/data/thumbs")
|
||||
# Try legacy flat path first.
|
||||
thumb_path = thumb_base / photo_id / f"{size}.webp"
|
||||
if not thumb_path.exists():
|
||||
# Try user-prefixed paths: /data/thumbs/*/photo_id/size.webp
|
||||
matches = list(thumb_base.glob(f"*/{photo_id}/{size}.webp"))
|
||||
if matches:
|
||||
thumb_path = matches[0]
|
||||
@@ -79,7 +57,7 @@ def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None:
|
||||
return None
|
||||
try:
|
||||
img = Image.open(thumb_path).convert("RGB")
|
||||
img.load() # force decode to catch corruption early
|
||||
img.load()
|
||||
arr = np.array(img)
|
||||
img.close()
|
||||
return arr
|
||||
@@ -88,207 +66,25 @@ def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None:
|
||||
return None
|
||||
|
||||
|
||||
@shared_task(name='embed_photo', queue='vision', bind=True, max_retries=3)
|
||||
def embed_photo(self, photo_id: str):
|
||||
"""Generate CLIP embedding for a photo and store in pgvector."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED):
|
||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium") # 640px
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
vector = embedder.embed_image(image)
|
||||
except Exception as exc:
|
||||
logger.exception("embed_photo failed for %s", photo_id)
|
||||
raise self.retry(exc=exc, countdown=60)
|
||||
|
||||
model_name = settings.vision.embedder.name
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(
|
||||
delete(Embedding).where(
|
||||
Embedding.photo_id == photo_id,
|
||||
Embedding.model == model_name,
|
||||
)
|
||||
)
|
||||
emb = Embedding(
|
||||
photo_id=photo_id,
|
||||
model=model_name,
|
||||
vector=vector.tolist(),
|
||||
)
|
||||
session.add(emb)
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("[%s] Embedded photo %s with %s", self.request.id, photo_id, model_name)
|
||||
return {'status': 'success', 'photo_id': photo_id}
|
||||
|
||||
|
||||
@shared_task(name='vision_fanout', queue='vision')
|
||||
def vision_fanout(photo_id: str):
|
||||
"""Dispatch all enabled vision tasks for a photo."""
|
||||
"""Dispatch vision work for a photo. Today this is just the binary
|
||||
classifier; the indirection stays so scanner/upload code keeps one
|
||||
entrypoint."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED):
|
||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||
|
||||
embed_photo.delay(photo_id)
|
||||
|
||||
if is_enabled(FLAG_OCR_ENABLED):
|
||||
ocr_photo.delay(photo_id)
|
||||
if is_enabled(FLAG_DETECTOR_ENABLED):
|
||||
detect_objects.delay(photo_id)
|
||||
if is_enabled(FLAG_FACES_ENABLED):
|
||||
extract_faces.delay(photo_id)
|
||||
if is_enabled(FLAG_CLASSIFIER_ENABLED):
|
||||
classify_content.delay(photo_id)
|
||||
|
||||
classify_content.delay(photo_id)
|
||||
return {'status': 'dispatched', 'photo_id': photo_id}
|
||||
|
||||
|
||||
@shared_task(name='ocr_photo', queue='vision', bind=True, max_retries=3)
|
||||
def ocr_photo(self, photo_id: str):
|
||||
"""Run OCR on a photo and store text regions."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_OCR_ENABLED):
|
||||
return {'status': 'skipped', 'reason': 'OCR disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "large") # 1280px for better OCR accuracy
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
ocr_engine = registry.get_ocr()
|
||||
results = ocr_engine.run(image)
|
||||
except Exception as exc:
|
||||
logger.exception("ocr_photo failed for %s", photo_id)
|
||||
raise self.retry(exc=exc, countdown=60)
|
||||
|
||||
if not results:
|
||||
logger.info("No OCR text found for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'regions': 0}
|
||||
|
||||
from app.models.ocr_text import OCRText
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(delete(OCRText).where(OCRText.photo_id == photo_id))
|
||||
for r in results:
|
||||
session.add(OCRText(
|
||||
photo_id=photo_id,
|
||||
text=r.text,
|
||||
language=r.language,
|
||||
confidence=r.confidence,
|
||||
bbox=r.bbox,
|
||||
))
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("[%s] OCR: %d text regions for photo %s", self.request.id, len(results), photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'regions': len(results)}
|
||||
|
||||
|
||||
@shared_task(name='detect_objects', queue='vision', bind=True, max_retries=3)
|
||||
def detect_objects(self, photo_id: str):
|
||||
"""Detect objects in a photo, create Tag(kind=object) rows, and
|
||||
link via photo_tags with confidence/bbox/source."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_DETECTOR_ENABLED):
|
||||
return {'status': 'skipped', 'reason': 'detection disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium") # 640px
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
detector = registry.get_detector()
|
||||
detections = detector.detect(image)
|
||||
except Exception as exc:
|
||||
logger.exception("detect_objects failed for %s", photo_id)
|
||||
raise self.retry(exc=exc, countdown=60)
|
||||
|
||||
if not detections:
|
||||
logger.info("No objects detected for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'objects': 0}
|
||||
|
||||
from app.models import Photo
|
||||
from app.models.tags import Tag, photo_tags
|
||||
|
||||
source_name = "vision:yolov8n"
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
# Get the photo's user_id so tags inherit ownership.
|
||||
photo = session.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
).scalar_one_or_none()
|
||||
owner_id = photo.user_id if photo else None
|
||||
|
||||
# Wipe previous detection results for this photo from this model
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.photo_id == photo_id,
|
||||
photo_tags.c.source == source_name,
|
||||
)
|
||||
)
|
||||
|
||||
# Group detections by label, keep highest confidence per label
|
||||
best_per_label: dict[str, tuple[float, list]] = {}
|
||||
for det in detections:
|
||||
if det.label not in best_per_label or det.confidence > best_per_label[det.label][0]:
|
||||
best_per_label[det.label] = (det.confidence, det.bbox)
|
||||
|
||||
for label, (confidence, bbox) in best_per_label.items():
|
||||
# Find or create the object tag (scoped to user)
|
||||
tag = session.execute(
|
||||
select(Tag).where(Tag.name == label, Tag.kind == 'object', Tag.user_id == owner_id)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if not tag:
|
||||
tag = Tag(name=label, kind='object', source=source_name, user_id=owner_id)
|
||||
session.add(tag)
|
||||
session.flush() # get tag.id
|
||||
|
||||
# Insert photo_tags association with ML metadata
|
||||
session.execute(
|
||||
photo_tags.insert().values(
|
||||
photo_id=photo_id,
|
||||
tag_id=tag.id,
|
||||
confidence=confidence,
|
||||
bbox=bbox,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
labels = [d.label for d in detections]
|
||||
logger.info("[%s] Detected %d objects in photo %s: %s", self.request.id, len(detections), photo_id, labels)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'objects': len(detections)}
|
||||
|
||||
|
||||
@shared_task(name='classify_content', queue='vision', bind=True, max_retries=3)
|
||||
def classify_content(self, photo_id: str):
|
||||
"""Classify image content type (screenshot, document, artwork, etc.)
|
||||
using CLIP zero-shot classification. Writes Tag(kind=content_type)."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_CLASSIFIER_ENABLED):
|
||||
return {'status': 'skipped', 'reason': 'classifier disabled'}
|
||||
"""Run the binary classifier and write:
|
||||
- a Tag(kind='content_type', name IN ('photography','other'))
|
||||
- Photo.needs_review = (label == 'other')
|
||||
"""
|
||||
if not is_enabled(FLAG_VISION_ENABLED):
|
||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium")
|
||||
if image is None:
|
||||
@@ -297,30 +93,28 @@ def classify_content(self, photo_id: str):
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
classifier = registry.get_classifier()
|
||||
results = classifier.classify(image)
|
||||
result = classifier.classify(image)
|
||||
except Exception as exc:
|
||||
logger.exception("classify_content failed for %s", photo_id)
|
||||
raise self.retry(exc=exc, countdown=60)
|
||||
|
||||
if not results:
|
||||
logger.info("No confident classification for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'content_type': None}
|
||||
|
||||
from app.models import Photo
|
||||
from app.models.tags import Tag, photo_tags
|
||||
|
||||
source_name = "vision:clip_classifier"
|
||||
best = results[0]
|
||||
label = result.label
|
||||
confidence = result.confidence
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
# Get the photo's user_id so tags inherit ownership.
|
||||
photo = session.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
).scalar_one_or_none()
|
||||
owner_id = photo.user_id if photo else None
|
||||
if photo is None:
|
||||
return {'status': 'error', 'message': 'photo not found'}
|
||||
owner_id = photo.user_id
|
||||
|
||||
# Wipe previous classification for this photo
|
||||
# Drop any previous classification for this photo.
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.photo_id == photo_id,
|
||||
@@ -328,13 +122,13 @@ def classify_content(self, photo_id: str):
|
||||
)
|
||||
)
|
||||
|
||||
# Find or create content_type tag (scoped to user)
|
||||
tag = session.execute(
|
||||
select(Tag).where(Tag.name == best.label, Tag.kind == 'content_type', Tag.user_id == owner_id)
|
||||
select(Tag).where(
|
||||
Tag.name == label, Tag.kind == 'content_type', Tag.user_id == owner_id
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if not tag:
|
||||
tag = Tag(name=best.label, kind='content_type', source=source_name, user_id=owner_id)
|
||||
tag = Tag(name=label, kind='content_type', source=source_name, user_id=owner_id)
|
||||
session.add(tag)
|
||||
session.flush()
|
||||
|
||||
@@ -342,232 +136,16 @@ def classify_content(self, photo_id: str):
|
||||
photo_tags.insert().values(
|
||||
photo_id=photo_id,
|
||||
tag_id=tag.id,
|
||||
confidence=best.confidence,
|
||||
confidence=confidence,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("[%s] Classified photo %s as '%s' (%.2f)", self.request.id, photo_id, best.label, best.confidence)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'content_type': best.label}
|
||||
|
||||
|
||||
def _load_original(photo_id: str) -> np.ndarray | None:
|
||||
"""Load the original photo file as an RGB numpy array, resized to
|
||||
max 1280px on the longest edge for face detection."""
|
||||
from sqlalchemy import create_engine, select as sa_select, text as sa_text
|
||||
from app.models import Photo
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
photo = session.execute(
|
||||
sa_select(Photo).where(Photo.id == photo_id)
|
||||
).scalar_one_or_none()
|
||||
if not photo or not photo.filepath:
|
||||
return None
|
||||
filepath = photo.filepath
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if not Path(filepath).exists():
|
||||
logger.warning("Original file not found: %s", filepath)
|
||||
return None
|
||||
|
||||
try:
|
||||
img = Image.open(filepath).convert("RGB")
|
||||
# Cap at 4000px on longest edge to avoid OOM, but keep as large
|
||||
# as possible for face detection accuracy
|
||||
max_dim = 4000
|
||||
w, h = img.size
|
||||
if max(w, h) > max_dim:
|
||||
scale = max_dim / max(w, h)
|
||||
resized = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC)
|
||||
img.close()
|
||||
img = resized
|
||||
arr = np.array(img)
|
||||
img.close()
|
||||
return arr
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load original %s: %s", filepath, e)
|
||||
return None
|
||||
|
||||
|
||||
@shared_task(name='extract_faces', queue='vision', bind=True, max_retries=3)
|
||||
def extract_faces(self, photo_id: str):
|
||||
"""Detect faces and store recognition embeddings using InsightFace
|
||||
(RetinaFace + ArcFace). No YOLO workaround needed — RetinaFace has
|
||||
strong human-vs-non-human precision on its own."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_FACES_ENABLED):
|
||||
return {'status': 'skipped', 'reason': 'faces disabled'}
|
||||
|
||||
image = _load_original(photo_id)
|
||||
if image is None:
|
||||
image = _load_thumb(photo_id, "large")
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'no image available'}
|
||||
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
face_proc = registry.get_face_processor()
|
||||
faces = face_proc.process(image)
|
||||
except Exception as exc:
|
||||
logger.exception("extract_faces failed for %s", photo_id)
|
||||
raise self.retry(exc=exc, countdown=60)
|
||||
|
||||
if not faces:
|
||||
logger.info("No faces detected for photo %s", photo_id)
|
||||
|
||||
return _save_faces(photo_id, faces)
|
||||
|
||||
|
||||
def _save_faces(photo_id: str, faces) -> dict:
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(delete(FaceEmbedding).where(FaceEmbedding.photo_id == photo_id))
|
||||
for face in faces:
|
||||
session.add(FaceEmbedding(
|
||||
photo_id=photo_id,
|
||||
bbox=face.bbox,
|
||||
vector=face.embedding.tolist(),
|
||||
quality=face.quality,
|
||||
cluster_id=None,
|
||||
))
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if faces:
|
||||
logger.info("Extracted %d verified face(s) from photo %s", len(faces), photo_id)
|
||||
_schedule_recluster_debounced()
|
||||
return {'status': 'success', 'photo_id': photo_id, 'faces': len(faces)}
|
||||
|
||||
|
||||
RECLUSTER_DEBOUNCE_KEY = "mule:recluster_faces:pending"
|
||||
RECLUSTER_DELAY = 120 # seconds after last face extraction
|
||||
|
||||
|
||||
def _schedule_recluster_debounced():
|
||||
"""Schedule a recluster_faces run, debounced so rapid-fire face
|
||||
extractions don't spawn hundreds of redundant cluster jobs."""
|
||||
try:
|
||||
import redis as _redis
|
||||
r = _redis.from_url(settings.redis_url)
|
||||
already_pending = r.set(RECLUSTER_DEBOUNCE_KEY, "1",
|
||||
ex=RECLUSTER_DELAY, nx=True)
|
||||
if already_pending:
|
||||
recluster_faces.apply_async(countdown=RECLUSTER_DELAY)
|
||||
logger.info("Scheduled debounced recluster_faces in %ds", RECLUSTER_DELAY)
|
||||
except Exception as e:
|
||||
logger.debug("recluster debounce check failed: %s", e)
|
||||
|
||||
|
||||
@shared_task(name='recluster_faces', queue='vision', bind=True, max_retries=10)
|
||||
def recluster_faces(self):
|
||||
"""Run DBSCAN clustering over all face embeddings and assign/create
|
||||
Tag(kind=face_cluster) entries."""
|
||||
# Clear debounce key so new face extractions can schedule another round.
|
||||
try:
|
||||
import redis as _redis
|
||||
_redis.from_url(settings.redis_url).delete(RECLUSTER_DEBOUNCE_KEY)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not _vision_worker_ready():
|
||||
logger.info("Vision worker not ready yet — retrying in 30s")
|
||||
raise self.retry(countdown=30)
|
||||
|
||||
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_FACES_ENABLED):
|
||||
return {'status': 'skipped', 'reason': 'faces disabled'}
|
||||
|
||||
from app.models import Photo
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
from app.models.tags import Tag, photo_tags
|
||||
from app.services.vision.clustering import cluster_faces
|
||||
|
||||
source_name = "vision:sface"
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
face_rows = session.execute(
|
||||
select(FaceEmbedding).order_by(FaceEmbedding.created_at)
|
||||
).scalars().all()
|
||||
|
||||
if len(face_rows) < 2:
|
||||
logger.info("Not enough faces for clustering (%d)", len(face_rows))
|
||||
return {'status': 'success', 'clusters': 0}
|
||||
|
||||
embeddings = np.array([f.vector for f in face_rows], dtype=np.float32)
|
||||
labels = cluster_faces(embeddings, eps=settings.vision.faces.cluster_eps)
|
||||
|
||||
# Clean up old face_cluster tags and their photo_tags
|
||||
old_cluster_tags = session.execute(
|
||||
select(Tag).where(Tag.kind == 'face_cluster', Tag.source == source_name)
|
||||
).scalars().all()
|
||||
for old_tag in old_cluster_tags:
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.tag_id == old_tag.id,
|
||||
photo_tags.c.source == source_name,
|
||||
)
|
||||
)
|
||||
session.delete(old_tag)
|
||||
session.flush()
|
||||
|
||||
# Build new clusters
|
||||
cluster_tag_map: dict[int, str] = {}
|
||||
# Track which photos belong to which cluster
|
||||
cluster_photos: dict[int, set[str]] = {}
|
||||
|
||||
for i, label in enumerate(labels):
|
||||
if label == -1:
|
||||
face_rows[i].cluster_id = None
|
||||
continue
|
||||
|
||||
if label not in cluster_photos:
|
||||
cluster_photos[label] = set()
|
||||
cluster_photos[label].add(face_rows[i].photo_id)
|
||||
|
||||
if label not in cluster_tag_map:
|
||||
cluster_name = f"Person {label + 1}"
|
||||
# Inherit user_id from the representative photo.
|
||||
rep_photo = session.execute(
|
||||
select(Photo.user_id).where(Photo.id == face_rows[i].photo_id)
|
||||
).scalar_one_or_none()
|
||||
tag = Tag(
|
||||
name=cluster_name,
|
||||
kind='face_cluster',
|
||||
source=source_name,
|
||||
representative_photo_id=face_rows[i].photo_id,
|
||||
user_id=rep_photo,
|
||||
)
|
||||
session.add(tag)
|
||||
session.flush()
|
||||
cluster_tag_map[label] = tag.id
|
||||
|
||||
face_rows[i].cluster_id = cluster_tag_map[label]
|
||||
|
||||
# Write photo_tags associations so the tag count and tag_ids
|
||||
# filter work for face clusters
|
||||
for label, photo_ids in cluster_photos.items():
|
||||
tag_id = cluster_tag_map[label]
|
||||
for pid in photo_ids:
|
||||
session.execute(
|
||||
photo_tags.insert().values(
|
||||
photo_id=pid,
|
||||
tag_id=tag_id,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
session.execute(
|
||||
update(Photo)
|
||||
.where(Photo.id == photo_id)
|
||||
.values(needs_review=(label == 'other'))
|
||||
)
|
||||
|
||||
session.commit()
|
||||
except Exception:
|
||||
@@ -576,103 +154,41 @@ def recluster_faces(self):
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
n_clusters = len(cluster_tag_map)
|
||||
logger.info("Face clustering: %d clusters from %d faces", n_clusters, len(face_rows))
|
||||
return {'status': 'success', 'clusters': n_clusters, 'faces': len(face_rows)}
|
||||
logger.info("[%s] Classified %s as %s (%.2f)", self.request.id, photo_id, label, confidence)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'label': label}
|
||||
|
||||
|
||||
@shared_task(name='backfill_vision', bind=True, max_retries=10)
|
||||
def backfill_vision(self, task: str | None = None, limit: int | None = None):
|
||||
"""Queue vision tasks for photos that haven't been processed yet.
|
||||
Uses a sync DB connection to avoid asyncpg conflicts in Celery."""
|
||||
def backfill_vision(self, limit: int | None = None, **_ignored):
|
||||
"""Queue classify_content for photos without a content_type tag."""
|
||||
if not _vision_worker_ready():
|
||||
logger.info("Vision worker not ready yet — retrying in 30s")
|
||||
raise self.retry(countdown=30)
|
||||
|
||||
model_name = settings.vision.embedder.name
|
||||
ordering = "ORDER BY p.taken_at DESC NULLS LAST, p.added_at DESC NULLS LAST"
|
||||
limit_clause = " LIMIT :lim" if limit else ""
|
||||
params: dict = {"model": model_name}
|
||||
params: dict = {}
|
||||
if limit:
|
||||
params["lim"] = int(limit)
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
# Each query finds photos missing a specific pipeline output so
|
||||
# enabling a new processor after import still back-fills.
|
||||
embed_ids = []
|
||||
if task in ('embed', None):
|
||||
sql = f"""
|
||||
SELECT p.id FROM photos p
|
||||
LEFT JOIN embeddings e ON e.photo_id = p.id AND e.model = :model
|
||||
WHERE e.photo_id IS NULL AND p.processing_status = 'completed'
|
||||
{ordering}{limit_clause}
|
||||
"""
|
||||
embed_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||
|
||||
ocr_ids = []
|
||||
if task in ('ocr', None) and is_enabled(FLAG_OCR_ENABLED):
|
||||
sql = f"""
|
||||
SELECT p.id FROM photos p
|
||||
LEFT JOIN ocr_text o ON o.photo_id = p.id
|
||||
WHERE o.photo_id IS NULL AND p.processing_status = 'completed'
|
||||
{ordering}{limit_clause}
|
||||
"""
|
||||
ocr_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||
|
||||
detect_ids = []
|
||||
if task in ('detect', None) and is_enabled(FLAG_DETECTOR_ENABLED):
|
||||
sql = f"""
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.processing_status = 'completed'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM photo_tags pt WHERE pt.photo_id = p.id
|
||||
AND pt.source = 'vision:yolov8n'
|
||||
)
|
||||
{ordering}{limit_clause}
|
||||
"""
|
||||
detect_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||
|
||||
face_ids = []
|
||||
if task in ('faces', None) and is_enabled(FLAG_FACES_ENABLED):
|
||||
sql = f"""
|
||||
SELECT p.id FROM photos p
|
||||
LEFT JOIN face_embeddings fe ON fe.photo_id = p.id
|
||||
WHERE fe.photo_id IS NULL AND p.processing_status = 'completed'
|
||||
{ordering}{limit_clause}
|
||||
"""
|
||||
face_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||
|
||||
classify_ids = []
|
||||
if task in ('classify', None) and is_enabled(FLAG_CLASSIFIER_ENABLED):
|
||||
sql = f"""
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.processing_status = 'completed'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM photo_tags pt WHERE pt.photo_id = p.id
|
||||
AND pt.source = 'vision:clip_classifier'
|
||||
)
|
||||
{ordering}{limit_clause}
|
||||
"""
|
||||
classify_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
sql = f"""
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.processing_status = 'completed'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM photo_tags pt
|
||||
WHERE pt.photo_id = p.id
|
||||
AND pt.source = 'vision:clip_classifier'
|
||||
)
|
||||
{ordering}{limit_clause}
|
||||
"""
|
||||
ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
# Dispatch — deduplicate across query results.
|
||||
all_ids = set(embed_ids) | set(ocr_ids) | set(detect_ids) | set(face_ids) | set(classify_ids)
|
||||
for pid in embed_ids:
|
||||
embed_photo.delay(pid)
|
||||
for pid in ocr_ids:
|
||||
ocr_photo.delay(pid)
|
||||
for pid in detect_ids:
|
||||
detect_objects.delay(pid)
|
||||
for pid in face_ids:
|
||||
extract_faces.delay(pid)
|
||||
for pid in classify_ids:
|
||||
for pid in ids:
|
||||
classify_content.delay(pid)
|
||||
|
||||
logger.info("Backfill queued %d photos for vision processing", len(all_ids))
|
||||
return {'status': 'queued', 'count': len(all_ids)}
|
||||
logger.info("Backfill queued %d photos for classification", len(ids))
|
||||
return {'status': 'queued', 'count': len(ids)}
|
||||
|
||||
@@ -8,7 +8,6 @@ sqlalchemy[asyncio]==2.0.25
|
||||
aiosqlite==0.19.0 # SQLite escape hatch (docker-compose.sqlite.yml override)
|
||||
asyncpg==0.29.0 # async Postgres driver (default)
|
||||
psycopg2-binary==2.9.9 # sync Postgres driver, used by Alembic CLI
|
||||
pgvector==0.2.5 # pgvector SQLAlchemy types
|
||||
alembic==1.13.1
|
||||
|
||||
# Redis and Celery
|
||||
@@ -42,11 +41,6 @@ watchfiles==0.21.0
|
||||
# Vision pipeline (ONNX Runtime CPU inference)
|
||||
onnxruntime==1.18.1
|
||||
open-clip-torch==2.24.0 # tokenizer + export helper; inference via ONNX
|
||||
transformers>=4.37.0 # HuggingFace tokenizer for SigLIP models
|
||||
ultralytics==8.4.37 # YOLOv8n export helper; inference via ONNX
|
||||
rapidocr-onnxruntime==1.3.22
|
||||
scikit-learn==1.4.0 # DBSCAN for face clustering
|
||||
insightface>=0.7.3 # RetinaFace + ArcFace face detection/recognition
|
||||
numpy>=1.26.0,<2.0
|
||||
|
||||
# Utilities
|
||||
|
||||
@@ -3,7 +3,6 @@ import { Timeline } from './components/timeline/Timeline'
|
||||
import { DuplicatesView } from './components/duplicates/DuplicatesView'
|
||||
import { MapView } from './components/map/MapView'
|
||||
import { MemoriesView } from './components/memories/MemoriesView'
|
||||
import { PeopleView } from './components/people/PeopleView'
|
||||
import { TagsView } from './components/tags/TagsView'
|
||||
import { ColorsView } from './components/colors/ColorsView'
|
||||
import { RatedView } from './components/rated/RatedView'
|
||||
@@ -92,8 +91,6 @@ function MainApp() {
|
||||
<MemoriesView />
|
||||
) : currentSection === 'duplicates' ? (
|
||||
<DuplicatesView />
|
||||
) : currentSection === 'people' ? (
|
||||
<PeopleView />
|
||||
) : currentSection === 'tags' ? (
|
||||
<TagsView />
|
||||
) : currentSection === 'colors' ? (
|
||||
|
||||
@@ -16,10 +16,6 @@ import {
|
||||
FolderSearch,
|
||||
Shield,
|
||||
Brain,
|
||||
ScanText,
|
||||
UserSquare2,
|
||||
Boxes,
|
||||
Tags as TagsIcon,
|
||||
RotateCcw,
|
||||
} from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
@@ -1107,56 +1103,20 @@ interface AiFeaturesTabProps {
|
||||
) => Promise<void>
|
||||
}
|
||||
|
||||
// Flags are keyed by the backend's canonical name ("vision.enabled",
|
||||
// "vision.ocr.enabled", ...). The metadata here just adds presentation
|
||||
// (label, short description, icon) so the tab layout stays data-driven.
|
||||
const FLAG_META: Array<{
|
||||
id: string
|
||||
label: string
|
||||
description: string
|
||||
icon: React.ReactNode
|
||||
// Optional "run this backfill" hook — lets the user kick off a stage's
|
||||
// backfill right from the toggle row without hopping to a separate UI.
|
||||
backfillTask?: 'embed' | 'ocr' | 'detect' | 'faces' | 'classify'
|
||||
}> = [
|
||||
{
|
||||
id: 'vision.enabled',
|
||||
label: 'Vision pipeline (master switch)',
|
||||
label: 'Vision classifier',
|
||||
description:
|
||||
'When off, every AI stage below is skipped — including newly uploaded photos. ' +
|
||||
'Existing results stay intact.',
|
||||
'Binary photo-vs-other classifier. Flags screenshots, documents, memes and ' +
|
||||
'scans with "needs review" so they can be triaged.',
|
||||
icon: <Sparkles className="h-3.5 w-3.5" />,
|
||||
},
|
||||
{
|
||||
id: 'vision.ocr.enabled',
|
||||
label: 'Text recognition (OCR)',
|
||||
description: 'Extract printed / handwritten text from photos so it becomes searchable.',
|
||||
icon: <ScanText className="h-3.5 w-3.5" />,
|
||||
backfillTask: 'ocr',
|
||||
},
|
||||
{
|
||||
id: 'vision.detector.enabled',
|
||||
label: 'Object detection',
|
||||
description: 'Tag photos with detected objects (person, car, dog, …) via YOLOv8n.',
|
||||
icon: <Boxes className="h-3.5 w-3.5" />,
|
||||
backfillTask: 'detect',
|
||||
},
|
||||
{
|
||||
id: 'vision.faces.enabled',
|
||||
label: 'Face recognition',
|
||||
description:
|
||||
'Find and cluster faces across the library (RetinaFace + ArcFace). ' +
|
||||
'Expensive on big libraries — disable if you don\'t need the People view.',
|
||||
icon: <UserSquare2 className="h-3.5 w-3.5" />,
|
||||
backfillTask: 'faces',
|
||||
},
|
||||
{
|
||||
id: 'vision.classifier.enabled',
|
||||
label: 'Content classification',
|
||||
description: 'Zero-shot CLIP tags for scenes / activities (beach, wedding, …).',
|
||||
icon: <TagsIcon className="h-3.5 w-3.5" />,
|
||||
backfillTask: 'classify',
|
||||
},
|
||||
]
|
||||
|
||||
function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) {
|
||||
@@ -1183,12 +1143,11 @@ function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) {
|
||||
queryClient.invalidateQueries({ queryKey: ['features'] })
|
||||
}
|
||||
|
||||
type BackfillTask = 'embed' | 'ocr' | 'detect' | 'faces' | 'classify' | null
|
||||
const runBackfill = (task: BackfillTask) =>
|
||||
const runBackfill = () =>
|
||||
runAction(
|
||||
`ai-backfill:${task ?? 'all'}`,
|
||||
() => adminApi.triggerAiBackfill({ task }),
|
||||
task ? `Backfill queued for ${task}` : 'Full backfill queued',
|
||||
`ai-backfill:all`,
|
||||
() => adminApi.triggerAiBackfill({}),
|
||||
'Classifier backfill queued',
|
||||
(r) => `Celery task ${r.task_id}`,
|
||||
)
|
||||
|
||||
@@ -1286,17 +1245,6 @@ function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{meta.backfillTask && state.effective && !masterOff && (
|
||||
<div className="mt-2">
|
||||
<ActionButton
|
||||
loading={!!busy[`ai-backfill:${meta.backfillTask}`]}
|
||||
onClick={() => runBackfill(meta.backfillTask!)}
|
||||
>
|
||||
<RefreshCw className="h-3.5 w-3.5" />
|
||||
Run {meta.backfillTask} backfill
|
||||
</ActionButton>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
@@ -1306,35 +1254,17 @@ function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) {
|
||||
|
||||
<Section icon={<Cpu className="h-4 w-4" />} title="Manual pipeline triggers">
|
||||
<p className="text-xs text-text-muted">
|
||||
Run a full pass across the enabled stages, recompute face
|
||||
clusters, or force a fresh filesystem scan. All three are safe
|
||||
to run repeatedly — the backfill only touches photos that
|
||||
don\'t yet have a given output, and the rescan skips files
|
||||
that are already indexed.
|
||||
Run the classifier over any photos that haven't been classified yet,
|
||||
or force a fresh filesystem scan. Both are safe to run repeatedly.
|
||||
</p>
|
||||
<div className="mt-3 flex flex-wrap gap-2">
|
||||
<ActionButton
|
||||
loading={!!busy['ai-backfill:all']}
|
||||
onClick={() => runBackfill(null)}
|
||||
onClick={() => runBackfill()}
|
||||
disabled={masterOff}
|
||||
>
|
||||
<Sparkles className="h-4 w-4" />
|
||||
Run full vision backfill
|
||||
</ActionButton>
|
||||
<ActionButton
|
||||
loading={!!busy['recluster']}
|
||||
onClick={() =>
|
||||
runAction(
|
||||
'recluster',
|
||||
() => adminApi.triggerFaceRecluster(),
|
||||
'Face recluster queued',
|
||||
(r) => `Celery task ${r.task_id}`,
|
||||
)
|
||||
}
|
||||
disabled={masterOff || !flags['vision.faces.enabled']?.effective}
|
||||
>
|
||||
<UserSquare2 className="h-4 w-4" />
|
||||
Recluster faces
|
||||
Run classifier backfill
|
||||
</ActionButton>
|
||||
<ActionButton
|
||||
loading={!!busy['rescan-full']}
|
||||
|
||||
@@ -46,6 +46,8 @@ export function FilterBar() {
|
||||
const sortBy = useFilterStore((s) => s.sortBy)
|
||||
const sortOrder = useFilterStore((s) => s.sortOrder)
|
||||
const tagIds = useFilterStore((s) => s.tagIds)
|
||||
const needsReview = useFilterStore((s) => s.needsReview)
|
||||
const setNeedsReview = useFilterStore((s) => s.setNeedsReview)
|
||||
const currentSection = useFilterStore((s) => s.currentSection)
|
||||
|
||||
// Only the Flag pill is hidden inside the Discarded section. Flag has
|
||||
@@ -129,251 +131,263 @@ export function FilterBar() {
|
||||
const anyActive = hasActiveFilters(filterState)
|
||||
|
||||
return (
|
||||
// Fixed bar height + py-0 so neither the active filter pills nor the
|
||||
// clear-all button can stretch the bar vertically. The fixed h-11
|
||||
// matches the h-7 pills + 8px symmetric vertical padding.
|
||||
<div className="flex h-11 items-center gap-3 border-b border-border bg-surface px-3 py-0">
|
||||
<div className="flex h-9 items-center gap-3 border-b border-border bg-surface px-3 py-0">
|
||||
{/* Pills — left side, scroll horizontally if they overflow. */}
|
||||
<div className="flex min-w-0 flex-1 items-center gap-1.5 overflow-x-auto">
|
||||
{/* Date */}
|
||||
<FilterPill
|
||||
label="Date"
|
||||
value={dateValue}
|
||||
isActive={dateActive}
|
||||
onClear={() => {
|
||||
setDateFrom(null)
|
||||
setDateTo(null)
|
||||
}}
|
||||
>
|
||||
<div className="space-y-2">
|
||||
<div>
|
||||
<label className="mb-1 block text-[11px] text-text-muted">From</label>
|
||||
<input
|
||||
type="date"
|
||||
value={dateFrom ?? ''}
|
||||
onChange={(e) => setDateFrom(e.target.value || null)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text"
|
||||
/>
|
||||
{/* Date */}
|
||||
<FilterPill
|
||||
label="Date"
|
||||
value={dateValue}
|
||||
isActive={dateActive}
|
||||
onClear={() => {
|
||||
setDateFrom(null)
|
||||
setDateTo(null)
|
||||
}}
|
||||
>
|
||||
<div className="space-y-2">
|
||||
<div>
|
||||
<label className="mb-1 block text-[11px] text-text-muted">From</label>
|
||||
<input
|
||||
type="date"
|
||||
value={dateFrom ?? ''}
|
||||
onChange={(e) => setDateFrom(e.target.value || null)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label className="mb-1 block text-[11px] text-text-muted">To</label>
|
||||
<input
|
||||
type="date"
|
||||
value={dateTo ?? ''}
|
||||
onChange={(e) => setDateTo(e.target.value || null)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<label className="mb-1 block text-[11px] text-text-muted">To</label>
|
||||
<input
|
||||
type="date"
|
||||
value={dateTo ?? ''}
|
||||
onChange={(e) => setDateTo(e.target.value || null)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</FilterPill>
|
||||
</FilterPill>
|
||||
|
||||
{/* Type */}
|
||||
<FilterPill
|
||||
label="Type"
|
||||
value={typeValue}
|
||||
isActive={typeActive}
|
||||
onClear={() => mediaTypes.forEach((t) => toggleMediaType(t))}
|
||||
>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{MEDIA_TYPES.map(({ value, label }) => {
|
||||
const active = mediaTypes.includes(value)
|
||||
return (
|
||||
{/* Type */}
|
||||
<FilterPill
|
||||
label="Type"
|
||||
value={typeValue}
|
||||
isActive={typeActive}
|
||||
onClear={() => mediaTypes.forEach((t) => toggleMediaType(t))}
|
||||
>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{MEDIA_TYPES.map(({ value, label }) => {
|
||||
const active = mediaTypes.includes(value)
|
||||
return (
|
||||
<button
|
||||
key={value}
|
||||
onClick={() => toggleMediaType(value)}
|
||||
className={clsx(
|
||||
'rounded px-2 py-1 text-xs transition-colors',
|
||||
active
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Rating */}
|
||||
<FilterPill
|
||||
label="Rating"
|
||||
value={ratingValue}
|
||||
isActive={ratingActive}
|
||||
onClear={() => setRatingMin(0)}
|
||||
>
|
||||
<div>
|
||||
<p className="mb-1 text-[11px] text-text-muted">Minimum</p>
|
||||
<div className="flex gap-1">
|
||||
{[1, 2, 3, 4, 5].map((n) => (
|
||||
<button
|
||||
key={n}
|
||||
onClick={() => setRatingMin(ratingMin === n ? 0 : n)}
|
||||
className="p-0.5"
|
||||
title={`At least ${n} star${n > 1 ? 's' : ''}`}
|
||||
>
|
||||
<Star
|
||||
className={clsx(
|
||||
'h-5 w-5 transition-colors',
|
||||
n <= ratingMin
|
||||
? 'fill-star text-star'
|
||||
: 'text-text-muted hover:text-star'
|
||||
)}
|
||||
/>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Color */}
|
||||
<FilterPill
|
||||
label="Color"
|
||||
value={colorValue}
|
||||
isActive={colorActive}
|
||||
onClear={() => setColorLabel(null)}
|
||||
>
|
||||
<div className="flex items-center gap-1.5">
|
||||
{COLOR_LABEL_OPTIONS.map(({ value, className }) => {
|
||||
const active = colorLabel === value
|
||||
return (
|
||||
<button
|
||||
key={value}
|
||||
onClick={() => setColorLabel(active ? null : value)}
|
||||
className={clsx(
|
||||
'h-5 w-5 rounded-full ring-offset-2 ring-offset-surface transition-all',
|
||||
className,
|
||||
active ? 'ring-2 ring-primary' : 'opacity-60 hover:opacity-100'
|
||||
)}
|
||||
title={value}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
{colorLabel && (
|
||||
<button
|
||||
key={value}
|
||||
onClick={() => toggleMediaType(value)}
|
||||
onClick={() => setColorLabel(null)}
|
||||
className="ml-1 rounded p-0.5 text-text-muted hover:bg-surface-2 hover:text-text"
|
||||
title="Clear color"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Flag — hidden in the Discarded section, where the flag is
|
||||
* pinned to "discarded" by the section preset. */}
|
||||
{!hideFlagPill && (
|
||||
<FilterPill
|
||||
label="Flag"
|
||||
value={flagValue}
|
||||
isActive={flagActive}
|
||||
onClear={() => setFlag('any')}
|
||||
>
|
||||
<div className="flex flex-col gap-1">
|
||||
<button
|
||||
onClick={() => setFlag('any')}
|
||||
className={clsx(
|
||||
'rounded px-2 py-1 text-xs transition-colors',
|
||||
active
|
||||
'rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'any'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
Any
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Rating */}
|
||||
<FilterPill
|
||||
label="Rating"
|
||||
value={ratingValue}
|
||||
isActive={ratingActive}
|
||||
onClear={() => setRatingMin(0)}
|
||||
>
|
||||
<div>
|
||||
<p className="mb-1 text-[11px] text-text-muted">Minimum</p>
|
||||
<div className="flex gap-1">
|
||||
{[1, 2, 3, 4, 5].map((n) => (
|
||||
<button
|
||||
key={n}
|
||||
onClick={() => setRatingMin(ratingMin === n ? 0 : n)}
|
||||
className="p-0.5"
|
||||
title={`At least ${n} star${n > 1 ? 's' : ''}`}
|
||||
>
|
||||
<Star
|
||||
className={clsx(
|
||||
'h-5 w-5 transition-colors',
|
||||
n <= ratingMin
|
||||
? 'fill-star text-star'
|
||||
: 'text-text-muted hover:text-star'
|
||||
)}
|
||||
/>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Color */}
|
||||
<FilterPill
|
||||
label="Color"
|
||||
value={colorValue}
|
||||
isActive={colorActive}
|
||||
onClear={() => setColorLabel(null)}
|
||||
>
|
||||
<div className="flex items-center gap-1.5">
|
||||
{COLOR_LABEL_OPTIONS.map(({ value, className }) => {
|
||||
const active = colorLabel === value
|
||||
return (
|
||||
<button
|
||||
key={value}
|
||||
onClick={() => setColorLabel(active ? null : value)}
|
||||
onClick={() => setFlag('discarded')}
|
||||
className={clsx(
|
||||
'h-5 w-5 rounded-full ring-offset-2 ring-offset-surface transition-all',
|
||||
className,
|
||||
active ? 'ring-2 ring-primary' : 'opacity-60 hover:opacity-100'
|
||||
'rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'discarded'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
title={value}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
{colorLabel && (
|
||||
<button
|
||||
onClick={() => setColorLabel(null)}
|
||||
className="ml-1 rounded p-0.5 text-text-muted hover:bg-surface-2 hover:text-text"
|
||||
title="Clear color"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</FilterPill>
|
||||
>
|
||||
Discarded
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setFlag('date_warning')}
|
||||
className={clsx(
|
||||
'flex items-center gap-1.5 rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'date_warning'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
title="Photos whose folder/filename suggests a different date than the stored taken_at"
|
||||
>
|
||||
<AlertTriangle className="h-3 w-3" />
|
||||
Date issues
|
||||
</button>
|
||||
</div>
|
||||
</FilterPill>
|
||||
)}
|
||||
|
||||
{/* Flag — hidden in the Discarded section, where the flag is
|
||||
* pinned to "discarded" by the section preset. */}
|
||||
{!hideFlagPill && (
|
||||
<FilterPill
|
||||
label="Flag"
|
||||
value={flagValue}
|
||||
isActive={flagActive}
|
||||
onClear={() => setFlag('any')}
|
||||
{/* Needs review — binary toggle */}
|
||||
<button
|
||||
onClick={() => setNeedsReview(!needsReview)}
|
||||
className={clsx(
|
||||
'flex h-6 items-center gap-1 whitespace-nowrap rounded-full border px-2.5 text-xs transition-colors',
|
||||
needsReview
|
||||
? 'border-primary bg-primary text-white'
|
||||
: 'border-border bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
title="Show only photos classified as non-photographs (screenshots, documents, memes)"
|
||||
>
|
||||
<div className="flex flex-col gap-1">
|
||||
<button
|
||||
onClick={() => setFlag('any')}
|
||||
className={clsx(
|
||||
'rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'any'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
<AlertTriangle className="h-3 w-3" />
|
||||
Needs review
|
||||
</button>
|
||||
|
||||
{/* Tags */}
|
||||
{allTags.length > 0 && (
|
||||
<FilterPill
|
||||
label="Tags"
|
||||
value={tagValue}
|
||||
isActive={tagActive}
|
||||
onClear={() => setTagIds([])}
|
||||
>
|
||||
<TagFilterPopover
|
||||
allTags={allTags}
|
||||
selectedIds={tagIds}
|
||||
onToggle={toggleTagId}
|
||||
onClear={() => setTagIds([])}
|
||||
/>
|
||||
</FilterPill>
|
||||
)}
|
||||
|
||||
{/* Sort — always present, never "active/inactive" since there's
|
||||
always a value. */}
|
||||
<FilterPill label="Sort" value={sortValue} isActive>
|
||||
<div className="space-y-2">
|
||||
<select
|
||||
value={sortBy}
|
||||
onChange={(e) => setSortBy(e.target.value as SortField)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text focus:border-primary focus:outline-none"
|
||||
>
|
||||
Any
|
||||
</button>
|
||||
{SORT_OPTIONS.map((opt) => (
|
||||
<option key={opt.value} value={opt.value}>
|
||||
{opt.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<button
|
||||
onClick={() => setFlag('discarded')}
|
||||
className={clsx(
|
||||
'rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'discarded'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
onClick={toggleSortOrder}
|
||||
className="flex w-full items-center justify-center gap-1 rounded bg-surface-2 px-2 py-1 text-xs text-text-muted hover:bg-surface-offset hover:text-text"
|
||||
>
|
||||
Discarded
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setFlag('date_warning')}
|
||||
className={clsx(
|
||||
'flex items-center gap-1.5 rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'date_warning'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
{sortOrder === 'desc' ? (
|
||||
<>
|
||||
<ArrowDown className="h-3.5 w-3.5" />
|
||||
Descending
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<ArrowUp className="h-3.5 w-3.5" />
|
||||
Ascending
|
||||
</>
|
||||
)}
|
||||
title="Photos whose folder/filename suggests a different date than the stored taken_at"
|
||||
>
|
||||
<AlertTriangle className="h-3 w-3" />
|
||||
Date issues
|
||||
</button>
|
||||
</div>
|
||||
</FilterPill>
|
||||
)}
|
||||
|
||||
{/* Tags */}
|
||||
{allTags.length > 0 && (
|
||||
<FilterPill
|
||||
label="Tags"
|
||||
value={tagValue}
|
||||
isActive={tagActive}
|
||||
onClear={() => setTagIds([])}
|
||||
>
|
||||
<TagFilterPopover
|
||||
allTags={allTags}
|
||||
selectedIds={tagIds}
|
||||
onToggle={toggleTagId}
|
||||
onClear={() => setTagIds([])}
|
||||
/>
|
||||
</FilterPill>
|
||||
)}
|
||||
|
||||
{/* Sort — always present, never "active/inactive" since there's
|
||||
always a value. */}
|
||||
<FilterPill label="Sort" value={sortValue} isActive>
|
||||
<div className="space-y-2">
|
||||
<select
|
||||
value={sortBy}
|
||||
onChange={(e) => setSortBy(e.target.value as SortField)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text focus:border-primary focus:outline-none"
|
||||
>
|
||||
{SORT_OPTIONS.map((opt) => (
|
||||
<option key={opt.value} value={opt.value}>
|
||||
{opt.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<button
|
||||
onClick={toggleSortOrder}
|
||||
className="flex w-full items-center justify-center gap-1 rounded bg-surface-2 px-2 py-1 text-xs text-text-muted hover:bg-surface-offset hover:text-text"
|
||||
>
|
||||
{sortOrder === 'desc' ? (
|
||||
<>
|
||||
<ArrowDown className="h-3.5 w-3.5" />
|
||||
Descending
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<ArrowUp className="h-3.5 w-3.5" />
|
||||
Ascending
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Clear-all — borderless text affordance pinned next to the pill
|
||||
{/* Clear-all — borderless text affordance pinned next to the pill
|
||||
* cluster on the right. Lives inside the pills container so it
|
||||
* shares the same flex group and gap and reads as "another
|
||||
* pill". Only renders when any filter is active. */}
|
||||
{anyActive && (
|
||||
<button
|
||||
onClick={clearAll}
|
||||
className="ml-1 flex h-7 flex-shrink-0 items-center whitespace-nowrap px-1 text-xs text-text-muted underline-offset-2 hover:text-text hover:underline"
|
||||
title="Clear all filters in this section"
|
||||
>
|
||||
Clear all
|
||||
</button>
|
||||
)}
|
||||
{anyActive && (
|
||||
<button
|
||||
onClick={clearAll}
|
||||
className="ml-1 flex h-7 flex-shrink-0 items-center whitespace-nowrap px-1 text-xs text-text-muted underline-offset-2 hover:text-text hover:underline"
|
||||
title="Clear all filters in this section"
|
||||
>
|
||||
Clear all
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Search — pinned to the right edge of the bar. Same id as before
|
||||
|
||||
@@ -80,17 +80,10 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
||||
const navigateToSection = useFilterStore((s) => s.navigateToSection)
|
||||
const currentSection = useFilterStore((s) => s.currentSection)
|
||||
const { data: allTags = [] } = useTagsQuery()
|
||||
const { data: faceClusters = [] } = useTagsQuery('face_cluster')
|
||||
const { data: stats } = useLibraryStatsQuery()
|
||||
const { data: featuresMap } = useFeaturesQuery()
|
||||
const visionOn = featuresMap ? featuresMap['vision.enabled'] !== false : true
|
||||
const facesOn = visionOn && (featuresMap ? featuresMap['vision.faces.enabled'] !== false : true)
|
||||
const tagsOn =
|
||||
visionOn &&
|
||||
(featuresMap
|
||||
? featuresMap['vision.detector.enabled'] !== false ||
|
||||
featuresMap['vision.classifier.enabled'] !== false
|
||||
: true)
|
||||
const tagsOn = true
|
||||
const [dropTargetId, setDropTargetId] = useState<string | null>(null)
|
||||
|
||||
// Per-folder kebab menu open state. Stores the tree-item id ("folder-..."
|
||||
@@ -295,8 +288,8 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
||||
case 'tags':
|
||||
navigateToSection('tags', { groupBy: 'tag' })
|
||||
break
|
||||
case 'people':
|
||||
navigateToSection('people', { groupBy: 'tag' })
|
||||
case 'needs-review':
|
||||
navigateToSection('needs-review', { needsReview: true })
|
||||
break
|
||||
case 'colors':
|
||||
navigateToSection('colors', { groupBy: 'color' })
|
||||
@@ -425,7 +418,6 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
||||
// Total tag count for the badge on the Tags entry (user tags only).
|
||||
const userTags = allTags.filter((t) => t.kind === 'user')
|
||||
const tagsTotalCount = userTags.reduce((sum, t) => sum + (t.photo_count || 0), 0)
|
||||
const peopleTotalCount = faceClusters.reduce((sum, t) => sum + (t.photo_count || 0), 0)
|
||||
|
||||
const libraryTree: TreeItem[] = [
|
||||
{
|
||||
@@ -436,7 +428,7 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
||||
{ id: 'all-photos', label: 'All Photos', icon: <Image className="h-4 w-4" />, count: stats?.all_photos ?? 0 },
|
||||
{ id: 'rated', label: 'Rated', icon: <Star className="h-4 w-4" />, count: stats?.rated ?? 0 },
|
||||
...(tagsOn ? [{ id: 'tags', label: 'Tags', icon: <TagIcon className="h-4 w-4" />, count: tagsTotalCount }] : []),
|
||||
...(facesOn ? [{ id: 'people', label: 'People', icon: <Users className="h-4 w-4" />, count: peopleTotalCount }] : []),
|
||||
...(visionOn ? [{ id: 'needs-review', label: 'Needs Review', icon: <Users className="h-4 w-4" />, count: stats?.needs_review ?? 0 }] : []),
|
||||
{ id: 'colors', label: 'Colors', icon: <Palette className="h-4 w-4" />, count: stats?.colored ?? 0 },
|
||||
{ id: 'map', label: 'Map', icon: <MapPin className="h-4 w-4" />, count: stats?.with_gps ?? 0 },
|
||||
{ id: 'memories', label: 'Memories', icon: <Clock className="h-4 w-4" /> },
|
||||
@@ -516,7 +508,7 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
||||
style={
|
||||
isSectionHeader
|
||||
? undefined
|
||||
: { paddingLeft: `${8 + (depth - 1) * 12}px` }
|
||||
: { paddingLeft: `${depth * 20}px` }
|
||||
}
|
||||
onClick={() => {
|
||||
if (renamingId === item.id) return
|
||||
@@ -580,7 +572,7 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
||||
)}
|
||||
</button>
|
||||
) : (
|
||||
!isSectionHeader && <div className="w-3" />
|
||||
!isSectionHeader && <div className="h-4 w-4 flex-shrink-0" />
|
||||
)}
|
||||
|
||||
{/* Item Icon — section headers drop their icon in favor of the
|
||||
@@ -783,7 +775,7 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
||||
creatingUnder === item.id.slice('folder-'.length) && (
|
||||
<div
|
||||
className="flex items-center gap-1 px-2 py-1"
|
||||
style={{ paddingLeft: `${8 + (depth + 1) * 16 + 4}px` }}
|
||||
style={{ paddingLeft: `${(depth + 1) * 20}px` }}
|
||||
>
|
||||
<FolderPlus className="h-3 w-3 flex-shrink-0 text-text-muted" />
|
||||
<input
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
import { useState, useCallback, useMemo } from 'react'
|
||||
import { Users, Pencil, Check, X, Loader2, ArrowLeft } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { useTagsQuery } from '../../hooks/useTagsQuery'
|
||||
import {
|
||||
tags as tagsApi,
|
||||
photos as photosApi,
|
||||
type Tag,
|
||||
} from '../../services/api'
|
||||
import { useFilterStore } from '../../store/filterStore'
|
||||
import { useCardGridNav } from '../../hooks/useCardGridNav'
|
||||
import { Timeline } from '../timeline/Timeline'
|
||||
import { toast } from '../ToastContainer'
|
||||
|
||||
/**
|
||||
* People view — two states:
|
||||
* 1. Grid of face cluster cards (default) — arrow keys + Enter to browse
|
||||
* 2. Detail view showing a person's photos in the full Timeline — Esc to go back
|
||||
*/
|
||||
export function PeopleView() {
|
||||
const { data: rawClusters = [], isLoading } = useTagsQuery('face_cluster')
|
||||
const clusters = useMemo(
|
||||
() => [...rawClusters].sort((a, b) => b.photo_count - a.photo_count),
|
||||
[rawClusters]
|
||||
)
|
||||
const queryClient = useQueryClient()
|
||||
const setTagIds = useFilterStore((s) => s.setTagIds)
|
||||
|
||||
const [selectedPerson, setSelectedPerson] = useState<Tag | null>(null)
|
||||
const [editingId, setEditingId] = useState<string | null>(null)
|
||||
const [editName, setEditName] = useState('')
|
||||
|
||||
const renameMutation = useMutation({
|
||||
mutationFn: ({ id, name }: { id: string; name: string }) =>
|
||||
tagsApi.update(id, { name }),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ['tags'] })
|
||||
setEditingId(null)
|
||||
if (selectedPerson && editingId === selectedPerson.id) {
|
||||
setSelectedPerson({ ...selectedPerson, name: editName.trim() })
|
||||
}
|
||||
toast.success('Renamed')
|
||||
},
|
||||
onError: (e: any) =>
|
||||
toast.error('Rename failed', e?.response?.data?.detail || e.message),
|
||||
})
|
||||
|
||||
const startEditing = (tag: Tag) => {
|
||||
setEditingId(tag.id)
|
||||
setEditName(tag.name)
|
||||
}
|
||||
|
||||
const submitRename = () => {
|
||||
if (!editingId || !editName.trim()) return
|
||||
renameMutation.mutate({ id: editingId, name: editName.trim() })
|
||||
}
|
||||
|
||||
const enterDetail = useCallback(
|
||||
(person: Tag) => {
|
||||
setTagIds([person.id])
|
||||
setSelectedPerson(person)
|
||||
},
|
||||
[setTagIds]
|
||||
)
|
||||
|
||||
const exitDetail = useCallback(() => {
|
||||
setTagIds([])
|
||||
setSelectedPerson(null)
|
||||
}, [setTagIds])
|
||||
|
||||
const { activeIndex, gridRef } = useCardGridNav({
|
||||
items: clusters,
|
||||
inDetail: selectedPerson !== null,
|
||||
onEnter: enterDetail,
|
||||
onExit: exitDetail,
|
||||
})
|
||||
|
||||
// ── Detail view: a person's photos ─────────────────────────────────
|
||||
if (selectedPerson) {
|
||||
const isEditing = editingId === selectedPerson.id
|
||||
return (
|
||||
<div className="flex h-full flex-col overflow-hidden">
|
||||
<div className="flex items-center gap-3 border-b border-border px-4 py-3">
|
||||
<button
|
||||
onClick={exitDetail}
|
||||
className="rounded p-1 text-text-muted transition-colors hover:bg-surface-2 hover:text-text"
|
||||
title="Back to people"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</button>
|
||||
|
||||
{isEditing ? (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<input
|
||||
autoFocus
|
||||
value={editName}
|
||||
onChange={(e) => setEditName(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') submitRename()
|
||||
if (e.key === 'Escape') setEditingId(null)
|
||||
}}
|
||||
className="rounded border border-border bg-bg px-2 py-1 text-sm text-text focus:border-primary focus:outline-none"
|
||||
/>
|
||||
<button onClick={submitRename} className="rounded p-1 text-green-500 hover:bg-green-500/10">
|
||||
<Check className="h-4 w-4" />
|
||||
</button>
|
||||
<button onClick={() => setEditingId(null)} className="rounded p-1 text-text-muted hover:bg-surface-2">
|
||||
<X className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex items-center gap-2">
|
||||
<h2 className="text-sm font-semibold text-text">{selectedPerson.name}</h2>
|
||||
<button
|
||||
onClick={() => startEditing(selectedPerson)}
|
||||
className="rounded p-0.5 text-text-muted transition-colors hover:text-text"
|
||||
title="Rename"
|
||||
>
|
||||
<Pencil className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex-1 overflow-hidden">
|
||||
<Timeline />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ── Card grid ──────────────────────────────────────────────────────
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center text-text-muted">
|
||||
<Loader2 className="mr-2 h-5 w-5 animate-spin" />
|
||||
Loading people...
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (clusters.length === 0) {
|
||||
return (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-3 text-text-muted">
|
||||
<Users className="h-12 w-12 opacity-40" />
|
||||
<p className="text-sm">No people identified yet</p>
|
||||
<p className="max-w-xs text-center text-xs opacity-70">
|
||||
Face detection runs automatically when photos are scanned.
|
||||
People will appear here once faces are found and clustered.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto p-4 pb-20">
|
||||
<div className="mb-4 flex items-center gap-2 text-text-muted">
|
||||
<Users className="h-4 w-4" />
|
||||
<span className="text-sm font-medium">
|
||||
{clusters.length} {clusters.length === 1 ? 'person' : 'people'} identified
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
ref={gridRef}
|
||||
className="grid grid-cols-[repeat(auto-fill,minmax(140px,1fr))] gap-3"
|
||||
>
|
||||
{clusters.map((tag, i) => (
|
||||
<div
|
||||
key={tag.id}
|
||||
className={clsx(
|
||||
'group cursor-pointer overflow-hidden rounded-lg border bg-surface transition-all hover:border-primary/50 hover:shadow-md',
|
||||
i === activeIndex
|
||||
? 'border-primary ring-1 ring-primary/30'
|
||||
: editingId === tag.id
|
||||
? 'border-primary ring-1 ring-primary/30'
|
||||
: 'border-border'
|
||||
)}
|
||||
onClick={() => {
|
||||
if (editingId !== tag.id) enterDetail(tag)
|
||||
}}
|
||||
>
|
||||
<div className="relative aspect-square overflow-hidden bg-surface-2">
|
||||
{tag.representative_photo_id ? (
|
||||
<img
|
||||
src={photosApi.getThumbnailUrl(tag.representative_photo_id, 'small')}
|
||||
alt={tag.name}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center">
|
||||
<Users className="h-10 w-10 text-text-muted/30" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span className="absolute bottom-1.5 right-1.5 rounded-full bg-black/60 px-2 py-0.5 text-[11px] font-medium text-white backdrop-blur-sm">
|
||||
{tag.photo_count}
|
||||
</span>
|
||||
|
||||
<button
|
||||
className="absolute right-1.5 top-1.5 rounded-full bg-black/50 p-1 text-white opacity-0 transition-opacity group-hover:opacity-100"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
startEditing(tag)
|
||||
}}
|
||||
title="Rename"
|
||||
>
|
||||
<Pencil className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="px-2 py-1.5">
|
||||
{editingId === tag.id ? (
|
||||
<div className="flex items-center gap-1" onClick={(e) => e.stopPropagation()}>
|
||||
<input
|
||||
autoFocus
|
||||
value={editName}
|
||||
onChange={(e) => setEditName(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') submitRename()
|
||||
if (e.key === 'Escape') setEditingId(null)
|
||||
}}
|
||||
className="min-w-0 flex-1 rounded border border-border bg-bg px-1.5 py-0.5 text-xs text-text focus:border-primary focus:outline-none"
|
||||
/>
|
||||
<button onClick={submitRename} className="rounded p-0.5 text-green-500 hover:bg-green-500/10">
|
||||
<Check className="h-3 w-3" />
|
||||
</button>
|
||||
<button onClick={() => setEditingId(null)} className="rounded p-0.5 text-text-muted hover:bg-surface-2">
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<p className="truncate text-xs font-medium text-text">{tag.name}</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -17,11 +17,10 @@ export function TagsView() {
|
||||
const setTagIds = useFilterStore((s) => s.setTagIds)
|
||||
const [selectedTag, setSelectedTag] = useState<Tag | null>(null)
|
||||
|
||||
// Exclude face_cluster tags (those live in PeopleView)
|
||||
const tags = useMemo(
|
||||
() =>
|
||||
allTags
|
||||
.filter((t) => t.kind !== 'face_cluster')
|
||||
.filter((t) => t.kind === 'user')
|
||||
.sort((a, b) => b.photo_count - a.photo_count),
|
||||
[allTags]
|
||||
)
|
||||
|
||||
@@ -18,14 +18,7 @@ export function useFeaturesQuery() {
|
||||
})
|
||||
}
|
||||
|
||||
export function useIsFeatureEnabled(
|
||||
name:
|
||||
| 'vision.enabled'
|
||||
| 'vision.ocr.enabled'
|
||||
| 'vision.detector.enabled'
|
||||
| 'vision.faces.enabled'
|
||||
| 'vision.classifier.enabled',
|
||||
): boolean {
|
||||
export function useIsFeatureEnabled(name: 'vision.enabled'): boolean {
|
||||
const { data } = useFeaturesQuery()
|
||||
// Default to enabled while loading so we don't flash "feature off"
|
||||
// during a first-paint fetch. The backend is the source of truth;
|
||||
|
||||
@@ -89,6 +89,7 @@ function parseUrl(): HydratePayload {
|
||||
}
|
||||
|
||||
if (sp.get('duplicates') === 'true') out.duplicates = true
|
||||
if (sp.get('needs_review') === 'true') out.needsReview = true
|
||||
|
||||
const groupBy = sp.get('group')
|
||||
if (groupBy === 'date' || groupBy === 'tag') out.groupBy = groupBy
|
||||
@@ -123,6 +124,7 @@ function writeUrl(f: FilterState & { currentSection?: string }) {
|
||||
if (f.folderId) sp.set('folder_id', f.folderId)
|
||||
if (f.tagIds.length > 0) sp.set('tag_ids', f.tagIds.join(','))
|
||||
if (f.duplicates) sp.set('duplicates', 'true')
|
||||
if (f.needsReview) sp.set('needs_review', 'true')
|
||||
if (f.groupBy !== 'date') sp.set('group', f.groupBy)
|
||||
if (f.currentSection && f.currentSection !== 'all-photos')
|
||||
sp.set('section', f.currentSection)
|
||||
|
||||
@@ -52,6 +52,7 @@ export function usePhotosQuery() {
|
||||
const folderId = useFilterStore((s) => s.folderId)
|
||||
const tagIds = useFilterStore((s) => s.tagIds)
|
||||
const duplicates = useFilterStore((s) => s.duplicates)
|
||||
const needsReview = useFilterStore((s) => s.needsReview)
|
||||
const groupBy = useFilterStore((s) => s.groupBy)
|
||||
const sortBy = useFilterStore((s) => s.sortBy)
|
||||
const sortOrder = useFilterStore((s) => s.sortOrder)
|
||||
@@ -71,11 +72,12 @@ export function usePhotosQuery() {
|
||||
folderId,
|
||||
tagIds,
|
||||
duplicates,
|
||||
needsReview,
|
||||
groupBy,
|
||||
sortBy,
|
||||
sortOrder,
|
||||
}),
|
||||
[q, dateFrom, dateTo, mediaTypes, ratingMin, ratingMax, colorLabel, flag, heapId, folderId, tagIds, duplicates, groupBy, sortBy, sortOrder]
|
||||
[q, dateFrom, dateTo, mediaTypes, ratingMin, ratingMax, colorLabel, flag, heapId, folderId, tagIds, duplicates, needsReview, groupBy, sortBy, sortOrder]
|
||||
)
|
||||
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
@@ -438,7 +438,6 @@ export interface PipelineStage {
|
||||
export interface PipelineStats {
|
||||
total_photos: number
|
||||
total_images: number
|
||||
embedder_model: string
|
||||
stages: PipelineStage[]
|
||||
}
|
||||
|
||||
@@ -621,6 +620,7 @@ export interface LibraryStats {
|
||||
with_gps: number
|
||||
duplicates: number
|
||||
discarded: number
|
||||
needs_review: number
|
||||
total_photos: number
|
||||
total_videos: number
|
||||
total_size: number
|
||||
@@ -843,7 +843,7 @@ export const sharing = {
|
||||
}
|
||||
|
||||
// Tags API
|
||||
export type TagKind = 'user' | 'object' | 'scene' | 'face_cluster'
|
||||
export type TagKind = 'user' | 'content_type'
|
||||
|
||||
export interface Tag {
|
||||
id: string
|
||||
@@ -876,11 +876,6 @@ export const tags = {
|
||||
await api.delete(`/tags/${tagId}`)
|
||||
},
|
||||
|
||||
merge: async (sourceId: string, targetId: string): Promise<{ merged_into: string; target_name: string }> => {
|
||||
const response = await api.post(`/tags/${sourceId}/merge`, { target_id: targetId })
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Add one or more tags to a photo. */
|
||||
addToPhoto: async (photoId: string, tagIds: string[]) => {
|
||||
const response = await api.post(`/photos/${photoId}/tags`, { tag_ids: tagIds })
|
||||
@@ -1013,18 +1008,12 @@ export const admin = {
|
||||
},
|
||||
|
||||
triggerAiBackfill: async (body: {
|
||||
task?: 'embed' | 'ocr' | 'detect' | 'faces' | 'classify' | null
|
||||
limit?: number | null
|
||||
}): Promise<{ status: string; task_id: string }> => {
|
||||
const response = await api.post('/admin/ai/backfill', body)
|
||||
return response.data
|
||||
},
|
||||
|
||||
triggerFaceRecluster: async (): Promise<{ status: string; task_id: string }> => {
|
||||
const response = await api.post('/admin/ai/recluster-faces')
|
||||
return response.data
|
||||
},
|
||||
|
||||
triggerFullRescan: async (): Promise<{ status: string; task_id: string }> => {
|
||||
const response = await api.post('/admin/ai/rescan')
|
||||
return response.data
|
||||
|
||||
@@ -31,6 +31,8 @@ export interface FilterState {
|
||||
tagIds: string[]
|
||||
/** When true, restrict to photos flagged as duplicates by the scanner. */
|
||||
duplicates: boolean
|
||||
/** When true, restrict to photos classified as 'other' (needs_review). */
|
||||
needsReview: boolean
|
||||
/** Visual grouping mode. 'date' groups by month when sortBy is a date
|
||||
* field; 'tag' groups by photo tag membership. Independent of filters. */
|
||||
groupBy: GroupBy
|
||||
@@ -68,6 +70,7 @@ interface FilterStore extends FilterState {
|
||||
setTagIds: (ids: string[]) => void
|
||||
toggleTagId: (id: string) => void
|
||||
setDuplicates: (v: boolean) => void
|
||||
setNeedsReview: (v: boolean) => void
|
||||
setGroupBy: (mode: GroupBy) => void
|
||||
setSortBy: (field: SortField) => void
|
||||
setSortOrder: (order: SortOrder) => void
|
||||
@@ -102,6 +105,7 @@ export const INITIAL_FILTERS: FilterState = {
|
||||
folderId: null,
|
||||
tagIds: [],
|
||||
duplicates: false,
|
||||
needsReview: false,
|
||||
groupBy: 'date',
|
||||
sortBy: 'taken_at',
|
||||
sortOrder: 'desc',
|
||||
@@ -124,6 +128,7 @@ function snapshotFilters(s: FilterState): FilterState {
|
||||
folderId: s.folderId,
|
||||
tagIds: [...s.tagIds],
|
||||
duplicates: s.duplicates,
|
||||
needsReview: s.needsReview,
|
||||
groupBy: s.groupBy,
|
||||
sortBy: s.sortBy,
|
||||
sortOrder: s.sortOrder,
|
||||
@@ -159,6 +164,7 @@ export const useFilterStore = create<FilterStore>((set) => ({
|
||||
: [...s.tagIds, id],
|
||||
})),
|
||||
setDuplicates: (duplicates) => set({ duplicates }),
|
||||
setNeedsReview: (needsReview) => set({ needsReview }),
|
||||
setGroupBy: (groupBy) => set({ groupBy }),
|
||||
setSortBy: (sortBy) => set({ sortBy }),
|
||||
setSortOrder: (sortOrder) => set({ sortOrder }),
|
||||
@@ -218,6 +224,7 @@ export function filtersToParams(f: FilterState): Record<string, string | number>
|
||||
if (f.folderId) params.folder_id = f.folderId
|
||||
if (f.tagIds.length > 0) params.tag_ids = f.tagIds.join(',')
|
||||
if (f.duplicates) params.is_duplicate = 'true'
|
||||
if (f.needsReview) params.needs_review = 'true'
|
||||
params.sort = f.sortBy
|
||||
params.order = f.sortOrder
|
||||
return params
|
||||
@@ -237,6 +244,7 @@ export function hasActiveFilters(f: FilterState): boolean {
|
||||
f.heapId !== null ||
|
||||
f.folderId !== null ||
|
||||
f.tagIds.length > 0 ||
|
||||
f.duplicates
|
||||
f.duplicates ||
|
||||
f.needsReview
|
||||
)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ export interface Photo {
|
||||
color_label?: string | null
|
||||
is_discarded: boolean
|
||||
is_duplicate: boolean
|
||||
needs_review?: boolean
|
||||
has_date_warning?: boolean
|
||||
file_hash: string
|
||||
folder_id: string | null
|
||||
|
||||
24
mulita.yml
24
mulita.yml
@@ -23,30 +23,12 @@ performance:
|
||||
db_pool_size: 20
|
||||
db_pool_recycle: 3600
|
||||
|
||||
# AI vision pipeline — embedding, OCR, object detection, face recognition.
|
||||
# Runs on the dedicated `vision` Celery queue (PR4+). Set enabled: false
|
||||
# to disable all vision processing.
|
||||
# Vision pipeline — single binary classifier (photography vs other).
|
||||
# Photos landing in 'other' get needs_review=true.
|
||||
vision:
|
||||
enabled: true
|
||||
backend: onnx # "onnx" (CPU) | "rocm" (future GPU)
|
||||
backend: onnx
|
||||
models_dir: /data/models
|
||||
embedder:
|
||||
name: openclip_vitb32
|
||||
batch_size: 8
|
||||
ocr:
|
||||
enabled: true
|
||||
languages: [en]
|
||||
min_confidence: 0.5
|
||||
detector:
|
||||
enabled: true
|
||||
min_confidence: 0.35
|
||||
max_detections: 50
|
||||
faces:
|
||||
enabled: true
|
||||
min_face_size: 40
|
||||
recognition_threshold: 0.65
|
||||
cluster_eps: 0.5
|
||||
classifier:
|
||||
enabled: true
|
||||
min_confidence: 0.3
|
||||
worker_concurrency: 2
|
||||
|
||||
Reference in New Issue
Block a user