From 574d71371f9fcb0a127014af3e19397a5286fab8 Mon Sep 17 00:00:00 2001 From: dtoro Date: Tue, 14 Apr 2026 22:27:17 +0200 Subject: [PATCH] refactor: strip AI pipeline to binary photo/other classifier Drops face recognition, OCR, object detection, and semantic embeddings. The sole remaining vision task is a CLIP-based binary classifier (photography vs other); photos in "other" get needs_review=true so screenshots, documents, memes and scans can be triaged from a new filter pill in the UI. Co-Authored-By: Claude Opus 4.6 (1M context) --- backend/alembic/env.py | 1 - .../versions/0012_strip_ai_pipeline.py | 65 ++ .../versions/0013_drop_old_content_types.py | 35 ++ backend/app/config.py | 41 +- backend/app/database.py | 8 +- backend/app/models/__init__.py | 8 +- backend/app/models/embeddings.py | 24 - backend/app/models/face_embedding.py | 24 - backend/app/models/ocr_text.py | 20 - backend/app/models/photos.py | 6 + backend/app/models/tags.py | 13 +- backend/app/routers/admin.py | 47 +- backend/app/routers/library.py | 95 +-- backend/app/routers/photos.py | 3 + backend/app/routers/search.py | 8 +- backend/app/routers/tags.py | 62 +- backend/app/schemas/photos.py | 1 + backend/app/services/duplicates.py | 109 +--- backend/app/services/feature_flags.py | 204 +----- backend/app/services/search.py | 156 ++--- backend/app/services/vision/base.py | 94 +-- .../app/services/vision/bootstrap_models.py | 116 +--- backend/app/services/vision/classify.py | 169 +++-- backend/app/services/vision/clustering.py | 42 -- backend/app/services/vision/detect.py | 138 ---- backend/app/services/vision/embed.py | 142 +---- backend/app/services/vision/export_models.py | 244 +------- backend/app/services/vision/faces.py | 146 ----- .../services/vision/insightface_processor.py | 73 --- backend/app/services/vision/ocr.py | 45 -- backend/app/services/vision/onnx_backend.py | 46 -- backend/app/services/vision/registry.py | 69 +- backend/app/services/vision/rocm_backend.py | 25 - backend/app/tasks/celery.py | 7 +- backend/app/tasks/scan.py | 10 +- backend/app/tasks/vision.py | 590 ++---------------- backend/requirements.txt | 6 - frontend/src/App.tsx | 3 - .../src/components/dialogs/SettingsDialog.tsx | 92 +-- frontend/src/components/filter/FilterBar.tsx | 450 ++++++------- .../src/components/layout/LeftSidebar.tsx | 22 +- frontend/src/components/people/PeopleView.tsx | 241 ------- frontend/src/components/tags/TagsView.tsx | 3 +- frontend/src/hooks/useFeaturesQuery.ts | 9 +- frontend/src/hooks/useFilterUrlSync.ts | 2 + frontend/src/hooks/usePhotosQuery.ts | 4 +- frontend/src/services/api.ts | 15 +- frontend/src/store/filterStore.ts | 10 +- frontend/src/types/photo.ts | 1 + mulita.yml | 24 +- 50 files changed, 700 insertions(+), 3068 deletions(-) create mode 100644 backend/alembic/versions/0012_strip_ai_pipeline.py create mode 100644 backend/alembic/versions/0013_drop_old_content_types.py delete mode 100644 backend/app/models/embeddings.py delete mode 100644 backend/app/models/face_embedding.py delete mode 100644 backend/app/models/ocr_text.py delete mode 100644 backend/app/services/vision/clustering.py delete mode 100644 backend/app/services/vision/detect.py delete mode 100644 backend/app/services/vision/faces.py delete mode 100644 backend/app/services/vision/insightface_processor.py delete mode 100644 backend/app/services/vision/ocr.py delete mode 100644 backend/app/services/vision/onnx_backend.py delete mode 100644 backend/app/services/vision/rocm_backend.py delete mode 100644 frontend/src/components/people/PeopleView.tsx diff --git a/backend/alembic/env.py b/backend/alembic/env.py index efe903f..285044b 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -38,7 +38,6 @@ from app.models import ( # noqa: E402, F401 Tag, Heap, HeapPhoto, - Embedding, ) config = context.config diff --git a/backend/alembic/versions/0012_strip_ai_pipeline.py b/backend/alembic/versions/0012_strip_ai_pipeline.py new file mode 100644 index 0000000..98a9edd --- /dev/null +++ b/backend/alembic/versions/0012_strip_ai_pipeline.py @@ -0,0 +1,65 @@ +"""Strip AI pipeline to binary classifier only + +Revision ID: 0012_strip_ai +Revises: 0011_sharing +Create Date: 2026-04-14 + +Removes face recognition, OCR, object detection, and semantic embeddings. +The remaining AI is a single binary 'photography' vs 'other' classifier +whose output feeds Tag(kind='content_type') and a new Photo.needs_review +flag. + +Drops: embeddings, face_embeddings, ocr_text tables. +Drops: photo_tags rows produced by 'vision:yolov8n' and 'vision:sface'. +Drops: tags with kind IN ('object','scene','face_cluster'). +Drops: tags.representative_photo_id column. +Adds: photos.needs_review (bool, default false) + partial index. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0012_strip_ai" +down_revision: Union[str, None] = "0011_sharing" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Drop dropped-AI tables. CASCADE clears any lingering FKs/indices. + op.execute("DROP TABLE IF EXISTS embeddings CASCADE") + op.execute("DROP TABLE IF EXISTS face_embeddings CASCADE") + op.execute("DROP TABLE IF EXISTS ocr_text CASCADE") + + # Clear ML-produced photo_tags rows and their parent tags. + op.execute( + "DELETE FROM photo_tags WHERE source IN ('vision:yolov8n','vision:sface')" + ) + op.execute( + "DELETE FROM tags WHERE kind IN ('object','scene','face_cluster')" + ) + + # Drop the face-cluster representative column. + op.execute("ALTER TABLE tags DROP COLUMN IF EXISTS representative_photo_id") + + # Add the needs_review flag. + op.execute( + "ALTER TABLE photos ADD COLUMN IF NOT EXISTS needs_review " + "BOOLEAN NOT NULL DEFAULT false" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_photos_needs_review " + "ON photos(needs_review) WHERE needs_review" + ) + + +def downgrade() -> None: + # Data is not recoverable on downgrade — only the schema stubs are + # put back so a future reinstall of the old pipeline can re-populate. + op.execute("DROP INDEX IF EXISTS ix_photos_needs_review") + op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS needs_review") + + op.execute( + "ALTER TABLE tags ADD COLUMN IF NOT EXISTS representative_photo_id " + "VARCHAR REFERENCES photos(id) ON DELETE SET NULL" + ) diff --git a/backend/alembic/versions/0013_drop_old_content_types.py b/backend/alembic/versions/0013_drop_old_content_types.py new file mode 100644 index 0000000..a676501 --- /dev/null +++ b/backend/alembic/versions/0013_drop_old_content_types.py @@ -0,0 +1,35 @@ +"""Drop legacy content_type tags from the 6-category classifier + +Revision ID: 0013_drop_old_ct +Revises: 0012_strip_ai +Create Date: 2026-04-14 + +The previous classifier wrote Tag(kind='content_type', name IN +('photograph','screenshot','document','receipt','meme','artwork')). +The new binary classifier writes names ('photography','other'). Both +coexisted after the cutover so users saw duplicate groupings like +'photography' alongside 'photograph'. Drop the old names — photo_tags +rows cascade-delete via the FK. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0013_drop_old_ct" +down_revision: Union[str, None] = "0012_strip_ai" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +LEGACY_NAMES = ('photograph', 'screenshot', 'document', 'receipt', 'meme', 'artwork') + + +def upgrade() -> None: + op.execute( + "DELETE FROM tags WHERE kind = 'content_type' " + f"AND name IN {LEGACY_NAMES}" + ) + + +def downgrade() -> None: + pass diff --git a/backend/app/config.py b/backend/app/config.py index 7268702..2d54193 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -30,51 +30,16 @@ class PerformanceSettings(BaseModel): db_pool_max_overflow: int = 10 db_pool_recycle: int = 3600 -class EmbedderSettings(BaseModel): - """CLIP / SigLIP embedding model settings. - Supported: "openclip_vitb32" (512-d), "siglip2_vitb16" (768-d, default).""" - name: str = "openclip_vitb32" - batch_size: int = 8 - -class OCRSettings(BaseModel): - """PaddleOCR / rapidocr settings""" - enabled: bool = True - languages: list[str] = ["en"] - min_confidence: float = 0.5 - -class DetectorSettings(BaseModel): - """YOLOv8n object detection settings""" - enabled: bool = True - min_confidence: float = 0.35 - max_detections: int = 50 - -class FacesSettings(BaseModel): - """YuNet + SFace face detection/recognition settings""" - enabled: bool = True - min_face_size: int = 40 - recognition_threshold: float = 0.65 - cluster_eps: float = 0.5 - class ClassifierSettings(BaseModel): - """CLIP zero-shot content classification settings""" - enabled: bool = True + """Binary content classifier (photography vs other).""" min_confidence: float = 0.3 class VisionSettings(BaseModel): - """AI vision pipeline settings. Disabled when running on SQLite - (pgvector is required for embedding storage).""" + """Vision pipeline — one binary classifier (photography vs other).""" enabled: bool = True - backend: str = "onnx" # "onnx" | "rocm" (future) + backend: str = "onnx" models_dir: str = "/data/models" - # ONNX Runtime execution providers in priority order. - # Auto-detected at startup; falls back to CPU if GPU is unavailable. - # Options: "CUDAExecutionProvider", "ROCMExecutionProvider", - # "OpenVINOExecutionProvider", "CPUExecutionProvider" execution_providers: list[str] = ["CPUExecutionProvider"] - embedder: EmbedderSettings = EmbedderSettings() - ocr: OCRSettings = OCRSettings() - detector: DetectorSettings = DetectorSettings() - faces: FacesSettings = FacesSettings() classifier: ClassifierSettings = ClassifierSettings() worker_concurrency: int = 2 diff --git a/backend/app/database.py b/backend/app/database.py index 679b20d..e00c26b 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -111,13 +111,7 @@ async def init_db(): """Initialize database, create tables if they don't exist""" async with engine.begin() as conn: # Import all models to register them with Base - from app.models import User, Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto, Embedding - - # Postgres: ensure pgvector is available before create_all touches - # any Vector columns (added in later PRs but the extension is cheap - # and idempotent to create now). - if _is_postgres: - await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + from app.models import User, Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto # noqa: F401 # Create all tables. Note: create_all only creates *missing* tables — # it does NOT add new columns to existing tables when the model gains diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index d29e145..5737593 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -6,9 +6,6 @@ from app.models.photos import Photo from app.models.folders import Folder, SourceRoot from app.models.tags import Tag, PhotoTag from app.models.heaps import Heap, HeapPhoto -from app.models.embeddings import Embedding -from app.models.ocr_text import OCRText -from app.models.face_embedding import FaceEmbedding from app.models.sharing import HeapShare, FolderShare __all__ = [ @@ -20,9 +17,6 @@ __all__ = [ 'PhotoTag', 'Heap', 'HeapPhoto', - 'Embedding', - 'OCRText', - 'FaceEmbedding', 'HeapShare', 'FolderShare', -] \ No newline at end of file +] diff --git a/backend/app/models/embeddings.py b/backend/app/models/embeddings.py deleted file mode 100644 index 0693e22..0000000 --- a/backend/app/models/embeddings.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Embedding model — stores CLIP/SigLIP image embeddings via pgvector. - -Composite PK (photo_id, model) allows re-embedding with newer models -without clobbering old vectors. - -Vector dimension is 768 to support SigLIP2 ViT-B/16 (the default). -OpenCLIP ViT-B/32 (512-d) embeddings are zero-padded on insert so -both models coexist in the same column. The padding is invisible to -cosine similarity (zeros don't affect the angle). -""" -from sqlalchemy import Column, String, ForeignKey, DateTime, func -from pgvector.sqlalchemy import Vector - -from app.database import Base - - -class Embedding(Base): - __tablename__ = 'embeddings' - - photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True) - model = Column(String(64), primary_key=True) # e.g. 'siglip2_vitb16' - vector = Column(Vector(512)) # OpenCLIP ViT-B/32 → 512-d - created_at = Column(DateTime(timezone=True), server_default=func.now()) diff --git a/backend/app/models/face_embedding.py b/backend/app/models/face_embedding.py deleted file mode 100644 index 291d18c..0000000 --- a/backend/app/models/face_embedding.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Face embedding model — stores per-face detection + recognition vectors. - -cluster_id FKs to tags.id where kind='face_cluster'. Null means -unclustered (will be assigned by recluster_faces). -""" -from sqlalchemy import Column, String, Float, ForeignKey, DateTime, func -from sqlalchemy.dialects.postgresql import JSONB -from pgvector.sqlalchemy import Vector -import uuid - -from app.database import Base - - -class FaceEmbedding(Base): - __tablename__ = 'face_embeddings' - - id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) - photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), nullable=False, index=True) - bbox = Column(JSONB) # [x1, y1, x2, y2] normalized 0-1 - vector = Column(Vector(512)) # ArcFace → 512-d - cluster_id = Column(String, ForeignKey('tags.id', ondelete='SET NULL'), nullable=True, index=True) - quality = Column(Float) - created_at = Column(DateTime(timezone=True), server_default=func.now()) diff --git a/backend/app/models/ocr_text.py b/backend/app/models/ocr_text.py deleted file mode 100644 index 1cc51ca..0000000 --- a/backend/app/models/ocr_text.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -OCR text model — stores text regions extracted from photos via rapidocr. -""" -from sqlalchemy import Column, String, Float, ForeignKey, Text, DateTime, func -from sqlalchemy.dialects.postgresql import JSONB -import uuid - -from app.database import Base - - -class OCRText(Base): - __tablename__ = 'ocr_text' - - id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) - photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), nullable=False, index=True) - text = Column(Text, nullable=False) - language = Column(String(8), default='') - confidence = Column(Float) - bbox = Column(JSONB) # [x1, y1, x2, y2] normalized 0-1 - created_at = Column(DateTime(timezone=True), server_default=func.now()) diff --git a/backend/app/models/photos.py b/backend/app/models/photos.py index df76bae..f0153f1 100644 --- a/backend/app/models/photos.py +++ b/backend/app/models/photos.py @@ -50,6 +50,11 @@ class Photo(Base): # rows, and POST /folders/{id}/hide recomputes it on toggle. is_hidden = Column(Boolean, nullable=False, default=False, server_default='false', index=True) + # "Needs review" — set by the content classifier when a photo is + # classified as 'other' (screenshot, document, meme, scan, etc.) so + # the user can page through non-photographs in the UI and triage them. + needs_review = Column(Boolean, nullable=False, default=False, server_default='false', index=True) + # "Capture date is probably wrong" — denormalized from the folder/filename # date-guesser. Set at scan time and recomputed on every taken_at edit so # the filter bar can query it directly. See services/date_guess.py for @@ -118,4 +123,5 @@ class Photo(Base): Index('ix_photos_media_type', 'media_type'), Index('ix_photos_processing_status', 'processing_status'), Index('ix_photos_lat_lon', 'latitude', 'longitude'), + Index('ix_photos_needs_review', 'needs_review'), ) \ No newline at end of file diff --git a/backend/app/models/tags.py b/backend/app/models/tags.py index ff6fe8c..6391859 100644 --- a/backend/app/models/tags.py +++ b/backend/app/models/tags.py @@ -6,7 +6,7 @@ labels, and face clusters via the `kind` column. The `photo_tags` association carries per-photo ML metadata (confidence, bounding box, source model). """ -from sqlalchemy import Column, String, Float, ForeignKey, Table, Index, UniqueConstraint +from sqlalchemy import Column, String, Float, ForeignKey, Table, Index, UniqueConstraint # noqa: F401 from sqlalchemy.orm import relationship from sqlalchemy.dialects.postgresql import JSONB import uuid @@ -22,7 +22,7 @@ photo_tags = Table( # ML metadata — null for user-applied tags Column('confidence', Float, nullable=True), Column('bbox', JSONB, nullable=True), # [x1, y1, x2, y2] normalized 0-1 - Column('source', String, nullable=True), # e.g. "vision:yolov8n", "vision:sface" + Column('source', String, nullable=True), # e.g. "vision:clip_classifier" Index('ix_photo_tags_photo_id', 'photo_id'), Index('ix_photo_tags_tag_id', 'tag_id'), ) @@ -42,16 +42,11 @@ class Tag(Base): # Tag classification kind = Column(String, nullable=False, default='user', index=True) - # kind values: 'user' | 'object' | 'scene' | 'face_cluster' + # kind values: 'user' | 'content_type' # Which model produced this tag (null for user-created) source = Column(String, nullable=True) - # e.g. "vision:yolov8n", "vision:sface", null - - # For face clusters: the photo used as the cluster representative thumbnail - representative_photo_id = Column( - String, ForeignKey('photos.id', ondelete='SET NULL'), nullable=True - ) + # e.g. "vision:clip_classifier", null # Relationships photos = relationship("Photo", secondary=photo_tags, backref="tags") diff --git a/backend/app/routers/admin.py b/backend/app/routers/admin.py index 1e756a0..3ad5154 100644 --- a/backend/app/routers/admin.py +++ b/backend/app/routers/admin.py @@ -320,13 +320,8 @@ async def update_feature_flag( class BackfillVisionBody(BaseModel): - """POST body for triggering a vision backfill. ``task`` picks a - specific stage (``embed`` / ``ocr`` / ``detect`` / ``faces`` / - ``classify``); leaving it null runs every enabled stage. ``limit`` - caps how many photos per stage are queued — useful for smoke- - testing a newly-enabled feature before committing a full run. - """ - task: Optional[str] = None + """POST body for triggering a classifier backfill. ``limit`` caps how + many photos are queued.""" limit: Optional[int] = None @@ -335,61 +330,29 @@ async def trigger_ai_backfill( body: BackfillVisionBody, admin: User = Depends(require_admin), ): - """Queue a vision backfill pass. Identical code path as the automatic - post-scan backfill — just triggered manually from the UI.""" + """Queue a classifier backfill pass.""" if not is_enabled(FLAG_VISION_ENABLED): raise HTTPException( status_code=400, detail="Vision is currently disabled; enable it before running a backfill.", ) - valid_tasks = {'embed', 'ocr', 'detect', 'faces', 'classify'} - if body.task is not None and body.task not in valid_tasks: - raise HTTPException( - status_code=400, - detail=f"task must be one of {sorted(valid_tasks)} or null", - ) if body.limit is not None and body.limit <= 0: raise HTTPException(status_code=400, detail="limit must be positive") - # Import lazily so importing admin.py doesn't pull in the whole - # vision stack on startup (Celery task module loads numpy etc.). from app.tasks.vision import backfill_vision - result = backfill_vision.apply_async( - kwargs={'task': body.task, 'limit': body.limit} - ) + result = backfill_vision.apply_async(kwargs={'limit': body.limit}) logger.info( f"Admin '{admin.username}' queued vision backfill " - f"(task={body.task}, limit={body.limit}, celery_id={result.id})" + f"(limit={body.limit}, celery_id={result.id})" ) return { "status": "queued", "task_id": result.id, - "task": body.task, "limit": body.limit, } -@router.post("/ai/recluster-faces") -async def trigger_face_recluster(admin: User = Depends(require_admin)): - """Kick off face recluster. Normally auto-fires after a scan via a - debounced scheduler; this endpoint is for admins who want to force - a fresh clustering pass (e.g. after tweaking ``cluster_eps`` in - the YAML config).""" - if not is_enabled(FLAG_VISION_ENABLED): - raise HTTPException( - status_code=400, - detail="Vision is currently disabled; enable it before reclustering.", - ) - from app.tasks.vision import recluster_faces - - result = recluster_faces.apply_async() - logger.info( - f"Admin '{admin.username}' queued face recluster (celery_id={result.id})" - ) - return {"status": "queued", "task_id": result.id} - - @router.post("/ai/rescan") async def trigger_full_rescan(admin: User = Depends(require_admin)): """Dispatch the same scan_all_source_roots job the backend runs at diff --git a/backend/app/routers/library.py b/backend/app/routers/library.py index 86ff42d..b406b84 100644 --- a/backend/app/routers/library.py +++ b/backend/app/routers/library.py @@ -89,6 +89,12 @@ async def get_library_stats( ) ).scalar() or 0 + needs_review_count = ( + await db.execute( + select(func.count(Photo.id)).where(visible, Photo.needs_review.is_(True)) + ) + ).scalar() or 0 + # Legacy split (kept for the existing /stats consumers). photo_count = ( await db.execute( @@ -120,6 +126,7 @@ async def get_library_stats( "with_gps": with_gps_count, "duplicates": duplicates_count, "discarded": discarded_count, + "needs_review": needs_review_count, "total_photos": photo_count, "total_videos": video_count, "total_size": size, @@ -437,10 +444,7 @@ async def get_worker_status( r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0) r.ping() broker_ok = True - # `vision` is the big one — embed / classify / detect / ocr / - # extract_faces all land here, so it's where backlogs actually - # pile up. Leaving it off the dashboard made it look like the - # queue was always empty while the worker was clearly busy. + # `vision` runs the content classifier — the only heavy queue. for q in ('default', 'high', 'low', 'vision'): try: queue_depths[q] = int(r.llen(q) or 0) @@ -525,8 +529,6 @@ async def get_pipeline_stats( Keep the shape flat + serialisable; the frontend turns it straight into a list of rows without needing to know about the models. """ - from app.config import settings as _settings - from app.models import Embedding, FaceEmbedding, OCRText from app.models.tags import photo_tags # association Table, not a model owner = _owner_filter(current_user, scope) @@ -572,45 +574,15 @@ async def get_pipeline_stats( ) ) - # Embeddings: count distinct photos that have a row for the currently - # configured embedder model. A photo can have multiple model rows - # (historical re-embeds) so COUNT(DISTINCT) is the right thing here. - embedder_model = _settings.vision.embedder.name - embeddings_done = await scalar_count( - select(func.count(func.distinct(Embedding.photo_id))) - .select_from(Embedding) - .join(Photo, Photo.id == Embedding.photo_id) - .where(not_discarded, Embedding.model == embedder_model) - ) - - tagged_photos = await scalar_count( + # Classified: distinct photos with a content_type tag. + classified_done = await scalar_count( select(func.count(func.distinct(photo_tags.c.photo_id))) .select_from(photo_tags) .join(Photo, Photo.id == photo_tags.c.photo_id) - .where(not_discarded) + .where(not_discarded, photo_tags.c.source == 'vision:clip_classifier') ) - ocr_done = await scalar_count( - select(func.count(func.distinct(OCRText.photo_id))) - .select_from(OCRText) - .join(Photo, Photo.id == OCRText.photo_id) - .where(not_discarded) - ) - - # Faces: photos that have at least one face_embeddings row. A photo - # with no faces legitimately finishes face extraction with zero rows, - # so this undercounts by exactly "images with no visible people". We - # surface the photo-with-faces count rather than "images scanned for - # faces" because the latter isn't tracked anywhere. - faces_photos = await scalar_count( - select(func.count(func.distinct(FaceEmbedding.photo_id))) - .select_from(FaceEmbedding) - .join(Photo, Photo.id == FaceEmbedding.photo_id) - .where(not_discarded) - ) - face_rows = await scalar_count(select(func.count(FaceEmbedding.id))) - face_clusters = await scalar_count( - select(func.count(func.distinct(FaceEmbedding.cluster_id))) - .where(FaceEmbedding.cluster_id.is_not(None)) + needs_review_count = await scalar_count( + select(func.count(Photo.id)).where(not_discarded, Photo.needs_review.is_(True)) ) duplicate_groups = await scalar_count( @@ -657,43 +629,11 @@ async def get_pipeline_stats( "hint": "Feeds duplicate detection.", }, { - "key": "embeddings", - "label": f"Embeddings ({embedder_model})", - "done": embeddings_done, + "key": "classification", + "label": "Content classification (photo vs other)", + "done": classified_done, "total": total_images, - "hint": "Semantic search + content classification.", - }, - { - "key": "tags", - "label": "Object tags (YOLO)", - "done": tagged_photos, - "total": total_images, - "hint": "Auto-generated object labels. Not every photo has a detectable object.", - "partial": True, - }, - { - "key": "ocr", - "label": "OCR text", - "done": ocr_done, - "total": total_images, - "hint": "Extracted text from screenshots / documents. Many photos have none.", - "partial": True, - }, - { - "key": "faces", - "label": "Face detection", - "done": faces_photos, - "total": total_images, - "hint": f"{face_rows} face rows detected across {faces_photos} photos.", - "partial": True, - }, - { - "key": "face_clusters", - "label": "Face clusters", - "done": face_clusters, - "total": face_clusters, # no meaningful "total" — it's just the current count - "hint": "Built by recluster_faces. Run it after backfill to populate the People view.", - "standalone": True, + "hint": f"{needs_review_count} photos flagged for review.", }, { "key": "duplicates", @@ -708,7 +648,6 @@ async def get_pipeline_stats( return { "total_photos": total_photos, "total_images": total_images, - "embedder_model": embedder_model, "stages": stages, } diff --git a/backend/app/routers/photos.py b/backend/app/routers/photos.py index 4c203de..4fd5459 100644 --- a/backend/app/routers/photos.py +++ b/backend/app/routers/photos.py @@ -48,6 +48,7 @@ async def list_photos( color_label: Optional[str] = None, is_discarded: Optional[bool] = False, is_duplicate: Optional[bool] = None, + needs_review: Optional[bool] = None, has_date_warning: Optional[bool] = None, heap_id: Optional[str] = None, sort: str = "taken_at", @@ -196,6 +197,8 @@ async def list_photos( # view shows everything regardless of duplicate status. if is_duplicate is not None: filters.append(Photo.is_duplicate == is_duplicate) + if needs_review is not None: + filters.append(Photo.needs_review == needs_review) if has_date_warning is not None: filters.append(Photo.has_date_warning == has_date_warning) diff --git a/backend/app/routers/search.py b/backend/app/routers/search.py index 9d28d49..7d6c840 100644 --- a/backend/app/routers/search.py +++ b/backend/app/routers/search.py @@ -25,13 +25,7 @@ class SearchRequest(BaseModel): @router.post("") async def search_photos(body: SearchRequest, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)): - """Unified search endpoint. Every query runs hybrid (FTS + semantic) - by default — the user never picks a mode. - - Filters: - - tag_ids: list of tag IDs (any kind: user, object, face_cluster) - - date_from / date_to: ISO date strings - """ + """FTS search over photo metadata with optional tag and date filters.""" filters = body.filters or {} results = await hybrid_search( diff --git a/backend/app/routers/tags.py b/backend/app/routers/tags.py index d10a90b..d4fe3ab 100644 --- a/backend/app/routers/tags.py +++ b/backend/app/routers/tags.py @@ -1,10 +1,8 @@ """ Tags API router. -Unified across user tags, ML-detected objects, and face clusters via -the `kind` query parameter. Default behaviour (no kind filter) returns -all tags — the frontend's "Hide auto-generated tags" toggle filters -client-side or passes `kind=user`. +Unified across user tags and the binary content-type classifier +('photography' | 'other') via the `kind` column. """ from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query @@ -34,15 +32,11 @@ class TagUpdate(BaseModel): color: Optional[str] = None -class TagMerge(BaseModel): - target_id: str # tag to merge INTO - - # ── Endpoints ───────────────────────────────────────────────────────────── @router.get("") async def list_tags( - kind: Optional[str] = Query(None, description="Filter by kind: user, object, scene, face_cluster"), + kind: Optional[str] = Query(None, description="Filter by kind: user, content_type"), db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): @@ -90,7 +84,7 @@ async def list_tags( "color": tag.color, "kind": tag.kind, "source": tag.source, - "representative_photo_id": tag.representative_photo_id or first_photo_id, + "representative_photo_id": first_photo_id, "photo_count": int(count or 0), } for tag, count, first_photo_id in rows @@ -130,7 +124,7 @@ async def update_tag( tag_id: str, body: TagUpdate, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): - """Rename or recolor a tag (works for any kind — user, object, face_cluster).""" + """Rename or recolor a tag.""" result = await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id)) tag = result.scalar_one_or_none() if not tag: @@ -149,52 +143,6 @@ async def update_tag( return {"id": tag.id, "name": tag.name, "color": tag.color, "kind": tag.kind} -@router.post("/{tag_id}/merge") -async def merge_tag( - tag_id: str, body: TagMerge, db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Merge tag_id INTO target_id. All photo associations from the source - tag are moved to the target, then the source tag is deleted. - - Useful for merging auto-detected face clusters (e.g. "Person 3" → "Alice") - or merging duplicate object labels.""" - if tag_id == body.target_id: - raise HTTPException(status_code=400, detail="Cannot merge a tag into itself") - - source = (await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id))).scalar_one_or_none() - target = (await db.execute(select(Tag).where(Tag.id == body.target_id, Tag.user_id == current_user.id))).scalar_one_or_none() - if not source: - raise HTTPException(status_code=404, detail="Source tag not found") - if not target: - raise HTTPException(status_code=404, detail="Target tag not found") - - # Move photo associations: update tag_id from source → target. - # Skip any that would violate the PK (photo already tagged with target). - existing_target_photos = select(photo_tags.c.photo_id).where( - photo_tags.c.tag_id == body.target_id - ) - await db.execute( - update(photo_tags) - .where( - photo_tags.c.tag_id == tag_id, - photo_tags.c.photo_id.notin_(existing_target_photos), - ) - .values(tag_id=body.target_id) - ) - # Delete remaining source associations (duplicates that couldn't move) - from sqlalchemy import delete as sa_delete - await db.execute( - sa_delete(photo_tags).where(photo_tags.c.tag_id == tag_id) - ) - - # Delete source tag - await db.delete(source) - await db.commit() - - return {"merged_into": target.id, "target_name": target.name} - - @router.delete("/{tag_id}", status_code=204) async def delete_tag(tag_id: str, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)): """Delete a tag. Photo associations cascade-delete via the FK.""" diff --git a/backend/app/schemas/photos.py b/backend/app/schemas/photos.py index 0ad9fbb..41cebde 100644 --- a/backend/app/schemas/photos.py +++ b/backend/app/schemas/photos.py @@ -39,6 +39,7 @@ class PhotoResponse(PhotoBase): latitude: Optional[float] = None longitude: Optional[float] = None is_duplicate: bool = False + needs_review: bool = False has_date_warning: bool = False live_photo_video_id: Optional[str] = None owner_username: Optional[str] = None diff --git a/backend/app/services/duplicates.py b/backend/app/services/duplicates.py index 50b2207..4cb42ff 100644 --- a/backend/app/services/duplicates.py +++ b/backend/app/services/duplicates.py @@ -41,12 +41,10 @@ import uuid from datetime import datetime, timezone from typing import Optional -from sqlalchemy import select, update, text +from sqlalchemy import select, update from app.database import AsyncSessionLocal from app.models.photos import Photo -from app.models.embeddings import Embedding -from app.config import settings logger = logging.getLogger(__name__) @@ -54,11 +52,6 @@ logger = logging.getLogger(__name__) # pHash Hamming distance threshold (6 out of 64 bits). DEFAULT_PHASH_THRESHOLD = 6 -# CLIP cosine distance threshold. CLIP embeddings are L2-normalized, -# so cosine distance = 1 - dot(a, b). A threshold of 0.08 catches -# visually near-identical shots; 0.15 catches similar compositions. -DEFAULT_CLIP_THRESHOLD = 0.10 - def _hex_to_int(h: str) -> int: """Parse a 16-char hex pHash to a Python int. Returns -1 on bad input @@ -119,14 +112,12 @@ class _UnionFind: async def regroup_duplicates( phash_threshold: int = DEFAULT_PHASH_THRESHOLD, - clip_threshold: float = DEFAULT_CLIP_THRESHOLD, + **_ignored, ) -> dict: - """Full recompute of duplicate groups using pHash + CLIP similarity. + """Full recompute of duplicate groups using pHash similarity. Idempotent — safe to call as often as you like. Returns a summary dict. """ - embedder_model = settings.vision.embedder.name - async with AsyncSessionLocal() as session: # Pull all visible photos with a phash or embedding. rows = ( @@ -162,15 +153,6 @@ async def regroup_duplicates( if _hamming(hi, hj) <= phash_threshold: uf.union_by_key(phash_ids[i], phash_ids[j]) - # ── Phase 2: CLIP similarity via pgvector ── - # For each photo with an embedding, find its nearest neighbors - # within the cosine distance threshold using the HNSW index. - clip_matches = await _clip_neighbor_scan( - session, ids, embedder_model, clip_threshold - ) - for photo_id, neighbor_id in clip_matches: - uf.union_by_key(photo_id, neighbor_id) - # ── Write results ── await _clear_all_groups(session) @@ -202,16 +184,9 @@ async def regroup_duplicates( async def incremental_regroup( since: Optional[datetime] = None, phash_threshold: int = DEFAULT_PHASH_THRESHOLD, - clip_threshold: float = DEFAULT_CLIP_THRESHOLD, + **_ignored, ) -> dict: - """Incremental duplicate detection for newly added photos. - - Only photos added after `since` are compared against the full library. - Much faster than a full regroup for post-scan updates: - O(new × log N) via HNSW instead of O(N²). - """ - embedder_model = settings.vision.embedder.name - + """Incremental duplicate detection for newly added photos using pHash.""" async with AsyncSessionLocal() as session: # If no watermark, fall back to full regroup. if since is None: @@ -282,13 +257,6 @@ async def incremental_regroup( if _hamming(nh, eh) <= phash_threshold: uf.union_by_key(new_id, existing_id) - # ── Phase 2: CLIP — vector similarity for new photos only ── - clip_matches = await _clip_neighbor_scan( - session, new_ids, embedder_model, clip_threshold - ) - for photo_id, neighbor_id in clip_matches: - uf.union_by_key(photo_id, neighbor_id) - # ── Write results ── # Only update groups that contain at least one new photo. # Clear all groups first, then rewrite. @@ -323,73 +291,6 @@ async def incremental_regroup( } -async def _clip_neighbor_scan( - session, - photo_ids: list[str], - embedder_model: str, - threshold: float, -) -> list[tuple[str, str]]: - """For each photo in `photo_ids` that has a CLIP embedding, find - neighbors within cosine distance `threshold` using pgvector HNSW. - - Returns a list of (photo_id, neighbor_id) pairs. - """ - matches: list[tuple[str, str]] = [] - - if not photo_ids: - return matches - - # Batch: get all embeddings for the target photos. - target_embeddings = ( - await session.execute( - select(Embedding.photo_id, Embedding.vector) - .where(Embedding.photo_id.in_(photo_ids)) - .where(Embedding.model == embedder_model) - ) - ).all() - - if not target_embeddings: - return matches - - # For each target, query nearest neighbors via pgvector. - # We use raw SQL for the <=> cosine distance operator. - for photo_id, vector in target_embeddings: - # pgvector cosine distance: <=> operator - # Find top 20 nearest neighbors within threshold. - # Serialize the vector as "[a,b,c,...]" — pgvector's text - # format uses commas; numpy's default str() joins with spaces - # which Postgres rejects with "invalid input syntax for vector". - if hasattr(vector, 'tolist'): - vec_seq = vector.tolist() - else: - vec_seq = list(vector) - vec_text = '[' + ','.join(f'{float(x):.8f}' for x in vec_seq) + ']' - result = await session.execute( - text(""" - SELECT e.photo_id, (e.vector <=> :vec) AS distance - FROM embeddings e - JOIN photos p ON p.id = e.photo_id - WHERE e.model = :model - AND e.photo_id != :pid - AND p.is_trashed = false - AND p.is_hidden = false - AND (e.vector <=> :vec) < :threshold - ORDER BY e.vector <=> :vec - LIMIT 20 - """), - { - 'vec': vec_text, - 'pid': photo_id, - 'model': embedder_model, - 'threshold': threshold, - } - ) - for row in result.all(): - matches.append((photo_id, row[0])) - - return matches - - async def _clear_all_groups(session) -> None: """Reset duplicate_group_id / is_duplicate on every photo.""" await session.execute( diff --git a/backend/app/services/feature_flags.py b/backend/app/services/feature_flags.py index e630672..0b15cf3 100644 --- a/backend/app/services/feature_flags.py +++ b/backend/app/services/feature_flags.py @@ -1,23 +1,9 @@ """ -Runtime feature flags for expensive pipeline stages. +Runtime feature flags for the vision pipeline. -The YAML config (``mulita.yml``) ships reasonable defaults. Admins can -toggle these at runtime from the Settings → AI Features tab without -rebuilding the image or editing the bind-mounted YAML; the overrides -live in Redis so both the FastAPI backend and the Celery workers see -the same value within ~1s of the write. - -The key namespace is: - - mulita:flags: → "true" | "false" - -An unset key means "fall back to the YAML default" — so an admin who -has never touched the tab sees exactly the config-file behaviour. - -Only bool flags live here. Thresholds, batch sizes, model names etc. -stay in the YAML file because flipping them safely requires restarting -the vision workers (model reload, ONNX session re-init); that's not -something a single admin click should do. +Only one flag now — the master vision switch. Runtime overrides live in +Redis under ``mulita:flags:``; an unset key falls back to the +YAML default. """ from __future__ import annotations @@ -31,30 +17,16 @@ from app.config import settings logger = logging.getLogger(__name__) -# Feature identifiers. The public name is what the admin UI sends; the -# ``yaml_default`` getter returns the value the YAML would have set. -# Keep these in sync with the VisionSettings fields in ``config.py``. FLAG_VISION_ENABLED = 'vision.enabled' -FLAG_OCR_ENABLED = 'vision.ocr.enabled' -FLAG_DETECTOR_ENABLED = 'vision.detector.enabled' -FLAG_FACES_ENABLED = 'vision.faces.enabled' -FLAG_CLASSIFIER_ENABLED = 'vision.classifier.enabled' -ALL_FLAGS = ( - FLAG_VISION_ENABLED, - FLAG_OCR_ENABLED, - FLAG_DETECTOR_ENABLED, - FLAG_FACES_ENABLED, - FLAG_CLASSIFIER_ENABLED, -) +ALL_FLAGS = (FLAG_VISION_ENABLED,) +_VISION_QUEUE = 'vision' _REDIS: Optional[redis.Redis] = None def _redis() -> Optional[redis.Redis]: - """Lazy Redis client. Returns None if the broker is unreachable so - callers can fall back to YAML defaults instead of crashing.""" global _REDIS if _REDIS is None: try: @@ -69,19 +41,8 @@ def _redis() -> Optional[redis.Redis]: def _yaml_default(name: str) -> bool: - """Return the YAML-configured default for a flag. Used when Redis - has no value set (fresh install or admin never touched the tab).""" - v = settings.vision if name == FLAG_VISION_ENABLED: - return bool(v.enabled) - if name == FLAG_OCR_ENABLED: - return bool(v.ocr.enabled) - if name == FLAG_DETECTOR_ENABLED: - return bool(v.detector.enabled) - if name == FLAG_FACES_ENABLED: - return bool(v.faces.enabled) - if name == FLAG_CLASSIFIER_ENABLED: - return bool(v.classifier.enabled) + return bool(settings.vision.enabled) raise ValueError(f"Unknown feature flag: {name!r}") @@ -90,16 +51,6 @@ def _redis_key(name: str) -> str: def is_enabled(name: str) -> bool: - """Return True if feature ``name`` is currently enabled. - - Order of precedence: - 1. Redis override (set by PATCH /admin/feature-flags) - 2. YAML default - - Reads are cheap (~ms) and we intentionally do NOT add a local - process cache — the whole point of runtime flags is that a toggle - takes effect on the next task without a worker restart. - """ r = _redis() if r is not None: try: @@ -112,9 +63,6 @@ def is_enabled(name: str) -> bool: def set_flag(name: str, value: bool) -> None: - """Persist a flag override to Redis. No-op if Redis is unreachable - (we don't silently pretend to have written; raise so the admin - request returns a 500 instead of misleading success).""" if name not in ALL_FLAGS: raise ValueError(f"Unknown feature flag: {name!r}") r = _redis() @@ -125,9 +73,6 @@ def set_flag(name: str, value: bool) -> None: def reset_flag(name: str) -> None: - """Delete the Redis override so the flag falls back to its YAML - default. Useful if an admin wants a clean slate without guessing - what the config defaults are.""" if name not in ALL_FLAGS: raise ValueError(f"Unknown feature flag: {name!r}") r = _redis() @@ -137,150 +82,41 @@ def reset_flag(name: str) -> None: _apply_worker_side_effects(name) -# --------------------------------------------------------------------------- -# Worker-level side effects: when the admin flips a flag we don't just want -# gating at task-start (which still executes the message, it just returns -# 'skipped'). We also want queued work gone and the vision worker genuinely -# idle when the master switch is off. -# --------------------------------------------------------------------------- - -_VISION_QUEUE = 'vision' - -# Flag → celery task name(s) whose queued messages should be dropped when -# the flag goes off. Keeps the queue from replaying yesterday's work the -# moment someone re-enables the stage. -_TASKS_BY_FLAG: dict[str, tuple[str, ...]] = { - FLAG_VISION_ENABLED: ( - 'embed_photo', 'ocr_photo', 'detect_objects', 'extract_faces', - 'classify_content', 'vision_fanout', 'recluster_faces', - ), - FLAG_OCR_ENABLED: ('ocr_photo',), - FLAG_DETECTOR_ENABLED: ('detect_objects',), - FLAG_FACES_ENABLED: ('extract_faces', 'recluster_faces'), - FLAG_CLASSIFIER_ENABLED: ('classify_content',), -} - - def _apply_worker_side_effects(name: str) -> None: - """Bring the live workers in line with the new flag value. - - For the master ``vision.enabled`` flag we go beyond task gating and - actually stop consumption from the ``vision`` queue — flipping it - off puts the vision worker to sleep (no CPU, no model memory - churn) until it's flipped back on. For per-feature flags, the - running tasks already skip via ``is_enabled``; we just purge any - messages already sitting in the queue so the admin doesn't pay for - a backlog on re-enable. - - All operations are best-effort — if control messaging or a Redis - op fails, we log and return; the flag state itself is already - persisted so the gating path continues to work. - """ + """Attach or detach the vision consumer and purge queued work when + the master flag flips. Best-effort — state is already persisted.""" + if name != FLAG_VISION_ENABLED: + return try: - # Lazy import: avoids a circular dependency between the services - # module (imported from tasks.vision) and the celery app config. from app.tasks.celery import celery_app except Exception as e: logger.warning(f"feature_flags: celery app unavailable for side effects ({e})") return try: - if name == FLAG_VISION_ENABLED: - if is_enabled(FLAG_VISION_ENABLED): - # Re-attach the vision consumer so workers pick up tasks - # again. broadcast=True ensures every running worker - # receives the command. - celery_app.control.add_consumer(_VISION_QUEUE, reply=False) - logger.info("feature_flags: vision re-enabled; consumer added") - else: - celery_app.control.cancel_consumer(_VISION_QUEUE, reply=False) - _purge_queue(_VISION_QUEUE) - logger.info( - "feature_flags: vision disabled; consumer cancelled " - "and queue purged" - ) - return - - # Per-feature flag going off → drop pending tasks of its types. - if not is_enabled(name): - targets = _TASKS_BY_FLAG.get(name, ()) - if targets: - removed = _purge_queue_by_task_names(_VISION_QUEUE, targets) - logger.info( - f"feature_flags: {name} disabled; removed {removed} " - f"pending messages from {_VISION_QUEUE}" - ) + if is_enabled(FLAG_VISION_ENABLED): + celery_app.control.add_consumer(_VISION_QUEUE, reply=False) + logger.info("feature_flags: vision re-enabled; consumer added") + else: + celery_app.control.cancel_consumer(_VISION_QUEUE, reply=False) + _purge_queue(_VISION_QUEUE) + logger.info("feature_flags: vision disabled; consumer cancelled and queue purged") except Exception as e: - logger.warning(f"feature_flags: worker side effects failed for {name}: {e}") + logger.warning(f"feature_flags: worker side effects failed: {e}") def _purge_queue(queue: str) -> int: - """Drop every pending message from ``queue``. Returns the count - deleted. Celery's control.purge() purges the default queue only, - so we delete the Redis key directly (the broker's queue list).""" r = _redis() if r is None: return 0 try: - removed = r.delete(queue) - return int(removed or 0) + return int(r.delete(queue) or 0) except Exception as e: logger.warning(f"feature_flags: purge {queue} failed: {e}") return 0 -def _purge_queue_by_task_names(queue: str, task_names: tuple[str, ...]) -> int: - """Walk ``queue`` and drop any message whose Celery task name is in - ``task_names``. Other messages are preserved (pushed back in order) - so we don't flush embed tasks when the admin disabled only OCR. - - Celery stores each message as a JSON blob in a Redis list; the - task name lives at ``headers.task``. - """ - import json - r = _redis() - if r is None: - return 0 - try: - # Snapshot the queue, then rebuild it without the filtered names. - # Done inside a Redis transaction so a concurrent enqueue doesn't - # race with us (worst case it gets re-delivered after we release, - # which is the normal enqueue path anyway). - pipe = r.pipeline() - pipe.lrange(queue, 0, -1) - pipe.delete(queue) - raw_items, _ = pipe.execute() - kept: list[bytes | str] = [] - removed = 0 - for raw in raw_items or []: - try: - # Messages can be bytes or str depending on decode_responses. - payload = raw.decode() if isinstance(raw, bytes) else raw - msg = json.loads(payload) - task = ( - msg.get('headers', {}).get('task') - or msg.get('task') - ) - if task in task_names: - removed += 1 - continue - except Exception: - # Unparseable message — keep it, better to leak than - # to silently drop a message we can't identify. - pass - kept.append(raw) - if kept: - r.rpush(queue, *kept) - return removed - except Exception as e: - logger.warning(f"feature_flags: selective purge failed on {queue}: {e}") - return 0 - - def snapshot() -> dict[str, dict[str, object]]: - """Return every flag's current effective value, YAML default, and - whether it's overridden. Powers the admin UI tab. - """ r = _redis() out: dict[str, dict[str, object]] = {} for name in ALL_FLAGS: diff --git a/backend/app/services/search.py b/backend/app/services/search.py index 740148d..13d7fb1 100644 --- a/backend/app/services/search.py +++ b/backend/app/services/search.py @@ -1,19 +1,13 @@ """ -Unified search service — hybrid FTS + semantic (RRF) search. - -Phase 1 (PR4): semantic-only via pgvector cosine similarity. -Phase 2 (PR5): adds FTS via tsvector, enables RRF fusion. +FTS search over photos.search_vector with optional tag/date filters. """ import logging from typing import Optional -import numpy as np -from sqlalchemy import select, text, func +from sqlalchemy import select, text from sqlalchemy.ext.asyncio import AsyncSession from app.models import Photo -from app.models.embeddings import Embedding -from app.config import settings logger = logging.getLogger(__name__) @@ -27,135 +21,57 @@ async def hybrid_search( limit: int = 50, offset: int = 0, ) -> list[dict]: - """Run hybrid search (FTS + semantic) with RRF fusion. - - Currently semantic-only; FTS leg added in PR5. - """ - model_name = settings.vision.embedder.name - results = {} - - # ── Semantic search (CLIP text → pgvector cosine) ───────────────── + """Full-text search using photos.search_vector. No embeddings, no OCR.""" if q: try: - from app.services.vision.registry import registry - embedder = registry.get_embedder() - query_vec = embedder.embed_text(q) - - # pgvector cosine distance: <=> returns distance (lower = closer). - # Join photos so we can filter out discarded / hidden rows - # inside the same query — otherwise a hidden-folder photo - # can take a top-N rank and starve the visible results. - vec_str = "[" + ",".join(str(float(v)) for v in query_vec) + "]" - stmt = text(""" - SELECT e.photo_id, - (e.vector <=> :qvec::vector) AS distance - FROM embeddings e - JOIN photos p ON p.id = e.photo_id - WHERE e.model = :model - AND p.is_trashed = false - AND p.is_hidden = false - ORDER BY e.vector <=> :qvec::vector - LIMIT 200 - """) - rows = (await db.execute(stmt, {"qvec": vec_str, "model": model_name})).fetchall() - - for rank, (photo_id, distance) in enumerate(rows): - if photo_id not in results: - results[photo_id] = {"semantic_rank": rank, "fts_rank": None} - else: - results[photo_id]["semantic_rank"] = rank - - except Exception as e: - logger.warning("Semantic search failed (models may not be loaded): %s", e) - - # ── FTS search (photos.search_vector + ocr_text) ──────────────── - if q: - try: - # Same discarded/hidden filter as the semantic leg. - # The OCR branch joins photos (through photo_id) so we can - # filter there too; otherwise OCR hits in hidden folders - # would leak into results. fts_stmt = text(""" SELECT id, ts_rank(search_vector, plainto_tsquery('english', :q)) AS rank FROM photos WHERE search_vector @@ plainto_tsquery('english', :q) AND is_trashed = false AND is_hidden = false - UNION - SELECT o.photo_id AS id, - MAX(o.confidence) AS rank - FROM ocr_text o - JOIN photos p ON p.id = o.photo_id - WHERE to_tsvector('english', o.text) @@ plainto_tsquery('english', :q) - AND p.is_trashed = false - AND p.is_hidden = false - GROUP BY o.photo_id ORDER BY rank DESC - LIMIT 200 + LIMIT 500 """) - fts_rows = (await db.execute(fts_stmt, {"q": q})).fetchall() - for rank, (photo_id, score) in enumerate(fts_rows): - if photo_id not in results: - results[photo_id] = {"semantic_rank": None, "fts_rank": rank} - else: - results[photo_id]["fts_rank"] = rank + rows = (await db.execute(fts_stmt, {"q": q})).fetchall() except Exception as e: logger.warning("FTS search failed: %s", e) + rows = [] - # ── RRF fusion ──────────────────────────────────────────────────── - k = 60 - scored = [] - for photo_id, ranks in results.items(): - score = 0.0 - if ranks["semantic_rank"] is not None: - score += 1.0 / (k + ranks["semantic_rank"]) - if ranks.get("fts_rank") is not None: - score += 1.0 / (k + ranks["fts_rank"]) - scored.append((photo_id, score)) + scored = [(pid, float(rank)) for pid, rank in rows] - scored.sort(key=lambda x: -x[1]) - - # If no text query, fall back to recent photos. Always filter out - # discarded + hidden here — this path backs the "Tags" and "People" - # browse views, which should honor the folder hide flag. - if not q: if tag_ids: from app.models.tags import photo_tags - # Subquery to get distinct photo_ids matching the tag filter - sub = select(photo_tags.c.photo_id).where( - photo_tags.c.tag_id.in_(tag_ids) - ).distinct().subquery() - stmt = select(Photo.id).join(sub, Photo.id == sub.c.photo_id) - else: - stmt = select(Photo.id) - stmt = stmt.where( - Photo.is_discarded.is_(False), - Photo.is_hidden.is_(False), - ) - stmt = stmt.order_by(Photo.added_at.desc()) - if date_from: - stmt = stmt.where(Photo.taken_at >= date_from) - if date_to: - stmt = stmt.where(Photo.taken_at <= date_to) - stmt = stmt.offset(offset).limit(limit) - rows = (await db.execute(stmt)).fetchall() - return [{"photo_id": row[0], "score": 0.0} for row in rows] + photo_ids = [pid for pid, _ in scored] + if not photo_ids: + return [] + stmt = select(photo_tags.c.photo_id).where( + photo_tags.c.photo_id.in_(photo_ids), + photo_tags.c.tag_id.in_(tag_ids), + ).distinct() + valid = {row[0] for row in (await db.execute(stmt)).fetchall()} + scored = [(pid, s) for pid, s in scored if pid in valid] - # Apply filters to scored results - photo_ids = [pid for pid, _ in scored] - if not photo_ids: - return [] + page = scored[offset : offset + limit] + return [{"photo_id": pid, "score": s} for pid, s in page] - # Filter by tags if requested + # No text query — recent photos with tag/date filters. if tag_ids: from app.models.tags import photo_tags - stmt = select(photo_tags.c.photo_id).where( - photo_tags.c.photo_id.in_(photo_ids), - photo_tags.c.tag_id.in_(tag_ids), - ).distinct() - valid_ids = {row[0] for row in (await db.execute(stmt)).fetchall()} - scored = [(pid, s) for pid, s in scored if pid in valid_ids] - - # Paginate - page = scored[offset : offset + limit] - return [{"photo_id": pid, "score": score} for pid, score in page] + sub = select(photo_tags.c.photo_id).where( + photo_tags.c.tag_id.in_(tag_ids) + ).distinct().subquery() + stmt = select(Photo.id).join(sub, Photo.id == sub.c.photo_id) + else: + stmt = select(Photo.id) + stmt = stmt.where( + Photo.is_discarded.is_(False), + Photo.is_hidden.is_(False), + ) + if date_from: + stmt = stmt.where(Photo.taken_at >= date_from) + if date_to: + stmt = stmt.where(Photo.taken_at <= date_to) + stmt = stmt.order_by(Photo.added_at.desc()).offset(offset).limit(limit) + rows = (await db.execute(stmt)).fetchall() + return [{"photo_id": row[0], "score": 0.0} for row in rows] diff --git a/backend/app/services/vision/base.py b/backend/app/services/vision/base.py index 71a7856..1cdb0c7 100644 --- a/backend/app/services/vision/base.py +++ b/backend/app/services/vision/base.py @@ -1,105 +1,25 @@ """ -Abstract base classes for vision backends. +Abstract base classes for the vision backend. -Each ABC defines the contract a backend must satisfy. The default -implementation is ONNXBackend (onnx_backend.py). A ROCm backend can be -added later by subclassing these ABCs and registering via -settings.vision.backend. +The pipeline is now a single binary classifier: photography vs other. +Feature extraction is an internal detail of the classifier and is not +exposed as a separate service. """ from abc import ABC, abstractmethod from dataclasses import dataclass + import numpy as np -@dataclass -class DetectionBox: - """A single object detection result.""" - label: str - confidence: float - bbox: list[float] # [x1, y1, x2, y2] normalized 0-1 - - -@dataclass -class OCRResult: - """A single OCR text region.""" - text: str - confidence: float - bbox: list[float] # [x1, y1, x2, y2] normalized 0-1 - language: str = "" - - -@dataclass -class FaceDetection: - """A detected face with its recognition embedding.""" - bbox: list[float] # [x1, y1, x2, y2] normalized 0-1 - embedding: np.ndarray # float32 vector (128-d for SFace) - quality: float - - @dataclass class ClassificationResult: - """A content-type classification.""" label: str confidence: float -class Embedder(ABC): - """Generates image and text embeddings (e.g. OpenCLIP ViT-B/32).""" - - @abstractmethod - def embed_image(self, image: np.ndarray) -> np.ndarray: - """Return a normalized float32 embedding vector for an RGB image.""" - ... - - @abstractmethod - def embed_text(self, text: str) -> np.ndarray: - """Return a normalized float32 embedding vector for a text query.""" - ... - - @property - @abstractmethod - def dim(self) -> int: - """Dimensionality of the output embedding.""" - ... - - -class OCREngine(ABC): - """Extracts text from images (e.g. rapidocr-onnxruntime).""" - - @abstractmethod - def run(self, image: np.ndarray) -> list[OCRResult]: - """Return OCR results for an RGB image.""" - ... - - -class ObjectDetector(ABC): - """Detects objects in images (e.g. YOLOv8n).""" - - @abstractmethod - def detect(self, image: np.ndarray) -> list[DetectionBox]: - """Return detections for an RGB image.""" - ... - - class ContentClassifier(ABC): - """Classifies images into content types (screenshot, document, etc.).""" + """Classifies an image into 'photography' or 'other'.""" @abstractmethod - def classify(self, image: np.ndarray) -> list[ClassificationResult]: - """Return content type classifications for an RGB image.""" - ... - - -class FaceProcessor(ABC): - """Detects faces and extracts recognition embeddings (e.g. YuNet + SFace).""" - - @abstractmethod - def process(self, image: np.ndarray) -> list[FaceDetection]: - """Return face detections with embeddings for an RGB image.""" - ... - - @property - @abstractmethod - def embedding_dim(self) -> int: - """Dimensionality of face embedding vectors.""" + def classify(self, image: np.ndarray) -> ClassificationResult: ... diff --git a/backend/app/services/vision/bootstrap_models.py b/backend/app/services/vision/bootstrap_models.py index 425eb98..5df3631 100644 --- a/backend/app/services/vision/bootstrap_models.py +++ b/backend/app/services/vision/bootstrap_models.py @@ -1,124 +1,46 @@ """ -Download vision model weights on first worker boot. - -Run as: python -m app.services.vision.bootstrap_models - -Or called from the vision worker entrypoint before Celery starts. -Downloads are idempotent — existing files are skipped. - -For models that require export (OpenCLIP, YOLOv8n), see export_models.py. -Those must be exported once on any machine with pip, then placed in -the models volume before the worker starts. +Ensure the OpenCLIP ViT-B/32 visual encoder is present on worker boot. +Exported via export_models.py if missing. """ import logging -import os from pathlib import Path -from urllib.request import urlretrieve from app.config import settings logger = logging.getLogger(__name__) -# (relative_path, url, description) -# Models with url=None must be pre-exported via export_models.py. -# InsightFace (RetinaFace + ArcFace) auto-downloads via the insightface -# package on first use — no manual download entries needed. -DOWNLOADS = [] - -# Models that need manual export via export_models.py -EXPORTS = [ +REQUIRED = [ ("embed/visual.onnx", "OpenCLIP ViT-B/32 visual encoder"), - ("embed/textual.onnx", "OpenCLIP ViT-B/32 textual encoder"), - ("detect/yolov8n.onnx", "YOLOv8n object detector"), ] def bootstrap(models_dir: str | None = None): - """Ensure all model files are present. Download what we can, warn about - files that need manual export.""" base = Path(models_dir or settings.vision.models_dir) base.mkdir(parents=True, exist_ok=True) - # Download auto-downloadable models - for rel_path, url, desc in DOWNLOADS: - dest = base / rel_path - dest.parent.mkdir(parents=True, exist_ok=True) - - if dest.exists(): - logger.debug("Already exists: %s (%s)", dest, desc) - continue - - logger.info("Downloading %s → %s", desc, dest) - try: - urlretrieve(url, str(dest)) - size_kb = dest.stat().st_size / 1024 - logger.info("Downloaded %s (%.0f KB)", desc, size_kb) - except Exception as e: - logger.error("Failed to download %s: %s", desc, e) - if dest.exists(): - dest.unlink() - - # Check for manually-exported models - missing = [] - for rel_path, desc in EXPORTS: - dest = base / rel_path - if not dest.exists(): - missing.append((rel_path, desc)) - + missing = [(rel, desc) for rel, desc in REQUIRED if not (base / rel).exists()] if missing: - logger.warning( - "Missing %d model file(s); attempting automatic export:", - len(missing), - ) - for rel_path, desc in missing: - logger.warning(" %s — %s", base / rel_path, desc) + logger.warning("Missing %d model file(s); attempting automatic export", len(missing)) + try: + from app.services.vision import export_models + export_models.export_openclip_visual(base) + except Exception as e: + logger.error( + "Export failed: %s. Run `python -m app.services.vision.export_models " + "--models-dir %s` manually to retry.", + e, base, + ) - from app.services.vision import export_models - - missing_paths = {rel for rel, _ in missing} - - # Only run the export functions whose outputs are actually missing. - # Each function is mapped to the file(s) it produces. - export_map = [ - (export_models.export_openclip, ["embed/visual.onnx", "embed/textual.onnx"]), - (export_models.export_siglip2, ["embed_siglip2/visual.onnx", "embed_siglip2/textual.onnx"]), - (export_models.export_yolov8n, ["detect/yolov8n.onnx"]), - ] - for export_fn, outputs in export_map: - if not any(o in missing_paths for o in outputs): - continue - try: - export_fn(base) - except Exception as e: - logger.error( - "Export step %s failed: %s. " - "Run `python -m app.services.vision.export_models " - "--models-dir %s` manually to retry.", - export_fn.__name__, - e, - base, - ) - - # Re-check what's still missing after the export pass. - still_missing = [ - (rel_path, desc) - for rel_path, desc in EXPORTS - if not (base / rel_path).exists() - ] - if still_missing: - for rel_path, desc in still_missing: - logger.error(" still missing: %s — %s", base / rel_path, desc) - else: - logger.info("All model files present in %s", base) + still_missing = [(r, d) for r, d in REQUIRED if not (base / r).exists()] + if still_missing: + for rel, desc in still_missing: + logger.error(" still missing: %s — %s", base / rel, desc) else: logger.info("All model files present in %s", base) - # Signal readiness via Redis so the scan pipeline knows the vision - # worker can accept tasks. try: import redis as _redis - r = _redis.from_url(settings.redis_url) - r.set("mulita:vision:ready", "1") + _redis.from_url(settings.redis_url).set("mulita:vision:ready", "1") logger.info("Set mulita:vision:ready in Redis") except Exception as e: logger.warning("Could not set vision readiness flag in Redis: %s", e) diff --git a/backend/app/services/vision/classify.py b/backend/app/services/vision/classify.py index dcbcda7..d143154 100644 --- a/backend/app/services/vision/classify.py +++ b/backend/app/services/vision/classify.py @@ -1,117 +1,106 @@ """ -CLIP zero-shot content-type classifier. +Binary content classifier: 'photography' vs 'other'. -Uses the native OpenCLIP PyTorch text encoder for high-quality text -embeddings (the ONNX text encoder has degraded quality due to the -eot_indices workaround). Image embeddings use the ONNX visual encoder -which works well. +Uses OpenCLIP ViT-B/32 image features (ONNX) and two pre-computed text +prompt centroids. Text centroids are computed once with the native +open_clip text encoder and cached to {models_dir}/classifier/vectors.npz +so steady-state worker startup doesn't pay the PyTorch cost. """ +from __future__ import annotations + import logging +from pathlib import Path import numpy as np -import torch from app.config import VisionSettings -from app.services.vision.base import ContentClassifier, ClassificationResult +from app.services.vision.base import ClassificationResult, ContentClassifier +from app.services.vision.embed import CLIPVisualEncoder logger = logging.getLogger(__name__) -CATEGORY_PROMPTS = { - "screenshot": [ - "a screenshot of a computer screen", - "a screenshot of a phone screen", - "a screen capture of a user interface", - ], - "document": [ - "a scanned document", - "a photo of a document with printed text", - "a photo of a page of text on paper", - ], - "receipt": [ - "a photo of a receipt", - "a photo of a bill or invoice", - ], - "meme": [ - "an internet meme with text overlay", - "a funny image with caption text", - ], - "artwork": [ - "a painting or drawing", - "a sketch or illustration", - "digital art or graphic design", - ], - "photograph": [ + +PROMPTS = { + "photography": [ "a photograph taken with a camera", "a real photo of a real scene or person", "a candid photograph", + "a portrait photograph", + "a landscape photograph", + ], + "other": [ + "a screenshot of a computer screen", + "a screenshot of a phone screen", + "a screen capture of a user interface", + "a scanned document", + "a photo of a document with printed text", + "a photo of a receipt", + "a photo of a bill or invoice", + "an internet meme with text overlay", + "a funny image with caption text", + "a digital illustration or graphic design", ], } +def _compute_text_centroids() -> dict[str, np.ndarray]: + """Compute the 'photography' and 'other' centroid vectors using the + open_clip text encoder. Only called on the cache-miss path.""" + import open_clip + import torch + + logger.info("Computing CLIP text centroids for binary classifier") + model, _, _ = open_clip.create_model_and_transforms( + "ViT-B-32", pretrained="laion2b_s34b_b79k" + ) + model.eval() + tokenizer = open_clip.get_tokenizer("ViT-B-32") + + centroids: dict[str, np.ndarray] = {} + for label, prompts in PROMPTS.items(): + tokens = tokenizer(prompts) + with torch.no_grad(): + feats = model.encode_text(tokens) + feats = feats / feats.norm(dim=-1, keepdim=True) + avg = feats.mean(dim=0) + avg = avg / avg.norm() + centroids[label] = avg.numpy().astype(np.float32) + return centroids + + class CLIPContentClassifier(ContentClassifier): - """Zero-shot content classifier using CLIP text-image similarity. - Uses native PyTorch for text encoding, ONNX for image encoding.""" - def __init__(self, settings: VisionSettings): - import open_clip - self._min_confidence = settings.classifier.min_confidence + self._encoder = CLIPVisualEncoder(settings) - # Load native model for text encoding only. - # Use whichever model family the embedder is configured for so - # the classification text vectors live in the same space as the - # image embeddings. - embedder_name = settings.embedder.name - if embedder_name.startswith("siglip2"): - model_arch = "ViT-B-16-SigLIP-384" - pretrained = "webli" + cache_dir = Path(settings.models_dir) / "classifier" + cache_dir.mkdir(parents=True, exist_ok=True) + cache_path = cache_dir / "vectors.npz" + + if cache_path.exists(): + logger.info("Loading cached text centroids from %s", cache_path) + data = np.load(cache_path) + self._photo = data["photography"].astype(np.float32) + self._other = data["other"].astype(np.float32) else: - model_arch = "ViT-B-32" - pretrained = "laion2b_s34b_b79k" + centroids = _compute_text_centroids() + self._photo = centroids["photography"] + self._other = centroids["other"] + np.savez(cache_path, photography=self._photo, other=self._other) + logger.info("Cached text centroids to %s", cache_path) - logger.info("Loading %s text encoder for content classification", model_arch) - model, _, _ = open_clip.create_model_and_transforms( - model_arch, pretrained=pretrained - ) - model.eval() - self._model = model - self._tokenizer = open_clip.get_tokenizer(model_arch) + def classify(self, image: np.ndarray) -> ClassificationResult: + vec = self._encoder.encode(image) + s_photo = float(np.dot(vec, self._photo)) + s_other = float(np.dot(vec, self._other)) - # Get the ONNX image embedder from the registry - from app.services.vision.registry import registry - self._embedder = registry.get_embedder() + if s_photo >= s_other: + label = "photography" + margin = s_photo - s_other + else: + label = "other" + margin = s_other - s_photo - # Pre-compute text embeddings for each category - self._category_embeddings: dict[str, np.ndarray] = {} - for category, prompts in CATEGORY_PROMPTS.items(): - tokens = self._tokenizer(prompts) - with torch.no_grad(): - text_features = model.encode_text(tokens) - text_features /= text_features.norm(dim=-1, keepdim=True) - avg = text_features.mean(dim=0) - avg /= avg.norm() - self._category_embeddings[category] = avg.numpy().astype(np.float32) - - logger.info("Content classifier ready with %d categories", len(self._category_embeddings)) - - def classify(self, image: np.ndarray) -> list[ClassificationResult]: - img_vec = self._embedder.embed_image(image) - - # Cosine similarity against each category - scores = {} - for category, cat_vec in self._category_embeddings.items(): - scores[category] = float(np.dot(img_vec, cat_vec)) - - # Sort by score descending - ranked = sorted(scores.items(), key=lambda x: -x[1]) - best_cat, best_score = ranked[0] - second_score = ranked[1][1] - - margin = best_score - second_score - # Normalize: 0.01 margin → ~0.5 confidence, 0.03+ → ~1.0 + # 0.01 margin → ~0.3 conf, 0.03+ → ~1.0 confidence = min(1.0, margin * 30) - - if confidence >= self._min_confidence: - return [ClassificationResult(label=best_cat, confidence=confidence)] - - return [] + return ClassificationResult(label=label, confidence=confidence) diff --git a/backend/app/services/vision/clustering.py b/backend/app/services/vision/clustering.py deleted file mode 100644 index f4993c1..0000000 --- a/backend/app/services/vision/clustering.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Face embedding clustering using DBSCAN with cosine distance. - -Called by the periodic `recluster_faces` Celery task (PR7). -""" -import logging - -import numpy as np -from sklearn.cluster import DBSCAN - -logger = logging.getLogger(__name__) - - -def cluster_faces( - embeddings: np.ndarray, - eps: float = 0.35, - min_samples: int = 2, -) -> np.ndarray: - """Cluster face embeddings using DBSCAN with cosine metric. - - Args: - embeddings: (N, D) float32 array of L2-normalized face embeddings. - eps: Maximum cosine distance between two samples to be in the - same neighborhood. Lower = tighter clusters. - min_samples: Minimum cluster size. - - Returns: - (N,) int array of cluster labels. -1 = noise / unclustered. - """ - if len(embeddings) < min_samples: - return np.full(len(embeddings), -1, dtype=int) - - db = DBSCAN(eps=eps, min_samples=min_samples, metric="cosine") - labels = db.fit_predict(embeddings) - - n_clusters = len(set(labels) - {-1}) - n_noise = (labels == -1).sum() - logger.info( - "Face clustering: %d embeddings → %d clusters, %d noise", - len(embeddings), n_clusters, n_noise, - ) - return labels diff --git a/backend/app/services/vision/detect.py b/backend/app/services/vision/detect.py deleted file mode 100644 index f0d0ebb..0000000 --- a/backend/app/services/vision/detect.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -YOLOv8n object detector using raw ONNX Runtime. - -Expects {models_dir}/detect/yolov8n.onnx, exported from ultralytics -via bootstrap_models.py. We do NOT ship ultralytics at runtime to -avoid dragging in torch. -""" -import logging -from pathlib import Path - -import numpy as np -import onnxruntime as ort - -from app.config import VisionSettings -from app.services.vision.base import ObjectDetector, DetectionBox - -logger = logging.getLogger(__name__) - -_INPUT_SIZE = 640 - -# COCO class names (80 classes) -COCO_LABELS = [ - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", - "truck", "boat", "traffic light", "fire hydrant", "stop sign", - "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", - "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", - "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", - "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", - "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", - "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", - "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", - "couch", "potted plant", "bed", "dining table", "toilet", "tv", - "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", - "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush", -] - - -def _preprocess(image: np.ndarray) -> tuple[np.ndarray, float, float]: - """Letterbox-resize + normalize to NCHW float32. Returns input tensor - and scale factors for mapping boxes back to original coords.""" - from PIL import Image - - img = Image.fromarray(image).convert("RGB") - orig_w, orig_h = img.size - - scale = min(_INPUT_SIZE / orig_w, _INPUT_SIZE / orig_h) - new_w = int(orig_w * scale) - new_h = int(orig_h * scale) - img = img.resize((new_w, new_h), Image.BICUBIC) - - # Paste onto gray canvas - canvas = np.full((_INPUT_SIZE, _INPUT_SIZE, 3), 114, dtype=np.uint8) - pad_x = (_INPUT_SIZE - new_w) // 2 - pad_y = (_INPUT_SIZE - new_h) // 2 - canvas[pad_y : pad_y + new_h, pad_x : pad_x + new_w] = np.array(img) - - blob = canvas.astype(np.float32) / 255.0 - blob = blob.transpose(2, 0, 1)[np.newaxis] # NCHW - return blob, scale, pad_x, pad_y - - -def _postprocess( - outputs: np.ndarray, - scale: float, - pad_x: int, - pad_y: int, - orig_w: int, - orig_h: int, - conf_threshold: float, - max_detections: int, -) -> list[DetectionBox]: - """Parse YOLOv8 output (1, 84, N) → list of DetectionBox.""" - # outputs shape: (1, 84, N) where 84 = 4 box coords + 80 class scores - preds = outputs[0] # (84, N) - preds = preds.T # (N, 84) - - boxes_xywh = preds[:, :4] - scores = preds[:, 4:] - - class_ids = np.argmax(scores, axis=1) - confidences = scores[np.arange(len(scores)), class_ids] - - mask = confidences >= conf_threshold - boxes_xywh = boxes_xywh[mask] - class_ids = class_ids[mask] - confidences = confidences[mask] - - if len(confidences) == 0: - return [] - - # Sort by confidence, take top N - order = np.argsort(-confidences)[:max_detections] - boxes_xywh = boxes_xywh[order] - class_ids = class_ids[order] - confidences = confidences[order] - - results = [] - for i in range(len(confidences)): - cx, cy, w, h = boxes_xywh[i] - # Remove letterbox padding and rescale to original image - x1 = (cx - w / 2 - pad_x) / scale - y1 = (cy - h / 2 - pad_y) / scale - x2 = (cx + w / 2 - pad_x) / scale - y2 = (cy + h / 2 - pad_y) / scale - # Normalize to 0-1 - bbox = [ - max(0, x1 / orig_w), - max(0, y1 / orig_h), - min(1, x2 / orig_w), - min(1, y2 / orig_h), - ] - label = COCO_LABELS[class_ids[i]] if class_ids[i] < len(COCO_LABELS) else f"class_{class_ids[i]}" - results.append(DetectionBox(label=label, confidence=float(confidences[i]), bbox=bbox)) - - return results - - -class YOLOv8Detector(ObjectDetector): - def __init__(self, settings: VisionSettings): - model_path = Path(settings.models_dir) / "detect" / "yolov8n.onnx" - - from app.services.vision.providers import create_session - - logger.info("Loading YOLOv8n from %s", model_path) - self._session = create_session(str(model_path), configured_providers=settings.execution_providers) - self._conf_threshold = settings.detector.min_confidence - self._max_detections = settings.detector.max_detections - - def detect(self, image: np.ndarray) -> list[DetectionBox]: - orig_h, orig_w = image.shape[:2] - blob, scale, pad_x, pad_y = _preprocess(image) - input_name = self._session.get_inputs()[0].name - outputs = self._session.run(None, {input_name: blob})[0] - return _postprocess( - outputs, scale, pad_x, pad_y, orig_w, orig_h, - self._conf_threshold, self._max_detections, - ) diff --git a/backend/app/services/vision/embed.py b/backend/app/services/vision/embed.py index 6f42ba8..70e43a7 100644 --- a/backend/app/services/vision/embed.py +++ b/backend/app/services/vision/embed.py @@ -1,146 +1,58 @@ """ -CLIP / SigLIP2 embedder using ONNX Runtime. - -Supports two model families: - - OpenCLIP ViT-B/32 (512-d) — legacy, config name "openclip_vitb32" - - SigLIP2 ViT-B/16 (768-d) — default, config name "siglip2_vitb16" - -Expects two ONNX files under {models_dir}/embed/: - - visual.onnx (image encoder) - - textual.onnx (text encoder) - -These are exported from open_clip via export_models.py / bootstrap_models.py. +OpenCLIP ViT-B/32 visual encoder (ONNX). Produces 512-d image features +consumed by the content classifier. Not exposed as a standalone service; +the classifier owns the lifecycle. """ import logging from pathlib import Path import numpy as np -import onnxruntime as ort +import onnxruntime as ort # noqa: F401 (provider plumbing relies on this) from app.config import VisionSettings -from app.services.vision.base import Embedder logger = logging.getLogger(__name__) -# ── Model-specific constants ────────────────────────────────────────── - -# OpenCLIP ViT-B/32 (ImageNet norm, 224px) -_OPENCLIP_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32) -_OPENCLIP_STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32) -_OPENCLIP_SIZE = 224 - -# SigLIP2 ViT-B/16 (SigLIP norm, 384px) -_SIGLIP2_MEAN = np.array([0.5, 0.5, 0.5], dtype=np.float32) -_SIGLIP2_STD = np.array([0.5, 0.5, 0.5], dtype=np.float32) -_SIGLIP2_SIZE = 384 +_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32) +_STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32) +_SIZE = 224 -def _preprocess_image( - image: np.ndarray, - input_size: int, - mean: np.ndarray, - std: np.ndarray, -) -> np.ndarray: - """Resize, center-crop, normalize an RGB uint8 image to NCHW float32.""" +def _preprocess(image: np.ndarray) -> np.ndarray: from PIL import Image img = Image.fromarray(image).convert("RGB") w, h = img.size - scale = input_size / min(w, h) + scale = _SIZE / min(w, h) img = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC) w, h = img.size - left = (w - input_size) // 2 - top = (h - input_size) // 2 - img = img.crop((left, top, left + input_size, top + input_size)) + left = (w - _SIZE) // 2 + top = (h - _SIZE) // 2 + img = img.crop((left, top, left + _SIZE, top + _SIZE)) arr = np.array(img, dtype=np.float32) / 255.0 - arr = (arr - mean) / std - arr = arr.transpose(2, 0, 1) # HWC → CHW - return arr[np.newaxis] # NCHW + arr = (arr - _MEAN) / _STD + arr = arr.transpose(2, 0, 1) + return arr[np.newaxis] -class OpenCLIPEmbedder(Embedder): - """Legacy OpenCLIP ViT-B/32 embedder (512-d).""" +class CLIPVisualEncoder: + """OpenCLIP ViT-B/32 image encoder, 512-d normalized output.""" def __init__(self, settings: VisionSettings): - model_dir = Path(settings.models_dir) / "embed" - visual_path = model_dir / "visual.onnx" - textual_path = model_dir / "textual.onnx" - + model_path = Path(settings.models_dir) / "embed" / "visual.onnx" from app.services.vision.providers import create_session from app.config import settings as app_settings - providers = app_settings.vision.execution_providers - logger.info("Loading OpenCLIP visual encoder from %s", visual_path) - self._visual = create_session(str(visual_path), configured_providers=providers) + logger.info("Loading CLIP visual encoder from %s", model_path) + self._session = create_session( + str(model_path), + configured_providers=app_settings.vision.execution_providers, + ) - logger.info("Loading OpenCLIP textual encoder from %s", textual_path) - self._textual = create_session(str(textual_path), configured_providers=providers) - - def embed_image(self, image: np.ndarray) -> np.ndarray: - inp = _preprocess_image(image, _OPENCLIP_SIZE, _OPENCLIP_MEAN, _OPENCLIP_STD) - input_name = self._visual.get_inputs()[0].name - out = self._visual.run(None, {input_name: inp})[0][0] + def encode(self, image: np.ndarray) -> np.ndarray: + inp = _preprocess(image) + name = self._session.get_inputs()[0].name + out = self._session.run(None, {name: inp})[0][0] out = out / np.linalg.norm(out) return out.astype(np.float32) - - def embed_text(self, text: str) -> np.ndarray: - import open_clip - tokenizer = open_clip.get_tokenizer("ViT-B-32") - tokens = tokenizer([text]).numpy().astype(np.int64) - eot_indices = tokens.argmax(axis=-1).astype(np.int64) - inputs = self._textual.get_inputs() - out = self._textual.run(None, { - inputs[0].name: tokens, - inputs[1].name: eot_indices, - })[0][0] - out = out / np.linalg.norm(out) - return out.astype(np.float32) - - @property - def dim(self) -> int: - return 512 - - -class SigLIP2Embedder(Embedder): - """SigLIP2 ViT-B/16 embedder (768-d) — higher recall than OpenCLIP.""" - - def __init__(self, settings: VisionSettings): - model_dir = Path(settings.models_dir) / "embed_siglip2" - visual_path = model_dir / "visual.onnx" - textual_path = model_dir / "textual.onnx" - - from app.services.vision.providers import create_session - from app.config import settings as app_settings - providers = app_settings.vision.execution_providers - - logger.info("Loading SigLIP2 visual encoder from %s", visual_path) - self._visual = create_session(str(visual_path), configured_providers=providers) - - logger.info("Loading SigLIP2 textual encoder from %s", textual_path) - self._textual = create_session(str(textual_path), configured_providers=providers) - - def embed_image(self, image: np.ndarray) -> np.ndarray: - inp = _preprocess_image(image, _SIGLIP2_SIZE, _SIGLIP2_MEAN, _SIGLIP2_STD) - input_name = self._visual.get_inputs()[0].name - out = self._visual.run(None, {input_name: inp})[0][0] - out = out / np.linalg.norm(out) - return out.astype(np.float32) - - def embed_text(self, text: str) -> np.ndarray: - import open_clip - tokenizer = open_clip.get_tokenizer("ViT-B-16-SigLIP-384") - tokens = tokenizer([text]).numpy().astype(np.int64) - inputs = self._textual.get_inputs() - feed = {inputs[0].name: tokens} - # SigLIP2 text encoder may need attention mask - if len(inputs) > 1: - attention_mask = (tokens != 0).astype(np.int64) - feed[inputs[1].name] = attention_mask - out = self._textual.run(None, feed)[0][0] - out = out / np.linalg.norm(out) - return out.astype(np.float32) - - @property - def dim(self) -> int: - return 768 diff --git a/backend/app/services/vision/export_models.py b/backend/app/services/vision/export_models.py index e4822ae..d93a696 100644 --- a/backend/app/services/vision/export_models.py +++ b/backend/app/services/vision/export_models.py @@ -1,253 +1,61 @@ """ -Export / download all vision model weights to ONNX format. +Export the OpenCLIP ViT-B/32 visual encoder to ONNX. -Run ONCE on any machine with Python + pip (doesn't need GPU): +Run once on any machine with Python + pip (no GPU needed): - pip install open-clip-torch ultralytics onnx + pip install open-clip-torch onnx python -m app.services.vision.export_models [--models-dir /data/models] -This produces: - embed/visual.onnx (~350 MB) - embed/textual.onnx (~250 MB) - detect/yolov8n.onnx (~12 MB) - -YuNet and SFace are downloaded by bootstrap_models.py at worker boot -(Apache 2.0, lightweight, no export step needed). - -After export, copy the /data/models directory into your Docker volume: - docker cp /data/models mulita-worker:/data/models -Or mount a host path in docker-compose.yml. +Produces: + embed/visual.onnx (~350 MB) """ import argparse import logging -import sys from pathlib import Path logger = logging.getLogger(__name__) -def export_openclip(models_dir: Path): - """Export OpenCLIP ViT-B/32 to two ONNX files (visual + textual).""" +def export_openclip_visual(models_dir: Path): import torch import open_clip out_dir = models_dir / "embed" out_dir.mkdir(parents=True, exist_ok=True) - visual_path = out_dir / "visual.onnx" - textual_path = out_dir / "textual.onnx" - - if visual_path.exists() and textual_path.exists(): - logger.info("OpenCLIP ONNX files already exist, skipping export") + if visual_path.exists(): + logger.info("OpenCLIP visual.onnx already exists, skipping export") return logger.info("Loading OpenCLIP ViT-B-32 laion2b_s34b_b79k...") - model, _, preprocess = open_clip.create_model_and_transforms( + model, _, _ = open_clip.create_model_and_transforms( "ViT-B-32", pretrained="laion2b_s34b_b79k" ) model.eval() - # Use dynamo=False to get the legacy TorchScript exporter which - # produces IR version 9 (compatible with onnxruntime 1.17.x). - # The new torch.onnx.export default (dynamo=True) emits IR 10. - export_kwargs = dict(opset_version=14, dynamo=False) - - # ── Visual encoder ──────────────────────────────────────────────── - if not visual_path.exists(): - logger.info("Exporting visual encoder → %s", visual_path) - dummy_image = torch.randn(1, 3, 224, 224) - torch.onnx.export( - model.visual, - dummy_image, - str(visual_path), - input_names=["image"], - output_names=["embedding"], - dynamic_axes={"image": {0: "batch"}}, - **export_kwargs, - ) - size_mb = visual_path.stat().st_size / 1e6 - logger.info("Visual encoder exported (%.1f MB)", size_mb) - - # ── Textual encoder ─────────────────────────────────────────────── - if not textual_path.exists(): - logger.info("Exporting textual encoder → %s", textual_path) - tokenizer = open_clip.get_tokenizer("ViT-B-32") - dummy_text = tokenizer(["a photo"]).to(torch.int64) - - class TextEncoder(torch.nn.Module): - """Wrap the CLIP text encoder to avoid argmax in the ONNX graph. - OpenCLIP uses argmax to find the EOT token position, but ORT - ARM64 doesn't support ArgMax(13). We pre-compute the EOT index - from the token sequence and pass it directly.""" - def __init__(self, clip_model): - super().__init__() - self.transformer = clip_model.transformer - self.token_embedding = clip_model.token_embedding - self.positional_embedding = clip_model.positional_embedding - self.ln_final = clip_model.ln_final - self.text_projection = clip_model.text_projection - - def forward(self, text, eot_indices): - x = self.token_embedding(text) - x = x + self.positional_embedding - x = x.permute(1, 0, 2) # NLD -> LND - x = self.transformer(x) - x = x.permute(1, 0, 2) # LND -> NLD - x = self.ln_final(x) - # Take the feature at the EOT token. The EOT index is - # passed in as a separate input (computed outside ONNX) - # to avoid ArgMax(13) which ORT ARM64 doesn't support. - x = x[torch.arange(x.shape[0]), eot_indices] - x = x @ self.text_projection - return x - - text_enc = TextEncoder(model) - text_enc.eval() - - # Compute EOT indices from dummy tokens (argmax of token ids) - dummy_eot = dummy_text.argmax(dim=-1) - - torch.onnx.export( - text_enc, - (dummy_text, dummy_eot), - str(textual_path), - input_names=["text", "eot_indices"], - output_names=["embedding"], - dynamic_axes={"text": {0: "batch"}, "eot_indices": {0: "batch"}}, - **export_kwargs, - ) - size_mb = textual_path.stat().st_size / 1e6 - logger.info("Textual encoder exported (%.1f MB)", size_mb) - - -def export_siglip2(models_dir: Path): - """Export SigLIP2 ViT-B/16 to two ONNX files (visual + textual).""" - import torch - import open_clip - - out_dir = models_dir / "embed_siglip2" - out_dir.mkdir(parents=True, exist_ok=True) - - visual_path = out_dir / "visual.onnx" - textual_path = out_dir / "textual.onnx" - - if visual_path.exists() and textual_path.exists(): - logger.info("SigLIP2 ONNX files already exist, skipping export") - return - - logger.info("Loading SigLIP2 ViT-B-16-SigLIP-384 webli...") - model, _, preprocess = open_clip.create_model_and_transforms( - "ViT-B-16-SigLIP-384", pretrained="webli" + logger.info("Exporting visual encoder → %s", visual_path) + dummy = torch.randn(1, 3, 224, 224) + torch.onnx.export( + model.visual, + dummy, + str(visual_path), + input_names=["image"], + output_names=["embedding"], + dynamic_axes={"image": {0: "batch"}}, + opset_version=14, + dynamo=False, ) - model.eval() - - export_kwargs = dict(opset_version=14, dynamo=False) - - # ── Visual encoder ──────────────────────────────────────────────── - if not visual_path.exists(): - logger.info("Exporting SigLIP2 visual encoder → %s", visual_path) - dummy_image = torch.randn(1, 3, 384, 384) - torch.onnx.export( - model.visual, - dummy_image, - str(visual_path), - input_names=["image"], - output_names=["embedding"], - dynamic_axes={"image": {0: "batch"}}, - **export_kwargs, - ) - size_mb = visual_path.stat().st_size / 1e6 - logger.info("SigLIP2 visual encoder exported (%.1f MB)", size_mb) - - # ── Textual encoder ─────────────────────────────────────────────── - if not textual_path.exists(): - logger.info("Exporting SigLIP2 textual encoder → %s", textual_path) - tokenizer = open_clip.get_tokenizer("ViT-B-16-SigLIP-384") - dummy_text = tokenizer(["a photo"]).to(torch.int64) - - class SigLIP2TextEncoder(torch.nn.Module): - """Wrap the SigLIP2 text transformer for ONNX export.""" - def __init__(self, clip_model): - super().__init__() - self.text = clip_model.text - - def forward(self, text): - return self.text(text) - - text_enc = SigLIP2TextEncoder(model) - text_enc.eval() - - torch.onnx.export( - text_enc, - dummy_text, - str(textual_path), - input_names=["text"], - output_names=["embedding"], - dynamic_axes={"text": {0: "batch"}}, - **export_kwargs, - ) - size_mb = textual_path.stat().st_size / 1e6 - logger.info("SigLIP2 textual encoder exported (%.1f MB)", size_mb) - - -def export_yolov8n(models_dir: Path): - """Export YOLOv8n to ONNX.""" - out_dir = models_dir / "detect" - out_dir.mkdir(parents=True, exist_ok=True) - - onnx_path = out_dir / "yolov8n.onnx" - - if onnx_path.exists(): - logger.info("YOLOv8n ONNX already exists, skipping export") - return - - logger.info("Exporting YOLOv8n → %s", onnx_path) - - from ultralytics import YOLO - - model = YOLO("yolov8n.pt") - model.export(format="onnx", imgsz=640, simplify=True) - - # ultralytics exports to cwd as yolov8n.onnx — move to target. Use - # shutil.move rather than Path.rename so it works across filesystems - # (the cwd is typically /app inside the container, while the target - # /data/models is a separately-mounted volume — Path.rename raises - # "Invalid cross-device link" in that case). - import shutil - - exported = Path("yolov8n.onnx") - if exported.exists(): - shutil.move(str(exported), str(onnx_path)) - - size_mb = onnx_path.stat().st_size / 1e6 - logger.info("YOLOv8n exported (%.1f MB)", size_mb) + logger.info("Visual encoder exported (%.1f MB)", visual_path.stat().st_size / 1e6) def main(): - parser = argparse.ArgumentParser(description="Export vision model weights to ONNX") - parser.add_argument( - "--models-dir", - type=Path, - default=Path("/data/models"), - help="Directory to write model files (default: /data/models)", - ) + parser = argparse.ArgumentParser() + parser.add_argument("--models-dir", type=Path, default=Path("/data/models")) args = parser.parse_args() - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - ) - - models_dir = args.models_dir - models_dir.mkdir(parents=True, exist_ok=True) - - logger.info("Exporting models to %s", models_dir) - - export_openclip(models_dir) - export_siglip2(models_dir) - export_yolov8n(models_dir) - - logger.info("Done. Run bootstrap_models.py next to download YuNet + SFace.") + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + args.models_dir.mkdir(parents=True, exist_ok=True) + export_openclip_visual(args.models_dir) if __name__ == "__main__": diff --git a/backend/app/services/vision/faces.py b/backend/app/services/vision/faces.py deleted file mode 100644 index 4c3e860..0000000 --- a/backend/app/services/vision/faces.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -Face detection (YuNet) + recognition (SFace) using OpenCV DNN. - -YuNet is loaded via cv2.FaceDetectorYN which handles the multi-scale -anchor decoding and NMS internally. SFace recognition uses raw ONNX -Runtime for the 128-d embedding. - -Both models are from opencv_zoo (Apache 2.0 license). -Expects {models_dir}/face/: - - yunet.onnx (~233 KB) - - sface.onnx (~37 MB, 128-d embeddings) -""" -import logging -from pathlib import Path - -import numpy as np -import cv2 -import onnxruntime as ort - -from app.config import VisionSettings -from app.services.vision.base import FaceProcessor, FaceDetection - -logger = logging.getLogger(__name__) - - -def _align_face(image: np.ndarray, landmarks: np.ndarray) -> np.ndarray: - """Align and crop a 112x112 face patch using 5-point landmarks.""" - left_eye = landmarks[0] - right_eye = landmarks[1] - - dx = right_eye[0] - left_eye[0] - dy = right_eye[1] - left_eye[1] - angle = np.degrees(np.arctan2(dy, dx)) - eye_center = ((left_eye[0] + right_eye[0]) / 2, (left_eye[1] + right_eye[1]) / 2) - eye_dist = np.sqrt(dx * dx + dy * dy) - - M = cv2.getRotationMatrix2D(eye_center, angle, 1.0) - rotated = cv2.warpAffine(image, M, (image.shape[1], image.shape[0])) - - # Crop around face center - scale = 64.0 / max(eye_dist, 1e-6) - cx, cy = eye_center - half = 56.0 / scale - x1 = max(0, int(cx - half)) - y1 = max(0, int(cy - half * 0.8)) - x2 = min(rotated.shape[1], int(cx + half)) - y2 = min(rotated.shape[0], int(cy + half * 1.2)) - crop = rotated[y1:y2, x1:x2] - - if crop.size == 0: - return np.zeros((112, 112, 3), dtype=np.float32) - - return cv2.resize(crop, (112, 112)).astype(np.float32) - - -class YuNetSFaceProcessor(FaceProcessor): - def __init__(self, settings: VisionSettings): - face_dir = Path(settings.models_dir) / "face" - yunet_path = str(face_dir / "yunet.onnx") - sface_path = str(face_dir / "sface.onnx") - - # YuNet via OpenCV's FaceDetectorYN — handles anchor decoding + NMS - self._detector = cv2.FaceDetectorYN.create( - yunet_path, - "", - (640, 640), - settings.faces.recognition_threshold, - 0.3, # NMS threshold - 5000, # top_k - ) - logger.info("YuNet face detector loaded via OpenCV") - - # SFace via ONNX Runtime - from app.services.vision.providers import create_session - ort.set_default_logger_severity(3) - self._recognizer = create_session(sface_path, configured_providers=settings.execution_providers) - logger.info("SFace recognizer loaded via ONNX Runtime") - - self._min_face_size = settings.faces.min_face_size - - def process(self, image: np.ndarray) -> list[FaceDetection]: - orig_h, orig_w = image.shape[:2] - - # Convert RGB → BGR for OpenCV - bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - - # Set input size to actual image dimensions - self._detector.setInputSize((orig_w, orig_h)) - - # Detect faces - _, faces_raw = self._detector.detect(bgr) - - if faces_raw is None or len(faces_raw) == 0: - return [] - - results = [] - for face in faces_raw: - # face: [x, y, w, h, right_eye_x, right_eye_y, left_eye_x, left_eye_y, - # nose_x, nose_y, right_mouth_x, right_mouth_y, left_mouth_x, left_mouth_y, score] - x, y, w, h = int(face[0]), int(face[1]), int(face[2]), int(face[3]) - score = float(face[14]) - - # Filter small faces - face_size = max(w, h) - if face_size < self._min_face_size: - continue - - # Normalized bbox - bbox = [ - max(0, x / orig_w), - max(0, y / orig_h), - min(1, (x + w) / orig_w), - min(1, (y + h) / orig_h), - ] - - # Extract 5-point landmarks for alignment - landmarks = np.array([ - [face[4], face[5]], # right eye - [face[6], face[7]], # left eye - [face[8], face[9]], # nose - [face[10], face[11]], # right mouth - [face[12], face[13]], # left mouth - ], dtype=np.float32) - - # Align face for recognition - face_crop = _align_face(image, landmarks) - - # SFace expects (1, 3, 112, 112) float32, BGR - face_bgr = cv2.cvtColor(face_crop.astype(np.uint8), cv2.COLOR_RGB2BGR) - face_blob = (face_bgr.astype(np.float32) / 255.0).transpose(2, 0, 1)[np.newaxis] - - rec_input = self._recognizer.get_inputs()[0].name - embedding = self._recognizer.run(None, {rec_input: face_blob})[0][0] - embedding = embedding / np.linalg.norm(embedding) - - results.append(FaceDetection( - bbox=bbox, - embedding=embedding.astype(np.float32), - quality=score, - )) - - return results - - @property - def embedding_dim(self) -> int: - return 128 diff --git a/backend/app/services/vision/insightface_processor.py b/backend/app/services/vision/insightface_processor.py deleted file mode 100644 index 14683c0..0000000 --- a/backend/app/services/vision/insightface_processor.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -Face detection + recognition using InsightFace (RetinaFace + ArcFace). - -Uses the buffalo_l model pack which auto-downloads on first use (~300MB). -Produces 512-d ArcFace embeddings. Non-commercial research license — -fine for homelab self-hosting. -""" -import logging -from pathlib import Path - -import numpy as np - -from app.config import VisionSettings -from app.services.vision.base import FaceProcessor, FaceDetection - -logger = logging.getLogger(__name__) - - -class InsightFaceProcessor(FaceProcessor): - def __init__(self, settings: VisionSettings): - from insightface.app import FaceAnalysis - - model_root = str(Path(settings.models_dir) / "face" / "insightface") - logger.info("Loading InsightFace buffalo_l from %s", model_root) - - from app.services.vision.providers import get_providers - providers = get_providers(settings.execution_providers) - - self._app = FaceAnalysis( - name="buffalo_l", - root=model_root, - providers=providers, - ) - self._app.prepare(ctx_id=-1, det_size=(640, 640)) - self._min_det_score = settings.faces.recognition_threshold - - def process(self, image: np.ndarray) -> list[FaceDetection]: - orig_h, orig_w = image.shape[:2] - - # InsightFace expects BGR - bgr = image[:, :, ::-1].copy() - - faces = self._app.get(bgr) - - if not faces: - return [] - - results = [] - for face in faces: - if face.det_score < self._min_det_score: - continue - - # face.bbox is [x1, y1, x2, y2] in pixel coords - x1, y1, x2, y2 = face.bbox - bbox = [ - max(0, float(x1) / orig_w), - max(0, float(y1) / orig_h), - min(1, float(x2) / orig_w), - min(1, float(y2) / orig_h), - ] - - embedding = face.normed_embedding # already L2-normalized, 512-d - results.append(FaceDetection( - bbox=bbox, - embedding=embedding.astype(np.float32), - quality=float(face.det_score), - )) - - return results - - @property - def embedding_dim(self) -> int: - return 512 diff --git a/backend/app/services/vision/ocr.py b/backend/app/services/vision/ocr.py deleted file mode 100644 index 6809d53..0000000 --- a/backend/app/services/vision/ocr.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -OCR engine using rapidocr-onnxruntime (PP-OCRv4 weights). - -No PaddlePaddle dependency — pure ONNX Runtime. Language packs are -downloaded automatically by rapidocr on first use. -""" -import logging - -import numpy as np - -from app.config import VisionSettings -from app.services.vision.base import OCREngine, OCRResult - -logger = logging.getLogger(__name__) - - -class RapidOCREngine(OCREngine): - def __init__(self, settings: VisionSettings): - from rapidocr_onnxruntime import RapidOCR - - self._min_confidence = settings.ocr.min_confidence - self._engine = RapidOCR() - logger.info("RapidOCR engine initialized") - - def run(self, image: np.ndarray) -> list[OCRResult]: - result, _ = self._engine(image) - if not result: - return [] - - out = [] - for box, text, score in result: - if score < self._min_confidence: - continue - # box is [[x1,y1],[x2,y2],[x3,y3],[x4,y4]] — take bounding rect - xs = [p[0] for p in box] - ys = [p[1] for p in box] - h, w = image.shape[:2] - bbox = [ - min(xs) / w, - min(ys) / h, - max(xs) / w, - max(ys) / h, - ] - out.append(OCRResult(text=text, confidence=float(score), bbox=bbox)) - return out diff --git a/backend/app/services/vision/onnx_backend.py b/backend/app/services/vision/onnx_backend.py deleted file mode 100644 index 1cad8b1..0000000 --- a/backend/app/services/vision/onnx_backend.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -ONNX Runtime backend — default CPU inference for all vision models. - -Each create_* method returns a concrete implementation of the -corresponding ABC from base.py. Models are loaded from ONNX files -under settings.vision.models_dir, downloaded on first boot by -bootstrap_models.py. -""" -import logging - -from app.config import VisionSettings -from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor, ContentClassifier - -logger = logging.getLogger(__name__) - - -class ONNXBackend: - """Factory for ONNX Runtime-based vision model instances.""" - - def __init__(self, vision_settings: VisionSettings): - self._settings = vision_settings - - def create_embedder(self) -> Embedder: - model_name = self._settings.embedder.name - if model_name.startswith("siglip2"): - from app.services.vision.embed import SigLIP2Embedder - return SigLIP2Embedder(self._settings) - else: - from app.services.vision.embed import OpenCLIPEmbedder - return OpenCLIPEmbedder(self._settings) - - def create_ocr(self) -> OCREngine: - from app.services.vision.ocr import RapidOCREngine - return RapidOCREngine(self._settings) - - def create_detector(self) -> ObjectDetector: - from app.services.vision.detect import YOLOv8Detector - return YOLOv8Detector(self._settings) - - def create_face_processor(self) -> FaceProcessor: - from app.services.vision.insightface_processor import InsightFaceProcessor - return InsightFaceProcessor(self._settings) - - def create_classifier(self) -> ContentClassifier: - from app.services.vision.classify import CLIPContentClassifier - return CLIPContentClassifier(self._settings) diff --git a/backend/app/services/vision/registry.py b/backend/app/services/vision/registry.py index f5ff51b..46ff572 100644 --- a/backend/app/services/vision/registry.py +++ b/backend/app/services/vision/registry.py @@ -1,84 +1,29 @@ """ -ModelRegistry — singleton that lazy-loads vision models per worker process. - -Usage from Celery tasks: - - from app.services.vision.registry import registry - embedder = registry.get_embedder() - vec = embedder.embed_image(img) - -Models are created on first access and cached for the worker's lifetime. -The registry reads settings.vision to decide which backend to use and -where model weights live. +ModelRegistry — lazy-loads the single content classifier per worker. """ import logging from functools import lru_cache from app.config import settings -from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor, ContentClassifier +from app.services.vision.base import ContentClassifier logger = logging.getLogger(__name__) class ModelRegistry: - """Central access point for all vision models.""" - def __init__(self): self._vision = settings.vision - @lru_cache(maxsize=1) - def get_embedder(self) -> Embedder: - logger.info("Loading embedder: %s (backend=%s)", self._vision.embedder.name, self._vision.backend) - return self._load_backend().create_embedder() - - @lru_cache(maxsize=1) - def get_ocr(self) -> OCREngine: - logger.info("Loading OCR engine (backend=%s)", self._vision.backend) - return self._load_backend().create_ocr() - - @lru_cache(maxsize=1) - def get_detector(self) -> ObjectDetector: - logger.info("Loading object detector (backend=%s)", self._vision.backend) - return self._load_backend().create_detector() - - @lru_cache(maxsize=1) - def get_face_processor(self) -> FaceProcessor: - logger.info("Loading face processor (backend=%s)", self._vision.backend) - return self._load_backend().create_face_processor() - @lru_cache(maxsize=1) def get_classifier(self) -> ContentClassifier: logger.info("Loading content classifier (backend=%s)", self._vision.backend) - return self._load_backend().create_classifier() - - @lru_cache(maxsize=1) - def _load_backend(self): - """Import and instantiate the configured backend.""" - backend_name = self._vision.backend - if backend_name == "onnx": - from app.services.vision.onnx_backend import ONNXBackend - return ONNXBackend(self._vision) - elif backend_name == "rocm": - from app.services.vision.rocm_backend import ROCmBackend - return ROCmBackend(self._vision) - else: - raise ValueError(f"Unknown vision backend: {backend_name}") + from app.services.vision.classify import CLIPContentClassifier + return CLIPContentClassifier(self._vision) def warmup(self): - """Pre-load all enabled models. Called from Celery worker_process_init - on the vision queue to avoid cold-start latency on the first task.""" - logger.info("Warming up vision models...") - self.get_embedder() - if self._vision.ocr.enabled: - self.get_ocr() - if self._vision.detector.enabled: - self.get_detector() - if self._vision.faces.enabled: - self.get_face_processor() - if self._vision.classifier.enabled: - self.get_classifier() - logger.info("Vision model warmup complete") + logger.info("Warming up vision classifier...") + self.get_classifier() + logger.info("Vision warmup complete") -# Module-level singleton. Import this from tasks. registry = ModelRegistry() diff --git a/backend/app/services/vision/rocm_backend.py b/backend/app/services/vision/rocm_backend.py deleted file mode 100644 index 878a805..0000000 --- a/backend/app/services/vision/rocm_backend.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -ROCm backend — GPU-accelerated inference for Radeon 760M-class hardware. - -Stub: raises NotImplementedError on all factory methods. To enable, -set `vision.backend: rocm` in mulita.yml once ROCm support is implemented. -""" -from app.config import VisionSettings -from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor - - -class ROCmBackend: - def __init__(self, vision_settings: VisionSettings): - self._settings = vision_settings - - def create_embedder(self) -> Embedder: - raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'") - - def create_ocr(self) -> OCREngine: - raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'") - - def create_detector(self) -> ObjectDetector: - raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'") - - def create_face_processor(self) -> FaceProcessor: - raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'") diff --git a/backend/app/tasks/celery.py b/backend/app/tasks/celery.py index 514c5aa..ad5118e 100644 --- a/backend/app/tasks/celery.py +++ b/backend/app/tasks/celery.py @@ -39,14 +39,9 @@ celery_app.conf.update( # Explicit routes for every task name. Wildcard patterns don't match # short names produced by @shared_task(name='...'). task_routes={ - # Vision queue — GPU/CPU-bound inference - 'embed_photo': {'queue': 'vision'}, - 'ocr_photo': {'queue': 'vision'}, - 'detect_objects': {'queue': 'vision'}, - 'extract_faces': {'queue': 'vision'}, + # Vision queue — CPU-bound binary classification 'classify_content': {'queue': 'vision'}, 'vision_fanout': {'queue': 'vision'}, - 'recluster_faces': {'queue': 'vision'}, # High-priority queue — thumbnails & duplicates 'generate_thumbnails': {'queue': 'high'}, 'regenerate_all_thumbnails': {'queue': 'high'}, diff --git a/backend/app/tasks/scan.py b/backend/app/tasks/scan.py index 13e027f..0b82b8b 100644 --- a/backend/app/tasks/scan.py +++ b/backend/app/tasks/scan.py @@ -422,7 +422,7 @@ async def _scan_all_source_roots_async(): still hit Settings → Re-detect duplicates to force a fresh pass. """ from app.tasks.thumbs import incremental_regroup_duplicates_task - from app.tasks.vision import backfill_vision, recluster_faces + from app.tasks.vision import backfill_vision async with AsyncSessionLocal() as session: result = await session.execute( @@ -461,14 +461,6 @@ async def _scan_all_source_roots_async(): except Exception as e: logger.warning(f"Could not queue post-scan vision backfill: {e}") - # 300s gives face extraction time to run before reclustering. - # Fires even if some faces are still in-flight — the task is - # idempotent and the user can re-trigger from Settings. - try: - recluster_faces.apply_async(countdown=300) - except Exception as e: - logger.warning(f"Could not queue post-scan face recluster: {e}") - # Re-extract metadata for photos missing GPS coordinates. # Runs on every startup so photos scanned before the GPS fix # eventually get their coordinates populated. diff --git a/backend/app/tasks/vision.py b/backend/app/tasks/vision.py index 112655d..7976842 100644 --- a/backend/app/tasks/vision.py +++ b/backend/app/tasks/vision.py @@ -1,33 +1,21 @@ """ -Celery tasks for the vision pipeline — embedding, OCR, object detection, -face recognition. +Celery tasks for the vision pipeline. -All tasks run on the dedicated `vision` queue with limited concurrency -(memory-bound CPU inference). They read thumbnails generated by -generate_thumbnails, so they MUST run after thumbs complete. - -DB access uses sync psycopg2 sessions (not asyncpg) because Celery -forks workers and asyncpg connections can't be shared across forks. +A single binary classifier decides whether a photo is 'photography' or +'other'. Photos classified as 'other' get needs_review=true so the user +can triage screenshots / documents / memes in the UI. """ import logging from pathlib import Path import numpy as np from celery import shared_task -from sqlalchemy import create_engine, text as sa_text, select, delete +from sqlalchemy import create_engine, text as sa_text, select, delete, update from sqlalchemy.orm import Session, sessionmaker from PIL import Image -from app.models.embeddings import Embedding from app.config import settings -from app.services.feature_flags import ( - is_enabled, - FLAG_VISION_ENABLED, - FLAG_OCR_ENABLED, - FLAG_DETECTOR_ENABLED, - FLAG_FACES_ENABLED, - FLAG_CLASSIFIER_ENABLED, -) +from app.services.feature_flags import is_enabled, FLAG_VISION_ENABLED logger = logging.getLogger(__name__) @@ -35,7 +23,6 @@ VISION_READY_KEY = "mulita:vision:ready" def _vision_worker_ready() -> bool: - """Check whether the vision worker has finished model bootstrap.""" try: import redis as _redis return bool(_redis.from_url(settings.redis_url).exists(VISION_READY_KEY)) @@ -47,7 +34,6 @@ _sync_engine = None def _get_sync_engine(): - """Return a module-level singleton engine (one per worker process).""" global _sync_engine if _sync_engine is None: sync_url = settings.database_url.replace("+asyncpg", "+psycopg2").replace("+aiosqlite", "") @@ -56,21 +42,13 @@ def _get_sync_engine(): def _get_sync_session() -> Session: - """Create a sync DB session backed by the shared engine.""" return sessionmaker(bind=_get_sync_engine())() def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None: - """Load a thumbnail as an RGB numpy array. - - Thumbnails may live at ``/data/thumbs/{photo_id}/`` (legacy) or - ``/data/thumbs/{user_id}/{photo_id}/`` (multi-user). Try both. - """ thumb_base = Path("/data/thumbs") - # Try legacy flat path first. thumb_path = thumb_base / photo_id / f"{size}.webp" if not thumb_path.exists(): - # Try user-prefixed paths: /data/thumbs/*/photo_id/size.webp matches = list(thumb_base.glob(f"*/{photo_id}/{size}.webp")) if matches: thumb_path = matches[0] @@ -79,7 +57,7 @@ def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None: return None try: img = Image.open(thumb_path).convert("RGB") - img.load() # force decode to catch corruption early + img.load() arr = np.array(img) img.close() return arr @@ -88,207 +66,25 @@ def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None: return None -@shared_task(name='embed_photo', queue='vision', bind=True, max_retries=3) -def embed_photo(self, photo_id: str): - """Generate CLIP embedding for a photo and store in pgvector.""" - if not is_enabled(FLAG_VISION_ENABLED): - return {'status': 'skipped', 'reason': 'vision disabled'} - - image = _load_thumb(photo_id, "medium") # 640px - if image is None: - return {'status': 'error', 'message': 'thumbnail not found'} - - try: - from app.services.vision.registry import registry - embedder = registry.get_embedder() - vector = embedder.embed_image(image) - except Exception as exc: - logger.exception("embed_photo failed for %s", photo_id) - raise self.retry(exc=exc, countdown=60) - - model_name = settings.vision.embedder.name - - session = _get_sync_session() - try: - session.execute( - delete(Embedding).where( - Embedding.photo_id == photo_id, - Embedding.model == model_name, - ) - ) - emb = Embedding( - photo_id=photo_id, - model=model_name, - vector=vector.tolist(), - ) - session.add(emb) - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - - logger.info("[%s] Embedded photo %s with %s", self.request.id, photo_id, model_name) - return {'status': 'success', 'photo_id': photo_id} - - @shared_task(name='vision_fanout', queue='vision') def vision_fanout(photo_id: str): - """Dispatch all enabled vision tasks for a photo.""" + """Dispatch vision work for a photo. Today this is just the binary + classifier; the indirection stays so scanner/upload code keeps one + entrypoint.""" if not is_enabled(FLAG_VISION_ENABLED): return {'status': 'skipped', 'reason': 'vision disabled'} - - embed_photo.delay(photo_id) - - if is_enabled(FLAG_OCR_ENABLED): - ocr_photo.delay(photo_id) - if is_enabled(FLAG_DETECTOR_ENABLED): - detect_objects.delay(photo_id) - if is_enabled(FLAG_FACES_ENABLED): - extract_faces.delay(photo_id) - if is_enabled(FLAG_CLASSIFIER_ENABLED): - classify_content.delay(photo_id) - + classify_content.delay(photo_id) return {'status': 'dispatched', 'photo_id': photo_id} -@shared_task(name='ocr_photo', queue='vision', bind=True, max_retries=3) -def ocr_photo(self, photo_id: str): - """Run OCR on a photo and store text regions.""" - if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_OCR_ENABLED): - return {'status': 'skipped', 'reason': 'OCR disabled'} - - image = _load_thumb(photo_id, "large") # 1280px for better OCR accuracy - if image is None: - return {'status': 'error', 'message': 'thumbnail not found'} - - try: - from app.services.vision.registry import registry - ocr_engine = registry.get_ocr() - results = ocr_engine.run(image) - except Exception as exc: - logger.exception("ocr_photo failed for %s", photo_id) - raise self.retry(exc=exc, countdown=60) - - if not results: - logger.info("No OCR text found for photo %s", photo_id) - return {'status': 'success', 'photo_id': photo_id, 'regions': 0} - - from app.models.ocr_text import OCRText - - session = _get_sync_session() - try: - session.execute(delete(OCRText).where(OCRText.photo_id == photo_id)) - for r in results: - session.add(OCRText( - photo_id=photo_id, - text=r.text, - language=r.language, - confidence=r.confidence, - bbox=r.bbox, - )) - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - - logger.info("[%s] OCR: %d text regions for photo %s", self.request.id, len(results), photo_id) - return {'status': 'success', 'photo_id': photo_id, 'regions': len(results)} - - -@shared_task(name='detect_objects', queue='vision', bind=True, max_retries=3) -def detect_objects(self, photo_id: str): - """Detect objects in a photo, create Tag(kind=object) rows, and - link via photo_tags with confidence/bbox/source.""" - if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_DETECTOR_ENABLED): - return {'status': 'skipped', 'reason': 'detection disabled'} - - image = _load_thumb(photo_id, "medium") # 640px - if image is None: - return {'status': 'error', 'message': 'thumbnail not found'} - - try: - from app.services.vision.registry import registry - detector = registry.get_detector() - detections = detector.detect(image) - except Exception as exc: - logger.exception("detect_objects failed for %s", photo_id) - raise self.retry(exc=exc, countdown=60) - - if not detections: - logger.info("No objects detected for photo %s", photo_id) - return {'status': 'success', 'photo_id': photo_id, 'objects': 0} - - from app.models import Photo - from app.models.tags import Tag, photo_tags - - source_name = "vision:yolov8n" - - session = _get_sync_session() - try: - # Get the photo's user_id so tags inherit ownership. - photo = session.execute( - select(Photo).where(Photo.id == photo_id) - ).scalar_one_or_none() - owner_id = photo.user_id if photo else None - - # Wipe previous detection results for this photo from this model - session.execute( - delete(photo_tags).where( - photo_tags.c.photo_id == photo_id, - photo_tags.c.source == source_name, - ) - ) - - # Group detections by label, keep highest confidence per label - best_per_label: dict[str, tuple[float, list]] = {} - for det in detections: - if det.label not in best_per_label or det.confidence > best_per_label[det.label][0]: - best_per_label[det.label] = (det.confidence, det.bbox) - - for label, (confidence, bbox) in best_per_label.items(): - # Find or create the object tag (scoped to user) - tag = session.execute( - select(Tag).where(Tag.name == label, Tag.kind == 'object', Tag.user_id == owner_id) - ).scalar_one_or_none() - - if not tag: - tag = Tag(name=label, kind='object', source=source_name, user_id=owner_id) - session.add(tag) - session.flush() # get tag.id - - # Insert photo_tags association with ML metadata - session.execute( - photo_tags.insert().values( - photo_id=photo_id, - tag_id=tag.id, - confidence=confidence, - bbox=bbox, - source=source_name, - ) - ) - - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - - labels = [d.label for d in detections] - logger.info("[%s] Detected %d objects in photo %s: %s", self.request.id, len(detections), photo_id, labels) - return {'status': 'success', 'photo_id': photo_id, 'objects': len(detections)} - - @shared_task(name='classify_content', queue='vision', bind=True, max_retries=3) def classify_content(self, photo_id: str): - """Classify image content type (screenshot, document, artwork, etc.) - using CLIP zero-shot classification. Writes Tag(kind=content_type).""" - if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_CLASSIFIER_ENABLED): - return {'status': 'skipped', 'reason': 'classifier disabled'} + """Run the binary classifier and write: + - a Tag(kind='content_type', name IN ('photography','other')) + - Photo.needs_review = (label == 'other') + """ + if not is_enabled(FLAG_VISION_ENABLED): + return {'status': 'skipped', 'reason': 'vision disabled'} image = _load_thumb(photo_id, "medium") if image is None: @@ -297,30 +93,28 @@ def classify_content(self, photo_id: str): try: from app.services.vision.registry import registry classifier = registry.get_classifier() - results = classifier.classify(image) + result = classifier.classify(image) except Exception as exc: logger.exception("classify_content failed for %s", photo_id) raise self.retry(exc=exc, countdown=60) - if not results: - logger.info("No confident classification for photo %s", photo_id) - return {'status': 'success', 'photo_id': photo_id, 'content_type': None} - from app.models import Photo from app.models.tags import Tag, photo_tags source_name = "vision:clip_classifier" - best = results[0] + label = result.label + confidence = result.confidence session = _get_sync_session() try: - # Get the photo's user_id so tags inherit ownership. photo = session.execute( select(Photo).where(Photo.id == photo_id) ).scalar_one_or_none() - owner_id = photo.user_id if photo else None + if photo is None: + return {'status': 'error', 'message': 'photo not found'} + owner_id = photo.user_id - # Wipe previous classification for this photo + # Drop any previous classification for this photo. session.execute( delete(photo_tags).where( photo_tags.c.photo_id == photo_id, @@ -328,13 +122,13 @@ def classify_content(self, photo_id: str): ) ) - # Find or create content_type tag (scoped to user) tag = session.execute( - select(Tag).where(Tag.name == best.label, Tag.kind == 'content_type', Tag.user_id == owner_id) + select(Tag).where( + Tag.name == label, Tag.kind == 'content_type', Tag.user_id == owner_id + ) ).scalar_one_or_none() - if not tag: - tag = Tag(name=best.label, kind='content_type', source=source_name, user_id=owner_id) + tag = Tag(name=label, kind='content_type', source=source_name, user_id=owner_id) session.add(tag) session.flush() @@ -342,232 +136,16 @@ def classify_content(self, photo_id: str): photo_tags.insert().values( photo_id=photo_id, tag_id=tag.id, - confidence=best.confidence, + confidence=confidence, source=source_name, ) ) - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - logger.info("[%s] Classified photo %s as '%s' (%.2f)", self.request.id, photo_id, best.label, best.confidence) - return {'status': 'success', 'photo_id': photo_id, 'content_type': best.label} - - -def _load_original(photo_id: str) -> np.ndarray | None: - """Load the original photo file as an RGB numpy array, resized to - max 1280px on the longest edge for face detection.""" - from sqlalchemy import create_engine, select as sa_select, text as sa_text - from app.models import Photo - - session = _get_sync_session() - try: - photo = session.execute( - sa_select(Photo).where(Photo.id == photo_id) - ).scalar_one_or_none() - if not photo or not photo.filepath: - return None - filepath = photo.filepath - finally: - session.close() - - if not Path(filepath).exists(): - logger.warning("Original file not found: %s", filepath) - return None - - try: - img = Image.open(filepath).convert("RGB") - # Cap at 4000px on longest edge to avoid OOM, but keep as large - # as possible for face detection accuracy - max_dim = 4000 - w, h = img.size - if max(w, h) > max_dim: - scale = max_dim / max(w, h) - resized = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC) - img.close() - img = resized - arr = np.array(img) - img.close() - return arr - except Exception as e: - logger.warning("Failed to load original %s: %s", filepath, e) - return None - - -@shared_task(name='extract_faces', queue='vision', bind=True, max_retries=3) -def extract_faces(self, photo_id: str): - """Detect faces and store recognition embeddings using InsightFace - (RetinaFace + ArcFace). No YOLO workaround needed — RetinaFace has - strong human-vs-non-human precision on its own.""" - if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_FACES_ENABLED): - return {'status': 'skipped', 'reason': 'faces disabled'} - - image = _load_original(photo_id) - if image is None: - image = _load_thumb(photo_id, "large") - if image is None: - return {'status': 'error', 'message': 'no image available'} - - try: - from app.services.vision.registry import registry - face_proc = registry.get_face_processor() - faces = face_proc.process(image) - except Exception as exc: - logger.exception("extract_faces failed for %s", photo_id) - raise self.retry(exc=exc, countdown=60) - - if not faces: - logger.info("No faces detected for photo %s", photo_id) - - return _save_faces(photo_id, faces) - - -def _save_faces(photo_id: str, faces) -> dict: - from app.models.face_embedding import FaceEmbedding - - session = _get_sync_session() - try: - session.execute(delete(FaceEmbedding).where(FaceEmbedding.photo_id == photo_id)) - for face in faces: - session.add(FaceEmbedding( - photo_id=photo_id, - bbox=face.bbox, - vector=face.embedding.tolist(), - quality=face.quality, - cluster_id=None, - )) - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - - if faces: - logger.info("Extracted %d verified face(s) from photo %s", len(faces), photo_id) - _schedule_recluster_debounced() - return {'status': 'success', 'photo_id': photo_id, 'faces': len(faces)} - - -RECLUSTER_DEBOUNCE_KEY = "mule:recluster_faces:pending" -RECLUSTER_DELAY = 120 # seconds after last face extraction - - -def _schedule_recluster_debounced(): - """Schedule a recluster_faces run, debounced so rapid-fire face - extractions don't spawn hundreds of redundant cluster jobs.""" - try: - import redis as _redis - r = _redis.from_url(settings.redis_url) - already_pending = r.set(RECLUSTER_DEBOUNCE_KEY, "1", - ex=RECLUSTER_DELAY, nx=True) - if already_pending: - recluster_faces.apply_async(countdown=RECLUSTER_DELAY) - logger.info("Scheduled debounced recluster_faces in %ds", RECLUSTER_DELAY) - except Exception as e: - logger.debug("recluster debounce check failed: %s", e) - - -@shared_task(name='recluster_faces', queue='vision', bind=True, max_retries=10) -def recluster_faces(self): - """Run DBSCAN clustering over all face embeddings and assign/create - Tag(kind=face_cluster) entries.""" - # Clear debounce key so new face extractions can schedule another round. - try: - import redis as _redis - _redis.from_url(settings.redis_url).delete(RECLUSTER_DEBOUNCE_KEY) - except Exception: - pass - - if not _vision_worker_ready(): - logger.info("Vision worker not ready yet — retrying in 30s") - raise self.retry(countdown=30) - - if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_FACES_ENABLED): - return {'status': 'skipped', 'reason': 'faces disabled'} - - from app.models import Photo - from app.models.face_embedding import FaceEmbedding - from app.models.tags import Tag, photo_tags - from app.services.vision.clustering import cluster_faces - - source_name = "vision:sface" - - session = _get_sync_session() - try: - face_rows = session.execute( - select(FaceEmbedding).order_by(FaceEmbedding.created_at) - ).scalars().all() - - if len(face_rows) < 2: - logger.info("Not enough faces for clustering (%d)", len(face_rows)) - return {'status': 'success', 'clusters': 0} - - embeddings = np.array([f.vector for f in face_rows], dtype=np.float32) - labels = cluster_faces(embeddings, eps=settings.vision.faces.cluster_eps) - - # Clean up old face_cluster tags and their photo_tags - old_cluster_tags = session.execute( - select(Tag).where(Tag.kind == 'face_cluster', Tag.source == source_name) - ).scalars().all() - for old_tag in old_cluster_tags: - session.execute( - delete(photo_tags).where( - photo_tags.c.tag_id == old_tag.id, - photo_tags.c.source == source_name, - ) - ) - session.delete(old_tag) - session.flush() - - # Build new clusters - cluster_tag_map: dict[int, str] = {} - # Track which photos belong to which cluster - cluster_photos: dict[int, set[str]] = {} - - for i, label in enumerate(labels): - if label == -1: - face_rows[i].cluster_id = None - continue - - if label not in cluster_photos: - cluster_photos[label] = set() - cluster_photos[label].add(face_rows[i].photo_id) - - if label not in cluster_tag_map: - cluster_name = f"Person {label + 1}" - # Inherit user_id from the representative photo. - rep_photo = session.execute( - select(Photo.user_id).where(Photo.id == face_rows[i].photo_id) - ).scalar_one_or_none() - tag = Tag( - name=cluster_name, - kind='face_cluster', - source=source_name, - representative_photo_id=face_rows[i].photo_id, - user_id=rep_photo, - ) - session.add(tag) - session.flush() - cluster_tag_map[label] = tag.id - - face_rows[i].cluster_id = cluster_tag_map[label] - - # Write photo_tags associations so the tag count and tag_ids - # filter work for face clusters - for label, photo_ids in cluster_photos.items(): - tag_id = cluster_tag_map[label] - for pid in photo_ids: - session.execute( - photo_tags.insert().values( - photo_id=pid, - tag_id=tag_id, - source=source_name, - ) - ) + session.execute( + update(Photo) + .where(Photo.id == photo_id) + .values(needs_review=(label == 'other')) + ) session.commit() except Exception: @@ -576,103 +154,41 @@ def recluster_faces(self): finally: session.close() - n_clusters = len(cluster_tag_map) - logger.info("Face clustering: %d clusters from %d faces", n_clusters, len(face_rows)) - return {'status': 'success', 'clusters': n_clusters, 'faces': len(face_rows)} + logger.info("[%s] Classified %s as %s (%.2f)", self.request.id, photo_id, label, confidence) + return {'status': 'success', 'photo_id': photo_id, 'label': label} @shared_task(name='backfill_vision', bind=True, max_retries=10) -def backfill_vision(self, task: str | None = None, limit: int | None = None): - """Queue vision tasks for photos that haven't been processed yet. - Uses a sync DB connection to avoid asyncpg conflicts in Celery.""" +def backfill_vision(self, limit: int | None = None, **_ignored): + """Queue classify_content for photos without a content_type tag.""" if not _vision_worker_ready(): logger.info("Vision worker not ready yet — retrying in 30s") raise self.retry(countdown=30) - model_name = settings.vision.embedder.name ordering = "ORDER BY p.taken_at DESC NULLS LAST, p.added_at DESC NULLS LAST" limit_clause = " LIMIT :lim" if limit else "" - params: dict = {"model": model_name} + params: dict = {} if limit: params["lim"] = int(limit) session = _get_sync_session() try: - # Each query finds photos missing a specific pipeline output so - # enabling a new processor after import still back-fills. - embed_ids = [] - if task in ('embed', None): - sql = f""" - SELECT p.id FROM photos p - LEFT JOIN embeddings e ON e.photo_id = p.id AND e.model = :model - WHERE e.photo_id IS NULL AND p.processing_status = 'completed' - {ordering}{limit_clause} - """ - embed_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()] - - ocr_ids = [] - if task in ('ocr', None) and is_enabled(FLAG_OCR_ENABLED): - sql = f""" - SELECT p.id FROM photos p - LEFT JOIN ocr_text o ON o.photo_id = p.id - WHERE o.photo_id IS NULL AND p.processing_status = 'completed' - {ordering}{limit_clause} - """ - ocr_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()] - - detect_ids = [] - if task in ('detect', None) and is_enabled(FLAG_DETECTOR_ENABLED): - sql = f""" - SELECT p.id FROM photos p - WHERE p.processing_status = 'completed' - AND NOT EXISTS ( - SELECT 1 FROM photo_tags pt WHERE pt.photo_id = p.id - AND pt.source = 'vision:yolov8n' - ) - {ordering}{limit_clause} - """ - detect_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()] - - face_ids = [] - if task in ('faces', None) and is_enabled(FLAG_FACES_ENABLED): - sql = f""" - SELECT p.id FROM photos p - LEFT JOIN face_embeddings fe ON fe.photo_id = p.id - WHERE fe.photo_id IS NULL AND p.processing_status = 'completed' - {ordering}{limit_clause} - """ - face_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()] - - classify_ids = [] - if task in ('classify', None) and is_enabled(FLAG_CLASSIFIER_ENABLED): - sql = f""" - SELECT p.id FROM photos p - WHERE p.processing_status = 'completed' - AND NOT EXISTS ( - SELECT 1 FROM photo_tags pt WHERE pt.photo_id = p.id - AND pt.source = 'vision:clip_classifier' - ) - {ordering}{limit_clause} - """ - classify_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()] - except Exception: - session.rollback() - raise + sql = f""" + SELECT p.id FROM photos p + WHERE p.processing_status = 'completed' + AND NOT EXISTS ( + SELECT 1 FROM photo_tags pt + WHERE pt.photo_id = p.id + AND pt.source = 'vision:clip_classifier' + ) + {ordering}{limit_clause} + """ + ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()] finally: session.close() - # Dispatch — deduplicate across query results. - all_ids = set(embed_ids) | set(ocr_ids) | set(detect_ids) | set(face_ids) | set(classify_ids) - for pid in embed_ids: - embed_photo.delay(pid) - for pid in ocr_ids: - ocr_photo.delay(pid) - for pid in detect_ids: - detect_objects.delay(pid) - for pid in face_ids: - extract_faces.delay(pid) - for pid in classify_ids: + for pid in ids: classify_content.delay(pid) - logger.info("Backfill queued %d photos for vision processing", len(all_ids)) - return {'status': 'queued', 'count': len(all_ids)} + logger.info("Backfill queued %d photos for classification", len(ids)) + return {'status': 'queued', 'count': len(ids)} diff --git a/backend/requirements.txt b/backend/requirements.txt index e9db2bc..28ea36b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -8,7 +8,6 @@ sqlalchemy[asyncio]==2.0.25 aiosqlite==0.19.0 # SQLite escape hatch (docker-compose.sqlite.yml override) asyncpg==0.29.0 # async Postgres driver (default) psycopg2-binary==2.9.9 # sync Postgres driver, used by Alembic CLI -pgvector==0.2.5 # pgvector SQLAlchemy types alembic==1.13.1 # Redis and Celery @@ -42,11 +41,6 @@ watchfiles==0.21.0 # Vision pipeline (ONNX Runtime CPU inference) onnxruntime==1.18.1 open-clip-torch==2.24.0 # tokenizer + export helper; inference via ONNX -transformers>=4.37.0 # HuggingFace tokenizer for SigLIP models -ultralytics==8.4.37 # YOLOv8n export helper; inference via ONNX -rapidocr-onnxruntime==1.3.22 -scikit-learn==1.4.0 # DBSCAN for face clustering -insightface>=0.7.3 # RetinaFace + ArcFace face detection/recognition numpy>=1.26.0,<2.0 # Utilities diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 646cd66..e3f9427 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -3,7 +3,6 @@ import { Timeline } from './components/timeline/Timeline' import { DuplicatesView } from './components/duplicates/DuplicatesView' import { MapView } from './components/map/MapView' import { MemoriesView } from './components/memories/MemoriesView' -import { PeopleView } from './components/people/PeopleView' import { TagsView } from './components/tags/TagsView' import { ColorsView } from './components/colors/ColorsView' import { RatedView } from './components/rated/RatedView' @@ -92,8 +91,6 @@ function MainApp() { ) : currentSection === 'duplicates' ? ( - ) : currentSection === 'people' ? ( - ) : currentSection === 'tags' ? ( ) : currentSection === 'colors' ? ( diff --git a/frontend/src/components/dialogs/SettingsDialog.tsx b/frontend/src/components/dialogs/SettingsDialog.tsx index dcb0f31..30cc1eb 100644 --- a/frontend/src/components/dialogs/SettingsDialog.tsx +++ b/frontend/src/components/dialogs/SettingsDialog.tsx @@ -16,10 +16,6 @@ import { FolderSearch, Shield, Brain, - ScanText, - UserSquare2, - Boxes, - Tags as TagsIcon, RotateCcw, } from 'lucide-react' import clsx from 'clsx' @@ -1107,56 +1103,20 @@ interface AiFeaturesTabProps { ) => Promise } -// Flags are keyed by the backend's canonical name ("vision.enabled", -// "vision.ocr.enabled", ...). The metadata here just adds presentation -// (label, short description, icon) so the tab layout stays data-driven. const FLAG_META: Array<{ id: string label: string description: string icon: React.ReactNode - // Optional "run this backfill" hook — lets the user kick off a stage's - // backfill right from the toggle row without hopping to a separate UI. - backfillTask?: 'embed' | 'ocr' | 'detect' | 'faces' | 'classify' }> = [ { id: 'vision.enabled', - label: 'Vision pipeline (master switch)', + label: 'Vision classifier', description: - 'When off, every AI stage below is skipped — including newly uploaded photos. ' + - 'Existing results stay intact.', + 'Binary photo-vs-other classifier. Flags screenshots, documents, memes and ' + + 'scans with "needs review" so they can be triaged.', icon: , }, - { - id: 'vision.ocr.enabled', - label: 'Text recognition (OCR)', - description: 'Extract printed / handwritten text from photos so it becomes searchable.', - icon: , - backfillTask: 'ocr', - }, - { - id: 'vision.detector.enabled', - label: 'Object detection', - description: 'Tag photos with detected objects (person, car, dog, …) via YOLOv8n.', - icon: , - backfillTask: 'detect', - }, - { - id: 'vision.faces.enabled', - label: 'Face recognition', - description: - 'Find and cluster faces across the library (RetinaFace + ArcFace). ' + - 'Expensive on big libraries — disable if you don\'t need the People view.', - icon: , - backfillTask: 'faces', - }, - { - id: 'vision.classifier.enabled', - label: 'Content classification', - description: 'Zero-shot CLIP tags for scenes / activities (beach, wedding, …).', - icon: , - backfillTask: 'classify', - }, ] function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) { @@ -1183,12 +1143,11 @@ function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) { queryClient.invalidateQueries({ queryKey: ['features'] }) } - type BackfillTask = 'embed' | 'ocr' | 'detect' | 'faces' | 'classify' | null - const runBackfill = (task: BackfillTask) => + const runBackfill = () => runAction( - `ai-backfill:${task ?? 'all'}`, - () => adminApi.triggerAiBackfill({ task }), - task ? `Backfill queued for ${task}` : 'Full backfill queued', + `ai-backfill:all`, + () => adminApi.triggerAiBackfill({}), + 'Classifier backfill queued', (r) => `Celery task ${r.task_id}`, ) @@ -1286,17 +1245,6 @@ function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) { - {meta.backfillTask && state.effective && !masterOff && ( -
- runBackfill(meta.backfillTask!)} - > - - Run {meta.backfillTask} backfill - -
- )} ) })} @@ -1306,35 +1254,17 @@ function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) {
} title="Manual pipeline triggers">

- Run a full pass across the enabled stages, recompute face - clusters, or force a fresh filesystem scan. All three are safe - to run repeatedly — the backfill only touches photos that - don\'t yet have a given output, and the rescan skips files - that are already indexed. + Run the classifier over any photos that haven't been classified yet, + or force a fresh filesystem scan. Both are safe to run repeatedly.

runBackfill(null)} + onClick={() => runBackfill()} disabled={masterOff} > - Run full vision backfill - - - runAction( - 'recluster', - () => adminApi.triggerFaceRecluster(), - 'Face recluster queued', - (r) => `Celery task ${r.task_id}`, - ) - } - disabled={masterOff || !flags['vision.faces.enabled']?.effective} - > - - Recluster faces + Run classifier backfill s.sortBy) const sortOrder = useFilterStore((s) => s.sortOrder) const tagIds = useFilterStore((s) => s.tagIds) + const needsReview = useFilterStore((s) => s.needsReview) + const setNeedsReview = useFilterStore((s) => s.setNeedsReview) const currentSection = useFilterStore((s) => s.currentSection) // Only the Flag pill is hidden inside the Discarded section. Flag has @@ -129,251 +131,263 @@ export function FilterBar() { const anyActive = hasActiveFilters(filterState) return ( - // Fixed bar height + py-0 so neither the active filter pills nor the - // clear-all button can stretch the bar vertically. The fixed h-11 - // matches the h-7 pills + 8px symmetric vertical padding. -
+
{/* Pills — left side, scroll horizontally if they overflow. */}
- {/* Date */} - { - setDateFrom(null) - setDateTo(null) - }} - > -
-
- - setDateFrom(e.target.value || null)} - className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text" - /> + {/* Date */} + { + setDateFrom(null) + setDateTo(null) + }} + > +
+
+ + setDateFrom(e.target.value || null)} + className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text" + /> +
+
+ + setDateTo(e.target.value || null)} + className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text" + /> +
-
- - setDateTo(e.target.value || null)} - className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text" - /> -
-
- + - {/* Type */} - mediaTypes.forEach((t) => toggleMediaType(t))} - > -
- {MEDIA_TYPES.map(({ value, label }) => { - const active = mediaTypes.includes(value) - return ( + {/* Type */} + mediaTypes.forEach((t) => toggleMediaType(t))} + > +
+ {MEDIA_TYPES.map(({ value, label }) => { + const active = mediaTypes.includes(value) + return ( + + ) + })} +
+ + + {/* Rating */} + setRatingMin(0)} + > +
+

Minimum

+
+ {[1, 2, 3, 4, 5].map((n) => ( + + ))} +
+
+ + + {/* Color */} + setColorLabel(null)} + > +
+ {COLOR_LABEL_OPTIONS.map(({ value, className }) => { + const active = colorLabel === value + return ( + + )} +
+ + + {/* Flag — hidden in the Discarded section, where the flag is + * pinned to "discarded" by the section preset. */} + {!hideFlagPill && ( + setFlag('any')} + > +
+ - ) - })} -
- - - {/* Rating */} - setRatingMin(0)} - > -
-

Minimum

-
- {[1, 2, 3, 4, 5].map((n) => ( - ))} -
-
- - - {/* Color */} - setColorLabel(null)} - > -
- {COLOR_LABEL_OPTIONS.map(({ value, className }) => { - const active = colorLabel === value - return ( - - )} -
- + > + Discarded + + +
+ + )} - {/* Flag — hidden in the Discarded section, where the flag is - * pinned to "discarded" by the section preset. */} - {!hideFlagPill && ( - setFlag('any')} + {/* Needs review — binary toggle */} + + + {/* Tags */} + {allTags.length > 0 && ( + setTagIds([])} + > + setTagIds([])} + /> + + )} + + {/* Sort — always present, never "active/inactive" since there's + always a value. */} + +
+ -
- )} - {/* Tags */} - {allTags.length > 0 && ( - setTagIds([])} - > - setTagIds([])} - /> - - )} - - {/* Sort — always present, never "active/inactive" since there's - always a value. */} - -
- - -
- - - {/* Clear-all — borderless text affordance pinned next to the pill + {/* Clear-all — borderless text affordance pinned next to the pill * cluster on the right. Lives inside the pills container so it * shares the same flex group and gap and reads as "another * pill". Only renders when any filter is active. */} - {anyActive && ( - - )} + {anyActive && ( + + )}
{/* Search — pinned to the right edge of the bar. Same id as before diff --git a/frontend/src/components/layout/LeftSidebar.tsx b/frontend/src/components/layout/LeftSidebar.tsx index f79dc47..0a882d8 100644 --- a/frontend/src/components/layout/LeftSidebar.tsx +++ b/frontend/src/components/layout/LeftSidebar.tsx @@ -80,17 +80,10 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) { const navigateToSection = useFilterStore((s) => s.navigateToSection) const currentSection = useFilterStore((s) => s.currentSection) const { data: allTags = [] } = useTagsQuery() - const { data: faceClusters = [] } = useTagsQuery('face_cluster') const { data: stats } = useLibraryStatsQuery() const { data: featuresMap } = useFeaturesQuery() const visionOn = featuresMap ? featuresMap['vision.enabled'] !== false : true - const facesOn = visionOn && (featuresMap ? featuresMap['vision.faces.enabled'] !== false : true) - const tagsOn = - visionOn && - (featuresMap - ? featuresMap['vision.detector.enabled'] !== false || - featuresMap['vision.classifier.enabled'] !== false - : true) + const tagsOn = true const [dropTargetId, setDropTargetId] = useState(null) // Per-folder kebab menu open state. Stores the tree-item id ("folder-..." @@ -295,8 +288,8 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) { case 'tags': navigateToSection('tags', { groupBy: 'tag' }) break - case 'people': - navigateToSection('people', { groupBy: 'tag' }) + case 'needs-review': + navigateToSection('needs-review', { needsReview: true }) break case 'colors': navigateToSection('colors', { groupBy: 'color' }) @@ -425,7 +418,6 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) { // Total tag count for the badge on the Tags entry (user tags only). const userTags = allTags.filter((t) => t.kind === 'user') const tagsTotalCount = userTags.reduce((sum, t) => sum + (t.photo_count || 0), 0) - const peopleTotalCount = faceClusters.reduce((sum, t) => sum + (t.photo_count || 0), 0) const libraryTree: TreeItem[] = [ { @@ -436,7 +428,7 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) { { id: 'all-photos', label: 'All Photos', icon: , count: stats?.all_photos ?? 0 }, { id: 'rated', label: 'Rated', icon: , count: stats?.rated ?? 0 }, ...(tagsOn ? [{ id: 'tags', label: 'Tags', icon: , count: tagsTotalCount }] : []), - ...(facesOn ? [{ id: 'people', label: 'People', icon: , count: peopleTotalCount }] : []), + ...(visionOn ? [{ id: 'needs-review', label: 'Needs Review', icon: , count: stats?.needs_review ?? 0 }] : []), { id: 'colors', label: 'Colors', icon: , count: stats?.colored ?? 0 }, { id: 'map', label: 'Map', icon: , count: stats?.with_gps ?? 0 }, { id: 'memories', label: 'Memories', icon: }, @@ -516,7 +508,7 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) { style={ isSectionHeader ? undefined - : { paddingLeft: `${8 + (depth - 1) * 12}px` } + : { paddingLeft: `${depth * 20}px` } } onClick={() => { if (renamingId === item.id) return @@ -580,7 +572,7 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) { )} ) : ( - !isSectionHeader &&
+ !isSectionHeader &&
)} {/* Item Icon — section headers drop their icon in favor of the @@ -783,7 +775,7 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) { creatingUnder === item.id.slice('folder-'.length) && (
[...rawClusters].sort((a, b) => b.photo_count - a.photo_count), - [rawClusters] - ) - const queryClient = useQueryClient() - const setTagIds = useFilterStore((s) => s.setTagIds) - - const [selectedPerson, setSelectedPerson] = useState(null) - const [editingId, setEditingId] = useState(null) - const [editName, setEditName] = useState('') - - const renameMutation = useMutation({ - mutationFn: ({ id, name }: { id: string; name: string }) => - tagsApi.update(id, { name }), - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ['tags'] }) - setEditingId(null) - if (selectedPerson && editingId === selectedPerson.id) { - setSelectedPerson({ ...selectedPerson, name: editName.trim() }) - } - toast.success('Renamed') - }, - onError: (e: any) => - toast.error('Rename failed', e?.response?.data?.detail || e.message), - }) - - const startEditing = (tag: Tag) => { - setEditingId(tag.id) - setEditName(tag.name) - } - - const submitRename = () => { - if (!editingId || !editName.trim()) return - renameMutation.mutate({ id: editingId, name: editName.trim() }) - } - - const enterDetail = useCallback( - (person: Tag) => { - setTagIds([person.id]) - setSelectedPerson(person) - }, - [setTagIds] - ) - - const exitDetail = useCallback(() => { - setTagIds([]) - setSelectedPerson(null) - }, [setTagIds]) - - const { activeIndex, gridRef } = useCardGridNav({ - items: clusters, - inDetail: selectedPerson !== null, - onEnter: enterDetail, - onExit: exitDetail, - }) - - // ── Detail view: a person's photos ───────────────────────────────── - if (selectedPerson) { - const isEditing = editingId === selectedPerson.id - return ( -
-
- - - {isEditing ? ( -
- setEditName(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter') submitRename() - if (e.key === 'Escape') setEditingId(null) - }} - className="rounded border border-border bg-bg px-2 py-1 text-sm text-text focus:border-primary focus:outline-none" - /> - - -
- ) : ( -
-

{selectedPerson.name}

- -
- )} -
-
- -
-
- ) - } - - // ── Card grid ────────────────────────────────────────────────────── - if (isLoading) { - return ( -
- - Loading people... -
- ) - } - - if (clusters.length === 0) { - return ( -
- -

No people identified yet

-

- Face detection runs automatically when photos are scanned. - People will appear here once faces are found and clustered. -

-
- ) - } - - return ( -
-
- - - {clusters.length} {clusters.length === 1 ? 'person' : 'people'} identified - -
- -
- {clusters.map((tag, i) => ( -
{ - if (editingId !== tag.id) enterDetail(tag) - }} - > -
- {tag.representative_photo_id ? ( - {tag.name} - ) : ( -
- -
- )} - - - {tag.photo_count} - - - -
- -
- {editingId === tag.id ? ( -
e.stopPropagation()}> - setEditName(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter') submitRename() - if (e.key === 'Escape') setEditingId(null) - }} - className="min-w-0 flex-1 rounded border border-border bg-bg px-1.5 py-0.5 text-xs text-text focus:border-primary focus:outline-none" - /> - - -
- ) : ( -

{tag.name}

- )} -
-
- ))} -
-
- ) -} diff --git a/frontend/src/components/tags/TagsView.tsx b/frontend/src/components/tags/TagsView.tsx index 28f15f3..0f70867 100644 --- a/frontend/src/components/tags/TagsView.tsx +++ b/frontend/src/components/tags/TagsView.tsx @@ -17,11 +17,10 @@ export function TagsView() { const setTagIds = useFilterStore((s) => s.setTagIds) const [selectedTag, setSelectedTag] = useState(null) - // Exclude face_cluster tags (those live in PeopleView) const tags = useMemo( () => allTags - .filter((t) => t.kind !== 'face_cluster') + .filter((t) => t.kind === 'user') .sort((a, b) => b.photo_count - a.photo_count), [allTags] ) diff --git a/frontend/src/hooks/useFeaturesQuery.ts b/frontend/src/hooks/useFeaturesQuery.ts index 201ce2d..a3d6501 100644 --- a/frontend/src/hooks/useFeaturesQuery.ts +++ b/frontend/src/hooks/useFeaturesQuery.ts @@ -18,14 +18,7 @@ export function useFeaturesQuery() { }) } -export function useIsFeatureEnabled( - name: - | 'vision.enabled' - | 'vision.ocr.enabled' - | 'vision.detector.enabled' - | 'vision.faces.enabled' - | 'vision.classifier.enabled', -): boolean { +export function useIsFeatureEnabled(name: 'vision.enabled'): boolean { const { data } = useFeaturesQuery() // Default to enabled while loading so we don't flash "feature off" // during a first-paint fetch. The backend is the source of truth; diff --git a/frontend/src/hooks/useFilterUrlSync.ts b/frontend/src/hooks/useFilterUrlSync.ts index db0c206..7bbf369 100644 --- a/frontend/src/hooks/useFilterUrlSync.ts +++ b/frontend/src/hooks/useFilterUrlSync.ts @@ -89,6 +89,7 @@ function parseUrl(): HydratePayload { } if (sp.get('duplicates') === 'true') out.duplicates = true + if (sp.get('needs_review') === 'true') out.needsReview = true const groupBy = sp.get('group') if (groupBy === 'date' || groupBy === 'tag') out.groupBy = groupBy @@ -123,6 +124,7 @@ function writeUrl(f: FilterState & { currentSection?: string }) { if (f.folderId) sp.set('folder_id', f.folderId) if (f.tagIds.length > 0) sp.set('tag_ids', f.tagIds.join(',')) if (f.duplicates) sp.set('duplicates', 'true') + if (f.needsReview) sp.set('needs_review', 'true') if (f.groupBy !== 'date') sp.set('group', f.groupBy) if (f.currentSection && f.currentSection !== 'all-photos') sp.set('section', f.currentSection) diff --git a/frontend/src/hooks/usePhotosQuery.ts b/frontend/src/hooks/usePhotosQuery.ts index 8460330..7e98901 100644 --- a/frontend/src/hooks/usePhotosQuery.ts +++ b/frontend/src/hooks/usePhotosQuery.ts @@ -52,6 +52,7 @@ export function usePhotosQuery() { const folderId = useFilterStore((s) => s.folderId) const tagIds = useFilterStore((s) => s.tagIds) const duplicates = useFilterStore((s) => s.duplicates) + const needsReview = useFilterStore((s) => s.needsReview) const groupBy = useFilterStore((s) => s.groupBy) const sortBy = useFilterStore((s) => s.sortBy) const sortOrder = useFilterStore((s) => s.sortOrder) @@ -71,11 +72,12 @@ export function usePhotosQuery() { folderId, tagIds, duplicates, + needsReview, groupBy, sortBy, sortOrder, }), - [q, dateFrom, dateTo, mediaTypes, ratingMin, ratingMax, colorLabel, flag, heapId, folderId, tagIds, duplicates, groupBy, sortBy, sortOrder] + [q, dateFrom, dateTo, mediaTypes, ratingMin, ratingMax, colorLabel, flag, heapId, folderId, tagIds, duplicates, needsReview, groupBy, sortBy, sortOrder] ) const queryClient = useQueryClient() diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 0d33764..d253e85 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -438,7 +438,6 @@ export interface PipelineStage { export interface PipelineStats { total_photos: number total_images: number - embedder_model: string stages: PipelineStage[] } @@ -621,6 +620,7 @@ export interface LibraryStats { with_gps: number duplicates: number discarded: number + needs_review: number total_photos: number total_videos: number total_size: number @@ -843,7 +843,7 @@ export const sharing = { } // Tags API -export type TagKind = 'user' | 'object' | 'scene' | 'face_cluster' +export type TagKind = 'user' | 'content_type' export interface Tag { id: string @@ -876,11 +876,6 @@ export const tags = { await api.delete(`/tags/${tagId}`) }, - merge: async (sourceId: string, targetId: string): Promise<{ merged_into: string; target_name: string }> => { - const response = await api.post(`/tags/${sourceId}/merge`, { target_id: targetId }) - return response.data - }, - /** Add one or more tags to a photo. */ addToPhoto: async (photoId: string, tagIds: string[]) => { const response = await api.post(`/photos/${photoId}/tags`, { tag_ids: tagIds }) @@ -1013,18 +1008,12 @@ export const admin = { }, triggerAiBackfill: async (body: { - task?: 'embed' | 'ocr' | 'detect' | 'faces' | 'classify' | null limit?: number | null }): Promise<{ status: string; task_id: string }> => { const response = await api.post('/admin/ai/backfill', body) return response.data }, - triggerFaceRecluster: async (): Promise<{ status: string; task_id: string }> => { - const response = await api.post('/admin/ai/recluster-faces') - return response.data - }, - triggerFullRescan: async (): Promise<{ status: string; task_id: string }> => { const response = await api.post('/admin/ai/rescan') return response.data diff --git a/frontend/src/store/filterStore.ts b/frontend/src/store/filterStore.ts index 13ccce9..e2bc72b 100644 --- a/frontend/src/store/filterStore.ts +++ b/frontend/src/store/filterStore.ts @@ -31,6 +31,8 @@ export interface FilterState { tagIds: string[] /** When true, restrict to photos flagged as duplicates by the scanner. */ duplicates: boolean + /** When true, restrict to photos classified as 'other' (needs_review). */ + needsReview: boolean /** Visual grouping mode. 'date' groups by month when sortBy is a date * field; 'tag' groups by photo tag membership. Independent of filters. */ groupBy: GroupBy @@ -68,6 +70,7 @@ interface FilterStore extends FilterState { setTagIds: (ids: string[]) => void toggleTagId: (id: string) => void setDuplicates: (v: boolean) => void + setNeedsReview: (v: boolean) => void setGroupBy: (mode: GroupBy) => void setSortBy: (field: SortField) => void setSortOrder: (order: SortOrder) => void @@ -102,6 +105,7 @@ export const INITIAL_FILTERS: FilterState = { folderId: null, tagIds: [], duplicates: false, + needsReview: false, groupBy: 'date', sortBy: 'taken_at', sortOrder: 'desc', @@ -124,6 +128,7 @@ function snapshotFilters(s: FilterState): FilterState { folderId: s.folderId, tagIds: [...s.tagIds], duplicates: s.duplicates, + needsReview: s.needsReview, groupBy: s.groupBy, sortBy: s.sortBy, sortOrder: s.sortOrder, @@ -159,6 +164,7 @@ export const useFilterStore = create((set) => ({ : [...s.tagIds, id], })), setDuplicates: (duplicates) => set({ duplicates }), + setNeedsReview: (needsReview) => set({ needsReview }), setGroupBy: (groupBy) => set({ groupBy }), setSortBy: (sortBy) => set({ sortBy }), setSortOrder: (sortOrder) => set({ sortOrder }), @@ -218,6 +224,7 @@ export function filtersToParams(f: FilterState): Record if (f.folderId) params.folder_id = f.folderId if (f.tagIds.length > 0) params.tag_ids = f.tagIds.join(',') if (f.duplicates) params.is_duplicate = 'true' + if (f.needsReview) params.needs_review = 'true' params.sort = f.sortBy params.order = f.sortOrder return params @@ -237,6 +244,7 @@ export function hasActiveFilters(f: FilterState): boolean { f.heapId !== null || f.folderId !== null || f.tagIds.length > 0 || - f.duplicates + f.duplicates || + f.needsReview ) } diff --git a/frontend/src/types/photo.ts b/frontend/src/types/photo.ts index e65c4db..862cb52 100644 --- a/frontend/src/types/photo.ts +++ b/frontend/src/types/photo.ts @@ -17,6 +17,7 @@ export interface Photo { color_label?: string | null is_discarded: boolean is_duplicate: boolean + needs_review?: boolean has_date_warning?: boolean file_hash: string folder_id: string | null diff --git a/mulita.yml b/mulita.yml index 334b0ff..35f555f 100644 --- a/mulita.yml +++ b/mulita.yml @@ -23,30 +23,12 @@ performance: db_pool_size: 20 db_pool_recycle: 3600 -# AI vision pipeline — embedding, OCR, object detection, face recognition. -# Runs on the dedicated `vision` Celery queue (PR4+). Set enabled: false -# to disable all vision processing. +# Vision pipeline — single binary classifier (photography vs other). +# Photos landing in 'other' get needs_review=true. vision: enabled: true - backend: onnx # "onnx" (CPU) | "rocm" (future GPU) + backend: onnx models_dir: /data/models - embedder: - name: openclip_vitb32 - batch_size: 8 - ocr: - enabled: true - languages: [en] - min_confidence: 0.5 - detector: - enabled: true - min_confidence: 0.35 - max_detections: 50 - faces: - enabled: true - min_face_size: 40 - recognition_threshold: 0.65 - cluster_eps: 0.5 classifier: - enabled: true min_confidence: 0.3 worker_concurrency: 2