feat: CLIP-powered incremental duplicate detection

Replace O(N²) pHash-only duplicate detection with a hybrid approach:
- pHash Hamming distance for exact/near-exact copies
- CLIP embedding cosine similarity via pgvector HNSW for visually
  similar photos (crops, format changes, screenshots)

Post-scan now uses incremental mode: only newly added photos are
compared against the full library — O(new × log N) via HNSW index
instead of O(N²). Full regroup remains available from Settings.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

backend/app/tasks/scan.py

@@ -5,7 +5,7 @@ import os
 import hashlib
 import asyncio
 from pathlib import Path
-from datetime import datetime
+from datetime import datetime, timezone
 import logging
 import json
 from typing import List, Dict, Optional
@@ -419,7 +419,7 @@ async def _scan_all_source_roots_async():
     countdown is a best-effort hint — on a big library the user can
     still hit Settings → Re-detect duplicates to force a fresh pass.
     """
-    from app.tasks.thumbs import regroup_duplicates_task
+    from app.tasks.thumbs import incremental_regroup_duplicates_task
     from app.tasks.vision import backfill_vision, recluster_faces
 
     async with AsyncSessionLocal() as session:
@@ -441,7 +441,14 @@ async def _scan_all_source_roots_async():
         # firing too early just means the next manual run picks up the
         # late arrivals — no corrupted state.
         try:
-            regroup_duplicates_task.apply_async(countdown=60)
+            # Use incremental mode: only compare newly added photos
+            # against the full library via CLIP HNSW + pHash.
+            # O(new × log N) instead of O(N²).
+            scan_start = datetime.now(timezone.utc).isoformat()
+            incremental_regroup_duplicates_task.apply_async(
+                kwargs={'since_iso': scan_start},
+                countdown=60,
+            )
         except Exception as e:
             logger.warning(f"Could not queue post-scan regroup: {e}")
 

backend/app/tasks/thumbs.py

@@ -467,10 +467,23 @@ async def _backfill_phashes_async():
 
 @shared_task(name='regroup_duplicates')
 def regroup_duplicates_task():
-    """Celery wrapper around app.services.duplicates.regroup_duplicates.
+    """Full recompute of duplicate groups (pHash + CLIP similarity).
 
-    Importing the service inside the task body avoids a circular import
-    at worker boot (the service uses AsyncSessionLocal which is also
-    imported here at module top)."""
+    Used by the Settings → Re-detect duplicates button."""
     from app.services.duplicates import regroup_duplicates
-    return asyncio.run(regroup_duplicates())
+    return asyncio.run(regroup_duplicates())
+
+
+@shared_task(name='incremental_regroup_duplicates')
+def incremental_regroup_duplicates_task(since_iso: str | None = None):
+    """Incremental duplicate detection for newly added photos.
+
+    Compares only photos added after `since_iso` against the full library
+    using CLIP vector similarity (O(new × log N) via HNSW) plus pHash.
+    Default post-scan path — much faster than a full regroup."""
+    from app.services.duplicates import incremental_regroup
+    from datetime import datetime, timezone
+    since = None
+    if since_iso:
+        since = datetime.fromisoformat(since_iso)
+    return asyncio.run(incremental_regroup(since=since))