feat: CLIP-powered incremental duplicate detection

Replace O(N²) pHash-only duplicate detection with a hybrid approach:
- pHash Hamming distance for exact/near-exact copies
- CLIP embedding cosine similarity via pgvector HNSW for visually
  similar photos (crops, format changes, screenshots)

Post-scan now uses incremental mode: only newly added photos are
compared against the full library — O(new × log N) via HNSW index
instead of O(N²). Full regroup remains available from Settings.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-12 22:03:04 +02:00
parent 8f41a23c41
commit c7dd03ade2
3 changed files with 295 additions and 122 deletions

View File

@@ -5,7 +5,7 @@ import os
import hashlib
import asyncio
from pathlib import Path
from datetime import datetime
from datetime import datetime, timezone
import logging
import json
from typing import List, Dict, Optional
@@ -419,7 +419,7 @@ async def _scan_all_source_roots_async():
countdown is a best-effort hint — on a big library the user can
still hit Settings → Re-detect duplicates to force a fresh pass.
"""
from app.tasks.thumbs import regroup_duplicates_task
from app.tasks.thumbs import incremental_regroup_duplicates_task
from app.tasks.vision import backfill_vision, recluster_faces
async with AsyncSessionLocal() as session:
@@ -441,7 +441,14 @@ async def _scan_all_source_roots_async():
# firing too early just means the next manual run picks up the
# late arrivals — no corrupted state.
try:
regroup_duplicates_task.apply_async(countdown=60)
# Use incremental mode: only compare newly added photos
# against the full library via CLIP HNSW + pHash.
# O(new × log N) instead of O(N²).
scan_start = datetime.now(timezone.utc).isoformat()
incremental_regroup_duplicates_task.apply_async(
kwargs={'since_iso': scan_start},
countdown=60,
)
except Exception as e:
logger.warning(f"Could not queue post-scan regroup: {e}")