feat: CLIP-powered incremental duplicate detection
Replace O(N²) pHash-only duplicate detection with a hybrid approach: - pHash Hamming distance for exact/near-exact copies - CLIP embedding cosine similarity via pgvector HNSW for visually similar photos (crops, format changes, screenshots) Post-scan now uses incremental mode: only newly added photos are compared against the full library — O(new × log N) via HNSW index instead of O(N²). Full regroup remains available from Settings. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,7 @@ import os
|
||||
import hashlib
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
import logging
|
||||
import json
|
||||
from typing import List, Dict, Optional
|
||||
@@ -419,7 +419,7 @@ async def _scan_all_source_roots_async():
|
||||
countdown is a best-effort hint — on a big library the user can
|
||||
still hit Settings → Re-detect duplicates to force a fresh pass.
|
||||
"""
|
||||
from app.tasks.thumbs import regroup_duplicates_task
|
||||
from app.tasks.thumbs import incremental_regroup_duplicates_task
|
||||
from app.tasks.vision import backfill_vision, recluster_faces
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
@@ -441,7 +441,14 @@ async def _scan_all_source_roots_async():
|
||||
# firing too early just means the next manual run picks up the
|
||||
# late arrivals — no corrupted state.
|
||||
try:
|
||||
regroup_duplicates_task.apply_async(countdown=60)
|
||||
# Use incremental mode: only compare newly added photos
|
||||
# against the full library via CLIP HNSW + pHash.
|
||||
# O(new × log N) instead of O(N²).
|
||||
scan_start = datetime.now(timezone.utc).isoformat()
|
||||
incremental_regroup_duplicates_task.apply_async(
|
||||
kwargs={'since_iso': scan_start},
|
||||
countdown=60,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue post-scan regroup: {e}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user