feat: perceptual-hash duplicate detection + grouped picker view
The Duplicates section was useless: SHA-256-only detection only caught
byte-identical files, not the actual duplicates a real library
accumulates (re-encoded JPEGs, screenshots, resized exports), and the
view was a flat date-sorted list with no grouping or actions. This
replaces the whole flow.
Detection
- New phash + duplicate_group_id columns on Photo, added via an
idempotent ALTER TABLE pass in init_db (the project has no Alembic).
- Thumbs worker computes a 64-bit pHash from the original-resolution
decoded frame just before the destructive thumbnail loop. Falls back
silently — phash is nice-to-have, not a blocker for thumbnails.
- backfill_phashes Celery task fills in phashes for photos that
predated the column, reading the existing thumb_large rather than
re-decoding the original.
- regroup_duplicates service runs union-find over Hamming distance
(threshold 6), persists duplicate_group_id, and maintains is_duplicate
as derived state so existing badges/counts keep working. Chained
after scan_all_source_roots with a 60s countdown.
API
- GET /library/duplicates/groups returns all groups with members,
bucketed in Python from one query. Each group has a reason ("exact"
iff every member shares a SHA-256, "similar" otherwise).
- POST /library/maintenance/{regroup-duplicates,backfill-phashes}.
Frontend
- New DuplicatesView (sectioned grid, one section per cluster) replaces
the timeline when the user is in the duplicates section. Each section
shows a "Keep best, discard N" button that picks the highest-pixel
copy and reuses the existing undoable bulk-discard so Cmd+Z works.
- Manual best override: hover any non-best thumbnail and click "Keep
this" (Crown icon, top-right) to override the auto-pick. The header
annotates "(manual)" so it's obvious which copy will be kept.
- Keyboard nav within the duplicates view walks the flat member list,
with ↑/↓ jumping by the measured column count and scrollIntoView on
every move. Timeline's keyboard handler now early-returns in the
duplicates section so the two don't fight.
- BEST pill / Keep-this button live at top-right with a ring outline so
they don't collide visually with the cyan selection ring around a
selected cell. Dimensions chip moved to bottom-left to free both
right corners for the keep affordances.
- New "Duplicates" section in SettingsDialog: shows group/member counts
and exposes both backfill + re-detect actions, sharing a query cache
with DuplicatesView via DUPLICATE_GROUPS_QUERY_KEY.
- PhotoInfoPanel "Basic Info" section now shows the photo's full file
path in monospace below the size/dimensions/date grid.
- New imagehash==4.3.1 dep in requirements.txt.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -447,4 +447,128 @@ async def run_data_integrity_cleanup():
|
||||
return {"status": "success"}
|
||||
except Exception as e:
|
||||
logger.error(f"Manual cleanup failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Duplicate detection
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/duplicates/groups")
|
||||
async def get_duplicate_groups(db: AsyncSession = Depends(get_db)):
|
||||
"""Return every duplicate group with its members.
|
||||
|
||||
Drives the frontend grouped grid view in the Duplicates section. One
|
||||
SQL query, bucketed in Python — no N+1, no per-member fetch. Groups
|
||||
are sorted by member_count DESC then earliest taken_at DESC so the
|
||||
biggest / most recent clusters bubble to the top.
|
||||
|
||||
Each group also carries a `reason` field:
|
||||
* "exact" — every member shares the same SHA-256 (true byte
|
||||
duplicates that the perceptual hash trivially caught)
|
||||
* "similar" — members differ at the byte level but match perceptually
|
||||
"""
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Photo.id,
|
||||
Photo.filename,
|
||||
Photo.taken_at,
|
||||
Photo.file_size,
|
||||
Photo.width,
|
||||
Photo.height,
|
||||
Photo.thumb_small,
|
||||
Photo.file_hash,
|
||||
Photo.folder_id,
|
||||
Photo.media_type,
|
||||
Photo.duplicate_group_id,
|
||||
)
|
||||
.where(Photo.duplicate_group_id.is_not(None))
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.order_by(Photo.duplicate_group_id)
|
||||
)
|
||||
).all()
|
||||
|
||||
# Bucket members by group_id.
|
||||
groups: dict[str, list[dict]] = {}
|
||||
for row in rows:
|
||||
member = {
|
||||
"id": row[0],
|
||||
"filename": row[1],
|
||||
"taken_at": row[2].isoformat() if row[2] else None,
|
||||
"file_size": row[3],
|
||||
"width": row[4],
|
||||
"height": row[5],
|
||||
"thumb_small": row[6],
|
||||
"file_hash": row[7],
|
||||
"folder_id": row[8],
|
||||
"media_type": row[9],
|
||||
}
|
||||
groups.setdefault(row[10], []).append(member)
|
||||
|
||||
def earliest(g: list[dict]) -> str:
|
||||
# Used as a secondary sort key. Photos with no taken_at sort last
|
||||
# by returning a far-future sentinel.
|
||||
taken = [m["taken_at"] for m in g if m["taken_at"]]
|
||||
return min(taken) if taken else "9999"
|
||||
|
||||
out = []
|
||||
for group_id, members in groups.items():
|
||||
if len(members) < 2:
|
||||
# Defensive: a regroup race could leave a singleton briefly.
|
||||
# Skip it so the UI never shows a "group of 1".
|
||||
continue
|
||||
# exact iff every member shares the same non-null file_hash
|
||||
# (true byte-identical copies that pHash also caught). Anything
|
||||
# else — different hashes, missing hashes — counts as "similar".
|
||||
all_hashes = [m["file_hash"] for m in members]
|
||||
reason = (
|
||||
"exact"
|
||||
if len(set(all_hashes)) == 1 and all_hashes[0] is not None
|
||||
else "similar"
|
||||
)
|
||||
out.append({
|
||||
"group_id": group_id,
|
||||
"member_count": len(members),
|
||||
"reason": reason,
|
||||
"members": members,
|
||||
})
|
||||
|
||||
out.sort(key=lambda g: (-g["member_count"], earliest(g["members"])))
|
||||
return {
|
||||
"groups": out,
|
||||
"total_groups": len(out),
|
||||
"total_members": sum(g["member_count"] for g in out),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/maintenance/regroup-duplicates")
|
||||
async def trigger_regroup_duplicates():
|
||||
"""Recompute duplicate groups from current perceptual hashes.
|
||||
|
||||
Fires the celery `regroup_duplicates` task which walks every photo's
|
||||
phash, clusters by Hamming distance, and rewrites duplicate_group_id /
|
||||
is_duplicate columns. Idempotent."""
|
||||
from app.tasks.thumbs import regroup_duplicates_task
|
||||
try:
|
||||
regroup_duplicates_task.delay()
|
||||
return {"status": "queued"}
|
||||
except Exception as e:
|
||||
logger.error(f"Regroup queue failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@router.post("/maintenance/backfill-phashes")
|
||||
async def trigger_backfill_phashes():
|
||||
"""Compute perceptual hashes for every photo currently missing one.
|
||||
|
||||
One-shot recovery path for libraries that existed before the phash
|
||||
column was added — the thumbs worker computes phash for everything
|
||||
new, but old rows need a backfill pass."""
|
||||
from app.tasks.thumbs import backfill_phashes
|
||||
try:
|
||||
backfill_phashes.delay()
|
||||
return {"status": "queued"}
|
||||
except Exception as e:
|
||||
logger.error(f"Backfill queue failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
Reference in New Issue
Block a user