diff --git a/backend/app/database.py b/backend/app/database.py
index 70e281c..d3f4e12 100644
--- a/backend/app/database.py
+++ b/backend/app/database.py
@@ -58,17 +58,57 @@ async def init_db():
async with engine.begin() as conn:
# Import all models to register them with Base
from app.models import Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto, Embedding
-
- # Create all tables
+
+ # Create all tables. Note: create_all only creates *missing* tables —
+ # it does NOT add new columns to existing tables when the model gains
+ # them. Anything new on an existing table needs an explicit ALTER
+ # below.
await conn.run_sync(Base.metadata.create_all)
-
+
# Enable WAL mode for SQLite (better concurrency)
if "sqlite" in settings.database_url:
await conn.execute(text("PRAGMA journal_mode=WAL"))
await conn.execute(text("PRAGMA synchronous=NORMAL"))
await conn.execute(text("PRAGMA cache_size=10000"))
await conn.execute(text("PRAGMA temp_store=MEMORY"))
-
+
+ # ── Idempotent column adds ────────────────────────────────────────
+ # The project does not use Alembic; we lean on create_all + a small
+ # set of inline ALTER TABLE statements for the columns we've added
+ # post-launch. SQLite supports ADD COLUMN but not "IF NOT EXISTS"
+ # for columns, so introspect via PRAGMA first. Each entry is
+ # (column_name, ALTER statement). Add new columns at the bottom.
+ if "sqlite" in settings.database_url:
+ existing_cols = {
+ row[1]
+ for row in (
+ await conn.execute(text("PRAGMA table_info(photos)"))
+ ).fetchall()
+ }
+ pending_alters: list[tuple[str, str]] = [
+ ("phash", "ALTER TABLE photos ADD COLUMN phash VARCHAR(16)"),
+ (
+ "duplicate_group_id",
+ "ALTER TABLE photos ADD COLUMN duplicate_group_id VARCHAR",
+ ),
+ ]
+ for col_name, alter_sql in pending_alters:
+ if col_name not in existing_cols:
+ logger.info(f"Adding photos.{col_name} column")
+ await conn.execute(text(alter_sql))
+ # Indexes for the new duplicate-detection columns. CREATE INDEX
+ # IF NOT EXISTS is supported on SQLite so this is safe to run
+ # every startup.
+ await conn.execute(
+ text("CREATE INDEX IF NOT EXISTS ix_photos_phash ON photos(phash)")
+ )
+ await conn.execute(
+ text(
+ "CREATE INDEX IF NOT EXISTS ix_photos_duplicate_group_id "
+ "ON photos(duplicate_group_id)"
+ )
+ )
+
logger.info("Database initialized successfully")
async def create_fts_table():
diff --git a/backend/app/models/photos.py b/backend/app/models/photos.py
index 06e5752..a678c39 100644
--- a/backend/app/models/photos.py
+++ b/backend/app/models/photos.py
@@ -60,8 +60,24 @@ class Photo(Base):
# photo just means adding it to the active heap. Both DB columns may still
# exist on legacy installs but are no longer read or written.
- # Duplicate detection
+ # Duplicate detection.
+ #
+ # - file_hash (above): SHA-256 of the raw bytes. Catches byte-identical
+ # copies but not visually-identical re-encodes / resizes / screenshots.
+ # - phash: 16-char hex of a 64-bit perceptual hash, computed by the
+ # thumbs worker from the decoded original frame. Robust to resize and
+ # re-compression — this is what actually identifies "the same photo
+ # saved twice with different JPEG quality".
+ # - duplicate_group_id: shared by every photo in the same duplicate
+ # cluster. Maintained by app.services.duplicates.regroup_duplicates,
+ # not on individual writes — recomputed in batches after scans / on
+ # demand from the Settings panel.
+ # - is_duplicate: derived boolean (group_id IS NOT NULL). Kept as a real
+ # column so the existing PhotoThumbnail badge and /library/stats
+ # duplicates count don't have to change.
is_duplicate = Column(Boolean, default=False)
+ phash = Column(String(16), index=True)
+ duplicate_group_id = Column(String, index=True)
# Live photo support
live_photo_video_id = Column(String, ForeignKey('photos.id'))
diff --git a/backend/app/routers/library.py b/backend/app/routers/library.py
index a951062..4fbf93a 100644
--- a/backend/app/routers/library.py
+++ b/backend/app/routers/library.py
@@ -447,4 +447,128 @@ async def run_data_integrity_cleanup():
return {"status": "success"}
except Exception as e:
logger.error(f"Manual cleanup failed: {e}")
+ return {"status": "error", "message": str(e)}
+
+
+# ─────────────────────────────────────────────────────────────────────────
+# Duplicate detection
+# ─────────────────────────────────────────────────────────────────────────
+
+@router.get("/duplicates/groups")
+async def get_duplicate_groups(db: AsyncSession = Depends(get_db)):
+ """Return every duplicate group with its members.
+
+ Drives the frontend grouped grid view in the Duplicates section. One
+ SQL query, bucketed in Python — no N+1, no per-member fetch. Groups
+ are sorted by member_count DESC then earliest taken_at DESC so the
+ biggest / most recent clusters bubble to the top.
+
+ Each group also carries a `reason` field:
+ * "exact" — every member shares the same SHA-256 (true byte
+ duplicates that the perceptual hash trivially caught)
+ * "similar" — members differ at the byte level but match perceptually
+ """
+ rows = (
+ await db.execute(
+ select(
+ Photo.id,
+ Photo.filename,
+ Photo.taken_at,
+ Photo.file_size,
+ Photo.width,
+ Photo.height,
+ Photo.thumb_small,
+ Photo.file_hash,
+ Photo.folder_id,
+ Photo.media_type,
+ Photo.duplicate_group_id,
+ )
+ .where(Photo.duplicate_group_id.is_not(None))
+ .where(Photo.is_discarded.is_(False))
+ .order_by(Photo.duplicate_group_id)
+ )
+ ).all()
+
+ # Bucket members by group_id.
+ groups: dict[str, list[dict]] = {}
+ for row in rows:
+ member = {
+ "id": row[0],
+ "filename": row[1],
+ "taken_at": row[2].isoformat() if row[2] else None,
+ "file_size": row[3],
+ "width": row[4],
+ "height": row[5],
+ "thumb_small": row[6],
+ "file_hash": row[7],
+ "folder_id": row[8],
+ "media_type": row[9],
+ }
+ groups.setdefault(row[10], []).append(member)
+
+ def earliest(g: list[dict]) -> str:
+ # Used as a secondary sort key. Photos with no taken_at sort last
+ # by returning a far-future sentinel.
+ taken = [m["taken_at"] for m in g if m["taken_at"]]
+ return min(taken) if taken else "9999"
+
+ out = []
+ for group_id, members in groups.items():
+ if len(members) < 2:
+ # Defensive: a regroup race could leave a singleton briefly.
+ # Skip it so the UI never shows a "group of 1".
+ continue
+ # exact iff every member shares the same non-null file_hash
+ # (true byte-identical copies that pHash also caught). Anything
+ # else — different hashes, missing hashes — counts as "similar".
+ all_hashes = [m["file_hash"] for m in members]
+ reason = (
+ "exact"
+ if len(set(all_hashes)) == 1 and all_hashes[0] is not None
+ else "similar"
+ )
+ out.append({
+ "group_id": group_id,
+ "member_count": len(members),
+ "reason": reason,
+ "members": members,
+ })
+
+ out.sort(key=lambda g: (-g["member_count"], earliest(g["members"])))
+ return {
+ "groups": out,
+ "total_groups": len(out),
+ "total_members": sum(g["member_count"] for g in out),
+ }
+
+
+@router.post("/maintenance/regroup-duplicates")
+async def trigger_regroup_duplicates():
+ """Recompute duplicate groups from current perceptual hashes.
+
+ Fires the celery `regroup_duplicates` task which walks every photo's
+ phash, clusters by Hamming distance, and rewrites duplicate_group_id /
+ is_duplicate columns. Idempotent."""
+ from app.tasks.thumbs import regroup_duplicates_task
+ try:
+ regroup_duplicates_task.delay()
+ return {"status": "queued"}
+ except Exception as e:
+ logger.error(f"Regroup queue failed: {e}")
+ return {"status": "error", "message": str(e)}
+
+
+@router.post("/maintenance/backfill-phashes")
+async def trigger_backfill_phashes():
+ """Compute perceptual hashes for every photo currently missing one.
+
+ One-shot recovery path for libraries that existed before the phash
+ column was added — the thumbs worker computes phash for everything
+ new, but old rows need a backfill pass."""
+ from app.tasks.thumbs import backfill_phashes
+ try:
+ backfill_phashes.delay()
+ return {"status": "queued"}
+ except Exception as e:
+ logger.error(f"Backfill queue failed: {e}")
return {"status": "error", "message": str(e)}
\ No newline at end of file
diff --git a/backend/app/services/duplicates.py b/backend/app/services/duplicates.py
new file mode 100644
index 0000000..95a7abc
--- /dev/null
+++ b/backend/app/services/duplicates.py
@@ -0,0 +1,227 @@
+"""
+Duplicate detection: group photos by perceptual-hash similarity.
+
+Strategy
+--------
+Each photo carries a 16-char hex perceptual hash (`Photo.phash`) computed
+by the thumbnail worker from the original-resolution decoded frame
+(`app.tasks.thumbs._generate_thumbnails_async`). pHash is robust to
+resize / re-encoding / mild edits, so two photos that are visually the
+"same shot" land at small Hamming distance even when their bytes are
+completely different.
+
+This module turns those per-photo hashes into explicit *groups*. The
+result is persisted in two columns:
+
+ * `Photo.duplicate_group_id` — shared by every member of a group
+ * `Photo.is_duplicate` — derived: True iff group_id IS NOT NULL
+ (kept as a column so the existing
+ PhotoThumbnail badge and /library/stats
+ count don't have to change).
+
+The grouping is recomputed in batches by `regroup_duplicates`, NOT on
+individual writes — that lets us use a single in-memory pass instead of
+maintaining a per-row similarity index. Triggered automatically after
+each scan and on demand from the Settings panel.
+
+Complexity
+----------
+Pairwise O(N²) over photos with a non-null phash. At ~5 µs per Hamming
+distance in CPython this is roughly:
+
+ 1k photos → ~5 s
+ 5k photos → ~125 s
+ 10k photos → ~500 s
+
+That's the wrong shape for libraries past ~5k. The drop-in replacement
+is a BK-tree (e.g. `pybktree`) which gives O(log N) lookups for a fixed
+Hamming threshold; swap it in here when someone trips the limit. The
+public function signature stays the same.
+
+Out of scope (deferred)
+-----------------------
+* Dismissing a group / "intentional duplicates" — would need a per-group
+ or per-pair flag plus a skip-set in this function so re-grouping
+ doesn't bring them back. Add when there's a real user need.
+* Incremental updates on individual photo writes — currently we just
+ re-run the whole job after each scan, which is fine while the cost is
+ bounded.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Optional
+
+from sqlalchemy import select, update
+
+from app.database import AsyncSessionLocal
+from app.models.photos import Photo
+
+logger = logging.getLogger(__name__)
+
+
+# Hamming distance threshold under which two phashes are considered
+# "the same image". 6 bits out of 64 is the rule-of-thumb sweet spot for
+# pHash — tight enough to avoid false positives between unrelated photos,
+# loose enough to catch JPEG re-encodes, slight crops, and a screenshot
+# of a screenshot.
+DEFAULT_THRESHOLD = 6
+
+
+def _hex_to_int(h: str) -> int:
+ """Parse a 16-char hex pHash to a Python int. Returns -1 on bad input
+ so the pairwise loop can skip the row without raising."""
+ try:
+ return int(h, 16)
+ except (TypeError, ValueError):
+ return -1
+
+
+def _hamming(a: int, b: int) -> int:
+ """Population count of XOR — the canonical hash distance metric.
+ `int.bit_count()` is C-implemented in CPython 3.10+ and is by far
+ the fastest path; the `bin(...).count('1')` fallback is here only
+ so the function still works on older interpreters."""
+ x = a ^ b
+ try:
+ return x.bit_count() # type: ignore[attr-defined]
+ except AttributeError:
+ return bin(x).count('1')
+
+
+class _UnionFind:
+ """Tiny union-find / disjoint-set used to merge similar phashes into
+ connected components. Inlined here (rather than pulled from a dep)
+ because it's ~15 lines and we don't need anything fancy."""
+
+ def __init__(self, n: int) -> None:
+ self.parent = list(range(n))
+ self.rank = [0] * n
+
+ def find(self, x: int) -> int:
+ # Path compression — flattens the tree on lookup so subsequent
+ # finds are amortized O(α(N)) ≈ O(1).
+ while self.parent[x] != x:
+ self.parent[x] = self.parent[self.parent[x]]
+ x = self.parent[x]
+ return x
+
+ def union(self, a: int, b: int) -> None:
+ ra, rb = self.find(a), self.find(b)
+ if ra == rb:
+ return
+ if self.rank[ra] < self.rank[rb]:
+ ra, rb = rb, ra
+ self.parent[rb] = ra
+ if self.rank[ra] == self.rank[rb]:
+ self.rank[ra] += 1
+
+
+async def regroup_duplicates(threshold: int = DEFAULT_THRESHOLD) -> dict:
+ """Recompute every photo's duplicate_group_id from current phashes.
+
+ Idempotent — safe to call as often as you like. Returns a small
+ summary dict the maintenance endpoint surfaces back to the UI.
+
+ Photos that end up alone in a component (size 1) get
+ `duplicate_group_id=NULL` and `is_duplicate=False`. This is what
+ cleans up "dead" groups after the user discards N-1 members from
+ one.
+ """
+ async with AsyncSessionLocal() as session:
+ # Pull (id, phash) for every non-discarded photo with a hash.
+ # Discarded photos are excluded so we don't keep showing groups
+ # made up of trashed copies.
+ rows = (
+ await session.execute(
+ select(Photo.id, Photo.phash)
+ .where(Photo.phash.is_not(None))
+ .where(Photo.is_discarded.is_(False))
+ )
+ ).all()
+
+ n = len(rows)
+ if n == 0:
+ # Still need to clear stale group_ids in case the user just
+ # discarded the last surviving member of every group.
+ await _clear_all_groups(session)
+ await session.commit()
+ return {
+ 'photos_considered': 0,
+ 'groups': 0,
+ 'members': 0,
+ }
+
+ ids: list[str] = [row[0] for row in rows]
+ hashes: list[int] = [_hex_to_int(row[1]) for row in rows]
+
+ uf = _UnionFind(n)
+
+ # O(N²) pairwise comparison. See module docstring for the
+ # scaling analysis and the BK-tree upgrade path.
+ for i in range(n):
+ hi = hashes[i]
+ if hi < 0:
+ continue
+ for j in range(i + 1, n):
+ hj = hashes[j]
+ if hj < 0:
+ continue
+ if _hamming(hi, hj) <= threshold:
+ uf.union(i, j)
+
+ # Collect components. Each connected component of size >= 2 gets
+ # a fresh group id; size-1 components are intentionally dropped.
+ components: dict[int, list[int]] = {}
+ for i in range(n):
+ root = uf.find(i)
+ components.setdefault(root, []).append(i)
+
+ # First pass: clear EVERY photo's group_id so survivors of an
+ # earlier grouping that no longer match anyone end up clean. This
+ # is one bulk UPDATE rather than per-photo to keep the cost low
+ # even on big libraries.
+ await _clear_all_groups(session)
+
+ # Second pass: write the new group ids for components of size 2+.
+ groups_created = 0
+ members_total = 0
+ for members in components.values():
+ if len(members) < 2:
+ continue
+ group_id = str(uuid.uuid4())
+ member_ids = [ids[i] for i in members]
+ await session.execute(
+ update(Photo)
+ .where(Photo.id.in_(member_ids))
+ .values(
+ duplicate_group_id=group_id,
+ is_duplicate=True,
+ )
+ )
+ groups_created += 1
+ members_total += len(member_ids)
+
+ await session.commit()
+
+ logger.info(
+ f"regroup_duplicates: considered {n} photos, "
+ f"created {groups_created} group(s) covering {members_total} member(s)"
+ )
+
+ return {
+ 'photos_considered': n,
+ 'groups': groups_created,
+ 'members': members_total,
+ }
+
+
+async def _clear_all_groups(session) -> None:
+ """Reset duplicate_group_id / is_duplicate on every photo. Used as
+ the first half of a regroup pass so photos that no longer cluster
+ with anyone end up clean instead of carrying a stale group id."""
+ await session.execute(
+ update(Photo).values(duplicate_group_id=None, is_duplicate=False)
+ )
diff --git a/backend/app/tasks/scan.py b/backend/app/tasks/scan.py
index 97efa54..507d4eb 100644
--- a/backend/app/tasks/scan.py
+++ b/backend/app/tasks/scan.py
@@ -350,18 +350,39 @@ def scan_all_source_roots():
async def _scan_all_source_roots_async():
"""Read every active SourceRoot from the DB and queue a scan_folder task
for each. Source roots whose path no longer exists on disk are skipped
- with a warning (the cleanup service surfaces those at startup too)."""
+ with a warning (the cleanup service surfaces those at startup too).
+
+ After dispatching the scans, queue a delayed `regroup_duplicates`
+ pass so duplicate clusters are recomputed once the new photos have
+ finished thumbnailing (and therefore picked up phashes). The
+ countdown is a best-effort hint — on a big library the user can
+ still hit Settings → Re-detect duplicates to force a fresh pass.
+ """
+ from app.tasks.thumbs import regroup_duplicates_task
+
async with AsyncSessionLocal() as session:
result = await session.execute(
select(SourceRoot).where(SourceRoot.is_active == True) # noqa: E712
)
source_roots = result.scalars().all()
+ dispatched = 0
for sr in source_roots:
if os.path.exists(sr.path):
scan_folder.delay(sr.path, sr.id)
+ dispatched += 1
else:
logger.warning(f"Source root path does not exist: {sr.path}")
+ if dispatched > 0:
+ # 60s gives the thumbs worker a window to compute phashes for
+ # the new photos before regrouping. The task is idempotent, so
+ # firing too early just means the next manual run picks up the
+ # late arrivals — no corrupted state.
+ try:
+ regroup_duplicates_task.apply_async(countdown=60)
+ except Exception as e:
+ logger.warning(f"Could not queue post-scan regroup: {e}")
+
@shared_task(name='watch_folders')
def watch_folders():
diff --git a/backend/app/tasks/thumbs.py b/backend/app/tasks/thumbs.py
index 5242354..a25bf7e 100644
--- a/backend/app/tasks/thumbs.py
+++ b/backend/app/tasks/thumbs.py
@@ -280,11 +280,23 @@ async def _generate_thumbnails_async(photo_id: str, task):
# Auto-rotate based on EXIF
image = auto_rotate_image(image)
-
+
# Store original dimensions
photo.width = image.width
photo.height = image.height
-
+
+ # Perceptual hash from the original-resolution decoded frame.
+ # pHash is robust to resize/recompression but the thumbnail
+ # loop below mutates `image` in place, so this MUST run before
+ # the loop sees it. Failures are non-fatal — phash is a
+ # nice-to-have, not a blocker for thumbnail generation.
+ try:
+ import imagehash
+ photo.phash = str(imagehash.phash(image)) # 16-char hex
+ except Exception as e:
+ logger.warning(f"phash failed for {photo_id}: {e}")
+ photo.phash = None
+
# Generate thumbnails for each size
for size_name, size_value in THUMB_SIZES.items():
thumb_path = get_thumb_path(photo_id, size_name)
@@ -333,10 +345,83 @@ async def _regenerate_all_thumbnails_async():
)
)
photos = result.scalars().all()
-
+
logger.info(f"Regenerating thumbnails for {len(photos)} photos")
-
+
for photo in photos:
generate_thumbnails.delay(photo.id)
-
- return {'status': 'queued', 'count': len(photos)}
\ No newline at end of file
+
+ return {'status': 'queued', 'count': len(photos)}
+
+
+# ── Perceptual hash backfill ────────────────────────────────────────────
+#
+# When phash was added post-launch, every existing photo has phash=NULL.
+# This task fills them in by reading the existing thumb_large (the cheap
+# option — pHash is robust to scale, and the thumb is already on local
+# disk so we avoid re-decoding the original RAW/HEIC). Falls back to the
+# original filepath if the thumb isn't available for some reason. Runs
+# in batches to keep memory bounded and to give the user incremental
+# progress visible in the worker logs.
+
+@shared_task(name='backfill_phashes')
+def backfill_phashes():
+ """Compute and persist phash for every photo currently missing one."""
+ return asyncio.run(_backfill_phashes_async())
+
+
+async def _backfill_phashes_async():
+ import imagehash
+ from PIL import Image as _PILImage
+
+ BATCH = 100
+ total_done = 0
+ total_failed = 0
+
+ async with AsyncSessionLocal() as session:
+ while True:
+ result = await session.execute(
+ select(Photo)
+ .where(Photo.phash.is_(None))
+ .where(Photo.processing_status == 'completed')
+ .limit(BATCH)
+ )
+ batch = result.scalars().all()
+ if not batch:
+ break
+
+ for photo in batch:
+ source = photo.thumb_large or photo.filepath
+ try:
+ if not source or not os.path.exists(source):
+ photo.phash = None
+ total_failed += 1
+ continue
+ with _PILImage.open(source) as im:
+ photo.phash = str(imagehash.phash(im))
+ total_done += 1
+ except Exception as e:
+ logger.warning(f"phash backfill failed for {photo.id}: {e}")
+ total_failed += 1
+
+ await session.commit()
+ logger.info(
+ f"Backfilled phashes: {total_done} done, {total_failed} failed"
+ )
+
+ return {
+ 'status': 'success',
+ 'computed': total_done,
+ 'failed': total_failed,
+ }
+
+
+@shared_task(name='regroup_duplicates')
+def regroup_duplicates_task():
+ """Celery wrapper around app.services.duplicates.regroup_duplicates.
+
+ Importing the service inside the task body avoids a circular import
+ at worker boot (the service uses AsyncSessionLocal which is also
+ imported here at module top)."""
+ from app.services.duplicates import regroup_duplicates
+ return asyncio.run(regroup_duplicates())
\ No newline at end of file
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 3e526df..0aaa005 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -18,6 +18,7 @@ flower==2.0.1
# rawpy==0.19.0 # Optional - numpy compatibility issues, using Pillow as fallback
pillow==10.2.0
pillow-heif==0.15.0
+imagehash==4.3.1 # perceptual hash for duplicate detection
imageio==2.33.1
imageio-ffmpeg==0.4.9
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index dd979fb..7119e8a 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -1,5 +1,6 @@
import { useState } from 'react'
import { Timeline } from './components/timeline/Timeline'
+import { DuplicatesView } from './components/duplicates/DuplicatesView'
import { LeftSidebar } from './components/layout/LeftSidebar'
import { RightSidebar } from './components/layout/RightSidebar'
import { TopBar } from './components/layout/TopBar'
@@ -12,6 +13,7 @@ import { FilterBar } from './components/filter/FilterBar'
import { DiscardActionBar } from './components/discard/DiscardActionBar'
import { SettingsDialog } from './components/dialogs/SettingsDialog'
import { usePhotoStore } from './store/photoStore'
+import { useFilterStore } from './store/filterStore'
import { useKeyboardShortcuts } from './hooks/useKeyboardShortcuts'
import { useFilterUrlSync } from './hooks/useFilterUrlSync'
import { usePhotosQuery } from './hooks/usePhotosQuery'
@@ -21,6 +23,7 @@ function App() {
const [rightSidebarOpen, setRightSidebarOpen] = useState(true)
const [settingsOpen, setSettingsOpen] = useState(false)
const viewMode = usePhotoStore((state) => state.viewMode)
+ const currentSection = useFilterStore((s) => s.currentSection)
// Bidirectional sync of filter store with URL query params.
useFilterUrlSync()
@@ -71,7 +74,11 @@ function App() {
-
+ {/* The Duplicates section gets its own grouped grid view —
+ * the regular timeline can't represent groups, and a flat
+ * filtered list of "is_duplicate=true" photos was the old
+ * half-broken UX. */}
+ {currentSection === 'duplicates' ? : }
{/* Floating keyboard hints — bottom-center of the main column,
* glassy. Mounted here so it's centered against the timeline,
diff --git a/frontend/src/components/dialogs/SettingsDialog.tsx b/frontend/src/components/dialogs/SettingsDialog.tsx
index 9256a66..41d6f34 100644
--- a/frontend/src/components/dialogs/SettingsDialog.tsx
+++ b/frontend/src/components/dialogs/SettingsDialog.tsx
@@ -11,6 +11,8 @@ import {
Cpu,
AlertCircle,
CheckCircle2,
+ Copy,
+ Sparkles,
} from 'lucide-react'
import clsx from 'clsx'
import { useQuery, useQueryClient } from '@tanstack/react-query'
@@ -27,6 +29,9 @@ const SETTINGS_THUMB_STATS_KEY = ['settings', 'thumbnail-stats'] as const
const SETTINGS_LIB_STATS_KEY = ['settings', 'library-stats'] as const
const SETTINGS_WORKER_STATUS_KEY = ['settings', 'worker-status'] as const
const SETTINGS_MISSING_STATS_KEY = ['settings', 'missing-stats'] as const
+// Shared with the DuplicatesView so a regroup invalidates the same cache
+// the grid renders from. Imported via the canonical hook key.
+import { DUPLICATE_GROUPS_QUERY_KEY } from '../../hooks/useDuplicateGroupsQuery'
interface SettingsDialogProps {
isOpen: boolean
@@ -86,6 +91,14 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
refetchInterval: isOpen ? 5000 : false,
staleTime: 0,
})
+ // Duplicates: shares the cache with DuplicatesView so a regroup
+ // triggered from Settings updates the grid view immediately.
+ const duplicatesQuery = useQuery({
+ queryKey: DUPLICATE_GROUPS_QUERY_KEY,
+ queryFn: library.duplicates.groups,
+ enabled: isOpen,
+ staleTime: 0,
+ })
const thumbStats = thumbStatsQuery.data
const libStats = libStatsQuery.data
@@ -103,6 +116,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
const refreshStats = useCallback(() => {
queryClient.invalidateQueries({ queryKey: SETTINGS_THUMB_STATS_KEY })
queryClient.invalidateQueries({ queryKey: SETTINGS_LIB_STATS_KEY })
+ queryClient.invalidateQueries({ queryKey: DUPLICATE_GROUPS_QUERY_KEY })
}, [queryClient])
const refreshWorkers = useCallback(() => {
queryClient.invalidateQueries({ queryKey: SETTINGS_WORKER_STATUS_KEY })
@@ -249,6 +263,60 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
+ {/* ----------------------------------------------------- */}
+ {/* Duplicate detection */}
+ {/* ----------------------------------------------------- */}
+ }
+ title="Duplicates"
+ >
+
+
+
+
+
+ Duplicates are detected by perceptual hash (pHash), which catches
+ visually-identical photos even when their bytes differ — re-encoded
+ JPEGs, screenshots, resized exports. Backfill computes hashes for
+ photos that existed before pHash was added; Re-detect re-runs
+ clustering across the whole library.
+
+ Nothing in your library matches another photo at the perceptual-hash
+ level. If you've just added new photos, give the worker a minute and
+ re-run "Re-detect duplicates" from Settings.
+
+
+ )
+ }
+
+ return (
+
+
+
+
+ {data?.total_groups} group{data?.total_groups === 1 ? '' : 's'} ·{' '}
+ {data?.total_members} photo{data?.total_members === 1 ? '' : 's'}.
+ Click "Keep best" to auto-discard all but the highest-resolution
+ copy of each group. Cmd+Z to undo.
+
+
+
+
+ {groups.map((group, idx) => (
+ discardMutation.mutate(discardIds)}
+ onPreviewMember={(memberId) => openPreview(memberId, allMemberIds)}
+ onSelectMember={(memberId) => selectPhoto(memberId)}
+ selectedPhotos={selectedPhotos}
+ isPending={discardMutation.isPending}
+ // Hand the column-measurement ref to the first section only
+ // — every section's grid uses the same auto-fill rule so any
+ // one is representative of the rendered column count.
+ gridRef={idx === 0 ? sampleGridRef : undefined}
+ />
+ ))}
+
+
+ )
+}
+
+interface DuplicateGroupSectionProps {
+ group: DuplicateGroup
+ onKeepBest: (discardIds: string[]) => void
+ onPreviewMember: (memberId: string) => void
+ onSelectMember: (memberId: string) => void
+ selectedPhotos: string[]
+ isPending: boolean
+ /** Optional callback ref attached to this section's grid container.
+ * Used by DuplicatesView to measure the rendered column count for
+ * ↑/↓ keyboard navigation. Only the first section gets one. */
+ gridRef?: (el: HTMLDivElement | null) => void
+}
+
+function DuplicateGroupSection({
+ group,
+ onKeepBest,
+ onPreviewMember,
+ onSelectMember,
+ selectedPhotos,
+ isPending,
+ gridRef,
+}: DuplicateGroupSectionProps) {
+ // Auto-pick "best" copy: highest pixel count, ties broken by file_size,
+ // then earliest taken_at, then id for determinism. This is just the
+ // default — the user can override it by clicking the crown button on
+ // any other thumbnail (see `manualBestId`).
+ const autoBest = useMemo(() => pickBestMember(group.members), [group.members])
+ // When the user clicks "make this the best" on a non-default thumb,
+ // we override the auto-pick. Local to the section so different groups
+ // remember independent overrides; resets if the group itself changes.
+ const [manualBestId, setManualBestId] = useState(null)
+ const bestId =
+ manualBestId && group.members.some((m) => m.id === manualBestId)
+ ? manualBestId
+ : autoBest.id
+ const best = group.members.find((m) => m.id === bestId) ?? autoBest
+ const discardCount = group.member_count - 1
+ const isExact = group.reason === 'exact'
+
+ return (
+
+
+
+ onSelectMember(member.id)}
+ onDoubleClick={() => onPreviewMember(member.id)}
+ />
+ {/* "BEST" pill marks the photo that will be KEPT when the
+ * user clicks the discard button. Lives at top-right where
+ * it doesn't collide with the cyan selection ring (which
+ * draws around the cell perimeter); kept inset by 6px so
+ * the ring's outer edge has clearance on either side. */}
+ {isBest && (
+
+
+ Best
+
+ )}
+ {/* "Make this best" affordance — shown on hover for non-best
+ * thumbnails. Mirrors the BEST pill's top-right placement
+ * so the eye doesn't have to retarget when the user is
+ * scanning a row of thumbnails. Stops propagation so it
+ * doesn't double-fire as a selection click. */}
+ {!isBest && (
+
+ )}
+ {/* Dimensions chip — bottom-LEFT (the BEST/Keep affordances
+ * own the top-right corner). */}
+
+ {formatDimensions(member)}
+
+
+ )
+ })}
+
+
+ )
+}
+
+// ── Helpers ──────────────────────────────────────────────────────────────
+
+/** Score a member by (pixels, file_size, -taken_at) and return the winner.
+ * Larger pixel count wins; ties broken by file_size; final tie by earliest
+ * taken_at (more likely the original capture). */
+function pickBestMember(members: DuplicateGroupMember[]): DuplicateGroupMember {
+ return members.reduce((best, m) => {
+ const bestPixels = (best.width ?? 0) * (best.height ?? 0)
+ const mPixels = (m.width ?? 0) * (m.height ?? 0)
+ if (mPixels !== bestPixels) return mPixels > bestPixels ? m : best
+ const bestSize = best.file_size ?? 0
+ const mSize = m.file_size ?? 0
+ if (mSize !== bestSize) return mSize > bestSize ? m : best
+ // Earliest taken_at wins (treat null as far-future).
+ const bestTaken = best.taken_at ?? '9999'
+ const mTaken = m.taken_at ?? '9999'
+ if (mTaken !== bestTaken) return mTaken < bestTaken ? m : best
+ return best
+ })
+}
+
+function formatDimensions(m: DuplicateGroupMember): string {
+ if (!m.width || !m.height) return '?'
+ const mp = (m.width * m.height) / 1_000_000
+ if (mp >= 1) return `${mp.toFixed(1)}MP`
+ return `${m.width}×${m.height}`
+}
+
+function formatBytes(n: number): string {
+ if (n >= 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(1)}MB`
+ if (n >= 1024) return `${(n / 1024).toFixed(0)}KB`
+ return `${n}B`
+}
+
+/** Adapt a DuplicateGroupMember (the slim API shape) to a Photo, which
+ * is what PhotoThumbnail expects. We deliberately set is_duplicate=false
+ * on the synthetic Photo so the duplicate badge isn't drawn on every
+ * cell — the entire view is duplicates, the badge would be redundant. */
+function memberToPhoto(m: DuplicateGroupMember): Photo {
+ return {
+ id: m.id,
+ filepath: m.filename, // good enough for the RAW/video extension regex
+ filename: m.filename,
+ media_type: m.media_type,
+ width: m.width,
+ height: m.height,
+ taken_at: m.taken_at,
+ rating: 0,
+ is_discarded: false,
+ is_duplicate: false,
+ file_hash: m.file_hash ?? '',
+ folder_id: m.folder_id,
+ added_at: null,
+ thumb_small: m.thumb_small ?? undefined,
+ }
+}
diff --git a/frontend/src/components/sidebar/PhotoInfoPanel.tsx b/frontend/src/components/sidebar/PhotoInfoPanel.tsx
index d45fbef..9a2fe81 100644
--- a/frontend/src/components/sidebar/PhotoInfoPanel.tsx
+++ b/frontend/src/components/sidebar/PhotoInfoPanel.tsx
@@ -481,6 +481,16 @@ export function PhotoInfoPanel({ photoId, darkTheme = false }: PhotoInfoPanelPro
}
/>
+ {/* Filepath spans the full sidebar width — most paths are long
+ * enough that the two-column grid above wraps them painfully.
+ * Mono so each character lines up under the next, break-all
+ * so we never overflow horizontally on a long basename. */}
+
+ Path:
+
+ {photo.filepath || '—'}
+
+
s.sortBy)
const groupBy = useFilterStore((s) => s.groupBy)
+ const currentSection = useFilterStore((s) => s.currentSection)
const viewMode = usePhotoStore((s) => s.viewMode)
// Calculate number of columns + actual cell size based on container
@@ -393,6 +394,11 @@ export function Timeline() {
// moment the user closes preview.
useEffect(() => {
if (viewMode !== 'grid') return
+ // The duplicates section mounts its own grouped view (DuplicatesView)
+ // with its own keyboard nav — bail out so we don't double-handle
+ // arrow keys and try to navigate against a photoRows snapshot that
+ // doesn't match what the user actually sees on screen.
+ if (currentSection === 'duplicates') return
const handleKeyDown = (e: KeyboardEvent) => {
if (photoRows.length === 0) return
const target = e.target as HTMLElement | null
@@ -512,7 +518,7 @@ export function Timeline() {
window.addEventListener('keydown', handleKeyDown)
return () => window.removeEventListener('keydown', handleKeyDown)
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [viewMode, photoRows, photos, selectedPhotos, activePhotoId, photoRowItemIndex, items, cellSize])
+ }, [viewMode, photoRows, photos, selectedPhotos, activePhotoId, photoRowItemIndex, items, cellSize, currentSection])
if (isLoading) {
return (
diff --git a/frontend/src/hooks/useDuplicateGroupsQuery.ts b/frontend/src/hooks/useDuplicateGroupsQuery.ts
new file mode 100644
index 0000000..3768975
--- /dev/null
+++ b/frontend/src/hooks/useDuplicateGroupsQuery.ts
@@ -0,0 +1,17 @@
+import { useQuery } from '@tanstack/react-query'
+import { library, type DuplicateGroupsResponse } from '../services/api'
+
+export const DUPLICATE_GROUPS_QUERY_KEY = ['library', 'duplicates'] as const
+
+/**
+ * Duplicate groups computed by the backend regroup_duplicates task. The
+ * cache is shared between DuplicatesView and the Settings panel stats so
+ * the user only ever pays for one fetch per refresh.
+ */
+export function useDuplicateGroupsQuery() {
+ return useQuery({
+ queryKey: DUPLICATE_GROUPS_QUERY_KEY,
+ queryFn: library.duplicates.groups,
+ staleTime: 30_000,
+ })
+}
diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts
index 9f3de85..5cc2a93 100644
--- a/frontend/src/services/api.ts
+++ b/frontend/src/services/api.ts
@@ -349,7 +349,61 @@ export const library = {
const response = await api.post('/library/maintenance/cleanup')
return response.data
},
+
+ /** Recompute duplicate groups from current perceptual hashes.
+ * Idempotent — safe to fire repeatedly. */
+ regroupDuplicates: async (): Promise<{ status: string; message?: string }> => {
+ const response = await api.post('/library/maintenance/regroup-duplicates')
+ return response.data
+ },
+
+ /** Compute pHash for every photo currently missing one. One-shot
+ * recovery path for libraries that existed before the phash column
+ * was added. */
+ backfillPhashes: async (): Promise<{ status: string; message?: string }> => {
+ const response = await api.post('/library/maintenance/backfill-phashes')
+ return response.data
+ },
},
+
+ /** Duplicate groups computed by app.services.duplicates.regroup_duplicates.
+ * Drives the grouped grid view in the Duplicates section. */
+ duplicates: {
+ groups: async (): Promise => {
+ const response = await api.get('/library/duplicates/groups')
+ return response.data
+ },
+ },
+}
+
+// ── Duplicate groups ─────────────────────────────────────────────────────
+
+export interface DuplicateGroupMember {
+ id: string
+ filename: string
+ taken_at: string | null
+ file_size: number | null
+ width: number | null
+ height: number | null
+ thumb_small: string | null
+ file_hash: string | null
+ folder_id: string | null
+ media_type: string
+}
+
+export interface DuplicateGroup {
+ group_id: string
+ member_count: number
+ /** "exact" iff every member shares the same SHA-256 (true byte
+ * duplicates that pHash also caught). "similar" otherwise. */
+ reason: 'exact' | 'similar'
+ members: DuplicateGroupMember[]
+}
+
+export interface DuplicateGroupsResponse {
+ groups: DuplicateGroup[]
+ total_groups: number
+ total_members: number
}
export interface LibraryStats {