The Duplicates section was useless: SHA-256-only detection only caught
byte-identical files, not the actual duplicates a real library
accumulates (re-encoded JPEGs, screenshots, resized exports), and the
view was a flat date-sorted list with no grouping or actions. This
replaces the whole flow.
Detection
- New phash + duplicate_group_id columns on Photo, added via an
idempotent ALTER TABLE pass in init_db (the project has no Alembic).
- Thumbs worker computes a 64-bit pHash from the original-resolution
decoded frame just before the destructive thumbnail loop. Falls back
silently — phash is nice-to-have, not a blocker for thumbnails.
- backfill_phashes Celery task fills in phashes for photos that
predated the column, reading the existing thumb_large rather than
re-decoding the original.
- regroup_duplicates service runs union-find over Hamming distance
(threshold 6), persists duplicate_group_id, and maintains is_duplicate
as derived state so existing badges/counts keep working. Chained
after scan_all_source_roots with a 60s countdown.
API
- GET /library/duplicates/groups returns all groups with members,
bucketed in Python from one query. Each group has a reason ("exact"
iff every member shares a SHA-256, "similar" otherwise).
- POST /library/maintenance/{regroup-duplicates,backfill-phashes}.
Frontend
- New DuplicatesView (sectioned grid, one section per cluster) replaces
the timeline when the user is in the duplicates section. Each section
shows a "Keep best, discard N" button that picks the highest-pixel
copy and reuses the existing undoable bulk-discard so Cmd+Z works.
- Manual best override: hover any non-best thumbnail and click "Keep
this" (Crown icon, top-right) to override the auto-pick. The header
annotates "(manual)" so it's obvious which copy will be kept.
- Keyboard nav within the duplicates view walks the flat member list,
with ↑/↓ jumping by the measured column count and scrollIntoView on
every move. Timeline's keyboard handler now early-returns in the
duplicates section so the two don't fight.
- BEST pill / Keep-this button live at top-right with a ring outline so
they don't collide visually with the cyan selection ring around a
selected cell. Dimensions chip moved to bottom-left to free both
right corners for the keep affordances.
- New "Duplicates" section in SettingsDialog: shows group/member counts
and exposes both backfill + re-detect actions, sharing a query cache
with DuplicatesView via DUPLICATE_GROUPS_QUERY_KEY.
- PhotoInfoPanel "Basic Info" section now shows the photo's full file
path in monospace below the size/dimensions/date grid.
- New imagehash==4.3.1 dep in requirements.txt.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
409 lines
16 KiB
TypeScript
409 lines
16 KiB
TypeScript
import { useMemo, useState, useEffect, useCallback } from 'react'
|
||
import { Copy, Layers, Sparkles, Trash2, Loader2, Info, Crown } from 'lucide-react'
|
||
import clsx from 'clsx'
|
||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||
import {
|
||
useDuplicateGroupsQuery,
|
||
DUPLICATE_GROUPS_QUERY_KEY,
|
||
} from '../../hooks/useDuplicateGroupsQuery'
|
||
import {
|
||
photos as photosApi,
|
||
type DuplicateGroup,
|
||
type DuplicateGroupMember,
|
||
} from '../../services/api'
|
||
import { PhotoThumbnail } from '../timeline/PhotoThumbnail'
|
||
import { usePhotoStore } from '../../store/photoStore'
|
||
import { registerUndoable } from '../../store/undoStore'
|
||
import { LIBRARY_STATS_QUERY_KEY } from '../../hooks/useLibraryStatsQuery'
|
||
import { toast } from '../ToastContainer'
|
||
import type { Photo } from '../../types/photo'
|
||
|
||
/**
|
||
* Sectioned grid view of duplicate clusters. Replaces the old flat
|
||
* "is_duplicate=true" timeline. Each section is one cluster the
|
||
* regroup_duplicates task identified — header on top with a count and a
|
||
* "keep best, discard rest" button, members rendered as PhotoThumbnail
|
||
* cards below.
|
||
*
|
||
* Mounted from App.tsx in place of <Timeline /> when the user is in the
|
||
* duplicates section. Touches no filter store state.
|
||
*/
|
||
export function DuplicatesView() {
|
||
const { data, isLoading, isError, error } = useDuplicateGroupsQuery()
|
||
const queryClient = useQueryClient()
|
||
const openPreview = usePhotoStore((s) => s.openPreview)
|
||
const selectPhoto = usePhotoStore((s) => s.selectPhoto)
|
||
const selectedPhotos = usePhotoStore((s) => s.selectedPhotos)
|
||
const activePhotoId = usePhotoStore((s) => s.activePhotoId)
|
||
|
||
// Bulk discard with the same undoable wrapper the timeline uses, so
|
||
// Cmd+Z restores the discarded copies. invalidate ['library', 'duplicates']
|
||
// so the group disappears from the view immediately.
|
||
const discardMutation = useMutation({
|
||
mutationFn: (ids: string[]) => photosApi.bulkDiscard(ids),
|
||
onSuccess: (_, ids) => {
|
||
registerUndoable(
|
||
`Discarded ${ids.length} duplicate${ids.length === 1 ? '' : 's'}`,
|
||
async () => {
|
||
await photosApi.bulkRestore(ids)
|
||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||
queryClient.invalidateQueries({ queryKey: DUPLICATE_GROUPS_QUERY_KEY })
|
||
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||
}
|
||
)
|
||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||
queryClient.invalidateQueries({ queryKey: DUPLICATE_GROUPS_QUERY_KEY })
|
||
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||
},
|
||
onError: (e: any) =>
|
||
toast.error('Discard failed', e?.message || 'Unknown error'),
|
||
})
|
||
|
||
// Hooks below this point must run on every render — rules of hooks
|
||
// forbid early returns above any useState/useEffect/useMemo. The early
|
||
// loading/error/empty branches sit AFTER the hook block.
|
||
const groups = data?.groups ?? []
|
||
|
||
// Flat sequence of member ids in visual order. Drives both preview
|
||
// navigation and the in-grid keyboard walker. useMemo so the keyboard
|
||
// effect doesn't tear down on every render.
|
||
const allMemberIds = useMemo(
|
||
() => groups.flatMap((g) => g.members.map((m) => m.id)),
|
||
[groups]
|
||
)
|
||
|
||
// Track the rendered column count of the duplicates grid so ↑/↓ can
|
||
// skip a row instead of jumping a single cell. The grid uses
|
||
// `repeat(auto-fill, minmax(180px, 1fr))` so columns = floor(width/180).
|
||
// We measure the FIRST section's grid container — every section uses
|
||
// the same auto-fill rule so any one is representative.
|
||
const [columns, setColumns] = useState(4)
|
||
const sampleGridRef = useCallback((el: HTMLDivElement | null) => {
|
||
if (!el) return
|
||
const measure = () => {
|
||
const cols = Math.max(1, Math.floor(el.clientWidth / 180))
|
||
setColumns(cols)
|
||
}
|
||
measure()
|
||
const ro = new ResizeObserver(measure)
|
||
ro.observe(el)
|
||
// Caller doesn't get the cleanup hook but ResizeObserver disconnects
|
||
// when the element unmounts, which is fine for our lifecycle.
|
||
}, [])
|
||
|
||
// Window-level keyboard nav. Mirrors Timeline's handler but walks
|
||
// `allMemberIds` directly — duplicate groups don't have a uniform row
|
||
// grid so we approximate ↑/↓ via the measured `columns` count and
|
||
// wrap ←/→ across group boundaries.
|
||
useEffect(() => {
|
||
if (allMemberIds.length === 0) return
|
||
|
||
const onKeyDown = (e: KeyboardEvent) => {
|
||
const target = e.target as HTMLElement | null
|
||
if (target && (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA')) {
|
||
return
|
||
}
|
||
const key = e.key
|
||
if (
|
||
key !== 'ArrowLeft' &&
|
||
key !== 'ArrowRight' &&
|
||
key !== 'ArrowUp' &&
|
||
key !== 'ArrowDown'
|
||
) {
|
||
return
|
||
}
|
||
e.preventDefault()
|
||
const currentIdx = activePhotoId ? allMemberIds.indexOf(activePhotoId) : -1
|
||
const startIdx = currentIdx >= 0 ? currentIdx : 0
|
||
let nextIdx = startIdx
|
||
if (key === 'ArrowLeft') nextIdx = startIdx - 1
|
||
else if (key === 'ArrowRight') nextIdx = startIdx + 1
|
||
else if (key === 'ArrowUp') nextIdx = startIdx - columns
|
||
else if (key === 'ArrowDown') nextIdx = startIdx + columns
|
||
// Clamp to bounds — we don't wrap on out-of-range vertical moves
|
||
// since the grid is partitioned into groups and a "wrap" would
|
||
// skip across visually unrelated content.
|
||
nextIdx = Math.max(0, Math.min(allMemberIds.length - 1, nextIdx))
|
||
const nextId = allMemberIds[nextIdx]
|
||
if (!nextId) return
|
||
selectPhoto(nextId)
|
||
// Scroll the now-active cell into view if it's off-screen. The
|
||
// PhotoThumbnail wrapper carries data-dup-id so we can find it
|
||
// without threading refs through every cell.
|
||
const el = document.querySelector<HTMLElement>(
|
||
`[data-dup-id="${nextId}"]`
|
||
)
|
||
el?.scrollIntoView({ block: 'nearest', inline: 'nearest' })
|
||
}
|
||
|
||
window.addEventListener('keydown', onKeyDown)
|
||
return () => window.removeEventListener('keydown', onKeyDown)
|
||
}, [allMemberIds, activePhotoId, columns, selectPhoto])
|
||
|
||
if (isLoading) {
|
||
return (
|
||
<div className="flex h-full items-center justify-center text-text-muted">
|
||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||
Loading duplicate groups…
|
||
</div>
|
||
)
|
||
}
|
||
|
||
if (isError) {
|
||
return (
|
||
<div className="flex h-full items-center justify-center text-text-muted">
|
||
Could not load duplicate groups: {(error as any)?.message ?? 'unknown error'}
|
||
</div>
|
||
)
|
||
}
|
||
|
||
if (groups.length === 0) {
|
||
return (
|
||
<div className="flex h-full flex-col items-center justify-center gap-2 px-6 text-center text-text-muted">
|
||
<Sparkles className="h-8 w-8" />
|
||
<div className="text-sm font-medium text-text">No duplicates found</div>
|
||
<p className="max-w-sm text-xs">
|
||
Nothing in your library matches another photo at the perceptual-hash
|
||
level. If you've just added new photos, give the worker a minute and
|
||
re-run "Re-detect duplicates" from Settings.
|
||
</p>
|
||
</div>
|
||
)
|
||
}
|
||
|
||
return (
|
||
<div className="h-full overflow-auto bg-bg p-4">
|
||
<div className="mb-4 flex items-center gap-2 text-xs text-text-muted">
|
||
<Info className="h-3.5 w-3.5" />
|
||
<span>
|
||
{data?.total_groups} group{data?.total_groups === 1 ? '' : 's'} ·{' '}
|
||
{data?.total_members} photo{data?.total_members === 1 ? '' : 's'}.
|
||
Click "Keep best" to auto-discard all but the highest-resolution
|
||
copy of each group. Cmd+Z to undo.
|
||
</span>
|
||
</div>
|
||
|
||
<div className="space-y-6">
|
||
{groups.map((group, idx) => (
|
||
<DuplicateGroupSection
|
||
key={group.group_id}
|
||
group={group}
|
||
onKeepBest={(discardIds) => discardMutation.mutate(discardIds)}
|
||
onPreviewMember={(memberId) => openPreview(memberId, allMemberIds)}
|
||
onSelectMember={(memberId) => selectPhoto(memberId)}
|
||
selectedPhotos={selectedPhotos}
|
||
isPending={discardMutation.isPending}
|
||
// Hand the column-measurement ref to the first section only
|
||
// — every section's grid uses the same auto-fill rule so any
|
||
// one is representative of the rendered column count.
|
||
gridRef={idx === 0 ? sampleGridRef : undefined}
|
||
/>
|
||
))}
|
||
</div>
|
||
</div>
|
||
)
|
||
}
|
||
|
||
interface DuplicateGroupSectionProps {
|
||
group: DuplicateGroup
|
||
onKeepBest: (discardIds: string[]) => void
|
||
onPreviewMember: (memberId: string) => void
|
||
onSelectMember: (memberId: string) => void
|
||
selectedPhotos: string[]
|
||
isPending: boolean
|
||
/** Optional callback ref attached to this section's grid container.
|
||
* Used by DuplicatesView to measure the rendered column count for
|
||
* ↑/↓ keyboard navigation. Only the first section gets one. */
|
||
gridRef?: (el: HTMLDivElement | null) => void
|
||
}
|
||
|
||
function DuplicateGroupSection({
|
||
group,
|
||
onKeepBest,
|
||
onPreviewMember,
|
||
onSelectMember,
|
||
selectedPhotos,
|
||
isPending,
|
||
gridRef,
|
||
}: DuplicateGroupSectionProps) {
|
||
// Auto-pick "best" copy: highest pixel count, ties broken by file_size,
|
||
// then earliest taken_at, then id for determinism. This is just the
|
||
// default — the user can override it by clicking the crown button on
|
||
// any other thumbnail (see `manualBestId`).
|
||
const autoBest = useMemo(() => pickBestMember(group.members), [group.members])
|
||
// When the user clicks "make this the best" on a non-default thumb,
|
||
// we override the auto-pick. Local to the section so different groups
|
||
// remember independent overrides; resets if the group itself changes.
|
||
const [manualBestId, setManualBestId] = useState<string | null>(null)
|
||
const bestId =
|
||
manualBestId && group.members.some((m) => m.id === manualBestId)
|
||
? manualBestId
|
||
: autoBest.id
|
||
const best = group.members.find((m) => m.id === bestId) ?? autoBest
|
||
const discardCount = group.member_count - 1
|
||
const isExact = group.reason === 'exact'
|
||
|
||
return (
|
||
<section className="rounded-lg border border-border bg-surface">
|
||
<header className="flex items-center justify-between gap-3 border-b border-border px-3 py-2">
|
||
<div className="flex items-center gap-2 text-sm">
|
||
{isExact ? (
|
||
<Copy className="h-4 w-4 text-text-muted" />
|
||
) : (
|
||
<Layers className="h-4 w-4 text-text-muted" />
|
||
)}
|
||
<span className="font-medium text-text">
|
||
{group.member_count} {isExact ? 'exact' : 'similar'} photos
|
||
</span>
|
||
<span className="text-xs text-text-faint">
|
||
keeping: {formatDimensions(best)}
|
||
{best.file_size != null && ` · ${formatBytes(best.file_size)}`}
|
||
{manualBestId && manualBestId !== autoBest.id && (
|
||
<span className="ml-1 text-text-muted">(manual)</span>
|
||
)}
|
||
</span>
|
||
</div>
|
||
<button
|
||
onClick={() => {
|
||
const discardIds = group.members
|
||
.filter((m) => m.id !== best.id)
|
||
.map((m) => m.id)
|
||
if (discardIds.length === 0) return
|
||
onKeepBest(discardIds)
|
||
}}
|
||
disabled={isPending}
|
||
className={clsx(
|
||
'flex items-center gap-1.5 rounded border border-border px-2 py-1 text-xs font-medium transition-colors',
|
||
'hover:border-reject/50 hover:bg-reject/10 hover:text-reject',
|
||
isPending && 'cursor-not-allowed opacity-50'
|
||
)}
|
||
title="Keep the highest-resolution copy and discard the rest"
|
||
>
|
||
<Trash2 className="h-3 w-3" />
|
||
Keep best, discard {discardCount}
|
||
</button>
|
||
</header>
|
||
|
||
<div
|
||
ref={gridRef}
|
||
className="grid gap-1 p-2"
|
||
style={{
|
||
gridTemplateColumns:
|
||
'repeat(auto-fill, minmax(180px, 1fr))',
|
||
}}
|
||
>
|
||
{group.members.map((member) => {
|
||
const isBest = member.id === best.id
|
||
return (
|
||
<div
|
||
key={member.id}
|
||
data-dup-id={member.id}
|
||
className="group/dup relative"
|
||
>
|
||
<PhotoThumbnail
|
||
photo={memberToPhoto(member)}
|
||
size={180}
|
||
fill
|
||
isSelected={selectedPhotos.includes(member.id)}
|
||
onClick={() => onSelectMember(member.id)}
|
||
onDoubleClick={() => onPreviewMember(member.id)}
|
||
/>
|
||
{/* "BEST" pill marks the photo that will be KEPT when the
|
||
* user clicks the discard button. Lives at top-right where
|
||
* it doesn't collide with the cyan selection ring (which
|
||
* draws around the cell perimeter); kept inset by 6px so
|
||
* the ring's outer edge has clearance on either side. */}
|
||
{isBest && (
|
||
<span className="pointer-events-none absolute right-1.5 top-1.5 z-10 flex items-center gap-1 rounded bg-pick px-1.5 py-0.5 text-[10px] font-bold uppercase text-white shadow ring-1 ring-black/30">
|
||
<Crown className="h-3 w-3" />
|
||
Best
|
||
</span>
|
||
)}
|
||
{/* "Make this best" affordance — shown on hover for non-best
|
||
* thumbnails. Mirrors the BEST pill's top-right placement
|
||
* so the eye doesn't have to retarget when the user is
|
||
* scanning a row of thumbnails. Stops propagation so it
|
||
* doesn't double-fire as a selection click. */}
|
||
{!isBest && (
|
||
<button
|
||
onClick={(e) => {
|
||
e.stopPropagation()
|
||
setManualBestId(member.id)
|
||
}}
|
||
className="absolute right-1.5 top-1.5 z-10 hidden items-center gap-1 rounded bg-black/70 px-1.5 py-0.5 text-[10px] font-medium text-white shadow ring-1 ring-white/20 transition hover:bg-pick hover:text-white group-hover/dup:flex"
|
||
title="Keep this one instead"
|
||
>
|
||
<Crown className="h-3 w-3" />
|
||
Keep this
|
||
</button>
|
||
)}
|
||
{/* Dimensions chip — bottom-LEFT (the BEST/Keep affordances
|
||
* own the top-right corner). */}
|
||
<div className="pointer-events-none absolute bottom-1 left-1 z-10 rounded bg-black/70 px-1.5 py-0.5 text-[10px] font-mono text-white">
|
||
{formatDimensions(member)}
|
||
</div>
|
||
</div>
|
||
)
|
||
})}
|
||
</div>
|
||
</section>
|
||
)
|
||
}
|
||
|
||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||
|
||
/** Score a member by (pixels, file_size, -taken_at) and return the winner.
|
||
* Larger pixel count wins; ties broken by file_size; final tie by earliest
|
||
* taken_at (more likely the original capture). */
|
||
function pickBestMember(members: DuplicateGroupMember[]): DuplicateGroupMember {
|
||
return members.reduce((best, m) => {
|
||
const bestPixels = (best.width ?? 0) * (best.height ?? 0)
|
||
const mPixels = (m.width ?? 0) * (m.height ?? 0)
|
||
if (mPixels !== bestPixels) return mPixels > bestPixels ? m : best
|
||
const bestSize = best.file_size ?? 0
|
||
const mSize = m.file_size ?? 0
|
||
if (mSize !== bestSize) return mSize > bestSize ? m : best
|
||
// Earliest taken_at wins (treat null as far-future).
|
||
const bestTaken = best.taken_at ?? '9999'
|
||
const mTaken = m.taken_at ?? '9999'
|
||
if (mTaken !== bestTaken) return mTaken < bestTaken ? m : best
|
||
return best
|
||
})
|
||
}
|
||
|
||
function formatDimensions(m: DuplicateGroupMember): string {
|
||
if (!m.width || !m.height) return '?'
|
||
const mp = (m.width * m.height) / 1_000_000
|
||
if (mp >= 1) return `${mp.toFixed(1)}MP`
|
||
return `${m.width}×${m.height}`
|
||
}
|
||
|
||
function formatBytes(n: number): string {
|
||
if (n >= 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(1)}MB`
|
||
if (n >= 1024) return `${(n / 1024).toFixed(0)}KB`
|
||
return `${n}B`
|
||
}
|
||
|
||
/** Adapt a DuplicateGroupMember (the slim API shape) to a Photo, which
|
||
* is what PhotoThumbnail expects. We deliberately set is_duplicate=false
|
||
* on the synthetic Photo so the duplicate badge isn't drawn on every
|
||
* cell — the entire view is duplicates, the badge would be redundant. */
|
||
function memberToPhoto(m: DuplicateGroupMember): Photo {
|
||
return {
|
||
id: m.id,
|
||
filepath: m.filename, // good enough for the RAW/video extension regex
|
||
filename: m.filename,
|
||
media_type: m.media_type,
|
||
width: m.width,
|
||
height: m.height,
|
||
taken_at: m.taken_at,
|
||
rating: 0,
|
||
is_discarded: false,
|
||
is_duplicate: false,
|
||
file_hash: m.file_hash ?? '',
|
||
folder_id: m.folder_id,
|
||
added_at: null,
|
||
thumb_small: m.thumb_small ?? undefined,
|
||
}
|
||
}
|