feat: prune orphaned photo rows + retry-pending action
Adds /api/v1/library/maintenance/{missing-stats,prune-missing} backed
by a new cleanup helper that deletes Photo rows whose files no longer
exist on disk under a *mounted* source root. Skips photos under
unmounted roots so a temporarily-disconnected drive doesn't get
silently nuked.
Settings panel surfaces the orphan count with a destructive Prune
button, plus a "Kick pending" action that re-queues photos stuck in
processing_status='pending' (typically left behind when the scanner
created the row but the worker never picked up the thumbnail task).
Common trigger: PHOTO_DIRS in .env was repointed at a different
library root, leaving every old row dangling.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -142,6 +142,12 @@ class RegenerateThumbnailsRequest(BaseModel):
|
||||
default=False,
|
||||
description="If true, only re-queue photos whose processing_status is 'failed'.",
|
||||
)
|
||||
only_pending: bool = Field(
|
||||
default=False,
|
||||
description="If true, only (re-)queue photos whose processing_status is 'pending'. "
|
||||
"Useful for kicking rows that were created by a scan but never had "
|
||||
"their thumbnail task picked up.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/maintenance/thumbnail-stats")
|
||||
@@ -207,6 +213,8 @@ async def regenerate_thumbnails(
|
||||
query = query.where(Photo.media_type.in_(media_types))
|
||||
if body.only_failed:
|
||||
query = query.where(Photo.processing_status == 'failed')
|
||||
if body.only_pending:
|
||||
query = query.where(Photo.processing_status == 'pending')
|
||||
|
||||
photos = (await db.execute(query)).scalars().all()
|
||||
|
||||
@@ -390,6 +398,29 @@ async def get_worker_status(db: AsyncSession = Depends(get_db)):
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/missing-stats")
|
||||
async def get_missing_stats():
|
||||
"""Count photos whose files no longer exist on disk under a mounted
|
||||
source root. Surfaced in Settings so the user can see a number before
|
||||
pulling the trigger on prune-missing. Cheap enough to call freely."""
|
||||
from app.services.cleanup import prune_missing_photos
|
||||
return await prune_missing_photos(dry_run=True)
|
||||
|
||||
|
||||
@router.post("/maintenance/prune-missing")
|
||||
async def run_prune_missing():
|
||||
"""Actually delete the orphaned photo rows reported by /missing-stats.
|
||||
Common cause: PHOTO_DIRS in .env was repointed at a different library
|
||||
leaving every old row dangling. Skips any photo whose source root
|
||||
isn't currently mounted (almost always means an unmounted drive)."""
|
||||
from app.services.cleanup import prune_missing_photos
|
||||
try:
|
||||
return {"status": "success", **(await prune_missing_photos(dry_run=False))}
|
||||
except Exception as e:
|
||||
logger.error(f"Prune missing failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@router.post("/maintenance/cleanup")
|
||||
async def run_data_integrity_cleanup():
|
||||
"""Re-run the source-roots / folders / photos data-integrity cleanup
|
||||
|
||||
@@ -139,6 +139,73 @@ async def _warn_stale_source_roots(session: AsyncSession) -> int:
|
||||
return stale
|
||||
|
||||
|
||||
async def find_missing_photos(session: AsyncSession) -> tuple[list[str], list[str]]:
|
||||
"""Walk every non-discarded photo and check whether its file is still
|
||||
on disk. Returns (deletable_ids, skipped_under_unmounted_roots).
|
||||
|
||||
Skipped rows are photos whose owning source_root path itself doesn't
|
||||
resolve — that's almost always an unmounted drive, and silently
|
||||
deleting those rows would be data loss. The caller can surface the
|
||||
skip count separately so the user knows the cleanup wasn't a no-op
|
||||
by accident.
|
||||
"""
|
||||
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
||||
sr_mounted: dict[str, bool] = {sr.id: os.path.isdir(sr.path) for sr in sr_rows}
|
||||
|
||||
photos = (await session.execute(
|
||||
select(Photo.id, Photo.filepath, Photo.folder_id)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
)).all()
|
||||
|
||||
# folder -> source_root lookup
|
||||
folders = (await session.execute(select(Folder.id, Folder.source_root_id))).all()
|
||||
folder_to_sr = {fid: srid for fid, srid in folders}
|
||||
|
||||
deletable: list[str] = []
|
||||
skipped: list[str] = []
|
||||
for pid, fp, folder_id in photos:
|
||||
sr_id = folder_to_sr.get(folder_id)
|
||||
if sr_id is None or not sr_mounted.get(sr_id, False):
|
||||
skipped.append(pid)
|
||||
continue
|
||||
if not os.path.exists(fp):
|
||||
deletable.append(pid)
|
||||
return deletable, skipped
|
||||
|
||||
|
||||
async def prune_missing_photos(dry_run: bool = True) -> dict:
|
||||
"""Delete photo rows whose files are no longer on disk *and* whose
|
||||
source root is currently mounted. Common cause: PHOTO_DIRS in .env
|
||||
was repointed at a different library, leaving every old row orphaned.
|
||||
|
||||
Set dry_run=False to actually delete. The default is intentionally
|
||||
safe so the matching count can be surfaced in the UI before the
|
||||
user commits to it.
|
||||
"""
|
||||
from sqlalchemy import delete
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
deletable, skipped = await find_missing_photos(session)
|
||||
if not dry_run and deletable:
|
||||
# Chunked delete to keep the IN clause within SQLite limits.
|
||||
CHUNK = 500
|
||||
for i in range(0, len(deletable), CHUNK):
|
||||
await session.execute(
|
||||
delete(Photo).where(Photo.id.in_(deletable[i:i + CHUNK]))
|
||||
)
|
||||
await session.commit()
|
||||
logger.info(f"Pruned {len(deletable)} orphaned photo rows")
|
||||
return {
|
||||
"would_delete" if dry_run else "deleted": len(deletable),
|
||||
"skipped_unmounted": len(skipped),
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"prune_missing_photos failed: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def cleanup_data_integrity() -> dict:
|
||||
"""Top-level entry point. Runs the dedupe + count refresh in a single
|
||||
transaction. Returns a small summary dict for logging."""
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
type LibraryStats,
|
||||
type MediaType,
|
||||
type WorkerStatus,
|
||||
type MissingStats,
|
||||
} from '../../services/api'
|
||||
import { toast } from '../ToastContainer'
|
||||
|
||||
@@ -42,6 +43,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
const [thumbStats, setThumbStats] = useState<ThumbnailStats | null>(null)
|
||||
const [libStats, setLibStats] = useState<LibraryStats | null>(null)
|
||||
const [workerStatus, setWorkerStatus] = useState<WorkerStatus | null>(null)
|
||||
const [missingStats, setMissingStats] = useState<MissingStats | null>(null)
|
||||
const [loadingStats, setLoadingStats] = useState(false)
|
||||
const [loadingWorkers, setLoadingWorkers] = useState(false)
|
||||
const [showAllErrors, setShowAllErrors] = useState(false)
|
||||
@@ -69,8 +71,12 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
const refreshWorkers = useCallback(async () => {
|
||||
setLoadingWorkers(true)
|
||||
try {
|
||||
const ws = await library.maintenance.workerStatus()
|
||||
const [ws, ms] = await Promise.all([
|
||||
library.maintenance.workerStatus(),
|
||||
library.maintenance.missingStats(),
|
||||
])
|
||||
setWorkerStatus(ws)
|
||||
setMissingStats(ms)
|
||||
} catch (e) {
|
||||
console.error('Failed to load worker status', e)
|
||||
toast.error('Could not load worker status')
|
||||
@@ -111,7 +117,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
try {
|
||||
const result = await fn()
|
||||
toast.success(successTitle, describe?.(result))
|
||||
await refreshStats()
|
||||
await Promise.all([refreshStats(), refreshWorkers()])
|
||||
} catch (e: unknown) {
|
||||
const message = e instanceof Error ? e.message : String(e)
|
||||
toast.error(`${successTitle} failed`, message)
|
||||
@@ -119,11 +125,18 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
setBusy((b) => ({ ...b, [key]: false }))
|
||||
}
|
||||
},
|
||||
[busy, refreshStats]
|
||||
[busy, refreshStats, refreshWorkers]
|
||||
)
|
||||
|
||||
const regenerate = useCallback(
|
||||
(key: string, body: { media_types?: MediaType[]; only_failed?: boolean }) =>
|
||||
(
|
||||
key: string,
|
||||
body: {
|
||||
media_types?: MediaType[]
|
||||
only_failed?: boolean
|
||||
only_pending?: boolean
|
||||
}
|
||||
) =>
|
||||
runAction(
|
||||
key,
|
||||
() => library.maintenance.regenerateThumbnails(body),
|
||||
@@ -263,6 +276,18 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
{thumbStats ? ` (${thumbStats.failed})` : ''}
|
||||
</ActionButton>
|
||||
|
||||
<ActionButton
|
||||
loading={busy['regen-pending']}
|
||||
onClick={() =>
|
||||
regenerate('regen-pending', { only_pending: true })
|
||||
}
|
||||
disabled={!!thumbStats && thumbStats.pending === 0}
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
Kick pending
|
||||
{thumbStats ? ` (${thumbStats.pending})` : ''}
|
||||
</ActionButton>
|
||||
|
||||
<ActionButton
|
||||
loading={busy['regen-all']}
|
||||
destructive
|
||||
@@ -394,6 +419,54 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Orphaned rows (files gone from disk) */}
|
||||
{missingStats && (missingStats.would_delete ?? 0) > 0 && (
|
||||
<div className="mt-3 rounded border border-star/40 bg-star/10 p-2">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div className="text-xs">
|
||||
<div className="flex items-center gap-1.5 font-medium text-text">
|
||||
<AlertTriangle className="h-3.5 w-3.5 text-star" />
|
||||
{missingStats.would_delete} orphaned photo rows
|
||||
</div>
|
||||
<div className="mt-0.5 text-[10px] text-text-muted">
|
||||
Files no longer exist on disk under a mounted source
|
||||
root. Usually means PHOTO_DIRS was repointed at a
|
||||
different library.
|
||||
{missingStats.skipped_unmounted > 0 && (
|
||||
<>
|
||||
{' '}
|
||||
{missingStats.skipped_unmounted} more rows are
|
||||
under unmounted roots and will not be touched.
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<ActionButton
|
||||
loading={busy['prune-missing']}
|
||||
destructive
|
||||
onClick={() => {
|
||||
if (
|
||||
!confirm(
|
||||
`Delete ${missingStats.would_delete} photo rows whose files are missing? ` +
|
||||
'This cannot be undone.'
|
||||
)
|
||||
)
|
||||
return
|
||||
runAction(
|
||||
'prune-missing',
|
||||
() => library.maintenance.pruneMissing(),
|
||||
'Orphans pruned',
|
||||
(r) => `${r.deleted ?? 0} rows deleted`
|
||||
)
|
||||
}}
|
||||
>
|
||||
<AlertTriangle className="h-4 w-4" />
|
||||
Prune
|
||||
</ActionButton>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Per-worker breakdown */}
|
||||
{workerStatus && workerStatus.workers.length > 0 && (
|
||||
<div className="mt-3 space-y-2">
|
||||
|
||||
@@ -230,6 +230,18 @@ export interface RegenerateResult {
|
||||
}
|
||||
}
|
||||
|
||||
export interface MissingStats {
|
||||
would_delete?: number
|
||||
deleted?: number
|
||||
skipped_unmounted: number
|
||||
dry_run: boolean
|
||||
}
|
||||
|
||||
export interface PruneResult extends MissingStats {
|
||||
status?: string
|
||||
message?: string
|
||||
}
|
||||
|
||||
export interface WorkerInfo {
|
||||
name: string
|
||||
status: 'online' | 'unreachable'
|
||||
@@ -295,7 +307,11 @@ export const library = {
|
||||
/** Reset on-disk thumbs and re-queue Celery generation. With no
|
||||
* filters, every photo in the library is re-queued. */
|
||||
regenerateThumbnails: async (
|
||||
body: { media_types?: MediaType[]; only_failed?: boolean } = {}
|
||||
body: {
|
||||
media_types?: MediaType[]
|
||||
only_failed?: boolean
|
||||
only_pending?: boolean
|
||||
} = {}
|
||||
): Promise<RegenerateResult> => {
|
||||
const response = await api.post(
|
||||
'/library/maintenance/regenerate-thumbnails',
|
||||
@@ -312,6 +328,19 @@ export const library = {
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Dry-run count of photo rows whose files are no longer on disk
|
||||
* (under a mounted source root). */
|
||||
missingStats: async (): Promise<MissingStats> => {
|
||||
const response = await api.get('/library/maintenance/missing-stats')
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Actually delete the orphaned photo rows. */
|
||||
pruneMissing: async (): Promise<PruneResult> => {
|
||||
const response = await api.post('/library/maintenance/prune-missing')
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Re-run the source-roots / folders / photos integrity cleanup that
|
||||
* normally runs on backend startup. */
|
||||
cleanup: async (): Promise<{ status: string; message?: string }> => {
|
||||
|
||||
Reference in New Issue
Block a user