feat: prune orphaned photo rows + retry-pending action
Adds /api/v1/library/maintenance/{missing-stats,prune-missing} backed
by a new cleanup helper that deletes Photo rows whose files no longer
exist on disk under a *mounted* source root. Skips photos under
unmounted roots so a temporarily-disconnected drive doesn't get
silently nuked.
Settings panel surfaces the orphan count with a destructive Prune
button, plus a "Kick pending" action that re-queues photos stuck in
processing_status='pending' (typically left behind when the scanner
created the row but the worker never picked up the thumbnail task).
Common trigger: PHOTO_DIRS in .env was repointed at a different
library root, leaving every old row dangling.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -142,6 +142,12 @@ class RegenerateThumbnailsRequest(BaseModel):
|
|||||||
default=False,
|
default=False,
|
||||||
description="If true, only re-queue photos whose processing_status is 'failed'.",
|
description="If true, only re-queue photos whose processing_status is 'failed'.",
|
||||||
)
|
)
|
||||||
|
only_pending: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description="If true, only (re-)queue photos whose processing_status is 'pending'. "
|
||||||
|
"Useful for kicking rows that were created by a scan but never had "
|
||||||
|
"their thumbnail task picked up.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/maintenance/thumbnail-stats")
|
@router.get("/maintenance/thumbnail-stats")
|
||||||
@@ -207,6 +213,8 @@ async def regenerate_thumbnails(
|
|||||||
query = query.where(Photo.media_type.in_(media_types))
|
query = query.where(Photo.media_type.in_(media_types))
|
||||||
if body.only_failed:
|
if body.only_failed:
|
||||||
query = query.where(Photo.processing_status == 'failed')
|
query = query.where(Photo.processing_status == 'failed')
|
||||||
|
if body.only_pending:
|
||||||
|
query = query.where(Photo.processing_status == 'pending')
|
||||||
|
|
||||||
photos = (await db.execute(query)).scalars().all()
|
photos = (await db.execute(query)).scalars().all()
|
||||||
|
|
||||||
@@ -390,6 +398,29 @@ async def get_worker_status(db: AsyncSession = Depends(get_db)):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/maintenance/missing-stats")
|
||||||
|
async def get_missing_stats():
|
||||||
|
"""Count photos whose files no longer exist on disk under a mounted
|
||||||
|
source root. Surfaced in Settings so the user can see a number before
|
||||||
|
pulling the trigger on prune-missing. Cheap enough to call freely."""
|
||||||
|
from app.services.cleanup import prune_missing_photos
|
||||||
|
return await prune_missing_photos(dry_run=True)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/maintenance/prune-missing")
|
||||||
|
async def run_prune_missing():
|
||||||
|
"""Actually delete the orphaned photo rows reported by /missing-stats.
|
||||||
|
Common cause: PHOTO_DIRS in .env was repointed at a different library
|
||||||
|
leaving every old row dangling. Skips any photo whose source root
|
||||||
|
isn't currently mounted (almost always means an unmounted drive)."""
|
||||||
|
from app.services.cleanup import prune_missing_photos
|
||||||
|
try:
|
||||||
|
return {"status": "success", **(await prune_missing_photos(dry_run=False))}
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Prune missing failed: {e}")
|
||||||
|
return {"status": "error", "message": str(e)}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/maintenance/cleanup")
|
@router.post("/maintenance/cleanup")
|
||||||
async def run_data_integrity_cleanup():
|
async def run_data_integrity_cleanup():
|
||||||
"""Re-run the source-roots / folders / photos data-integrity cleanup
|
"""Re-run the source-roots / folders / photos data-integrity cleanup
|
||||||
|
|||||||
@@ -139,6 +139,73 @@ async def _warn_stale_source_roots(session: AsyncSession) -> int:
|
|||||||
return stale
|
return stale
|
||||||
|
|
||||||
|
|
||||||
|
async def find_missing_photos(session: AsyncSession) -> tuple[list[str], list[str]]:
|
||||||
|
"""Walk every non-discarded photo and check whether its file is still
|
||||||
|
on disk. Returns (deletable_ids, skipped_under_unmounted_roots).
|
||||||
|
|
||||||
|
Skipped rows are photos whose owning source_root path itself doesn't
|
||||||
|
resolve — that's almost always an unmounted drive, and silently
|
||||||
|
deleting those rows would be data loss. The caller can surface the
|
||||||
|
skip count separately so the user knows the cleanup wasn't a no-op
|
||||||
|
by accident.
|
||||||
|
"""
|
||||||
|
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
||||||
|
sr_mounted: dict[str, bool] = {sr.id: os.path.isdir(sr.path) for sr in sr_rows}
|
||||||
|
|
||||||
|
photos = (await session.execute(
|
||||||
|
select(Photo.id, Photo.filepath, Photo.folder_id)
|
||||||
|
.where(Photo.is_discarded.is_(False))
|
||||||
|
)).all()
|
||||||
|
|
||||||
|
# folder -> source_root lookup
|
||||||
|
folders = (await session.execute(select(Folder.id, Folder.source_root_id))).all()
|
||||||
|
folder_to_sr = {fid: srid for fid, srid in folders}
|
||||||
|
|
||||||
|
deletable: list[str] = []
|
||||||
|
skipped: list[str] = []
|
||||||
|
for pid, fp, folder_id in photos:
|
||||||
|
sr_id = folder_to_sr.get(folder_id)
|
||||||
|
if sr_id is None or not sr_mounted.get(sr_id, False):
|
||||||
|
skipped.append(pid)
|
||||||
|
continue
|
||||||
|
if not os.path.exists(fp):
|
||||||
|
deletable.append(pid)
|
||||||
|
return deletable, skipped
|
||||||
|
|
||||||
|
|
||||||
|
async def prune_missing_photos(dry_run: bool = True) -> dict:
|
||||||
|
"""Delete photo rows whose files are no longer on disk *and* whose
|
||||||
|
source root is currently mounted. Common cause: PHOTO_DIRS in .env
|
||||||
|
was repointed at a different library, leaving every old row orphaned.
|
||||||
|
|
||||||
|
Set dry_run=False to actually delete. The default is intentionally
|
||||||
|
safe so the matching count can be surfaced in the UI before the
|
||||||
|
user commits to it.
|
||||||
|
"""
|
||||||
|
from sqlalchemy import delete
|
||||||
|
async with AsyncSessionLocal() as session:
|
||||||
|
try:
|
||||||
|
deletable, skipped = await find_missing_photos(session)
|
||||||
|
if not dry_run and deletable:
|
||||||
|
# Chunked delete to keep the IN clause within SQLite limits.
|
||||||
|
CHUNK = 500
|
||||||
|
for i in range(0, len(deletable), CHUNK):
|
||||||
|
await session.execute(
|
||||||
|
delete(Photo).where(Photo.id.in_(deletable[i:i + CHUNK]))
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
logger.info(f"Pruned {len(deletable)} orphaned photo rows")
|
||||||
|
return {
|
||||||
|
"would_delete" if dry_run else "deleted": len(deletable),
|
||||||
|
"skipped_unmounted": len(skipped),
|
||||||
|
"dry_run": dry_run,
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"prune_missing_photos failed: {e}")
|
||||||
|
await session.rollback()
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
async def cleanup_data_integrity() -> dict:
|
async def cleanup_data_integrity() -> dict:
|
||||||
"""Top-level entry point. Runs the dedupe + count refresh in a single
|
"""Top-level entry point. Runs the dedupe + count refresh in a single
|
||||||
transaction. Returns a small summary dict for logging."""
|
transaction. Returns a small summary dict for logging."""
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import {
|
|||||||
type LibraryStats,
|
type LibraryStats,
|
||||||
type MediaType,
|
type MediaType,
|
||||||
type WorkerStatus,
|
type WorkerStatus,
|
||||||
|
type MissingStats,
|
||||||
} from '../../services/api'
|
} from '../../services/api'
|
||||||
import { toast } from '../ToastContainer'
|
import { toast } from '../ToastContainer'
|
||||||
|
|
||||||
@@ -42,6 +43,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
|||||||
const [thumbStats, setThumbStats] = useState<ThumbnailStats | null>(null)
|
const [thumbStats, setThumbStats] = useState<ThumbnailStats | null>(null)
|
||||||
const [libStats, setLibStats] = useState<LibraryStats | null>(null)
|
const [libStats, setLibStats] = useState<LibraryStats | null>(null)
|
||||||
const [workerStatus, setWorkerStatus] = useState<WorkerStatus | null>(null)
|
const [workerStatus, setWorkerStatus] = useState<WorkerStatus | null>(null)
|
||||||
|
const [missingStats, setMissingStats] = useState<MissingStats | null>(null)
|
||||||
const [loadingStats, setLoadingStats] = useState(false)
|
const [loadingStats, setLoadingStats] = useState(false)
|
||||||
const [loadingWorkers, setLoadingWorkers] = useState(false)
|
const [loadingWorkers, setLoadingWorkers] = useState(false)
|
||||||
const [showAllErrors, setShowAllErrors] = useState(false)
|
const [showAllErrors, setShowAllErrors] = useState(false)
|
||||||
@@ -69,8 +71,12 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
|||||||
const refreshWorkers = useCallback(async () => {
|
const refreshWorkers = useCallback(async () => {
|
||||||
setLoadingWorkers(true)
|
setLoadingWorkers(true)
|
||||||
try {
|
try {
|
||||||
const ws = await library.maintenance.workerStatus()
|
const [ws, ms] = await Promise.all([
|
||||||
|
library.maintenance.workerStatus(),
|
||||||
|
library.maintenance.missingStats(),
|
||||||
|
])
|
||||||
setWorkerStatus(ws)
|
setWorkerStatus(ws)
|
||||||
|
setMissingStats(ms)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error('Failed to load worker status', e)
|
console.error('Failed to load worker status', e)
|
||||||
toast.error('Could not load worker status')
|
toast.error('Could not load worker status')
|
||||||
@@ -111,7 +117,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
|||||||
try {
|
try {
|
||||||
const result = await fn()
|
const result = await fn()
|
||||||
toast.success(successTitle, describe?.(result))
|
toast.success(successTitle, describe?.(result))
|
||||||
await refreshStats()
|
await Promise.all([refreshStats(), refreshWorkers()])
|
||||||
} catch (e: unknown) {
|
} catch (e: unknown) {
|
||||||
const message = e instanceof Error ? e.message : String(e)
|
const message = e instanceof Error ? e.message : String(e)
|
||||||
toast.error(`${successTitle} failed`, message)
|
toast.error(`${successTitle} failed`, message)
|
||||||
@@ -119,11 +125,18 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
|||||||
setBusy((b) => ({ ...b, [key]: false }))
|
setBusy((b) => ({ ...b, [key]: false }))
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[busy, refreshStats]
|
[busy, refreshStats, refreshWorkers]
|
||||||
)
|
)
|
||||||
|
|
||||||
const regenerate = useCallback(
|
const regenerate = useCallback(
|
||||||
(key: string, body: { media_types?: MediaType[]; only_failed?: boolean }) =>
|
(
|
||||||
|
key: string,
|
||||||
|
body: {
|
||||||
|
media_types?: MediaType[]
|
||||||
|
only_failed?: boolean
|
||||||
|
only_pending?: boolean
|
||||||
|
}
|
||||||
|
) =>
|
||||||
runAction(
|
runAction(
|
||||||
key,
|
key,
|
||||||
() => library.maintenance.regenerateThumbnails(body),
|
() => library.maintenance.regenerateThumbnails(body),
|
||||||
@@ -263,6 +276,18 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
|||||||
{thumbStats ? ` (${thumbStats.failed})` : ''}
|
{thumbStats ? ` (${thumbStats.failed})` : ''}
|
||||||
</ActionButton>
|
</ActionButton>
|
||||||
|
|
||||||
|
<ActionButton
|
||||||
|
loading={busy['regen-pending']}
|
||||||
|
onClick={() =>
|
||||||
|
regenerate('regen-pending', { only_pending: true })
|
||||||
|
}
|
||||||
|
disabled={!!thumbStats && thumbStats.pending === 0}
|
||||||
|
>
|
||||||
|
<RefreshCw className="h-4 w-4" />
|
||||||
|
Kick pending
|
||||||
|
{thumbStats ? ` (${thumbStats.pending})` : ''}
|
||||||
|
</ActionButton>
|
||||||
|
|
||||||
<ActionButton
|
<ActionButton
|
||||||
loading={busy['regen-all']}
|
loading={busy['regen-all']}
|
||||||
destructive
|
destructive
|
||||||
@@ -394,6 +419,54 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Orphaned rows (files gone from disk) */}
|
||||||
|
{missingStats && (missingStats.would_delete ?? 0) > 0 && (
|
||||||
|
<div className="mt-3 rounded border border-star/40 bg-star/10 p-2">
|
||||||
|
<div className="flex items-center justify-between gap-2">
|
||||||
|
<div className="text-xs">
|
||||||
|
<div className="flex items-center gap-1.5 font-medium text-text">
|
||||||
|
<AlertTriangle className="h-3.5 w-3.5 text-star" />
|
||||||
|
{missingStats.would_delete} orphaned photo rows
|
||||||
|
</div>
|
||||||
|
<div className="mt-0.5 text-[10px] text-text-muted">
|
||||||
|
Files no longer exist on disk under a mounted source
|
||||||
|
root. Usually means PHOTO_DIRS was repointed at a
|
||||||
|
different library.
|
||||||
|
{missingStats.skipped_unmounted > 0 && (
|
||||||
|
<>
|
||||||
|
{' '}
|
||||||
|
{missingStats.skipped_unmounted} more rows are
|
||||||
|
under unmounted roots and will not be touched.
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<ActionButton
|
||||||
|
loading={busy['prune-missing']}
|
||||||
|
destructive
|
||||||
|
onClick={() => {
|
||||||
|
if (
|
||||||
|
!confirm(
|
||||||
|
`Delete ${missingStats.would_delete} photo rows whose files are missing? ` +
|
||||||
|
'This cannot be undone.'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
runAction(
|
||||||
|
'prune-missing',
|
||||||
|
() => library.maintenance.pruneMissing(),
|
||||||
|
'Orphans pruned',
|
||||||
|
(r) => `${r.deleted ?? 0} rows deleted`
|
||||||
|
)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<AlertTriangle className="h-4 w-4" />
|
||||||
|
Prune
|
||||||
|
</ActionButton>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Per-worker breakdown */}
|
{/* Per-worker breakdown */}
|
||||||
{workerStatus && workerStatus.workers.length > 0 && (
|
{workerStatus && workerStatus.workers.length > 0 && (
|
||||||
<div className="mt-3 space-y-2">
|
<div className="mt-3 space-y-2">
|
||||||
|
|||||||
@@ -230,6 +230,18 @@ export interface RegenerateResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface MissingStats {
|
||||||
|
would_delete?: number
|
||||||
|
deleted?: number
|
||||||
|
skipped_unmounted: number
|
||||||
|
dry_run: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PruneResult extends MissingStats {
|
||||||
|
status?: string
|
||||||
|
message?: string
|
||||||
|
}
|
||||||
|
|
||||||
export interface WorkerInfo {
|
export interface WorkerInfo {
|
||||||
name: string
|
name: string
|
||||||
status: 'online' | 'unreachable'
|
status: 'online' | 'unreachable'
|
||||||
@@ -295,7 +307,11 @@ export const library = {
|
|||||||
/** Reset on-disk thumbs and re-queue Celery generation. With no
|
/** Reset on-disk thumbs and re-queue Celery generation. With no
|
||||||
* filters, every photo in the library is re-queued. */
|
* filters, every photo in the library is re-queued. */
|
||||||
regenerateThumbnails: async (
|
regenerateThumbnails: async (
|
||||||
body: { media_types?: MediaType[]; only_failed?: boolean } = {}
|
body: {
|
||||||
|
media_types?: MediaType[]
|
||||||
|
only_failed?: boolean
|
||||||
|
only_pending?: boolean
|
||||||
|
} = {}
|
||||||
): Promise<RegenerateResult> => {
|
): Promise<RegenerateResult> => {
|
||||||
const response = await api.post(
|
const response = await api.post(
|
||||||
'/library/maintenance/regenerate-thumbnails',
|
'/library/maintenance/regenerate-thumbnails',
|
||||||
@@ -312,6 +328,19 @@ export const library = {
|
|||||||
return response.data
|
return response.data
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/** Dry-run count of photo rows whose files are no longer on disk
|
||||||
|
* (under a mounted source root). */
|
||||||
|
missingStats: async (): Promise<MissingStats> => {
|
||||||
|
const response = await api.get('/library/maintenance/missing-stats')
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
|
||||||
|
/** Actually delete the orphaned photo rows. */
|
||||||
|
pruneMissing: async (): Promise<PruneResult> => {
|
||||||
|
const response = await api.post('/library/maintenance/prune-missing')
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
|
||||||
/** Re-run the source-roots / folders / photos integrity cleanup that
|
/** Re-run the source-roots / folders / photos integrity cleanup that
|
||||||
* normally runs on backend startup. */
|
* normally runs on backend startup. */
|
||||||
cleanup: async (): Promise<{ status: string; message?: string }> => {
|
cleanup: async (): Promise<{ status: string; message?: string }> => {
|
||||||
|
|||||||
Reference in New Issue
Block a user