4bb2c959a8
prune_missing_photos previously skipped every photo whose source root path didn't resolve, on the assumption that a missing path meant the underlying drive was unmounted (and silently deleting under those conditions would be data loss). That conflated 'drive unmounted' with 'user renamed the folder in their file manager'. A library with 4,154 orphaned photo rows from a since-renamed Nextcloud folder hit exactly this case: the /nextcloud-users mount was fine, but the source root path 'Taco and Muli - 2024 onward' no longer existed because the user had renamed it to 'Photo Archive 2004-2024'. Every photo under it was reported as skipped_unmounted forever. Classify source root state as present/renamed/unmounted by checking whether the immediate parent is readable. 'renamed' is now treated as prunable; 'unmounted' still skips. Warning messages differ so the user knows which fix to apply. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
324 lines
12 KiB
Python
324 lines
12 KiB
Python
"""
|
|
One-shot data integrity cleanup for source_roots / folders / photos.
|
|
|
|
Earlier versions of the scanner stored paths verbatim, so trailing slashes
|
|
and redundant separators produced duplicate SourceRoot and Folder rows for
|
|
the same physical directory. The watcher also auto-created source roots
|
|
when fired with a parent dir. This module merges the duplicates and
|
|
re-points photos to the canonical folder so the data lines up with the
|
|
post-fix scanner.
|
|
|
|
Idempotent: safe to run on every backend startup.
|
|
"""
|
|
import os
|
|
import logging
|
|
from datetime import datetime
|
|
from sqlalchemy import select, update, func
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import Photo, Folder, SourceRoot
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _normalize_path(path: str) -> str:
|
|
return os.path.normpath(path)
|
|
|
|
|
|
async def _dedupe_source_roots(session: AsyncSession) -> int:
|
|
"""Group source roots by normalized path and merge duplicates. Returns
|
|
the number of rows deleted."""
|
|
result = await session.execute(select(SourceRoot))
|
|
rows = result.scalars().all()
|
|
|
|
groups: dict[str, list[SourceRoot]] = {}
|
|
for sr in rows:
|
|
norm = _normalize_path(sr.path)
|
|
groups.setdefault(norm, []).append(sr)
|
|
|
|
deleted = 0
|
|
for norm, srs in groups.items():
|
|
if len(srs) == 1:
|
|
# Make sure the canonical row's path is normalized too.
|
|
if srs[0].path != norm:
|
|
srs[0].path = norm
|
|
continue
|
|
# Pick the canonical row: prefer one with a non-empty name and the
|
|
# earliest added_at (most likely the original).
|
|
canonical = sorted(
|
|
srs,
|
|
key=lambda s: (not bool(s.name), s.added_at or datetime.max),
|
|
)[0]
|
|
canonical.path = norm
|
|
for sr in srs:
|
|
if sr.id == canonical.id:
|
|
continue
|
|
# Re-point folders that referenced the duplicate root.
|
|
await session.execute(
|
|
update(Folder)
|
|
.where(Folder.source_root_id == sr.id)
|
|
.values(source_root_id=canonical.id)
|
|
)
|
|
await session.delete(sr)
|
|
deleted += 1
|
|
|
|
return deleted
|
|
|
|
|
|
async def _dedupe_folders(session: AsyncSession) -> int:
|
|
"""Group folders by normalized path and merge duplicates. Returns the
|
|
number of rows deleted."""
|
|
result = await session.execute(select(Folder))
|
|
rows = result.scalars().all()
|
|
|
|
groups: dict[str, list[Folder]] = {}
|
|
for f in rows:
|
|
norm = _normalize_path(f.path)
|
|
groups.setdefault(norm, []).append(f)
|
|
|
|
deleted = 0
|
|
for norm, folders in groups.items():
|
|
if len(folders) == 1:
|
|
if folders[0].path != norm:
|
|
folders[0].path = norm
|
|
continue
|
|
# Canonical = the one with the most photos already attached, then
|
|
# the lowest-id (deterministic tiebreaker).
|
|
canonical = sorted(
|
|
folders,
|
|
key=lambda f: (-(f.photo_count or 0), f.id),
|
|
)[0]
|
|
canonical.path = norm
|
|
for f in folders:
|
|
if f.id == canonical.id:
|
|
continue
|
|
# Re-point photos to the canonical folder.
|
|
await session.execute(
|
|
update(Photo)
|
|
.where(Photo.folder_id == f.id)
|
|
.values(folder_id=canonical.id)
|
|
)
|
|
await session.delete(f)
|
|
deleted += 1
|
|
|
|
return deleted
|
|
|
|
|
|
async def _recompute_folder_counts(session: AsyncSession) -> None:
|
|
"""Set folder.photo_count to the actual non-discarded photo count."""
|
|
result = await session.execute(select(Folder))
|
|
folders = result.scalars().all()
|
|
for f in folders:
|
|
count_result = await session.execute(
|
|
select(func.count(Photo.id)).where(
|
|
Photo.folder_id == f.id,
|
|
Photo.is_discarded == False, # noqa: E712
|
|
)
|
|
)
|
|
f.photo_count = int(count_result.scalar() or 0)
|
|
|
|
|
|
def _parent_is_accessible(path: str) -> bool:
|
|
"""True if the parent directory of `path` is readable. Used to
|
|
distinguish 'user renamed/deleted the source root folder' (parent
|
|
mount fine, leaf gone) from 'drive unmounted' (whole subtree
|
|
inaccessible). The former is safe to prune from; the latter is
|
|
not."""
|
|
parent = os.path.dirname(path.rstrip(os.sep))
|
|
if not parent:
|
|
return False
|
|
try:
|
|
os.listdir(parent)
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _sr_state(sr_path: str) -> str:
|
|
"""Classify a source root path as one of:
|
|
'present' — directory exists, business as usual
|
|
'renamed' — leaf missing but parent mount is accessible (user
|
|
renamed/deleted the folder in their file manager)
|
|
'unmounted'— parent itself inaccessible (drive not mounted)
|
|
"""
|
|
if os.path.isdir(sr_path):
|
|
return 'present'
|
|
if _parent_is_accessible(sr_path):
|
|
return 'renamed'
|
|
return 'unmounted'
|
|
|
|
|
|
async def _warn_stale_source_roots(session: AsyncSession) -> int:
|
|
"""Log a warning for any active source root whose path no longer exists
|
|
on disk. Doesn't delete — a missing path could be a temporarily
|
|
unmounted drive, and silently dropping user data is worse than
|
|
surfacing a noisy log line. Logs different hints for renamed-vs-
|
|
unmounted so the user knows which knob to turn.
|
|
"""
|
|
result = await session.execute(select(SourceRoot))
|
|
rows = result.scalars().all()
|
|
stale = 0
|
|
for sr in rows:
|
|
state = _sr_state(sr.path)
|
|
if state == 'present':
|
|
continue
|
|
stale += 1
|
|
if state == 'renamed':
|
|
logger.warning(
|
|
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
|
f"— parent mount is fine, looks like the folder was renamed "
|
|
f"or deleted. Photos under it can be cleared via "
|
|
f"POST /api/v1/library/maintenance/prune-missing."
|
|
)
|
|
else:
|
|
logger.warning(
|
|
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
|
f"— parent directory is also inaccessible; is the docker "
|
|
f"mount still in place? (Edit docker-compose.yml or "
|
|
f"PHOTO_DIRS in .env to fix.)"
|
|
)
|
|
return stale
|
|
|
|
|
|
async def find_missing(
|
|
session: AsyncSession,
|
|
) -> tuple[list[str], list[str], list[str]]:
|
|
"""Walk every non-discarded photo + every folder and check whether
|
|
they still resolve on disk. Returns
|
|
(deletable_photo_ids, deletable_folder_ids, skipped_photo_ids).
|
|
|
|
Skipped rows are photos/folders whose owning source_root is truly
|
|
inaccessible (parent mount missing) — that's almost always an
|
|
unmounted drive, and silently deleting those rows would be data
|
|
loss. Photos under a source root whose leaf is missing but whose
|
|
parent mount IS accessible (user renamed/deleted the folder) are
|
|
treated as deletable, since their files are genuinely gone from
|
|
the user's library.
|
|
"""
|
|
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
|
# "Available" = source root path exists OR parent mount is accessible.
|
|
# Only truly-unmounted source roots skip pruning.
|
|
sr_mounted: dict[str, bool] = {
|
|
sr.id: _sr_state(sr.path) != 'unmounted' for sr in sr_rows
|
|
}
|
|
|
|
photos = (await session.execute(
|
|
select(Photo.id, Photo.filepath, Photo.folder_id)
|
|
.where(Photo.is_discarded.is_(False))
|
|
)).all()
|
|
|
|
folders = (await session.execute(
|
|
select(Folder.id, Folder.path, Folder.source_root_id)
|
|
)).all()
|
|
folder_to_sr = {fid: srid for fid, _path, srid in folders}
|
|
|
|
deletable_photos: list[str] = []
|
|
skipped: list[str] = []
|
|
for pid, fp, folder_id in photos:
|
|
sr_id = folder_to_sr.get(folder_id)
|
|
if sr_id is None or not sr_mounted.get(sr_id, False):
|
|
skipped.append(pid)
|
|
continue
|
|
if not os.path.exists(fp):
|
|
deletable_photos.append(pid)
|
|
|
|
deletable_folders: list[str] = []
|
|
for fid, fpath, sr_id in folders:
|
|
if sr_id is None or not sr_mounted.get(sr_id, False):
|
|
continue
|
|
if not os.path.isdir(fpath):
|
|
deletable_folders.append(fid)
|
|
|
|
return deletable_photos, deletable_folders, skipped
|
|
|
|
|
|
async def prune_missing_photos(dry_run: bool = True) -> dict:
|
|
"""Delete photo + folder rows whose paths are no longer on disk *and*
|
|
whose source root is currently mounted. Common cause: PHOTO_DIRS in
|
|
.env was repointed at a different library, leaving every old row
|
|
orphaned.
|
|
|
|
Set dry_run=False to actually delete. The default is intentionally
|
|
safe so the matching count can be surfaced in the UI before the
|
|
user commits to it.
|
|
|
|
Function name kept for backwards compatibility — it now also prunes
|
|
folders, not just photos.
|
|
"""
|
|
from sqlalchemy import delete
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
deletable_photos, deletable_folders, skipped = await find_missing(session)
|
|
if not dry_run:
|
|
CHUNK = 500
|
|
# Photos first (folders may FK from them via folder_id).
|
|
for i in range(0, len(deletable_photos), CHUNK):
|
|
await session.execute(
|
|
delete(Photo).where(
|
|
Photo.id.in_(deletable_photos[i:i + CHUNK])
|
|
)
|
|
)
|
|
# Then drop folders that ALSO no longer have any photos
|
|
# pointing at them. We re-check after the photo delete so
|
|
# we don't strand a folder that legitimately exists on
|
|
# disk but happened to match the orphan list.
|
|
if deletable_folders:
|
|
for i in range(0, len(deletable_folders), CHUNK):
|
|
chunk = deletable_folders[i:i + CHUNK]
|
|
# Only delete folders that now have zero photos
|
|
# left attached (defensive — should always be 0
|
|
# if the path is gone, but a concurrent scan
|
|
# could re-create rows).
|
|
still_used = (await session.execute(
|
|
select(Photo.folder_id)
|
|
.where(Photo.folder_id.in_(chunk))
|
|
.distinct()
|
|
)).scalars().all()
|
|
safe = [f for f in chunk if f not in set(still_used)]
|
|
if safe:
|
|
await session.execute(
|
|
delete(Folder).where(Folder.id.in_(safe))
|
|
)
|
|
await session.commit()
|
|
logger.info(
|
|
f"Pruned {len(deletable_photos)} photo rows + "
|
|
f"{len(deletable_folders)} folder rows"
|
|
)
|
|
key_p = "would_delete" if dry_run else "deleted"
|
|
key_f = "would_delete_folders" if dry_run else "deleted_folders"
|
|
return {
|
|
key_p: len(deletable_photos),
|
|
key_f: len(deletable_folders),
|
|
"skipped_unmounted": len(skipped),
|
|
"dry_run": dry_run,
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"prune_missing_photos failed: {e}")
|
|
await session.rollback()
|
|
raise
|
|
|
|
|
|
async def cleanup_data_integrity() -> dict:
|
|
"""Top-level entry point. Runs the dedupe + count refresh in a single
|
|
transaction. Returns a small summary dict for logging."""
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
sr_deleted = await _dedupe_source_roots(session)
|
|
f_deleted = await _dedupe_folders(session)
|
|
await _recompute_folder_counts(session)
|
|
stale = await _warn_stale_source_roots(session)
|
|
await session.commit()
|
|
summary = {
|
|
"source_roots_merged": sr_deleted,
|
|
"folders_merged": f_deleted,
|
|
"source_roots_stale": stale,
|
|
}
|
|
if sr_deleted or f_deleted:
|
|
logger.info(f"Cleanup merged duplicates: {summary}")
|
|
return summary
|
|
except Exception as e:
|
|
logger.error(f"Cleanup failed: {e}")
|
|
await session.rollback()
|
|
raise
|