d27ec1af2e
prune_missing_photos() previously only deleted Photo rows whose files were gone, leaving every folder row from the old library in the DB — which made the sidebar tree wildly out of sync with the on-disk structure (still showing /photos/2024/, /photos/2026/03/, etc. that no longer exist). Now also drops Folder rows whose path doesn't resolve under a mounted source root, with the same defensive "skip if source root unmounted" guard. The Settings orphan card surfaces both counts. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
276 lines
10 KiB
Python
276 lines
10 KiB
Python
"""
|
|
One-shot data integrity cleanup for source_roots / folders / photos.
|
|
|
|
Earlier versions of the scanner stored paths verbatim, so trailing slashes
|
|
and redundant separators produced duplicate SourceRoot and Folder rows for
|
|
the same physical directory. The watcher also auto-created source roots
|
|
when fired with a parent dir. This module merges the duplicates and
|
|
re-points photos to the canonical folder so the data lines up with the
|
|
post-fix scanner.
|
|
|
|
Idempotent: safe to run on every backend startup.
|
|
"""
|
|
import os
|
|
import logging
|
|
from datetime import datetime
|
|
from sqlalchemy import select, update, func
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import Photo, Folder, SourceRoot
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _normalize_path(path: str) -> str:
|
|
return os.path.normpath(path)
|
|
|
|
|
|
async def _dedupe_source_roots(session: AsyncSession) -> int:
|
|
"""Group source roots by normalized path and merge duplicates. Returns
|
|
the number of rows deleted."""
|
|
result = await session.execute(select(SourceRoot))
|
|
rows = result.scalars().all()
|
|
|
|
groups: dict[str, list[SourceRoot]] = {}
|
|
for sr in rows:
|
|
norm = _normalize_path(sr.path)
|
|
groups.setdefault(norm, []).append(sr)
|
|
|
|
deleted = 0
|
|
for norm, srs in groups.items():
|
|
if len(srs) == 1:
|
|
# Make sure the canonical row's path is normalized too.
|
|
if srs[0].path != norm:
|
|
srs[0].path = norm
|
|
continue
|
|
# Pick the canonical row: prefer one with a non-empty name and the
|
|
# earliest added_at (most likely the original).
|
|
canonical = sorted(
|
|
srs,
|
|
key=lambda s: (not bool(s.name), s.added_at or datetime.max),
|
|
)[0]
|
|
canonical.path = norm
|
|
for sr in srs:
|
|
if sr.id == canonical.id:
|
|
continue
|
|
# Re-point folders that referenced the duplicate root.
|
|
await session.execute(
|
|
update(Folder)
|
|
.where(Folder.source_root_id == sr.id)
|
|
.values(source_root_id=canonical.id)
|
|
)
|
|
await session.delete(sr)
|
|
deleted += 1
|
|
|
|
return deleted
|
|
|
|
|
|
async def _dedupe_folders(session: AsyncSession) -> int:
|
|
"""Group folders by normalized path and merge duplicates. Returns the
|
|
number of rows deleted."""
|
|
result = await session.execute(select(Folder))
|
|
rows = result.scalars().all()
|
|
|
|
groups: dict[str, list[Folder]] = {}
|
|
for f in rows:
|
|
norm = _normalize_path(f.path)
|
|
groups.setdefault(norm, []).append(f)
|
|
|
|
deleted = 0
|
|
for norm, folders in groups.items():
|
|
if len(folders) == 1:
|
|
if folders[0].path != norm:
|
|
folders[0].path = norm
|
|
continue
|
|
# Canonical = the one with the most photos already attached, then
|
|
# the lowest-id (deterministic tiebreaker).
|
|
canonical = sorted(
|
|
folders,
|
|
key=lambda f: (-(f.photo_count or 0), f.id),
|
|
)[0]
|
|
canonical.path = norm
|
|
for f in folders:
|
|
if f.id == canonical.id:
|
|
continue
|
|
# Re-point photos to the canonical folder.
|
|
await session.execute(
|
|
update(Photo)
|
|
.where(Photo.folder_id == f.id)
|
|
.values(folder_id=canonical.id)
|
|
)
|
|
await session.delete(f)
|
|
deleted += 1
|
|
|
|
return deleted
|
|
|
|
|
|
async def _recompute_folder_counts(session: AsyncSession) -> None:
|
|
"""Set folder.photo_count to the actual non-discarded photo count."""
|
|
result = await session.execute(select(Folder))
|
|
folders = result.scalars().all()
|
|
for f in folders:
|
|
count_result = await session.execute(
|
|
select(func.count(Photo.id)).where(
|
|
Photo.folder_id == f.id,
|
|
Photo.is_discarded == False, # noqa: E712
|
|
)
|
|
)
|
|
f.photo_count = int(count_result.scalar() or 0)
|
|
|
|
|
|
async def _warn_stale_source_roots(session: AsyncSession) -> int:
|
|
"""Log a warning for any active source root whose path no longer exists
|
|
on disk. Doesn't delete — a missing path could be a temporarily
|
|
unmounted drive, and silently dropping user data is worse than
|
|
surfacing a noisy log line.
|
|
"""
|
|
result = await session.execute(select(SourceRoot))
|
|
rows = result.scalars().all()
|
|
stale = 0
|
|
for sr in rows:
|
|
if not os.path.isdir(sr.path):
|
|
stale += 1
|
|
logger.warning(
|
|
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
|
f"— is the docker mount still in place? "
|
|
f"(Edit docker-compose.yml or PHOTO_DIRS in .env to fix.)"
|
|
)
|
|
return stale
|
|
|
|
|
|
async def find_missing(
|
|
session: AsyncSession,
|
|
) -> tuple[list[str], list[str], list[str]]:
|
|
"""Walk every non-discarded photo + every folder and check whether
|
|
they still resolve on disk. Returns
|
|
(deletable_photo_ids, deletable_folder_ids, skipped_photo_ids).
|
|
|
|
Skipped rows are photos/folders whose owning source_root path itself
|
|
doesn't resolve — that's almost always an unmounted drive, and
|
|
silently deleting those rows would be data loss. The caller can
|
|
surface the skip count separately so the user knows the cleanup
|
|
wasn't a no-op by accident.
|
|
"""
|
|
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
|
sr_mounted: dict[str, bool] = {sr.id: os.path.isdir(sr.path) for sr in sr_rows}
|
|
|
|
photos = (await session.execute(
|
|
select(Photo.id, Photo.filepath, Photo.folder_id)
|
|
.where(Photo.is_discarded.is_(False))
|
|
)).all()
|
|
|
|
folders = (await session.execute(
|
|
select(Folder.id, Folder.path, Folder.source_root_id)
|
|
)).all()
|
|
folder_to_sr = {fid: srid for fid, _path, srid in folders}
|
|
|
|
deletable_photos: list[str] = []
|
|
skipped: list[str] = []
|
|
for pid, fp, folder_id in photos:
|
|
sr_id = folder_to_sr.get(folder_id)
|
|
if sr_id is None or not sr_mounted.get(sr_id, False):
|
|
skipped.append(pid)
|
|
continue
|
|
if not os.path.exists(fp):
|
|
deletable_photos.append(pid)
|
|
|
|
deletable_folders: list[str] = []
|
|
for fid, fpath, sr_id in folders:
|
|
if sr_id is None or not sr_mounted.get(sr_id, False):
|
|
continue
|
|
if not os.path.isdir(fpath):
|
|
deletable_folders.append(fid)
|
|
|
|
return deletable_photos, deletable_folders, skipped
|
|
|
|
|
|
async def prune_missing_photos(dry_run: bool = True) -> dict:
|
|
"""Delete photo + folder rows whose paths are no longer on disk *and*
|
|
whose source root is currently mounted. Common cause: PHOTO_DIRS in
|
|
.env was repointed at a different library, leaving every old row
|
|
orphaned.
|
|
|
|
Set dry_run=False to actually delete. The default is intentionally
|
|
safe so the matching count can be surfaced in the UI before the
|
|
user commits to it.
|
|
|
|
Function name kept for backwards compatibility — it now also prunes
|
|
folders, not just photos.
|
|
"""
|
|
from sqlalchemy import delete
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
deletable_photos, deletable_folders, skipped = await find_missing(session)
|
|
if not dry_run:
|
|
CHUNK = 500
|
|
# Photos first (folders may FK from them via folder_id).
|
|
for i in range(0, len(deletable_photos), CHUNK):
|
|
await session.execute(
|
|
delete(Photo).where(
|
|
Photo.id.in_(deletable_photos[i:i + CHUNK])
|
|
)
|
|
)
|
|
# Then drop folders that ALSO no longer have any photos
|
|
# pointing at them. We re-check after the photo delete so
|
|
# we don't strand a folder that legitimately exists on
|
|
# disk but happened to match the orphan list.
|
|
if deletable_folders:
|
|
for i in range(0, len(deletable_folders), CHUNK):
|
|
chunk = deletable_folders[i:i + CHUNK]
|
|
# Only delete folders that now have zero photos
|
|
# left attached (defensive — should always be 0
|
|
# if the path is gone, but a concurrent scan
|
|
# could re-create rows).
|
|
still_used = (await session.execute(
|
|
select(Photo.folder_id)
|
|
.where(Photo.folder_id.in_(chunk))
|
|
.distinct()
|
|
)).scalars().all()
|
|
safe = [f for f in chunk if f not in set(still_used)]
|
|
if safe:
|
|
await session.execute(
|
|
delete(Folder).where(Folder.id.in_(safe))
|
|
)
|
|
await session.commit()
|
|
logger.info(
|
|
f"Pruned {len(deletable_photos)} photo rows + "
|
|
f"{len(deletable_folders)} folder rows"
|
|
)
|
|
key_p = "would_delete" if dry_run else "deleted"
|
|
key_f = "would_delete_folders" if dry_run else "deleted_folders"
|
|
return {
|
|
key_p: len(deletable_photos),
|
|
key_f: len(deletable_folders),
|
|
"skipped_unmounted": len(skipped),
|
|
"dry_run": dry_run,
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"prune_missing_photos failed: {e}")
|
|
await session.rollback()
|
|
raise
|
|
|
|
|
|
async def cleanup_data_integrity() -> dict:
|
|
"""Top-level entry point. Runs the dedupe + count refresh in a single
|
|
transaction. Returns a small summary dict for logging."""
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
sr_deleted = await _dedupe_source_roots(session)
|
|
f_deleted = await _dedupe_folders(session)
|
|
await _recompute_folder_counts(session)
|
|
stale = await _warn_stale_source_roots(session)
|
|
await session.commit()
|
|
summary = {
|
|
"source_roots_merged": sr_deleted,
|
|
"folders_merged": f_deleted,
|
|
"source_roots_stale": stale,
|
|
}
|
|
if sr_deleted or f_deleted:
|
|
logger.info(f"Cleanup merged duplicates: {summary}")
|
|
return summary
|
|
except Exception as e:
|
|
logger.error(f"Cleanup failed: {e}")
|
|
await session.rollback()
|
|
raise
|