""" One-shot data integrity cleanup for source_roots / folders / photos. Earlier versions of the scanner stored paths verbatim, so trailing slashes and redundant separators produced duplicate SourceRoot and Folder rows for the same physical directory. The watcher also auto-created source roots when fired with a parent dir. This module merges the duplicates and re-points photos to the canonical folder so the data lines up with the post-fix scanner. Idempotent: safe to run on every backend startup. """ import os import logging from datetime import datetime from sqlalchemy import select, update, func from sqlalchemy.ext.asyncio import AsyncSession from app.database import AsyncSessionLocal from app.models import Photo, Folder, SourceRoot logger = logging.getLogger(__name__) def _normalize_path(path: str) -> str: return os.path.normpath(path) async def _dedupe_source_roots(session: AsyncSession) -> int: """Group source roots by normalized path and merge duplicates. Returns the number of rows deleted.""" result = await session.execute(select(SourceRoot)) rows = result.scalars().all() groups: dict[str, list[SourceRoot]] = {} for sr in rows: norm = _normalize_path(sr.path) groups.setdefault(norm, []).append(sr) deleted = 0 for norm, srs in groups.items(): if len(srs) == 1: # Make sure the canonical row's path is normalized too. if srs[0].path != norm: srs[0].path = norm continue # Pick the canonical row: prefer one with a non-empty name and the # earliest added_at (most likely the original). canonical = sorted( srs, key=lambda s: (not bool(s.name), s.added_at or datetime.max), )[0] canonical.path = norm for sr in srs: if sr.id == canonical.id: continue # Re-point folders that referenced the duplicate root. await session.execute( update(Folder) .where(Folder.source_root_id == sr.id) .values(source_root_id=canonical.id) ) await session.delete(sr) deleted += 1 return deleted async def _dedupe_folders(session: AsyncSession) -> int: """Group folders by normalized path and merge duplicates. Returns the number of rows deleted.""" result = await session.execute(select(Folder)) rows = result.scalars().all() groups: dict[str, list[Folder]] = {} for f in rows: norm = _normalize_path(f.path) groups.setdefault(norm, []).append(f) deleted = 0 for norm, folders in groups.items(): if len(folders) == 1: if folders[0].path != norm: folders[0].path = norm continue # Canonical = the one with the most photos already attached, then # the lowest-id (deterministic tiebreaker). canonical = sorted( folders, key=lambda f: (-(f.photo_count or 0), f.id), )[0] canonical.path = norm for f in folders: if f.id == canonical.id: continue # Re-point photos to the canonical folder. await session.execute( update(Photo) .where(Photo.folder_id == f.id) .values(folder_id=canonical.id) ) await session.delete(f) deleted += 1 return deleted async def _recompute_folder_counts(session: AsyncSession) -> None: """Set folder.photo_count to the actual non-discarded photo count.""" result = await session.execute(select(Folder)) folders = result.scalars().all() for f in folders: count_result = await session.execute( select(func.count(Photo.id)).where( Photo.folder_id == f.id, Photo.is_discarded == False, # noqa: E712 ) ) f.photo_count = int(count_result.scalar() or 0) def _parent_is_accessible(path: str) -> bool: """True if the parent directory of `path` is readable. Used to distinguish 'user renamed/deleted the source root folder' (parent mount fine, leaf gone) from 'drive unmounted' (whole subtree inaccessible). The former is safe to prune from; the latter is not.""" parent = os.path.dirname(path.rstrip(os.sep)) if not parent: return False try: os.listdir(parent) return True except OSError: return False def _sr_state(sr_path: str) -> str: """Classify a source root path as one of: 'present' — directory exists, business as usual 'renamed' — leaf missing but parent mount is accessible (user renamed/deleted the folder in their file manager) 'unmounted'— parent itself inaccessible (drive not mounted) """ if os.path.isdir(sr_path): return 'present' if _parent_is_accessible(sr_path): return 'renamed' return 'unmounted' async def _warn_stale_source_roots(session: AsyncSession) -> int: """Log a warning for any active source root whose path no longer exists on disk. Doesn't delete — a missing path could be a temporarily unmounted drive, and silently dropping user data is worse than surfacing a noisy log line. Logs different hints for renamed-vs- unmounted so the user knows which knob to turn. """ result = await session.execute(select(SourceRoot)) rows = result.scalars().all() stale = 0 for sr in rows: state = _sr_state(sr.path) if state == 'present': continue stale += 1 if state == 'renamed': logger.warning( f"Source root '{sr.name}' path is missing on disk: {sr.path} " f"— parent mount is fine, looks like the folder was renamed " f"or deleted. Photos under it can be cleared via " f"POST /api/v1/library/maintenance/prune-missing." ) else: logger.warning( f"Source root '{sr.name}' path is missing on disk: {sr.path} " f"— parent directory is also inaccessible; is the docker " f"mount still in place? (Edit docker-compose.yml or " f"PHOTO_DIRS in .env to fix.)" ) return stale async def find_missing( session: AsyncSession, ) -> tuple[list[str], list[str], list[str]]: """Walk every non-discarded photo + every folder and check whether they still resolve on disk. Returns (deletable_photo_ids, deletable_folder_ids, skipped_photo_ids). Skipped rows are photos/folders whose owning source_root is truly inaccessible (parent mount missing) — that's almost always an unmounted drive, and silently deleting those rows would be data loss. Photos under a source root whose leaf is missing but whose parent mount IS accessible (user renamed/deleted the folder) are treated as deletable, since their files are genuinely gone from the user's library. """ sr_rows = (await session.execute(select(SourceRoot))).scalars().all() # "Available" = source root path exists OR parent mount is accessible. # Only truly-unmounted source roots skip pruning. sr_mounted: dict[str, bool] = { sr.id: _sr_state(sr.path) != 'unmounted' for sr in sr_rows } photos = (await session.execute( select(Photo.id, Photo.filepath, Photo.folder_id) .where(Photo.is_discarded.is_(False)) )).all() folders = (await session.execute( select(Folder.id, Folder.path, Folder.source_root_id) )).all() folder_to_sr = {fid: srid for fid, _path, srid in folders} deletable_photos: list[str] = [] skipped: list[str] = [] for pid, fp, folder_id in photos: sr_id = folder_to_sr.get(folder_id) if sr_id is None or not sr_mounted.get(sr_id, False): skipped.append(pid) continue if not os.path.exists(fp): deletable_photos.append(pid) deletable_folders: list[str] = [] for fid, fpath, sr_id in folders: if sr_id is None or not sr_mounted.get(sr_id, False): continue if not os.path.isdir(fpath): deletable_folders.append(fid) return deletable_photos, deletable_folders, skipped async def prune_missing_photos(dry_run: bool = True) -> dict: """Delete photo + folder rows whose paths are no longer on disk *and* whose source root is currently mounted. Common cause: PHOTO_DIRS in .env was repointed at a different library, leaving every old row orphaned. Set dry_run=False to actually delete. The default is intentionally safe so the matching count can be surfaced in the UI before the user commits to it. Function name kept for backwards compatibility — it now also prunes folders, not just photos. """ from sqlalchemy import delete async with AsyncSessionLocal() as session: try: deletable_photos, deletable_folders, skipped = await find_missing(session) if not dry_run: CHUNK = 500 # Photos first (folders may FK from them via folder_id). for i in range(0, len(deletable_photos), CHUNK): await session.execute( delete(Photo).where( Photo.id.in_(deletable_photos[i:i + CHUNK]) ) ) # Then drop folders that ALSO no longer have any photos # pointing at them. We re-check after the photo delete so # we don't strand a folder that legitimately exists on # disk but happened to match the orphan list. if deletable_folders: for i in range(0, len(deletable_folders), CHUNK): chunk = deletable_folders[i:i + CHUNK] # Only delete folders that now have zero photos # left attached (defensive — should always be 0 # if the path is gone, but a concurrent scan # could re-create rows). still_used = (await session.execute( select(Photo.folder_id) .where(Photo.folder_id.in_(chunk)) .distinct() )).scalars().all() safe = [f for f in chunk if f not in set(still_used)] if safe: await session.execute( delete(Folder).where(Folder.id.in_(safe)) ) await session.commit() logger.info( f"Pruned {len(deletable_photos)} photo rows + " f"{len(deletable_folders)} folder rows" ) key_p = "would_delete" if dry_run else "deleted" key_f = "would_delete_folders" if dry_run else "deleted_folders" return { key_p: len(deletable_photos), key_f: len(deletable_folders), "skipped_unmounted": len(skipped), "dry_run": dry_run, } except Exception as e: logger.error(f"prune_missing_photos failed: {e}") await session.rollback() raise async def prune_orphan_thumbnails( thumbs_root: str = "/data/thumbs", dry_run: bool = True, ) -> dict: """Remove `/data/thumbs/{user_id}/{photo_id}/` directories whose photo_id no longer exists in the photos table. Layout was per-Phase-4 set up by app.tasks.thumbs and is keyed by `{user_id}/{photo_id}/`. The thumbs worker never deletes its own output on photo removal, so over the lifetime of a library these directories accumulate. Set dry_run=False to actually `rm -rf` each matched directory. Returns counts of matched / removed dirs and any per-dir errors. """ import shutil if not os.path.isdir(thumbs_root): return { "would_remove": 0, "removed": 0, "skipped_no_root": True, "dry_run": dry_run, } async with AsyncSessionLocal() as session: live_ids = { row[0] for row in (await session.execute(select(Photo.id))).all() } matched: list[str] = [] errors: list[str] = [] for user_dir in os.listdir(thumbs_root): user_path = os.path.join(thumbs_root, user_dir) if not os.path.isdir(user_path): continue for photo_dir in os.listdir(user_path): if photo_dir in live_ids: continue matched.append(os.path.join(user_path, photo_dir)) removed = 0 if not dry_run: for path in matched: try: shutil.rmtree(path) removed += 1 except OSError as e: errors.append(f"{path}: {e}") key = "would_remove" if dry_run else "removed" return { key: len(matched) if dry_run else removed, "errors": errors, "dry_run": dry_run, } async def discard_missing_photos() -> dict: """Soft variant of prune_missing_photos for the periodic beat catch-up. Walks every active source root that is currently `present` (not 'renamed' — the user-driven manual flow handles those — and not 'unmounted'), and for each Photo whose file is gone from disk, sets is_discarded=True so it shows up in the in-app trash. Idempotent: skips photos that are already discarded. Hard-deletion stays manual via prune_missing_photos so users can review the list before committing. """ async with AsyncSessionLocal() as session: try: sr_rows = (await session.execute(select(SourceRoot))).scalars().all() present_sr_ids = { sr.id for sr in sr_rows if _sr_state(sr.path) == 'present' } if not present_sr_ids: return {"discarded": 0, "checked": 0} folder_to_sr = { fid: srid for fid, _path, srid in (await session.execute( select(Folder.id, Folder.path, Folder.source_root_id) )).all() } photo_rows = (await session.execute( select(Photo.id, Photo.filepath, Photo.folder_id) .where(Photo.is_discarded.is_(False)) )).all() missing_ids: list[str] = [] checked = 0 for pid, fp, folder_id in photo_rows: sr_id = folder_to_sr.get(folder_id) if sr_id not in present_sr_ids: continue checked += 1 if not os.path.exists(fp): missing_ids.append(pid) if missing_ids: CHUNK = 500 now = datetime.utcnow() for i in range(0, len(missing_ids), CHUNK): await session.execute( update(Photo) .where(Photo.id.in_(missing_ids[i:i + CHUNK])) .values(is_discarded=True, discarded_at=now) ) await session.commit() logger.info( f"discard_missing_photos: discarded {len(missing_ids)} " f"of {checked} photos under {len(present_sr_ids)} present source roots" ) return {"discarded": len(missing_ids), "checked": checked} except Exception as e: logger.error(f"discard_missing_photos failed: {e}") await session.rollback() raise async def cleanup_data_integrity() -> dict: """Top-level entry point. Runs the dedupe + count refresh in a single transaction. Returns a small summary dict for logging.""" async with AsyncSessionLocal() as session: try: sr_deleted = await _dedupe_source_roots(session) f_deleted = await _dedupe_folders(session) await _recompute_folder_counts(session) stale = await _warn_stale_source_roots(session) await session.commit() summary = { "source_roots_merged": sr_deleted, "folders_merged": f_deleted, "source_roots_stale": stale, } if sr_deleted or f_deleted: logger.info(f"Cleanup merged duplicates: {summary}") return summary except Exception as e: logger.error(f"Cleanup failed: {e}") await session.rollback() raise