diff --git a/backend/app/main.py b/backend/app/main.py index 158891a..fa8cc73 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -13,6 +13,7 @@ from app.config import settings from app.database import init_db from app.routers import photos, folders, heaps, tags, discard, library from app.services.scanner import start_initial_scan +from app.services.cleanup import cleanup_data_integrity # Configure logging logging.basicConfig( @@ -25,17 +26,24 @@ logger = logging.getLogger(__name__) async def lifespan(app: FastAPI): """Manage application lifecycle""" logger.info("Starting Mulita application...") - + # Initialize database await init_db() - + + # One-shot cleanup of duplicate source_roots / folders left over from + # earlier scanner versions that didn't normalize paths. Idempotent. + try: + await cleanup_data_integrity() + except Exception as e: + logger.error(f"Startup cleanup failed (continuing): {e}") + # Start initial scan if configured if settings.scanner.initial_scan_on_start: logger.info("Starting initial library scan...") await start_initial_scan() - + yield - + logger.info("Shutting down Mulita application...") # Create FastAPI app diff --git a/backend/app/services/cleanup.py b/backend/app/services/cleanup.py new file mode 100644 index 0000000..d5261d5 --- /dev/null +++ b/backend/app/services/cleanup.py @@ -0,0 +1,141 @@ +""" +One-shot data integrity cleanup for source_roots / folders / photos. + +Earlier versions of the scanner stored paths verbatim, so trailing slashes +and redundant separators produced duplicate SourceRoot and Folder rows for +the same physical directory. The watcher also auto-created source roots +when fired with a parent dir. This module merges the duplicates and +re-points photos to the canonical folder so the data lines up with the +post-fix scanner. + +Idempotent: safe to run on every backend startup. +""" +import os +import logging +from datetime import datetime +from sqlalchemy import select, update, func +from sqlalchemy.ext.asyncio import AsyncSession + +from app.database import AsyncSessionLocal +from app.models import Photo, Folder, SourceRoot + +logger = logging.getLogger(__name__) + + +def _normalize_path(path: str) -> str: + return os.path.normpath(path) + + +async def _dedupe_source_roots(session: AsyncSession) -> int: + """Group source roots by normalized path and merge duplicates. Returns + the number of rows deleted.""" + result = await session.execute(select(SourceRoot)) + rows = result.scalars().all() + + groups: dict[str, list[SourceRoot]] = {} + for sr in rows: + norm = _normalize_path(sr.path) + groups.setdefault(norm, []).append(sr) + + deleted = 0 + for norm, srs in groups.items(): + if len(srs) == 1: + # Make sure the canonical row's path is normalized too. + if srs[0].path != norm: + srs[0].path = norm + continue + # Pick the canonical row: prefer one with a non-empty name and the + # earliest added_at (most likely the original). + canonical = sorted( + srs, + key=lambda s: (not bool(s.name), s.added_at or datetime.max), + )[0] + canonical.path = norm + for sr in srs: + if sr.id == canonical.id: + continue + # Re-point folders that referenced the duplicate root. + await session.execute( + update(Folder) + .where(Folder.source_root_id == sr.id) + .values(source_root_id=canonical.id) + ) + await session.delete(sr) + deleted += 1 + + return deleted + + +async def _dedupe_folders(session: AsyncSession) -> int: + """Group folders by normalized path and merge duplicates. Returns the + number of rows deleted.""" + result = await session.execute(select(Folder)) + rows = result.scalars().all() + + groups: dict[str, list[Folder]] = {} + for f in rows: + norm = _normalize_path(f.path) + groups.setdefault(norm, []).append(f) + + deleted = 0 + for norm, folders in groups.items(): + if len(folders) == 1: + if folders[0].path != norm: + folders[0].path = norm + continue + # Canonical = the one with the most photos already attached, then + # the lowest-id (deterministic tiebreaker). + canonical = sorted( + folders, + key=lambda f: (-(f.photo_count or 0), f.id), + )[0] + canonical.path = norm + for f in folders: + if f.id == canonical.id: + continue + # Re-point photos to the canonical folder. + await session.execute( + update(Photo) + .where(Photo.folder_id == f.id) + .values(folder_id=canonical.id) + ) + await session.delete(f) + deleted += 1 + + return deleted + + +async def _recompute_folder_counts(session: AsyncSession) -> None: + """Set folder.photo_count to the actual non-discarded photo count.""" + result = await session.execute(select(Folder)) + folders = result.scalars().all() + for f in folders: + count_result = await session.execute( + select(func.count(Photo.id)).where( + Photo.folder_id == f.id, + Photo.is_discarded == False, # noqa: E712 + ) + ) + f.photo_count = int(count_result.scalar() or 0) + + +async def cleanup_data_integrity() -> dict: + """Top-level entry point. Runs the dedupe + count refresh in a single + transaction. Returns a small summary dict for logging.""" + async with AsyncSessionLocal() as session: + try: + sr_deleted = await _dedupe_source_roots(session) + f_deleted = await _dedupe_folders(session) + await _recompute_folder_counts(session) + await session.commit() + summary = { + "source_roots_merged": sr_deleted, + "folders_merged": f_deleted, + } + if sr_deleted or f_deleted: + logger.info(f"Cleanup merged duplicates: {summary}") + return summary + except Exception as e: + logger.error(f"Cleanup failed: {e}") + await session.rollback() + raise diff --git a/backend/app/tasks/scan.py b/backend/app/tasks/scan.py index c21f19f..ad691eb 100644 --- a/backend/app/tasks/scan.py +++ b/backend/app/tasks/scan.py @@ -181,39 +181,49 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta await session.rollback() raise +def _normalize_path(path: str) -> str: + """Canonicalise a filesystem path so we don't get duplicate DB rows for + the same physical directory due to trailing slashes, redundant separators, + or `.` segments. Symlinks are NOT resolved (we want to keep mount paths + intact for cross-machine portability).""" + return os.path.normpath(path) + + async def get_or_create_source_root(session: AsyncSession, path: str) -> SourceRoot: - """Get or create a source root entry""" + """Get or create a source root entry, matching by normalized path.""" from sqlalchemy import select - + + norm = _normalize_path(path) result = await session.execute( - select(SourceRoot).where(SourceRoot.path == path) + select(SourceRoot).where(SourceRoot.path == norm) ) source_root = result.scalar_one_or_none() - + if not source_root: source_root = SourceRoot( - name=Path(path).name, - path=path + name=Path(norm).name, + path=norm, ) session.add(source_root) await session.flush() - + return source_root + async def get_or_create_folder(session: AsyncSession, path: str, source_root_id: str) -> Folder: - """Get or create a folder entry""" + """Get or create a folder entry, matching by normalized path.""" from sqlalchemy import select - + + norm = _normalize_path(path) result = await session.execute( - select(Folder).where(Folder.path == path) + select(Folder).where(Folder.path == norm) ) folder = result.scalar_one_or_none() - + if not folder: - parent_path = str(Path(path).parent) - parent = None - - if parent_path != path: # Not root folder + parent_path = _normalize_path(str(Path(norm).parent)) + + if parent_path != norm: # Not the filesystem root parent_result = await session.execute( select(Folder).where(Folder.path == parent_path) ) @@ -226,16 +236,16 @@ async def get_or_create_folder(session: AsyncSession, path: str, source_root_id: parent_id = parent.id else: parent_id = None - + folder = Folder( - name=Path(path).name, - path=path, + name=Path(norm).name, + path=norm, parent_id=parent_id, - source_root_id=source_root_id + source_root_id=source_root_id, ) session.add(folder) await session.flush() - + return folder @shared_task(name='scan_all_source_roots')