fix: deduplicate scanner-created source_roots and folders

Two related fixes:

1. Prevention — scanner now normalizes paths before lookup/insert
   in get_or_create_source_root and get_or_create_folder. Trailing
   slashes, redundant separators, and `.` segments all collapse to
   the same row. _normalize_path uses os.path.normpath; symlinks
   are intentionally NOT resolved so mount paths stay intact for
   cross-machine portability.

2. Cleanup — new app/services/cleanup.py runs on backend startup
   (idempotent) and merges any pre-existing duplicates left over
   from older scanner versions:
   - Groups source_roots by normalized path. Picks the canonical
     row (preferring one with a non-empty name and the earliest
     added_at), re-points child Folder rows via UPDATE, and
     deletes the duplicates.
   - Same for folders, with photo_count as the tiebreaker. Photos
     get re-pointed to the canonical folder via UPDATE.
   - Recomputes folder.photo_count from the actual non-discarded
     photo membership so the sidebar count matches reality.

Wired into main.py's lifespan handler. On the dev DB this merged
the empty-name "/host/Pictures/MulitaTest/" duplicate that was
showing up alongside the canonical MulitaTest source root.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-07 23:43:53 +02:00
parent 7dcfa8f30d
commit cf7c72d437
3 changed files with 183 additions and 24 deletions

View File

@@ -181,39 +181,49 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
await session.rollback()
raise
def _normalize_path(path: str) -> str:
"""Canonicalise a filesystem path so we don't get duplicate DB rows for
the same physical directory due to trailing slashes, redundant separators,
or `.` segments. Symlinks are NOT resolved (we want to keep mount paths
intact for cross-machine portability)."""
return os.path.normpath(path)
async def get_or_create_source_root(session: AsyncSession, path: str) -> SourceRoot:
"""Get or create a source root entry"""
"""Get or create a source root entry, matching by normalized path."""
from sqlalchemy import select
norm = _normalize_path(path)
result = await session.execute(
select(SourceRoot).where(SourceRoot.path == path)
select(SourceRoot).where(SourceRoot.path == norm)
)
source_root = result.scalar_one_or_none()
if not source_root:
source_root = SourceRoot(
name=Path(path).name,
path=path
name=Path(norm).name,
path=norm,
)
session.add(source_root)
await session.flush()
return source_root
async def get_or_create_folder(session: AsyncSession, path: str, source_root_id: str) -> Folder:
"""Get or create a folder entry"""
"""Get or create a folder entry, matching by normalized path."""
from sqlalchemy import select
norm = _normalize_path(path)
result = await session.execute(
select(Folder).where(Folder.path == path)
select(Folder).where(Folder.path == norm)
)
folder = result.scalar_one_or_none()
if not folder:
parent_path = str(Path(path).parent)
parent = None
if parent_path != path: # Not root folder
parent_path = _normalize_path(str(Path(norm).parent))
if parent_path != norm: # Not the filesystem root
parent_result = await session.execute(
select(Folder).where(Folder.path == parent_path)
)
@@ -226,16 +236,16 @@ async def get_or_create_folder(session: AsyncSession, path: str, source_root_id:
parent_id = parent.id
else:
parent_id = None
folder = Folder(
name=Path(path).name,
path=path,
name=Path(norm).name,
path=norm,
parent_id=parent_id,
source_root_id=source_root_id
source_root_id=source_root_id,
)
session.add(folder)
await session.flush()
return folder
@shared_task(name='scan_all_source_roots')