fix: deduplicate scanner-created source_roots and folders
Two related fixes:
1. Prevention — scanner now normalizes paths before lookup/insert
in get_or_create_source_root and get_or_create_folder. Trailing
slashes, redundant separators, and `.` segments all collapse to
the same row. _normalize_path uses os.path.normpath; symlinks
are intentionally NOT resolved so mount paths stay intact for
cross-machine portability.
2. Cleanup — new app/services/cleanup.py runs on backend startup
(idempotent) and merges any pre-existing duplicates left over
from older scanner versions:
- Groups source_roots by normalized path. Picks the canonical
row (preferring one with a non-empty name and the earliest
added_at), re-points child Folder rows via UPDATE, and
deletes the duplicates.
- Same for folders, with photo_count as the tiebreaker. Photos
get re-pointed to the canonical folder via UPDATE.
- Recomputes folder.photo_count from the actual non-discarded
photo membership so the sidebar count matches reality.
Wired into main.py's lifespan handler. On the dev DB this merged
the empty-name "/host/Pictures/MulitaTest/" duplicate that was
showing up alongside the canonical MulitaTest source root.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -181,39 +181,49 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
"""Canonicalise a filesystem path so we don't get duplicate DB rows for
|
||||
the same physical directory due to trailing slashes, redundant separators,
|
||||
or `.` segments. Symlinks are NOT resolved (we want to keep mount paths
|
||||
intact for cross-machine portability)."""
|
||||
return os.path.normpath(path)
|
||||
|
||||
|
||||
async def get_or_create_source_root(session: AsyncSession, path: str) -> SourceRoot:
|
||||
"""Get or create a source root entry"""
|
||||
"""Get or create a source root entry, matching by normalized path."""
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
norm = _normalize_path(path)
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.path == path)
|
||||
select(SourceRoot).where(SourceRoot.path == norm)
|
||||
)
|
||||
source_root = result.scalar_one_or_none()
|
||||
|
||||
|
||||
if not source_root:
|
||||
source_root = SourceRoot(
|
||||
name=Path(path).name,
|
||||
path=path
|
||||
name=Path(norm).name,
|
||||
path=norm,
|
||||
)
|
||||
session.add(source_root)
|
||||
await session.flush()
|
||||
|
||||
|
||||
return source_root
|
||||
|
||||
|
||||
async def get_or_create_folder(session: AsyncSession, path: str, source_root_id: str) -> Folder:
|
||||
"""Get or create a folder entry"""
|
||||
"""Get or create a folder entry, matching by normalized path."""
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
norm = _normalize_path(path)
|
||||
result = await session.execute(
|
||||
select(Folder).where(Folder.path == path)
|
||||
select(Folder).where(Folder.path == norm)
|
||||
)
|
||||
folder = result.scalar_one_or_none()
|
||||
|
||||
|
||||
if not folder:
|
||||
parent_path = str(Path(path).parent)
|
||||
parent = None
|
||||
|
||||
if parent_path != path: # Not root folder
|
||||
parent_path = _normalize_path(str(Path(norm).parent))
|
||||
|
||||
if parent_path != norm: # Not the filesystem root
|
||||
parent_result = await session.execute(
|
||||
select(Folder).where(Folder.path == parent_path)
|
||||
)
|
||||
@@ -226,16 +236,16 @@ async def get_or_create_folder(session: AsyncSession, path: str, source_root_id:
|
||||
parent_id = parent.id
|
||||
else:
|
||||
parent_id = None
|
||||
|
||||
|
||||
folder = Folder(
|
||||
name=Path(path).name,
|
||||
path=path,
|
||||
name=Path(norm).name,
|
||||
path=norm,
|
||||
parent_id=parent_id,
|
||||
source_root_id=source_root_id
|
||||
source_root_id=source_root_id,
|
||||
)
|
||||
session.add(folder)
|
||||
await session.flush()
|
||||
|
||||
|
||||
return folder
|
||||
|
||||
@shared_task(name='scan_all_source_roots')
|
||||
|
||||
Reference in New Issue
Block a user