a27267f7ad
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
- backend: app/services/vision/, app/tasks/vision.py,
app/services/feature_flags.py, app/routers/features.py — all
deleted; admin AI/feature-flag endpoints and the worker-vision
bootstrap call gone. Photo.needs_review and its index dropped.
- frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
types, "Needs Review" sidebar entry + filter, needs_review filter
URL param all gone.
- infra: worker-vision compose service + models_data volume deleted;
worker-light command no longer runs bootstrap_models; the db
image switches from pgvector/pgvector:pg16 to postgres:16; backend
Dockerfile drops the dedicated torch RUN layer; requirements.txt
drops torch/torchvision/open-clip-torch/onnxruntime.
Alembic 0019_drop_ai_remnants:
- drops photos.needs_review + ix_photos_needs_review
- DROP EXTENSION IF EXISTS vector (must run before the image swap;
the new postgres:16 doesn't ship pgvector)
New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
448 lines
16 KiB
Python
448 lines
16 KiB
Python
"""
|
|
One-shot data integrity cleanup for source_roots / folders / photos.
|
|
|
|
Earlier versions of the scanner stored paths verbatim, so trailing slashes
|
|
and redundant separators produced duplicate SourceRoot and Folder rows for
|
|
the same physical directory. The watcher also auto-created source roots
|
|
when fired with a parent dir. This module merges the duplicates and
|
|
re-points photos to the canonical folder so the data lines up with the
|
|
post-fix scanner.
|
|
|
|
Idempotent: safe to run on every backend startup.
|
|
"""
|
|
import os
|
|
import logging
|
|
from datetime import datetime
|
|
from sqlalchemy import select, update, func
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import Photo, Folder, SourceRoot
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _normalize_path(path: str) -> str:
|
|
return os.path.normpath(path)
|
|
|
|
|
|
async def _dedupe_source_roots(session: AsyncSession) -> int:
|
|
"""Group source roots by normalized path and merge duplicates. Returns
|
|
the number of rows deleted."""
|
|
result = await session.execute(select(SourceRoot))
|
|
rows = result.scalars().all()
|
|
|
|
groups: dict[str, list[SourceRoot]] = {}
|
|
for sr in rows:
|
|
norm = _normalize_path(sr.path)
|
|
groups.setdefault(norm, []).append(sr)
|
|
|
|
deleted = 0
|
|
for norm, srs in groups.items():
|
|
if len(srs) == 1:
|
|
# Make sure the canonical row's path is normalized too.
|
|
if srs[0].path != norm:
|
|
srs[0].path = norm
|
|
continue
|
|
# Pick the canonical row: prefer one with a non-empty name and the
|
|
# earliest added_at (most likely the original).
|
|
canonical = sorted(
|
|
srs,
|
|
key=lambda s: (not bool(s.name), s.added_at or datetime.max),
|
|
)[0]
|
|
canonical.path = norm
|
|
for sr in srs:
|
|
if sr.id == canonical.id:
|
|
continue
|
|
# Re-point folders that referenced the duplicate root.
|
|
await session.execute(
|
|
update(Folder)
|
|
.where(Folder.source_root_id == sr.id)
|
|
.values(source_root_id=canonical.id)
|
|
)
|
|
await session.delete(sr)
|
|
deleted += 1
|
|
|
|
return deleted
|
|
|
|
|
|
async def _dedupe_folders(session: AsyncSession) -> int:
|
|
"""Group folders by normalized path and merge duplicates. Returns the
|
|
number of rows deleted."""
|
|
result = await session.execute(select(Folder))
|
|
rows = result.scalars().all()
|
|
|
|
groups: dict[str, list[Folder]] = {}
|
|
for f in rows:
|
|
norm = _normalize_path(f.path)
|
|
groups.setdefault(norm, []).append(f)
|
|
|
|
deleted = 0
|
|
for norm, folders in groups.items():
|
|
if len(folders) == 1:
|
|
if folders[0].path != norm:
|
|
folders[0].path = norm
|
|
continue
|
|
# Canonical = the one with the most photos already attached, then
|
|
# the lowest-id (deterministic tiebreaker).
|
|
canonical = sorted(
|
|
folders,
|
|
key=lambda f: (-(f.photo_count or 0), f.id),
|
|
)[0]
|
|
canonical.path = norm
|
|
for f in folders:
|
|
if f.id == canonical.id:
|
|
continue
|
|
# Re-point photos to the canonical folder.
|
|
await session.execute(
|
|
update(Photo)
|
|
.where(Photo.folder_id == f.id)
|
|
.values(folder_id=canonical.id)
|
|
)
|
|
await session.delete(f)
|
|
deleted += 1
|
|
|
|
return deleted
|
|
|
|
|
|
async def _recompute_folder_counts(session: AsyncSession) -> None:
|
|
"""Set folder.photo_count to the actual non-discarded photo count."""
|
|
result = await session.execute(select(Folder))
|
|
folders = result.scalars().all()
|
|
for f in folders:
|
|
count_result = await session.execute(
|
|
select(func.count(Photo.id)).where(
|
|
Photo.folder_id == f.id,
|
|
Photo.is_discarded == False, # noqa: E712
|
|
)
|
|
)
|
|
f.photo_count = int(count_result.scalar() or 0)
|
|
|
|
|
|
def _parent_is_accessible(path: str) -> bool:
|
|
"""True if the parent directory of `path` is readable. Used to
|
|
distinguish 'user renamed/deleted the source root folder' (parent
|
|
mount fine, leaf gone) from 'drive unmounted' (whole subtree
|
|
inaccessible). The former is safe to prune from; the latter is
|
|
not."""
|
|
parent = os.path.dirname(path.rstrip(os.sep))
|
|
if not parent:
|
|
return False
|
|
try:
|
|
os.listdir(parent)
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _sr_state(sr_path: str) -> str:
|
|
"""Classify a source root path as one of:
|
|
'present' — directory exists, business as usual
|
|
'renamed' — leaf missing but parent mount is accessible (user
|
|
renamed/deleted the folder in their file manager)
|
|
'unmounted'— parent itself inaccessible (drive not mounted)
|
|
"""
|
|
if os.path.isdir(sr_path):
|
|
return 'present'
|
|
if _parent_is_accessible(sr_path):
|
|
return 'renamed'
|
|
return 'unmounted'
|
|
|
|
|
|
async def _warn_stale_source_roots(session: AsyncSession) -> int:
|
|
"""Log a warning for any active source root whose path no longer exists
|
|
on disk. Doesn't delete — a missing path could be a temporarily
|
|
unmounted drive, and silently dropping user data is worse than
|
|
surfacing a noisy log line. Logs different hints for renamed-vs-
|
|
unmounted so the user knows which knob to turn.
|
|
"""
|
|
result = await session.execute(select(SourceRoot))
|
|
rows = result.scalars().all()
|
|
stale = 0
|
|
for sr in rows:
|
|
state = _sr_state(sr.path)
|
|
if state == 'present':
|
|
continue
|
|
stale += 1
|
|
if state == 'renamed':
|
|
logger.warning(
|
|
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
|
f"— parent mount is fine, looks like the folder was renamed "
|
|
f"or deleted. Photos under it can be cleared via "
|
|
f"POST /api/v1/library/maintenance/prune-missing."
|
|
)
|
|
else:
|
|
logger.warning(
|
|
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
|
f"— parent directory is also inaccessible; is the docker "
|
|
f"mount still in place? (Edit docker-compose.yml or "
|
|
f"PHOTO_DIRS in .env to fix.)"
|
|
)
|
|
return stale
|
|
|
|
|
|
async def find_missing(
|
|
session: AsyncSession,
|
|
) -> tuple[list[str], list[str], list[str]]:
|
|
"""Walk every non-discarded photo + every folder and check whether
|
|
they still resolve on disk. Returns
|
|
(deletable_photo_ids, deletable_folder_ids, skipped_photo_ids).
|
|
|
|
Skipped rows are photos/folders whose owning source_root is truly
|
|
inaccessible (parent mount missing) — that's almost always an
|
|
unmounted drive, and silently deleting those rows would be data
|
|
loss. Photos under a source root whose leaf is missing but whose
|
|
parent mount IS accessible (user renamed/deleted the folder) are
|
|
treated as deletable, since their files are genuinely gone from
|
|
the user's library.
|
|
"""
|
|
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
|
# "Available" = source root path exists OR parent mount is accessible.
|
|
# Only truly-unmounted source roots skip pruning.
|
|
sr_mounted: dict[str, bool] = {
|
|
sr.id: _sr_state(sr.path) != 'unmounted' for sr in sr_rows
|
|
}
|
|
|
|
photos = (await session.execute(
|
|
select(Photo.id, Photo.filepath, Photo.folder_id)
|
|
.where(Photo.is_discarded.is_(False))
|
|
)).all()
|
|
|
|
folders = (await session.execute(
|
|
select(Folder.id, Folder.path, Folder.source_root_id)
|
|
)).all()
|
|
folder_to_sr = {fid: srid for fid, _path, srid in folders}
|
|
|
|
deletable_photos: list[str] = []
|
|
skipped: list[str] = []
|
|
for pid, fp, folder_id in photos:
|
|
sr_id = folder_to_sr.get(folder_id)
|
|
if sr_id is None or not sr_mounted.get(sr_id, False):
|
|
skipped.append(pid)
|
|
continue
|
|
if not os.path.exists(fp):
|
|
deletable_photos.append(pid)
|
|
|
|
deletable_folders: list[str] = []
|
|
for fid, fpath, sr_id in folders:
|
|
if sr_id is None or not sr_mounted.get(sr_id, False):
|
|
continue
|
|
if not os.path.isdir(fpath):
|
|
deletable_folders.append(fid)
|
|
|
|
return deletable_photos, deletable_folders, skipped
|
|
|
|
|
|
async def prune_missing_photos(dry_run: bool = True) -> dict:
|
|
"""Delete photo + folder rows whose paths are no longer on disk *and*
|
|
whose source root is currently mounted. Common cause: PHOTO_DIRS in
|
|
.env was repointed at a different library, leaving every old row
|
|
orphaned.
|
|
|
|
Set dry_run=False to actually delete. The default is intentionally
|
|
safe so the matching count can be surfaced in the UI before the
|
|
user commits to it.
|
|
|
|
Function name kept for backwards compatibility — it now also prunes
|
|
folders, not just photos.
|
|
"""
|
|
from sqlalchemy import delete
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
deletable_photos, deletable_folders, skipped = await find_missing(session)
|
|
if not dry_run:
|
|
CHUNK = 500
|
|
# Photos first (folders may FK from them via folder_id).
|
|
for i in range(0, len(deletable_photos), CHUNK):
|
|
await session.execute(
|
|
delete(Photo).where(
|
|
Photo.id.in_(deletable_photos[i:i + CHUNK])
|
|
)
|
|
)
|
|
# Then drop folders that ALSO no longer have any photos
|
|
# pointing at them. We re-check after the photo delete so
|
|
# we don't strand a folder that legitimately exists on
|
|
# disk but happened to match the orphan list.
|
|
if deletable_folders:
|
|
for i in range(0, len(deletable_folders), CHUNK):
|
|
chunk = deletable_folders[i:i + CHUNK]
|
|
# Only delete folders that now have zero photos
|
|
# left attached (defensive — should always be 0
|
|
# if the path is gone, but a concurrent scan
|
|
# could re-create rows).
|
|
still_used = (await session.execute(
|
|
select(Photo.folder_id)
|
|
.where(Photo.folder_id.in_(chunk))
|
|
.distinct()
|
|
)).scalars().all()
|
|
safe = [f for f in chunk if f not in set(still_used)]
|
|
if safe:
|
|
await session.execute(
|
|
delete(Folder).where(Folder.id.in_(safe))
|
|
)
|
|
await session.commit()
|
|
logger.info(
|
|
f"Pruned {len(deletable_photos)} photo rows + "
|
|
f"{len(deletable_folders)} folder rows"
|
|
)
|
|
key_p = "would_delete" if dry_run else "deleted"
|
|
key_f = "would_delete_folders" if dry_run else "deleted_folders"
|
|
return {
|
|
key_p: len(deletable_photos),
|
|
key_f: len(deletable_folders),
|
|
"skipped_unmounted": len(skipped),
|
|
"dry_run": dry_run,
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"prune_missing_photos failed: {e}")
|
|
await session.rollback()
|
|
raise
|
|
|
|
|
|
async def prune_orphan_thumbnails(
|
|
thumbs_root: str = "/data/thumbs",
|
|
dry_run: bool = True,
|
|
) -> dict:
|
|
"""Remove `/data/thumbs/{user_id}/{photo_id}/` directories whose
|
|
photo_id no longer exists in the photos table.
|
|
|
|
Layout was per-Phase-4 set up by app.tasks.thumbs and is keyed by
|
|
`{user_id}/{photo_id}/`. The thumbs worker never deletes its own
|
|
output on photo removal, so over the lifetime of a library these
|
|
directories accumulate.
|
|
|
|
Set dry_run=False to actually `rm -rf` each matched directory.
|
|
Returns counts of matched / removed dirs and any per-dir errors.
|
|
"""
|
|
import shutil
|
|
|
|
if not os.path.isdir(thumbs_root):
|
|
return {
|
|
"would_remove": 0,
|
|
"removed": 0,
|
|
"skipped_no_root": True,
|
|
"dry_run": dry_run,
|
|
}
|
|
|
|
async with AsyncSessionLocal() as session:
|
|
live_ids = {
|
|
row[0]
|
|
for row in (await session.execute(select(Photo.id))).all()
|
|
}
|
|
|
|
matched: list[str] = []
|
|
errors: list[str] = []
|
|
for user_dir in os.listdir(thumbs_root):
|
|
user_path = os.path.join(thumbs_root, user_dir)
|
|
if not os.path.isdir(user_path):
|
|
continue
|
|
for photo_dir in os.listdir(user_path):
|
|
if photo_dir in live_ids:
|
|
continue
|
|
matched.append(os.path.join(user_path, photo_dir))
|
|
|
|
removed = 0
|
|
if not dry_run:
|
|
for path in matched:
|
|
try:
|
|
shutil.rmtree(path)
|
|
removed += 1
|
|
except OSError as e:
|
|
errors.append(f"{path}: {e}")
|
|
|
|
key = "would_remove" if dry_run else "removed"
|
|
return {
|
|
key: len(matched) if dry_run else removed,
|
|
"errors": errors,
|
|
"dry_run": dry_run,
|
|
}
|
|
|
|
|
|
async def discard_missing_photos() -> dict:
|
|
"""Soft variant of prune_missing_photos for the periodic beat
|
|
catch-up. Walks every active source root that is currently
|
|
`present` (not 'renamed' — the user-driven manual flow handles
|
|
those — and not 'unmounted'), and for each Photo whose file is
|
|
gone from disk, sets is_discarded=True so it shows up in the
|
|
in-app trash. Idempotent: skips photos that are already
|
|
discarded.
|
|
|
|
Hard-deletion stays manual via prune_missing_photos so users
|
|
can review the list before committing.
|
|
"""
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
|
present_sr_ids = {
|
|
sr.id for sr in sr_rows if _sr_state(sr.path) == 'present'
|
|
}
|
|
if not present_sr_ids:
|
|
return {"discarded": 0, "checked": 0}
|
|
|
|
folder_to_sr = {
|
|
fid: srid
|
|
for fid, _path, srid in (await session.execute(
|
|
select(Folder.id, Folder.path, Folder.source_root_id)
|
|
)).all()
|
|
}
|
|
|
|
photo_rows = (await session.execute(
|
|
select(Photo.id, Photo.filepath, Photo.folder_id)
|
|
.where(Photo.is_discarded.is_(False))
|
|
)).all()
|
|
|
|
missing_ids: list[str] = []
|
|
checked = 0
|
|
for pid, fp, folder_id in photo_rows:
|
|
sr_id = folder_to_sr.get(folder_id)
|
|
if sr_id not in present_sr_ids:
|
|
continue
|
|
checked += 1
|
|
if not os.path.exists(fp):
|
|
missing_ids.append(pid)
|
|
|
|
if missing_ids:
|
|
CHUNK = 500
|
|
now = datetime.utcnow()
|
|
for i in range(0, len(missing_ids), CHUNK):
|
|
await session.execute(
|
|
update(Photo)
|
|
.where(Photo.id.in_(missing_ids[i:i + CHUNK]))
|
|
.values(is_discarded=True, discarded_at=now)
|
|
)
|
|
await session.commit()
|
|
logger.info(
|
|
f"discard_missing_photos: discarded {len(missing_ids)} "
|
|
f"of {checked} photos under {len(present_sr_ids)} present source roots"
|
|
)
|
|
|
|
return {"discarded": len(missing_ids), "checked": checked}
|
|
except Exception as e:
|
|
logger.error(f"discard_missing_photos failed: {e}")
|
|
await session.rollback()
|
|
raise
|
|
|
|
|
|
async def cleanup_data_integrity() -> dict:
|
|
"""Top-level entry point. Runs the dedupe + count refresh in a single
|
|
transaction. Returns a small summary dict for logging."""
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
sr_deleted = await _dedupe_source_roots(session)
|
|
f_deleted = await _dedupe_folders(session)
|
|
await _recompute_folder_counts(session)
|
|
stale = await _warn_stale_source_roots(session)
|
|
await session.commit()
|
|
summary = {
|
|
"source_roots_merged": sr_deleted,
|
|
"folders_merged": f_deleted,
|
|
"source_roots_stale": stale,
|
|
}
|
|
if sr_deleted or f_deleted:
|
|
logger.info(f"Cleanup merged duplicates: {summary}")
|
|
return summary
|
|
except Exception as e:
|
|
logger.error(f"Cleanup failed: {e}")
|
|
await session.rollback()
|
|
raise
|