refactor: drop AI/vision pipeline + plain Postgres + full-refresh script
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
- backend: app/services/vision/, app/tasks/vision.py,
app/services/feature_flags.py, app/routers/features.py — all
deleted; admin AI/feature-flag endpoints and the worker-vision
bootstrap call gone. Photo.needs_review and its index dropped.
- frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
types, "Needs Review" sidebar entry + filter, needs_review filter
URL param all gone.
- infra: worker-vision compose service + models_data volume deleted;
worker-light command no longer runs bootstrap_models; the db
image switches from pgvector/pgvector:pg16 to postgres:16; backend
Dockerfile drops the dedicated torch RUN layer; requirements.txt
drops torch/torchvision/open-clip-torch/onnxruntime.
Alembic 0019_drop_ai_remnants:
- drops photos.needs_review + ix_photos_needs_review
- DROP EXTENSION IF EXISTS vector (must run before the image swap;
the new postgres:16 doesn't ship pgvector)
New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -299,6 +299,65 @@ async def prune_missing_photos(dry_run: bool = True) -> dict:
|
||||
raise
|
||||
|
||||
|
||||
async def prune_orphan_thumbnails(
|
||||
thumbs_root: str = "/data/thumbs",
|
||||
dry_run: bool = True,
|
||||
) -> dict:
|
||||
"""Remove `/data/thumbs/{user_id}/{photo_id}/` directories whose
|
||||
photo_id no longer exists in the photos table.
|
||||
|
||||
Layout was per-Phase-4 set up by app.tasks.thumbs and is keyed by
|
||||
`{user_id}/{photo_id}/`. The thumbs worker never deletes its own
|
||||
output on photo removal, so over the lifetime of a library these
|
||||
directories accumulate.
|
||||
|
||||
Set dry_run=False to actually `rm -rf` each matched directory.
|
||||
Returns counts of matched / removed dirs and any per-dir errors.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
if not os.path.isdir(thumbs_root):
|
||||
return {
|
||||
"would_remove": 0,
|
||||
"removed": 0,
|
||||
"skipped_no_root": True,
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
live_ids = {
|
||||
row[0]
|
||||
for row in (await session.execute(select(Photo.id))).all()
|
||||
}
|
||||
|
||||
matched: list[str] = []
|
||||
errors: list[str] = []
|
||||
for user_dir in os.listdir(thumbs_root):
|
||||
user_path = os.path.join(thumbs_root, user_dir)
|
||||
if not os.path.isdir(user_path):
|
||||
continue
|
||||
for photo_dir in os.listdir(user_path):
|
||||
if photo_dir in live_ids:
|
||||
continue
|
||||
matched.append(os.path.join(user_path, photo_dir))
|
||||
|
||||
removed = 0
|
||||
if not dry_run:
|
||||
for path in matched:
|
||||
try:
|
||||
shutil.rmtree(path)
|
||||
removed += 1
|
||||
except OSError as e:
|
||||
errors.append(f"{path}: {e}")
|
||||
|
||||
key = "would_remove" if dry_run else "removed"
|
||||
return {
|
||||
key: len(matched) if dry_run else removed,
|
||||
"errors": errors,
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
|
||||
|
||||
async def discard_missing_photos() -> dict:
|
||||
"""Soft variant of prune_missing_photos for the periodic beat
|
||||
catch-up. Walks every active source root that is currently
|
||||
|
||||
Reference in New Issue
Block a user