refactor: drop AI/vision pipeline + plain Postgres + full-refresh script
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
- backend: app/services/vision/, app/tasks/vision.py,
app/services/feature_flags.py, app/routers/features.py — all
deleted; admin AI/feature-flag endpoints and the worker-vision
bootstrap call gone. Photo.needs_review and its index dropped.
- frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
types, "Needs Review" sidebar entry + filter, needs_review filter
URL param all gone.
- infra: worker-vision compose service + models_data volume deleted;
worker-light command no longer runs bootstrap_models; the db
image switches from pgvector/pgvector:pg16 to postgres:16; backend
Dockerfile drops the dedicated torch RUN layer; requirements.txt
drops torch/torchvision/open-clip-torch/onnxruntime.
Alembic 0019_drop_ai_remnants:
- drops photos.needs_review + ix_photos_needs_review
- DROP EXTENSION IF EXISTS vector (must run before the image swap;
the new postgres:16 doesn't ship pgvector)
New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -102,12 +102,6 @@ async def get_library_stats(
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
needs_review_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(visible, Photo.needs_review.is_(True))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
# Legacy split (kept for the existing /stats consumers).
|
||||
photo_count = (
|
||||
await db.execute(
|
||||
@@ -134,7 +128,6 @@ async def get_library_stats(
|
||||
"with_gps": with_gps_count,
|
||||
"duplicates": duplicates_count,
|
||||
"discarded": discarded_count,
|
||||
"needs_review": needs_review_count,
|
||||
"total_photos": photo_count,
|
||||
"total_videos": video_count,
|
||||
"total_size": size,
|
||||
@@ -465,8 +458,7 @@ async def get_worker_status(
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
r.ping()
|
||||
broker_ok = True
|
||||
# `vision` runs the content classifier — the only heavy queue.
|
||||
for q in ('default', 'high', 'low', 'vision'):
|
||||
for q in ('default', 'high', 'low'):
|
||||
try:
|
||||
queue_depths[q] = int(r.llen(q) or 0)
|
||||
except Exception:
|
||||
@@ -595,17 +587,6 @@ async def get_pipeline_stats(
|
||||
)
|
||||
)
|
||||
|
||||
# Classified: distinct photos with a content_type tag.
|
||||
classified_done = await scalar_count(
|
||||
select(func.count(func.distinct(photo_tags.c.photo_id)))
|
||||
.select_from(photo_tags)
|
||||
.join(Photo, Photo.id == photo_tags.c.photo_id)
|
||||
.where(not_discarded, photo_tags.c.source == 'vision:clip_classifier')
|
||||
)
|
||||
needs_review_count = await scalar_count(
|
||||
select(func.count(Photo.id)).where(not_discarded, Photo.needs_review.is_(True))
|
||||
)
|
||||
|
||||
duplicate_groups = await scalar_count(
|
||||
select(func.count(func.distinct(Photo.duplicate_group_id))).where(
|
||||
not_discarded, Photo.duplicate_group_id.is_not(None)
|
||||
@@ -649,13 +630,6 @@ async def get_pipeline_stats(
|
||||
"total": total_images,
|
||||
"hint": "Feeds duplicate detection.",
|
||||
},
|
||||
{
|
||||
"key": "classification",
|
||||
"label": "Content classification (photo vs other)",
|
||||
"done": classified_done,
|
||||
"total": total_images,
|
||||
"hint": f"{needs_review_count} photos flagged for review.",
|
||||
},
|
||||
{
|
||||
"key": "duplicates",
|
||||
"label": "Duplicate groups",
|
||||
|
||||
Reference in New Issue
Block a user