refactor: drop AI/vision pipeline + plain Postgres + full-refresh script
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
- backend: app/services/vision/, app/tasks/vision.py,
app/services/feature_flags.py, app/routers/features.py — all
deleted; admin AI/feature-flag endpoints and the worker-vision
bootstrap call gone. Photo.needs_review and its index dropped.
- frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
types, "Needs Review" sidebar entry + filter, needs_review filter
URL param all gone.
- infra: worker-vision compose service + models_data volume deleted;
worker-light command no longer runs bootstrap_models; the db
image switches from pgvector/pgvector:pg16 to postgres:16; backend
Dockerfile drops the dedicated torch RUN layer; requirements.txt
drops torch/torchvision/open-clip-torch/onnxruntime.
Alembic 0019_drop_ai_remnants:
- drops photos.needs_review + ix_photos_needs_review
- DROP EXTENSION IF EXISTS vector (must run before the image swap;
the new postgres:16 doesn't ship pgvector)
New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -2,15 +2,12 @@
|
||||
Celery configuration and app initialization
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
|
||||
from celery import Celery
|
||||
from celery.signals import worker_process_init
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Create Celery app
|
||||
celery_app = Celery(
|
||||
'mulita',
|
||||
broker=settings.celery_broker_url,
|
||||
@@ -19,43 +16,31 @@ celery_app = Celery(
|
||||
'app.tasks.scan',
|
||||
'app.tasks.thumbs',
|
||||
'app.tasks.video',
|
||||
'app.tasks.vision',
|
||||
'app.services.metadata', # extract_metadata lives here
|
||||
'app.services.metadata',
|
||||
]
|
||||
)
|
||||
|
||||
# Configure Celery
|
||||
celery_app.conf.update(
|
||||
task_serializer='json',
|
||||
accept_content=['json'],
|
||||
result_serializer='json',
|
||||
timezone='UTC',
|
||||
enable_utc=True,
|
||||
# Robust acknowledgment: keep message in broker until task succeeds.
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
# Global time limits — individual tasks can override via decorator.
|
||||
task_soft_time_limit=300, # 5 min — raises SoftTimeLimitExceeded
|
||||
task_time_limit=600, # 10 min — SIGKILL
|
||||
# Explicit routes for every task name. Wildcard patterns don't match
|
||||
# short names produced by @shared_task(name='...').
|
||||
task_soft_time_limit=300,
|
||||
task_time_limit=600,
|
||||
task_routes={
|
||||
# Vision queue — CPU-bound binary classification
|
||||
'classify_content': {'queue': 'vision'},
|
||||
'vision_fanout': {'queue': 'vision'},
|
||||
# High-priority queue — thumbnails & duplicates
|
||||
'generate_thumbnails': {'queue': 'high'},
|
||||
'regenerate_all_thumbnails': {'queue': 'high'},
|
||||
'backfill_phashes': {'queue': 'high'},
|
||||
'regroup_duplicates': {'queue': 'high'},
|
||||
'incremental_regroup_duplicates': {'queue': 'high'},
|
||||
# Low-priority queue — scans
|
||||
'scan_folder': {'queue': 'low'},
|
||||
'scan_all_source_roots': {'queue': 'low'},
|
||||
'backfill_gps': {'queue': 'low'},
|
||||
# Pre-transcode HEVC videos in the background so /playback is a
|
||||
# cache hit on first user click. CPU-heavy but tolerant of the
|
||||
# low-priority queue (it doesn't block any user-facing flow).
|
||||
# CPU-heavy but tolerant of the low-priority queue (doesn't block
|
||||
# any user-facing flow).
|
||||
'pretranscode_video': {'queue': 'low'},
|
||||
# `watch_folders` is retired (file events come from NC webhooks)
|
||||
# but the task definition still exists as a no-op shim for any
|
||||
@@ -79,21 +64,3 @@ celery_app.conf.update(
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@worker_process_init.connect
|
||||
def _warmup_vision_models(**kwargs):
|
||||
"""Pre-load vision models in the worker process so the first task
|
||||
doesn't pay cold-start latency. Only runs on the vision queue."""
|
||||
# The worker name contains the queue — only warm up vision workers.
|
||||
worker_queues = os.environ.get("CELERY_QUEUES", "")
|
||||
if "vision" not in worker_queues:
|
||||
# Heuristic: check the celery command line for -Q vision
|
||||
import sys
|
||||
if "vision" not in " ".join(sys.argv):
|
||||
return
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
registry.warmup()
|
||||
except Exception:
|
||||
logger.exception("Vision model warmup failed")
|
||||
Reference in New Issue
Block a user