refactor: drop AI/vision pipeline + plain Postgres + full-refresh script
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
- backend: app/services/vision/, app/tasks/vision.py,
app/services/feature_flags.py, app/routers/features.py — all
deleted; admin AI/feature-flag endpoints and the worker-vision
bootstrap call gone. Photo.needs_review and its index dropped.
- frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
types, "Needs Review" sidebar entry + filter, needs_review filter
URL param all gone.
- infra: worker-vision compose service + models_data volume deleted;
worker-light command no longer runs bootstrap_models; the db
image switches from pgvector/pgvector:pg16 to postgres:16; backend
Dockerfile drops the dedicated torch RUN layer; requirements.txt
drops torch/torchvision/open-clip-torch/onnxruntime.
Alembic 0019_drop_ai_remnants:
- drops photos.needs_review + ix_photos_needs_review
- DROP EXTENSION IF EXISTS vector (must run before the image swap;
the new postgres:16 doesn't ship pgvector)
New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -146,7 +146,7 @@ services:
|
||||
dockerfile: Dockerfile
|
||||
image: mule-image-worker
|
||||
container_name: mulita-worker-light
|
||||
command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --beat --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_LIGHT_CONCURRENCY:-2} -Q default,high,low -n light@%h"
|
||||
command: sh -c "celery -A app.tasks.celery worker --beat --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_LIGHT_CONCURRENCY:-2} -Q default,high,low -n light@%h"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
@@ -155,7 +155,6 @@ services:
|
||||
- proxies_data:/data/proxies
|
||||
- video_cache_data:/data/video-cache
|
||||
- db_data:/data/db
|
||||
- models_data:/data/models
|
||||
environment:
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
@@ -203,85 +202,8 @@ services:
|
||||
# * `--beat` was folded into worker-light's command so the
|
||||
# periodic discard_missing_photos_beat job still fires.
|
||||
|
||||
worker-vision:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
image: mule-image-worker
|
||||
container_name: mulita-worker-vision
|
||||
command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_VISION_CONCURRENCY:-5} -Q vision -n vision@%h"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
- ${NEXTCLOUD_USERS_HOST_PATH:-./photos}:/nextcloud-users:rw
|
||||
- thumbs_data:/data/thumbs
|
||||
- proxies_data:/data/proxies
|
||||
- video_cache_data:/data/video-cache
|
||||
- db_data:/data/db
|
||||
- models_data:/data/models
|
||||
environment:
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=/photos
|
||||
- NEXTCLOUD_USERS_ROOT=${NEXTCLOUD_USERS_ROOT:-/nextcloud-users}
|
||||
- NEXTCLOUD_BASE_URL=${NEXTCLOUD_BASE_URL:-}
|
||||
- NEXTCLOUD_WEBHOOK_SECRET=${NEXTCLOUD_WEBHOOK_SECRET:-}
|
||||
- SECRET_KEY=${SECRET_KEY:-mulita-dev-secret-change-me}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
- MULITA_CELERY_WORKER=1
|
||||
# ONNX Runtime execution providers. Set to "auto" to auto-detect
|
||||
# GPU (CUDA > ROCm > OpenVINO > CPU), or explicitly:
|
||||
# "CUDAExecutionProvider,CPUExecutionProvider"
|
||||
# "ROCMExecutionProvider,CPUExecutionProvider"
|
||||
# Default: CPU only. To enable GPU, also uncomment the deploy
|
||||
# section below and install nvidia-container-toolkit on the host.
|
||||
- VISION_EXECUTION_PROVIDERS=${VISION_EXECUTION_PROVIDERS:-CPUExecutionProvider}
|
||||
# Pin each ONNX session to one intra-op thread so N prefork children
|
||||
# × default-all-cores doesn't oversubscribe the box. With
|
||||
# concurrency=5 and OMP=1, vision peaks at 5 busy cores, leaving
|
||||
# one for worker-light + system. These env vars cover the three
|
||||
# threading runtimes ONNX Runtime might pick up on first use.
|
||||
- OMP_NUM_THREADS=1
|
||||
- OPENBLAS_NUM_THREADS=1
|
||||
- MKL_NUM_THREADS=1
|
||||
# Uncomment for NVIDIA GPU passthrough:
|
||||
# deploy:
|
||||
# resources:
|
||||
# reservations:
|
||||
# devices:
|
||||
# - driver: nvidia
|
||||
# count: all
|
||||
# capabilities: [gpu]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "celery -A app.tasks.celery inspect ping -d vision@$$HOSTNAME 2>/dev/null | grep -q OK"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 300s
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
backend:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- mulita-network
|
||||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||||
# docker DNS forwards the lookup to the host's resolver, which
|
||||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||||
# traefik exposure list, so TLS handshakes against it die with
|
||||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||||
extra_hosts:
|
||||
- "cloud.hubris.network:192.168.8.175"
|
||||
restart: unless-stopped
|
||||
|
||||
db:
|
||||
image: pgvector/pgvector:pg16
|
||||
image: postgres:16
|
||||
container_name: mulita-db
|
||||
environment:
|
||||
POSTGRES_USER: mulita
|
||||
@@ -343,5 +265,4 @@ volumes:
|
||||
video_cache_data:
|
||||
db_data:
|
||||
redis_data:
|
||||
pg_data:
|
||||
models_data:
|
||||
pg_data:
|
||||
Reference in New Issue
Block a user