refactor: drop AI/vision pipeline + plain Postgres + full-refresh script

Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
  - backend: app/services/vision/, app/tasks/vision.py,
    app/services/feature_flags.py, app/routers/features.py — all
    deleted; admin AI/feature-flag endpoints and the worker-vision
    bootstrap call gone. Photo.needs_review and its index dropped.
  - frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
    types, "Needs Review" sidebar entry + filter, needs_review filter
    URL param all gone.
  - infra: worker-vision compose service + models_data volume deleted;
    worker-light command no longer runs bootstrap_models; the db
    image switches from pgvector/pgvector:pg16 to postgres:16; backend
    Dockerfile drops the dedicated torch RUN layer; requirements.txt
    drops torch/torchvision/open-clip-torch/onnxruntime.

Alembic 0019_drop_ai_remnants:
  - drops photos.needs_review + ix_photos_needs_review
  - DROP EXTENSION IF EXISTS vector (must run before the image swap;
    the new postgres:16 doesn't ship pgvector)

New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
claudio
2026-05-14 00:20:38 +02:00
parent 6915c30911
commit a27267f7ad
39 changed files with 265 additions and 1573 deletions

View File

@@ -18,14 +18,6 @@ from app.models.user import User
from app.models.photos import Photo
from app.models.folders import SourceRoot
from app.config import settings
from app.services.feature_flags import (
ALL_FLAGS,
snapshot as flags_snapshot,
set_flag,
reset_flag,
is_enabled,
FLAG_VISION_ENABLED,
)
logger = logging.getLogger(__name__)
@@ -266,104 +258,3 @@ async def delete_user(
logger.info(f"Admin '{admin.username}' deactivated user '{user.username}'")
return {"status": "ok", "detail": f"User '{user.username}' deactivated"}
# ---------------------------------------------------------------------------
# AI / vision feature flags + manual triggers
# ---------------------------------------------------------------------------
class FeatureFlagUpdate(BaseModel):
"""PATCH body for toggling a feature flag.
``value`` sets an explicit override (true/false); omitting it clears
the override and reverts the flag to its YAML default.
"""
value: Optional[bool] = None
@router.get("/feature-flags")
async def get_feature_flags(admin: User = Depends(require_admin)):
"""Return every tunable feature flag with its current effective
value, YAML default, and whether an admin override is in effect."""
return {"flags": flags_snapshot()}
@router.patch("/feature-flags/{flag_name}")
async def update_feature_flag(
flag_name: str,
body: FeatureFlagUpdate,
admin: User = Depends(require_admin),
):
"""Set or clear an override for one flag. With ``value`` set, the
flag is pinned to that boolean; without it, the override is deleted
and the YAML default takes over again.
New value is observed by vision tasks on their next invocation —
there's no worker restart required.
"""
if flag_name not in ALL_FLAGS:
raise HTTPException(status_code=404, detail=f"Unknown flag: {flag_name}")
try:
if body.value is None:
reset_flag(flag_name)
action = "cleared override"
else:
set_flag(flag_name, bool(body.value))
action = f"set to {body.value}"
except RuntimeError as e:
# Redis unreachable — surface as 503 so the UI doesn't think it
# succeeded silently.
raise HTTPException(status_code=503, detail=str(e))
logger.info(f"Admin '{admin.username}' {action} for flag '{flag_name}'")
return {"flags": flags_snapshot()}
class BackfillVisionBody(BaseModel):
"""POST body for triggering a classifier backfill. ``limit`` caps how
many photos are queued."""
limit: Optional[int] = None
@router.post("/ai/backfill")
async def trigger_ai_backfill(
body: BackfillVisionBody,
admin: User = Depends(require_admin),
):
"""Queue a classifier backfill pass."""
if not is_enabled(FLAG_VISION_ENABLED):
raise HTTPException(
status_code=400,
detail="Vision is currently disabled; enable it before running a backfill.",
)
if body.limit is not None and body.limit <= 0:
raise HTTPException(status_code=400, detail="limit must be positive")
from app.tasks.vision import backfill_vision
result = backfill_vision.apply_async(kwargs={'limit': body.limit})
logger.info(
f"Admin '{admin.username}' queued vision backfill "
f"(limit={body.limit}, celery_id={result.id})"
)
return {
"status": "queued",
"task_id": result.id,
"limit": body.limit,
}
@router.post("/ai/rescan")
async def trigger_full_rescan(admin: User = Depends(require_admin)):
"""Dispatch the same scan_all_source_roots job the backend runs at
startup. Picks up any new files on disk and, through the
post-scan hook, queues a vision backfill for whatever still lacks
embeddings / OCR / etc.
"""
from app.tasks.scan import scan_all_source_roots
result = scan_all_source_roots.apply_async()
logger.info(
f"Admin '{admin.username}' queued full rescan (celery_id={result.id})"
)
return {"status": "queued", "task_id": result.id}