refactor: drop AI/vision pipeline + plain Postgres + full-refresh script
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
- backend: app/services/vision/, app/tasks/vision.py,
app/services/feature_flags.py, app/routers/features.py — all
deleted; admin AI/feature-flag endpoints and the worker-vision
bootstrap call gone. Photo.needs_review and its index dropped.
- frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
types, "Needs Review" sidebar entry + filter, needs_review filter
URL param all gone.
- infra: worker-vision compose service + models_data volume deleted;
worker-light command no longer runs bootstrap_models; the db
image switches from pgvector/pgvector:pg16 to postgres:16; backend
Dockerfile drops the dedicated torch RUN layer; requirements.txt
drops torch/torchvision/open-clip-torch/onnxruntime.
Alembic 0019_drop_ai_remnants:
- drops photos.needs_review + ix_photos_needs_review
- DROP EXTENSION IF EXISTS vector (must run before the image swap;
the new postgres:16 doesn't ship pgvector)
New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -18,14 +18,6 @@ from app.models.user import User
|
||||
from app.models.photos import Photo
|
||||
from app.models.folders import SourceRoot
|
||||
from app.config import settings
|
||||
from app.services.feature_flags import (
|
||||
ALL_FLAGS,
|
||||
snapshot as flags_snapshot,
|
||||
set_flag,
|
||||
reset_flag,
|
||||
is_enabled,
|
||||
FLAG_VISION_ENABLED,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -266,104 +258,3 @@ async def delete_user(
|
||||
|
||||
logger.info(f"Admin '{admin.username}' deactivated user '{user.username}'")
|
||||
return {"status": "ok", "detail": f"User '{user.username}' deactivated"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AI / vision feature flags + manual triggers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FeatureFlagUpdate(BaseModel):
|
||||
"""PATCH body for toggling a feature flag.
|
||||
|
||||
``value`` sets an explicit override (true/false); omitting it clears
|
||||
the override and reverts the flag to its YAML default.
|
||||
"""
|
||||
value: Optional[bool] = None
|
||||
|
||||
|
||||
@router.get("/feature-flags")
|
||||
async def get_feature_flags(admin: User = Depends(require_admin)):
|
||||
"""Return every tunable feature flag with its current effective
|
||||
value, YAML default, and whether an admin override is in effect."""
|
||||
return {"flags": flags_snapshot()}
|
||||
|
||||
|
||||
@router.patch("/feature-flags/{flag_name}")
|
||||
async def update_feature_flag(
|
||||
flag_name: str,
|
||||
body: FeatureFlagUpdate,
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Set or clear an override for one flag. With ``value`` set, the
|
||||
flag is pinned to that boolean; without it, the override is deleted
|
||||
and the YAML default takes over again.
|
||||
|
||||
New value is observed by vision tasks on their next invocation —
|
||||
there's no worker restart required.
|
||||
"""
|
||||
if flag_name not in ALL_FLAGS:
|
||||
raise HTTPException(status_code=404, detail=f"Unknown flag: {flag_name}")
|
||||
try:
|
||||
if body.value is None:
|
||||
reset_flag(flag_name)
|
||||
action = "cleared override"
|
||||
else:
|
||||
set_flag(flag_name, bool(body.value))
|
||||
action = f"set to {body.value}"
|
||||
except RuntimeError as e:
|
||||
# Redis unreachable — surface as 503 so the UI doesn't think it
|
||||
# succeeded silently.
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
|
||||
logger.info(f"Admin '{admin.username}' {action} for flag '{flag_name}'")
|
||||
return {"flags": flags_snapshot()}
|
||||
|
||||
|
||||
class BackfillVisionBody(BaseModel):
|
||||
"""POST body for triggering a classifier backfill. ``limit`` caps how
|
||||
many photos are queued."""
|
||||
limit: Optional[int] = None
|
||||
|
||||
|
||||
@router.post("/ai/backfill")
|
||||
async def trigger_ai_backfill(
|
||||
body: BackfillVisionBody,
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Queue a classifier backfill pass."""
|
||||
if not is_enabled(FLAG_VISION_ENABLED):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Vision is currently disabled; enable it before running a backfill.",
|
||||
)
|
||||
if body.limit is not None and body.limit <= 0:
|
||||
raise HTTPException(status_code=400, detail="limit must be positive")
|
||||
|
||||
from app.tasks.vision import backfill_vision
|
||||
|
||||
result = backfill_vision.apply_async(kwargs={'limit': body.limit})
|
||||
logger.info(
|
||||
f"Admin '{admin.username}' queued vision backfill "
|
||||
f"(limit={body.limit}, celery_id={result.id})"
|
||||
)
|
||||
return {
|
||||
"status": "queued",
|
||||
"task_id": result.id,
|
||||
"limit": body.limit,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/ai/rescan")
|
||||
async def trigger_full_rescan(admin: User = Depends(require_admin)):
|
||||
"""Dispatch the same scan_all_source_roots job the backend runs at
|
||||
startup. Picks up any new files on disk and, through the
|
||||
post-scan hook, queues a vision backfill for whatever still lacks
|
||||
embeddings / OCR / etc.
|
||||
"""
|
||||
from app.tasks.scan import scan_all_source_roots
|
||||
|
||||
result = scan_all_source_roots.apply_async()
|
||||
logger.info(
|
||||
f"Admin '{admin.username}' queued full rescan (celery_id={result.id})"
|
||||
)
|
||||
return {"status": "queued", "task_id": result.id}
|
||||
|
||||
Reference in New Issue
Block a user