dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: drop AI/vision pipeline + plain Postgres + full-refresh script

Browse Source 
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
  - backend: app/services/vision/, app/tasks/vision.py,
    app/services/feature_flags.py, app/routers/features.py — all
    deleted; admin AI/feature-flag endpoints and the worker-vision
    bootstrap call gone. Photo.needs_review and its index dropped.
  - frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
    types, "Needs Review" sidebar entry + filter, needs_review filter
    URL param all gone.
  - infra: worker-vision compose service + models_data volume deleted;
    worker-light command no longer runs bootstrap_models; the db
    image switches from pgvector/pgvector:pg16 to postgres:16; backend
    Dockerfile drops the dedicated torch RUN layer; requirements.txt
    drops torch/torchvision/open-clip-torch/onnxruntime.

Alembic 0019_drop_ai_remnants:
  - drops photos.needs_review + ix_photos_needs_review
  - DROP EXTENSION IF EXISTS vector (must run before the image swap;
    the new postgres:16 doesn't ship pgvector)

New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
claudio
2026-05-14 00:20:38 +02:00
parent 6915c30911
commit a27267f7ad
39 changed files with 265 additions and 1573 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
backend/app/routers/admin.py
View File

@@ -18,14 +18,6 @@ from app.models.user import User
from app.models.photos import Photo
from app.models.folders import SourceRoot
from app.config import settings
from app.services.feature_flags import (
ALL_FLAGS,
snapshot as flags_snapshot,
set_flag,
reset_flag,
is_enabled,
FLAG_VISION_ENABLED,
)
logger = logging.getLogger(__name__)
@@ -266,104 +258,3 @@ async def delete_user(
logger.info(f"Admin '{admin.username}' deactivated user '{user.username}'")
return {"status": "ok", "detail": f"User '{user.username}' deactivated"}
# ---------------------------------------------------------------------------
# AI / vision feature flags + manual triggers
# ---------------------------------------------------------------------------
class FeatureFlagUpdate(BaseModel):
"""PATCH body for toggling a feature flag.
``value`` sets an explicit override (true/false); omitting it clears
the override and reverts the flag to its YAML default.
"""
value: Optional[bool] = None
@router.get("/feature-flags")
async def get_feature_flags(admin: User = Depends(require_admin)):
"""Return every tunable feature flag with its current effective
value, YAML default, and whether an admin override is in effect."""
return {"flags": flags_snapshot()}
@router.patch("/feature-flags/{flag_name}")
async def update_feature_flag(
flag_name: str,
body: FeatureFlagUpdate,
admin: User = Depends(require_admin),
):
"""Set or clear an override for one flag. With ``value`` set, the
flag is pinned to that boolean; without it, the override is deleted
and the YAML default takes over again.
New value is observed by vision tasks on their next invocation —
there's no worker restart required.
"""
if flag_name not in ALL_FLAGS:
raise HTTPException(status_code=404, detail=f"Unknown flag: {flag_name}")
try:
if body.value is None:
reset_flag(flag_name)
action = "cleared override"
else:
set_flag(flag_name, bool(body.value))
action = f"set to {body.value}"
except RuntimeError as e:
# Redis unreachable — surface as 503 so the UI doesn't think it
# succeeded silently.
raise HTTPException(status_code=503, detail=str(e))
logger.info(f"Admin '{admin.username}' {action} for flag '{flag_name}'")
return {"flags": flags_snapshot()}
class BackfillVisionBody(BaseModel):
"""POST body for triggering a classifier backfill. ``limit`` caps how
many photos are queued."""
limit: Optional[int] = None
@router.post("/ai/backfill")
async def trigger_ai_backfill(
body: BackfillVisionBody,
admin: User = Depends(require_admin),
):
"""Queue a classifier backfill pass."""
if not is_enabled(FLAG_VISION_ENABLED):
raise HTTPException(
status_code=400,
detail="Vision is currently disabled; enable it before running a backfill.",
)
if body.limit is not None and body.limit <= 0:
raise HTTPException(status_code=400, detail="limit must be positive")
from app.tasks.vision import backfill_vision
result = backfill_vision.apply_async(kwargs={'limit': body.limit})
logger.info(
f"Admin '{admin.username}' queued vision backfill "
f"(limit={body.limit}, celery_id={result.id})"
)
return {
"status": "queued",
"task_id": result.id,
"limit": body.limit,
}
@router.post("/ai/rescan")
async def trigger_full_rescan(admin: User = Depends(require_admin)):
"""Dispatch the same scan_all_source_roots job the backend runs at
startup. Picks up any new files on disk and, through the
post-scan hook, queues a vision backfill for whatever still lacks
embeddings / OCR / etc.
"""
from app.tasks.scan import scan_all_source_roots
result = scan_all_source_roots.apply_async()
logger.info(
f"Admin '{admin.username}' queued full rescan (celery_id={result.id})"
)
return {"status": "queued", "task_id": result.id}

23
backend/app/routers/features.py
View File

@@ -1,23 +0,0 @@
"""
Public feature-flag read API — lets the authenticated frontend know
which AI-powered sections to render.
This is NOT the admin mutation endpoint (that's in ``admin.py`` and
gated by ``require_admin``). Here we only expose the effective boolean
state so the UI can hide things like the People view, Tags view, or
text-search affordances when the underlying pipeline stage is off.
"""
from fastapi import APIRouter, Depends
from app.dependencies import get_current_user
from app.models.user import User
from app.services.feature_flags import ALL_FLAGS, is_enabled
router = APIRouter()
@router.get("")
async def get_enabled_features(_: User = Depends(get_current_user)):
"""Return ``{flag_name: bool}`` for every known flag, reflecting
the currently effective value (admin override or YAML default)."""
return {name: is_enabled(name) for name in ALL_FLAGS}

28
backend/app/routers/library.py
View File

@@ -102,12 +102,6 @@ async def get_library_stats(
)
).scalar() or 0
needs_review_count = (
await db.execute(
select(func.count(Photo.id)).where(visible, Photo.needs_review.is_(True))
)
).scalar() or 0
# Legacy split (kept for the existing /stats consumers).
photo_count = (
await db.execute(
@@ -134,7 +128,6 @@ async def get_library_stats(
"with_gps": with_gps_count,
"duplicates": duplicates_count,
"discarded": discarded_count,
"needs_review": needs_review_count,
"total_photos": photo_count,
"total_videos": video_count,
"total_size": size,
@@ -465,8 +458,7 @@ async def get_worker_status(
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
r.ping()
broker_ok = True
# `vision` runs the content classifier — the only heavy queue.
for q in ('default', 'high', 'low', 'vision'):
for q in ('default', 'high', 'low'):
try:
queue_depths[q] = int(r.llen(q) or 0)
except Exception:
@@ -595,17 +587,6 @@ async def get_pipeline_stats(
)
)
# Classified: distinct photos with a content_type tag.
classified_done = await scalar_count(
select(func.count(func.distinct(photo_tags.c.photo_id)))
.select_from(photo_tags)
.join(Photo, Photo.id == photo_tags.c.photo_id)
.where(not_discarded, photo_tags.c.source == 'vision:clip_classifier')
)
needs_review_count = await scalar_count(
select(func.count(Photo.id)).where(not_discarded, Photo.needs_review.is_(True))
)
duplicate_groups = await scalar_count(
select(func.count(func.distinct(Photo.duplicate_group_id))).where(
not_discarded, Photo.duplicate_group_id.is_not(None)
@@ -649,13 +630,6 @@ async def get_pipeline_stats(
"total": total_images,
"hint": "Feeds duplicate detection.",
},
{
"key": "classification",
"label": "Content classification (photo vs other)",
"done": classified_done,
"total": total_images,
"hint": f"{needs_review_count} photos flagged for review.",
},
{
"key": "duplicates",
"label": "Duplicate groups",

3
backend/app/routers/photos.py
View File

@@ -73,7 +73,6 @@ async def list_photos(
color_label: Optional[str] = None,
is_discarded: Optional[bool] = False,
is_duplicate: Optional[bool] = None,
needs_review: Optional[bool] = None,
has_date_warning: Optional[bool] = None,
heap_id: Optional[str] = None,
sort: str = "taken_at",
@@ -244,8 +243,6 @@ async def list_photos(
# view shows everything regardless of duplicate status.
if is_duplicate is not None:
filters.append(Photo.is_duplicate == is_duplicate)
if needs_review is not None:
filters.append(Photo.needs_review == needs_review)
if has_date_warning is not None:
filters.append(Photo.has_date_warning == has_date_warning)