refactor: drop AI/vision pipeline + plain Postgres + full-refresh script
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
- backend: app/services/vision/, app/tasks/vision.py,
app/services/feature_flags.py, app/routers/features.py — all
deleted; admin AI/feature-flag endpoints and the worker-vision
bootstrap call gone. Photo.needs_review and its index dropped.
- frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
types, "Needs Review" sidebar entry + filter, needs_review filter
URL param all gone.
- infra: worker-vision compose service + models_data volume deleted;
worker-light command no longer runs bootstrap_models; the db
image switches from pgvector/pgvector:pg16 to postgres:16; backend
Dockerfile drops the dedicated torch RUN layer; requirements.txt
drops torch/torchvision/open-clip-torch/onnxruntime.
Alembic 0019_drop_ai_remnants:
- drops photos.needs_review + ix_photos_needs_review
- DROP EXTENSION IF EXISTS vector (must run before the image swap;
the new postgres:16 doesn't ship pgvector)
New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
115
backend/scripts/full_refresh.py
Normal file
115
backend/scripts/full_refresh.py
Normal file
@@ -0,0 +1,115 @@
|
||||
"""Bring the mule-image DB into 100% sync with Nextcloud + filesystem.
|
||||
|
||||
Multi-phase one-shot operation invoked via:
|
||||
|
||||
docker exec mulita-backend python scripts/full_refresh.py [--dry-run]
|
||||
|
||||
Phases:
|
||||
1. Data integrity (sync, ~1s): cleanup_data_integrity dedupes
|
||||
SourceRoots / Folders by normalized path and recomputes folder
|
||||
photo_count.
|
||||
2. Forward scan (async, minutes): walk every active SourceRoot on
|
||||
disk, create/update Photo rows for new files, resurrect any
|
||||
accidentally-discarded photos whose mtime advanced.
|
||||
3. Hard prune (sync, seconds): delete Photo + Folder rows for paths
|
||||
that no longer exist on disk under a *mounted* root. Skips
|
||||
unmounted roots — matches prune_missing_photos's existing
|
||||
refuse-when-empty behavior.
|
||||
4. Orphan thumbnail dirs (sync, seconds): remove
|
||||
/data/thumbs/{user_id}/{photo_id}/ for any photo_id that's no
|
||||
longer in the photos table.
|
||||
|
||||
Pass --dry-run to compute counts for phases 3+4 without making changes.
|
||||
Phases 1 and 2 always run for real — they're idempotent and additive.
|
||||
|
||||
Print a structured summary at the end. Exit non-zero on any phase
|
||||
error; partial completion still surfaces the counts gathered so far.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from app.services.cleanup import (
|
||||
cleanup_data_integrity,
|
||||
prune_missing_photos,
|
||||
prune_orphan_thumbnails,
|
||||
)
|
||||
from app.tasks.scan import _scan_folder_async
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.folders import SourceRoot
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("full_refresh")
|
||||
|
||||
|
||||
async def _scan_all_inline() -> int:
|
||||
"""Scan every active SourceRoot inline (not via celery). Returns the
|
||||
number of roots actually walked."""
|
||||
import os
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active.is_(True))
|
||||
)
|
||||
roots = result.scalars().all()
|
||||
|
||||
walked = 0
|
||||
for sr in roots:
|
||||
if not os.path.exists(sr.path):
|
||||
logger.warning("source root path missing, skipping: %s", sr.path)
|
||||
continue
|
||||
logger.info("scanning %s …", sr.path)
|
||||
await _scan_folder_async(sr.path, sr.id, task=None)
|
||||
walked += 1
|
||||
return walked
|
||||
|
||||
|
||||
async def main(dry_run: bool) -> dict:
|
||||
summary: dict = {"dry_run": dry_run}
|
||||
|
||||
logger.info("phase 1: cleanup_data_integrity")
|
||||
summary["phase1_cleanup"] = await cleanup_data_integrity()
|
||||
|
||||
logger.info("phase 2: scan_all_source_roots (inline)")
|
||||
summary["phase2_scan_roots_walked"] = await _scan_all_inline()
|
||||
|
||||
logger.info("phase 3: prune_missing_photos (dry_run=%s)", dry_run)
|
||||
summary["phase3_prune"] = await prune_missing_photos(dry_run=dry_run)
|
||||
|
||||
logger.info("phase 4: prune_orphan_thumbnails (dry_run=%s)", dry_run)
|
||||
summary["phase4_orphan_thumbs"] = await prune_orphan_thumbnails(
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def cli() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Phases 3+4 report counts without making changes",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
result = asyncio.run(main(dry_run=args.dry_run))
|
||||
except Exception:
|
||||
logger.exception("full_refresh failed")
|
||||
return 1
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(cli())
|
||||
Reference in New Issue
Block a user