dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: drop AI/vision pipeline + plain Postgres + full-refresh script

Browse Source 
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
  - backend: app/services/vision/, app/tasks/vision.py,
    app/services/feature_flags.py, app/routers/features.py — all
    deleted; admin AI/feature-flag endpoints and the worker-vision
    bootstrap call gone. Photo.needs_review and its index dropped.
  - frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
    types, "Needs Review" sidebar entry + filter, needs_review filter
    URL param all gone.
  - infra: worker-vision compose service + models_data volume deleted;
    worker-light command no longer runs bootstrap_models; the db
    image switches from pgvector/pgvector:pg16 to postgres:16; backend
    Dockerfile drops the dedicated torch RUN layer; requirements.txt
    drops torch/torchvision/open-clip-torch/onnxruntime.

Alembic 0019_drop_ai_remnants:
  - drops photos.needs_review + ix_photos_needs_review
  - DROP EXTENSION IF EXISTS vector (must run before the image swap;
    the new postgres:16 doesn't ship pgvector)

New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
claudio
2026-05-14 00:20:38 +02:00
parent 6915c30911
commit a27267f7ad
39 changed files with 265 additions and 1573 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
backend/app/services/cleanup.py
View File

@@ -299,6 +299,65 @@ async def prune_missing_photos(dry_run: bool = True) -> dict:
raise
async def prune_orphan_thumbnails(
thumbs_root: str = "/data/thumbs",
dry_run: bool = True,
) -> dict:
"""Remove `/data/thumbs/{user_id}/{photo_id}/` directories whose
photo_id no longer exists in the photos table.
Layout was per-Phase-4 set up by app.tasks.thumbs and is keyed by
`{user_id}/{photo_id}/`. The thumbs worker never deletes its own
output on photo removal, so over the lifetime of a library these
directories accumulate.
Set dry_run=False to actually `rm -rf` each matched directory.
Returns counts of matched / removed dirs and any per-dir errors.
"""
import shutil
if not os.path.isdir(thumbs_root):
return {
"would_remove": 0,
"removed": 0,
"skipped_no_root": True,
"dry_run": dry_run,
}
async with AsyncSessionLocal() as session:
live_ids = {
row[0]
for row in (await session.execute(select(Photo.id))).all()
}
matched: list[str] = []
errors: list[str] = []
for user_dir in os.listdir(thumbs_root):
user_path = os.path.join(thumbs_root, user_dir)
if not os.path.isdir(user_path):
continue
for photo_dir in os.listdir(user_path):
if photo_dir in live_ids:
continue
matched.append(os.path.join(user_path, photo_dir))
removed = 0
if not dry_run:
for path in matched:
try:
shutil.rmtree(path)
removed += 1
except OSError as e:
errors.append(f"{path}: {e}")
key = "would_remove" if dry_run else "removed"
return {
key: len(matched) if dry_run else removed,
"errors": errors,
"dry_run": dry_run,
}
async def discard_missing_photos() -> dict:
"""Soft variant of prune_missing_photos for the periodic beat
catch-up. Walks every active source root that is currently

137
backend/app/services/feature_flags.py
View File

@@ -1,137 +0,0 @@
"""
Runtime feature flags for the vision pipeline.
Only one flag now — the master vision switch. Runtime overrides live in
Redis under ``mulita:flags:<name>``; an unset key falls back to the
YAML default.
"""
from __future__ import annotations
import logging
from typing import Optional
import redis
from app.config import settings
logger = logging.getLogger(__name__)
FLAG_VISION_ENABLED = 'vision.enabled'
ALL_FLAGS = (FLAG_VISION_ENABLED,)
_VISION_QUEUE = 'vision'
_REDIS: Optional[redis.Redis] = None
def _redis() -> Optional[redis.Redis]:
global _REDIS
if _REDIS is None:
try:
_REDIS = redis.Redis.from_url(
settings.celery_broker_url, decode_responses=True
)
_REDIS.ping()
except Exception as e:
logger.warning(f"feature_flags: Redis unavailable, using YAML defaults ({e})")
_REDIS = None
return _REDIS
def _yaml_default(name: str) -> bool:
if name == FLAG_VISION_ENABLED:
return bool(settings.vision.enabled)
raise ValueError(f"Unknown feature flag: {name!r}")
def _redis_key(name: str) -> str:
return f"mulita:flags:{name}"
def is_enabled(name: str) -> bool:
r = _redis()
if r is not None:
try:
raw = r.get(_redis_key(name))
if raw is not None:
return raw.lower() == 'true'
except Exception as e:
logger.warning(f"feature_flags: Redis read failed for {name} ({e})")
return _yaml_default(name)
def set_flag(name: str, value: bool) -> None:
if name not in ALL_FLAGS:
raise ValueError(f"Unknown feature flag: {name!r}")
r = _redis()
if r is None:
raise RuntimeError("Redis unavailable; cannot update feature flags")
r.set(_redis_key(name), 'true' if value else 'false')
_apply_worker_side_effects(name)
def reset_flag(name: str) -> None:
if name not in ALL_FLAGS:
raise ValueError(f"Unknown feature flag: {name!r}")
r = _redis()
if r is None:
raise RuntimeError("Redis unavailable; cannot reset feature flags")
r.delete(_redis_key(name))
_apply_worker_side_effects(name)
def _apply_worker_side_effects(name: str) -> None:
"""Attach or detach the vision consumer and purge queued work when
the master flag flips. Best-effort — state is already persisted."""
if name != FLAG_VISION_ENABLED:
return
try:
from app.tasks.celery import celery_app
except Exception as e:
logger.warning(f"feature_flags: celery app unavailable for side effects ({e})")
return
try:
if is_enabled(FLAG_VISION_ENABLED):
celery_app.control.add_consumer(_VISION_QUEUE, reply=False)
logger.info("feature_flags: vision re-enabled; consumer added")
else:
celery_app.control.cancel_consumer(_VISION_QUEUE, reply=False)
_purge_queue(_VISION_QUEUE)
logger.info("feature_flags: vision disabled; consumer cancelled and queue purged")
except Exception as e:
logger.warning(f"feature_flags: worker side effects failed: {e}")
def _purge_queue(queue: str) -> int:
r = _redis()
if r is None:
return 0
try:
return int(r.delete(queue) or 0)
except Exception as e:
logger.warning(f"feature_flags: purge {queue} failed: {e}")
return 0
def snapshot() -> dict[str, dict[str, object]]:
r = _redis()
out: dict[str, dict[str, object]] = {}
for name in ALL_FLAGS:
default = _yaml_default(name)
override = None
if r is not None:
try:
raw = r.get(_redis_key(name))
if raw is not None:
override = raw.lower() == 'true'
except Exception:
pass
out[name] = {
'effective': override if override is not None else default,
'default': default,
'overridden': override is not None,
}
return out

2
backend/app/services/nextcloud_dav.py
View File

@@ -389,7 +389,7 @@ def get_preview_bytes(
user: User, fileid: int, x: int, y: int
) -> Optional[bytes]:
"""Sync sibling of `get_preview_async` for callers in non-async
contexts (e.g. the vision celery worker, which is sync).
contexts.
Returns the preview body on success, None on 404 / non-success /
missing credentials. Caller is expected to feed the bytes into

7
backend/app/services/vision/__init__.py
View File

@@ -1,7 +0,0 @@
"""
Vision pipeline services — embedding, OCR, object detection, face recognition.
All inference is done through the ModelRegistry singleton, which lazy-loads
ONNX Runtime sessions on first use and caches them for the lifetime of the
worker process.
"""

25
backend/app/services/vision/base.py
View File

@@ -1,25 +0,0 @@
"""
Abstract base classes for the vision backend.
The pipeline is now a single binary classifier: photography vs other.
Feature extraction is an internal detail of the classifier and is not
exposed as a separate service.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass
import numpy as np
@dataclass
class ClassificationResult:
label: str
confidence: float
class ContentClassifier(ABC):
"""Classifies an image into 'photography' or 'other'."""
@abstractmethod
def classify(self, image: np.ndarray) -> ClassificationResult:
...

51
backend/app/services/vision/bootstrap_models.py
View File

@@ -1,51 +0,0 @@
"""
Ensure the OpenCLIP ViT-B/32 visual encoder is present on worker boot.
Exported via export_models.py if missing.
"""
import logging
from pathlib import Path
from app.config import settings
logger = logging.getLogger(__name__)
REQUIRED = [
("embed/visual.onnx", "OpenCLIP ViT-B/32 visual encoder"),
]
def bootstrap(models_dir: str | None = None):
base = Path(models_dir or settings.vision.models_dir)
base.mkdir(parents=True, exist_ok=True)
missing = [(rel, desc) for rel, desc in REQUIRED if not (base / rel).exists()]
if missing:
logger.warning("Missing %d model file(s); attempting automatic export", len(missing))
try:
from app.services.vision import export_models
export_models.export_openclip_visual(base)
except Exception as e:
logger.error(
"Export failed: %s. Run `python -m app.services.vision.export_models "
"--models-dir %s` manually to retry.",
e, base,
)
still_missing = [(r, d) for r, d in REQUIRED if not (base / r).exists()]
if still_missing:
for rel, desc in still_missing:
logger.error(" still missing: %s — %s", base / rel, desc)
else:
logger.info("All model files present in %s", base)
try:
import redis as _redis
_redis.from_url(settings.redis_url).set("mulita:vision:ready", "1")
logger.info("Set mulita:vision:ready in Redis")
except Exception as e:
logger.warning("Could not set vision readiness flag in Redis: %s", e)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
bootstrap()

106
backend/app/services/vision/classify.py
View File

@@ -1,106 +0,0 @@
"""
Binary content classifier: 'photography' vs 'other'.
Uses OpenCLIP ViT-B/32 image features (ONNX) and two pre-computed text
prompt centroids. Text centroids are computed once with the native
open_clip text encoder and cached to {models_dir}/classifier/vectors.npz
so steady-state worker startup doesn't pay the PyTorch cost.
"""
from __future__ import annotations
import logging
from pathlib import Path
import numpy as np
from app.config import VisionSettings
from app.services.vision.base import ClassificationResult, ContentClassifier
from app.services.vision.embed import CLIPVisualEncoder
logger = logging.getLogger(__name__)
PROMPTS = {
"photography": [
"a photograph taken with a camera",
"a real photo of a real scene or person",
"a candid photograph",
"a portrait photograph",
"a landscape photograph",
],
"other": [
"a screenshot of a computer screen",
"a screenshot of a phone screen",
"a screen capture of a user interface",
"a scanned document",
"a photo of a document with printed text",
"a photo of a receipt",
"a photo of a bill or invoice",
"an internet meme with text overlay",
"a funny image with caption text",
"a digital illustration or graphic design",
],
}
def _compute_text_centroids() -> dict[str, np.ndarray]:
"""Compute the 'photography' and 'other' centroid vectors using the
open_clip text encoder. Only called on the cache-miss path."""
import open_clip
import torch
logger.info("Computing CLIP text centroids for binary classifier")
model, _, _ = open_clip.create_model_and_transforms(
"ViT-B-32", pretrained="laion2b_s34b_b79k"
)
model.eval()
tokenizer = open_clip.get_tokenizer("ViT-B-32")
centroids: dict[str, np.ndarray] = {}
for label, prompts in PROMPTS.items():
tokens = tokenizer(prompts)
with torch.no_grad():
feats = model.encode_text(tokens)
feats = feats / feats.norm(dim=-1, keepdim=True)
avg = feats.mean(dim=0)
avg = avg / avg.norm()
centroids[label] = avg.numpy().astype(np.float32)
return centroids
class CLIPContentClassifier(ContentClassifier):
def __init__(self, settings: VisionSettings):
self._min_confidence = settings.classifier.min_confidence
self._encoder = CLIPVisualEncoder(settings)
cache_dir = Path(settings.models_dir) / "classifier"
cache_dir.mkdir(parents=True, exist_ok=True)
cache_path = cache_dir / "vectors.npz"
if cache_path.exists():
logger.info("Loading cached text centroids from %s", cache_path)
data = np.load(cache_path)
self._photo = data["photography"].astype(np.float32)
self._other = data["other"].astype(np.float32)
else:
centroids = _compute_text_centroids()
self._photo = centroids["photography"]
self._other = centroids["other"]
np.savez(cache_path, photography=self._photo, other=self._other)
logger.info("Cached text centroids to %s", cache_path)
def classify(self, image: np.ndarray) -> ClassificationResult:
vec = self._encoder.encode(image)
s_photo = float(np.dot(vec, self._photo))
s_other = float(np.dot(vec, self._other))
if s_photo >= s_other:
label = "photography"
margin = s_photo - s_other
else:
label = "other"
margin = s_other - s_photo
# 0.01 margin → ~0.3 conf, 0.03+ → ~1.0
confidence = min(1.0, margin * 30)
return ClassificationResult(label=label, confidence=confidence)

58
backend/app/services/vision/embed.py
View File

@@ -1,58 +0,0 @@
"""
OpenCLIP ViT-B/32 visual encoder (ONNX). Produces 512-d image features
consumed by the content classifier. Not exposed as a standalone service;
the classifier owns the lifecycle.
"""
import logging
from pathlib import Path
import numpy as np
import onnxruntime as ort # noqa: F401 (provider plumbing relies on this)
from app.config import VisionSettings
logger = logging.getLogger(__name__)
_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
_STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32)
_SIZE = 224
def _preprocess(image: np.ndarray) -> np.ndarray:
from PIL import Image
img = Image.fromarray(image).convert("RGB")
w, h = img.size
scale = _SIZE / min(w, h)
img = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC)
w, h = img.size
left = (w - _SIZE) // 2
top = (h - _SIZE) // 2
img = img.crop((left, top, left + _SIZE, top + _SIZE))
arr = np.array(img, dtype=np.float32) / 255.0
arr = (arr - _MEAN) / _STD
arr = arr.transpose(2, 0, 1)
return arr[np.newaxis]
class CLIPVisualEncoder:
"""OpenCLIP ViT-B/32 image encoder, 512-d normalized output."""
def __init__(self, settings: VisionSettings):
model_path = Path(settings.models_dir) / "embed" / "visual.onnx"
from app.services.vision.providers import create_session
from app.config import settings as app_settings
logger.info("Loading CLIP visual encoder from %s", model_path)
self._session = create_session(
str(model_path),
configured_providers=app_settings.vision.execution_providers,
)
def encode(self, image: np.ndarray) -> np.ndarray:
inp = _preprocess(image)
name = self._session.get_inputs()[0].name
out = self._session.run(None, {name: inp})[0][0]
out = out / np.linalg.norm(out)
return out.astype(np.float32)

62
backend/app/services/vision/export_models.py
View File

@@ -1,62 +0,0 @@
"""
Export the OpenCLIP ViT-B/32 visual encoder to ONNX.
Run once on any machine with Python + pip (no GPU needed):
pip install open-clip-torch onnx
python -m app.services.vision.export_models [--models-dir /data/models]
Produces:
embed/visual.onnx (~350 MB)
"""
import argparse
import logging
from pathlib import Path
logger = logging.getLogger(__name__)
def export_openclip_visual(models_dir: Path):
import torch
import open_clip
out_dir = models_dir / "embed"
out_dir.mkdir(parents=True, exist_ok=True)
visual_path = out_dir / "visual.onnx"
if visual_path.exists():
logger.info("OpenCLIP visual.onnx already exists, skipping export")
return
logger.info("Loading OpenCLIP ViT-B-32 laion2b_s34b_b79k...")
model, _, _ = open_clip.create_model_and_transforms(
"ViT-B-32", pretrained="laion2b_s34b_b79k"
)
model.eval()
logger.info("Exporting visual encoder → %s", visual_path)
dummy = torch.randn(1, 3, 224, 224)
torch.onnx.export(
model.visual,
dummy,
str(visual_path),
input_names=["image"],
output_names=["embedding"],
dynamic_axes={"image": {0: "batch"}},
opset_version=14,
dynamo=False,
)
logger.info("Visual encoder exported (%.1f MB)", visual_path.stat().st_size / 1e6)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--models-dir", type=Path, default=Path("/data/models"))
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
args.models_dir.mkdir(parents=True, exist_ok=True)
export_openclip_visual(args.models_dir)
if __name__ == "__main__":
main()

86
backend/app/services/vision/providers.py
View File

@@ -1,86 +0,0 @@
"""
ONNX Runtime execution provider resolution with GPU auto-detection.
Resolves configured execution providers against what's actually available
in the current ONNX Runtime build. Falls back to CPU if no GPU provider
is available. Logs the selected provider so users can confirm GPU is active.
"""
import logging
import onnxruntime as ort
logger = logging.getLogger(__name__)
_resolved: list[str] | None = None
def get_providers(configured: list[str] | None = None) -> list[str]:
"""Return the best available execution providers.
1. If `configured` is provided, filter to only those that are
actually available in the current ORT build.
2. If none of the configured providers are available, fall back
to CPUExecutionProvider.
3. Auto-detect: if configured is ["auto"], probe for GPU providers.
Results are cached after first call.
"""
global _resolved
if _resolved is not None:
return _resolved
available = set(ort.get_available_providers())
logger.info("ONNX Runtime available providers: %s", sorted(available))
if configured is None or configured == ["CPUExecutionProvider"]:
_resolved = ["CPUExecutionProvider"]
return _resolved
if configured == ["auto"]:
# Auto-detect: prefer CUDA > ROCm > OpenVINO > CPU
priority = [
"CUDAExecutionProvider",
"ROCMExecutionProvider",
"OpenVINOExecutionProvider",
]
for p in priority:
if p in available:
_resolved = [p, "CPUExecutionProvider"]
logger.info("Auto-detected GPU provider: %s", p)
return _resolved
_resolved = ["CPUExecutionProvider"]
logger.info("No GPU provider detected, using CPU")
return _resolved
# Filter configured list to available providers.
resolved = [p for p in configured if p in available]
if not resolved:
logger.warning(
"None of the configured providers %s are available. "
"Falling back to CPU. Available: %s",
configured,
sorted(available),
)
resolved = ["CPUExecutionProvider"]
else:
# Always include CPU as fallback.
if "CPUExecutionProvider" not in resolved:
resolved.append("CPUExecutionProvider")
_resolved = resolved
logger.info("Using ONNX Runtime providers: %s", _resolved)
return _resolved
def create_session(
model_path: str,
opts: ort.SessionOptions | None = None,
configured_providers: list[str] | None = None,
) -> ort.InferenceSession:
"""Create an ONNX InferenceSession with the best available providers."""
providers = get_providers(configured_providers)
if opts is None:
opts = ort.SessionOptions()
opts.inter_op_num_threads = 2
opts.intra_op_num_threads = 2
return ort.InferenceSession(model_path, opts, providers=providers)

29
backend/app/services/vision/registry.py
View File

@@ -1,29 +0,0 @@
"""
ModelRegistry — lazy-loads the single content classifier per worker.
"""
import logging
from functools import lru_cache
from app.config import settings
from app.services.vision.base import ContentClassifier
logger = logging.getLogger(__name__)
class ModelRegistry:
def __init__(self):
self._vision = settings.vision
@lru_cache(maxsize=1)
def get_classifier(self) -> ContentClassifier:
logger.info("Loading content classifier (backend=%s)", self._vision.backend)
from app.services.vision.classify import CLIPContentClassifier
return CLIPContentClassifier(self._vision)
def warmup(self):
logger.info("Warming up vision classifier...")
self.get_classifier()
logger.info("Vision warmup complete")
registry = ModelRegistry()