feat: runtime feature flags, upload/download, RAW decoding
Adds Redis-backed feature flags for vision stages with admin UI toggles and manual backfill trigger, photo upload and download routers with frontend upload modal, and rawpy-based RAW decoding with JPEG fallback for misnamed DNGs. Fixes pgvector serialization, is_trashed filter, and naive-datetime bind in incremental duplicate regrouping; bumps Celery time limits on regroup tasks beyond the 5-minute default. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -11,7 +11,7 @@ import os
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.database import init_db
|
from app.database import init_db
|
||||||
from app.routers import photos, folders, heaps, tags, discard, library, search, auth, admin, sharing
|
from app.routers import photos, folders, heaps, tags, discard, library, search, auth, admin, sharing, upload, download, features
|
||||||
from app.services.scanner import start_initial_scan, bootstrap_default_source_root
|
from app.services.scanner import start_initial_scan, bootstrap_default_source_root
|
||||||
from app.services.cleanup import cleanup_data_integrity
|
from app.services.cleanup import cleanup_data_integrity
|
||||||
|
|
||||||
@@ -95,6 +95,9 @@ app.include_router(tags.router, prefix="/api/v1/tags", tags=["tags"])
|
|||||||
app.include_router(discard.router, prefix="/api/v1/discard", tags=["discard"])
|
app.include_router(discard.router, prefix="/api/v1/discard", tags=["discard"])
|
||||||
app.include_router(library.router, prefix="/api/v1/library", tags=["library"])
|
app.include_router(library.router, prefix="/api/v1/library", tags=["library"])
|
||||||
app.include_router(search.router, prefix="/api/v1/photos/search", tags=["search"])
|
app.include_router(search.router, prefix="/api/v1/photos/search", tags=["search"])
|
||||||
|
app.include_router(upload.router, prefix="/api/v1/upload", tags=["upload"])
|
||||||
|
app.include_router(download.router, prefix="/api/v1/download", tags=["download"])
|
||||||
|
app.include_router(features.router, prefix="/api/v1/features", tags=["features"])
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
async def root():
|
async def root():
|
||||||
|
|||||||
@@ -18,6 +18,14 @@ from app.models.user import User
|
|||||||
from app.models.photos import Photo
|
from app.models.photos import Photo
|
||||||
from app.models.folders import SourceRoot
|
from app.models.folders import SourceRoot
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
from app.services.feature_flags import (
|
||||||
|
ALL_FLAGS,
|
||||||
|
snapshot as flags_snapshot,
|
||||||
|
set_flag,
|
||||||
|
reset_flag,
|
||||||
|
is_enabled,
|
||||||
|
FLAG_VISION_ENABLED,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -258,3 +266,141 @@ async def delete_user(
|
|||||||
|
|
||||||
logger.info(f"Admin '{admin.username}' deactivated user '{user.username}'")
|
logger.info(f"Admin '{admin.username}' deactivated user '{user.username}'")
|
||||||
return {"status": "ok", "detail": f"User '{user.username}' deactivated"}
|
return {"status": "ok", "detail": f"User '{user.username}' deactivated"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# AI / vision feature flags + manual triggers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class FeatureFlagUpdate(BaseModel):
|
||||||
|
"""PATCH body for toggling a feature flag.
|
||||||
|
|
||||||
|
``value`` sets an explicit override (true/false); omitting it clears
|
||||||
|
the override and reverts the flag to its YAML default.
|
||||||
|
"""
|
||||||
|
value: Optional[bool] = None
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/feature-flags")
|
||||||
|
async def get_feature_flags(admin: User = Depends(require_admin)):
|
||||||
|
"""Return every tunable feature flag with its current effective
|
||||||
|
value, YAML default, and whether an admin override is in effect."""
|
||||||
|
return {"flags": flags_snapshot()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/feature-flags/{flag_name}")
|
||||||
|
async def update_feature_flag(
|
||||||
|
flag_name: str,
|
||||||
|
body: FeatureFlagUpdate,
|
||||||
|
admin: User = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""Set or clear an override for one flag. With ``value`` set, the
|
||||||
|
flag is pinned to that boolean; without it, the override is deleted
|
||||||
|
and the YAML default takes over again.
|
||||||
|
|
||||||
|
New value is observed by vision tasks on their next invocation —
|
||||||
|
there's no worker restart required.
|
||||||
|
"""
|
||||||
|
if flag_name not in ALL_FLAGS:
|
||||||
|
raise HTTPException(status_code=404, detail=f"Unknown flag: {flag_name}")
|
||||||
|
try:
|
||||||
|
if body.value is None:
|
||||||
|
reset_flag(flag_name)
|
||||||
|
action = "cleared override"
|
||||||
|
else:
|
||||||
|
set_flag(flag_name, bool(body.value))
|
||||||
|
action = f"set to {body.value}"
|
||||||
|
except RuntimeError as e:
|
||||||
|
# Redis unreachable — surface as 503 so the UI doesn't think it
|
||||||
|
# succeeded silently.
|
||||||
|
raise HTTPException(status_code=503, detail=str(e))
|
||||||
|
|
||||||
|
logger.info(f"Admin '{admin.username}' {action} for flag '{flag_name}'")
|
||||||
|
return {"flags": flags_snapshot()}
|
||||||
|
|
||||||
|
|
||||||
|
class BackfillVisionBody(BaseModel):
|
||||||
|
"""POST body for triggering a vision backfill. ``task`` picks a
|
||||||
|
specific stage (``embed`` / ``ocr`` / ``detect`` / ``faces`` /
|
||||||
|
``classify``); leaving it null runs every enabled stage. ``limit``
|
||||||
|
caps how many photos per stage are queued — useful for smoke-
|
||||||
|
testing a newly-enabled feature before committing a full run.
|
||||||
|
"""
|
||||||
|
task: Optional[str] = None
|
||||||
|
limit: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/ai/backfill")
|
||||||
|
async def trigger_ai_backfill(
|
||||||
|
body: BackfillVisionBody,
|
||||||
|
admin: User = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""Queue a vision backfill pass. Identical code path as the automatic
|
||||||
|
post-scan backfill — just triggered manually from the UI."""
|
||||||
|
if not is_enabled(FLAG_VISION_ENABLED):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail="Vision is currently disabled; enable it before running a backfill.",
|
||||||
|
)
|
||||||
|
valid_tasks = {'embed', 'ocr', 'detect', 'faces', 'classify'}
|
||||||
|
if body.task is not None and body.task not in valid_tasks:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"task must be one of {sorted(valid_tasks)} or null",
|
||||||
|
)
|
||||||
|
if body.limit is not None and body.limit <= 0:
|
||||||
|
raise HTTPException(status_code=400, detail="limit must be positive")
|
||||||
|
|
||||||
|
# Import lazily so importing admin.py doesn't pull in the whole
|
||||||
|
# vision stack on startup (Celery task module loads numpy etc.).
|
||||||
|
from app.tasks.vision import backfill_vision
|
||||||
|
|
||||||
|
result = backfill_vision.apply_async(
|
||||||
|
kwargs={'task': body.task, 'limit': body.limit}
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
f"Admin '{admin.username}' queued vision backfill "
|
||||||
|
f"(task={body.task}, limit={body.limit}, celery_id={result.id})"
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"status": "queued",
|
||||||
|
"task_id": result.id,
|
||||||
|
"task": body.task,
|
||||||
|
"limit": body.limit,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/ai/recluster-faces")
|
||||||
|
async def trigger_face_recluster(admin: User = Depends(require_admin)):
|
||||||
|
"""Kick off face recluster. Normally auto-fires after a scan via a
|
||||||
|
debounced scheduler; this endpoint is for admins who want to force
|
||||||
|
a fresh clustering pass (e.g. after tweaking ``cluster_eps`` in
|
||||||
|
the YAML config)."""
|
||||||
|
if not is_enabled(FLAG_VISION_ENABLED):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail="Vision is currently disabled; enable it before reclustering.",
|
||||||
|
)
|
||||||
|
from app.tasks.vision import recluster_faces
|
||||||
|
|
||||||
|
result = recluster_faces.apply_async()
|
||||||
|
logger.info(
|
||||||
|
f"Admin '{admin.username}' queued face recluster (celery_id={result.id})"
|
||||||
|
)
|
||||||
|
return {"status": "queued", "task_id": result.id}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/ai/rescan")
|
||||||
|
async def trigger_full_rescan(admin: User = Depends(require_admin)):
|
||||||
|
"""Dispatch the same scan_all_source_roots job the backend runs at
|
||||||
|
startup. Picks up any new files on disk and, through the
|
||||||
|
post-scan hook, queues a vision backfill for whatever still lacks
|
||||||
|
embeddings / OCR / etc.
|
||||||
|
"""
|
||||||
|
from app.tasks.scan import scan_all_source_roots
|
||||||
|
|
||||||
|
result = scan_all_source_roots.apply_async()
|
||||||
|
logger.info(
|
||||||
|
f"Admin '{admin.username}' queued full rescan (celery_id={result.id})"
|
||||||
|
)
|
||||||
|
return {"status": "queued", "task_id": result.id}
|
||||||
|
|||||||
247
backend/app/routers/download.py
Normal file
247
backend/app/routers/download.py
Normal file
@@ -0,0 +1,247 @@
|
|||||||
|
"""
|
||||||
|
Download router — streams a .zip of every photo in a folder (recursively)
|
||||||
|
or a heap back to the browser.
|
||||||
|
|
||||||
|
Auth: both endpoints accept the regular Authorization header *or* a
|
||||||
|
``?token=JWT`` query string, mirroring the media endpoints. That lets the
|
||||||
|
frontend trigger a download with a plain ``<a href>`` (which can't set a
|
||||||
|
header), keeping the client side a one-liner.
|
||||||
|
|
||||||
|
Implementation: we build the zip into a ``NamedTemporaryFile`` and then
|
||||||
|
stream its bytes back, deleting the temp file on the way out. Stored
|
||||||
|
(uncompressed) mode because photos and videos are already compressed —
|
||||||
|
deflating them again just burns CPU for a fraction of a percent. For
|
||||||
|
very large libraries the temp-file route is mildly wasteful vs. a true
|
||||||
|
streaming zip (zipstream-ng etc), but it avoids a new dependency and
|
||||||
|
handles arbitrary folder sizes without blowing out RAM.
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.database import get_db
|
||||||
|
from app.dependencies import get_current_user_media
|
||||||
|
from app.models import Folder, Heap, Photo, SourceRoot
|
||||||
|
from app.models.heaps import heap_photos
|
||||||
|
from app.models.user import User
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_filename(name: str) -> str:
|
||||||
|
"""Strip characters that Content-Disposition or Windows filesystems
|
||||||
|
would choke on. Keeps the download's filename readable without
|
||||||
|
needing any escaping on the client side."""
|
||||||
|
cleaned = re.sub(r'[\\/:*?"<>|\r\n\t]', '_', name).strip().strip('.')
|
||||||
|
return cleaned or 'download'
|
||||||
|
|
||||||
|
|
||||||
|
async def _collect_folder_photos(
|
||||||
|
folder_id: str,
|
||||||
|
user: User,
|
||||||
|
db: AsyncSession,
|
||||||
|
) -> tuple[str, str, List[Photo]]:
|
||||||
|
"""Resolve a folder or source-root id → (base_path, display_name,
|
||||||
|
photos). ``base_path`` is the prefix we strip off each photo's
|
||||||
|
filepath when naming zip entries, so the archive mirrors the user's
|
||||||
|
on-disk structure under that folder.
|
||||||
|
"""
|
||||||
|
folder = (await db.execute(
|
||||||
|
select(Folder).where(Folder.id == folder_id, Folder.user_id == user.id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
|
||||||
|
base_path: str
|
||||||
|
display_name: str
|
||||||
|
if folder is not None:
|
||||||
|
base_path = os.path.normpath(folder.path)
|
||||||
|
display_name = folder.name or os.path.basename(base_path)
|
||||||
|
else:
|
||||||
|
sr = (await db.execute(
|
||||||
|
select(SourceRoot).where(
|
||||||
|
SourceRoot.id == folder_id,
|
||||||
|
SourceRoot.user_id == user.id,
|
||||||
|
)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if sr is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Folder not found")
|
||||||
|
base_path = os.path.normpath(sr.path)
|
||||||
|
display_name = sr.name or os.path.basename(base_path)
|
||||||
|
|
||||||
|
# Every photo whose filepath is at or below the base path — matches
|
||||||
|
# the same prefix convention folders.py uses for recursive deletes.
|
||||||
|
descendant_prefix = base_path.rstrip(os.sep) + os.sep
|
||||||
|
result = await db.execute(
|
||||||
|
select(Photo).where(
|
||||||
|
Photo.user_id == user.id,
|
||||||
|
Photo.is_discarded == False, # noqa: E712
|
||||||
|
(Photo.filepath == base_path) | (Photo.filepath.like(descendant_prefix + '%')),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
photos = list(result.scalars().all())
|
||||||
|
return base_path, display_name, photos
|
||||||
|
|
||||||
|
|
||||||
|
def _build_zip(
|
||||||
|
photos: List[Photo],
|
||||||
|
arcname_fn,
|
||||||
|
) -> tempfile.NamedTemporaryFile:
|
||||||
|
"""Write ``photos`` into a fresh ZIP_STORED temp file.
|
||||||
|
|
||||||
|
``arcname_fn(photo, used_names)`` returns the entry name to use for
|
||||||
|
the given photo; the caller supplies it because folder downloads
|
||||||
|
want path-preserving names while heap downloads flatten to bare
|
||||||
|
filenames (with a collision suffix).
|
||||||
|
"""
|
||||||
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.zip')
|
||||||
|
try:
|
||||||
|
used: set[str] = set()
|
||||||
|
with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_STORED, allowZip64=True) as zf:
|
||||||
|
for p in photos:
|
||||||
|
if not p.filepath or not os.path.exists(p.filepath):
|
||||||
|
# Silent skip: the scanner may have indexed files
|
||||||
|
# that have since been moved / unlinked by a shell.
|
||||||
|
continue
|
||||||
|
name = arcname_fn(p, used)
|
||||||
|
used.add(name)
|
||||||
|
try:
|
||||||
|
zf.write(p.filepath, name)
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning(f"Skipping {p.filepath} in zip: {e}")
|
||||||
|
tmp.close()
|
||||||
|
return tmp
|
||||||
|
except Exception:
|
||||||
|
tmp.close()
|
||||||
|
try:
|
||||||
|
os.unlink(tmp.name)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _stream_and_cleanup(path: str):
|
||||||
|
"""Yield the temp zip in 1 MiB chunks and unlink it when the
|
||||||
|
iterator is exhausted (or GC'd, if the client disconnects early)."""
|
||||||
|
try:
|
||||||
|
with open(path, 'rb') as f:
|
||||||
|
while True:
|
||||||
|
chunk = f.read(1024 * 1024)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
yield chunk
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
os.unlink(path)
|
||||||
|
except OSError as e:
|
||||||
|
logger.debug(f"Temp zip cleanup failed for {path}: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def _dedupe(name: str, used: set[str]) -> str:
|
||||||
|
"""Return ``name`` (or ``name (2)``, ``name (3)`` ...) such that the
|
||||||
|
result doesn't collide with anything in ``used``. Needed for heap
|
||||||
|
downloads where two members can have identical filenames from
|
||||||
|
different folders."""
|
||||||
|
if name not in used:
|
||||||
|
return name
|
||||||
|
stem, ext = os.path.splitext(name)
|
||||||
|
n = 2
|
||||||
|
while True:
|
||||||
|
cand = f"{stem} ({n}){ext}"
|
||||||
|
if cand not in used:
|
||||||
|
return cand
|
||||||
|
n += 1
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/folders/{folder_id}")
|
||||||
|
async def download_folder(
|
||||||
|
folder_id: str,
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user_media),
|
||||||
|
):
|
||||||
|
"""Zip every (non-discarded) photo under a folder/source-root and
|
||||||
|
stream it back. Entries preserve the folder structure relative to
|
||||||
|
the downloaded root so the resulting archive is a faithful snapshot.
|
||||||
|
"""
|
||||||
|
base_path, display_name, photos = await _collect_folder_photos(
|
||||||
|
folder_id, current_user, db
|
||||||
|
)
|
||||||
|
if not photos:
|
||||||
|
raise HTTPException(status_code=404, detail="No photos to download")
|
||||||
|
|
||||||
|
def arcname(p: Photo, _used: set[str]) -> str:
|
||||||
|
# Relative path from the download root, falling back to the
|
||||||
|
# bare filename if the photo somehow lives outside base_path.
|
||||||
|
abs_path = os.path.normpath(p.filepath)
|
||||||
|
if abs_path.startswith(base_path + os.sep):
|
||||||
|
rel = abs_path[len(base_path) + 1:]
|
||||||
|
elif abs_path == base_path:
|
||||||
|
rel = os.path.basename(abs_path)
|
||||||
|
else:
|
||||||
|
rel = p.filename or os.path.basename(abs_path)
|
||||||
|
# Nest everything under display_name so users see one top-level
|
||||||
|
# folder inside the zip rather than loose files.
|
||||||
|
return os.path.join(_safe_filename(display_name), rel)
|
||||||
|
|
||||||
|
tmp = _build_zip(photos, arcname)
|
||||||
|
filename = _safe_filename(display_name) + '.zip'
|
||||||
|
return StreamingResponse(
|
||||||
|
_stream_and_cleanup(tmp.name),
|
||||||
|
media_type='application/zip',
|
||||||
|
headers={
|
||||||
|
'Content-Disposition': f'attachment; filename="{filename}"',
|
||||||
|
'Content-Length': str(os.path.getsize(tmp.name)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/heaps/{heap_id}")
|
||||||
|
async def download_heap(
|
||||||
|
heap_id: str,
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user_media),
|
||||||
|
):
|
||||||
|
"""Zip every photo in a heap. Heaps are flat collections, so entries
|
||||||
|
use the original filename (with a ``(2)`` collision suffix when
|
||||||
|
two members share a name)."""
|
||||||
|
heap = (await db.execute(
|
||||||
|
select(Heap).where(Heap.id == heap_id, Heap.user_id == current_user.id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if heap is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Heap not found")
|
||||||
|
|
||||||
|
result = await db.execute(
|
||||||
|
select(Photo)
|
||||||
|
.join(heap_photos, heap_photos.c.photo_id == Photo.id)
|
||||||
|
.where(
|
||||||
|
heap_photos.c.heap_id == heap_id,
|
||||||
|
Photo.is_discarded == False, # noqa: E712
|
||||||
|
)
|
||||||
|
)
|
||||||
|
photos = list(result.scalars().all())
|
||||||
|
if not photos:
|
||||||
|
raise HTTPException(status_code=404, detail="Heap is empty")
|
||||||
|
|
||||||
|
def arcname(p: Photo, used: set[str]) -> str:
|
||||||
|
bare = p.filename or os.path.basename(p.filepath or 'photo')
|
||||||
|
entry = os.path.join(_safe_filename(heap.name), _dedupe(bare, used))
|
||||||
|
return entry
|
||||||
|
|
||||||
|
tmp = _build_zip(photos, arcname)
|
||||||
|
filename = _safe_filename(heap.name) + '.zip'
|
||||||
|
return StreamingResponse(
|
||||||
|
_stream_and_cleanup(tmp.name),
|
||||||
|
media_type='application/zip',
|
||||||
|
headers={
|
||||||
|
'Content-Disposition': f'attachment; filename="{filename}"',
|
||||||
|
'Content-Length': str(os.path.getsize(tmp.name)),
|
||||||
|
},
|
||||||
|
)
|
||||||
23
backend/app/routers/features.py
Normal file
23
backend/app/routers/features.py
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
"""
|
||||||
|
Public feature-flag read API — lets the authenticated frontend know
|
||||||
|
which AI-powered sections to render.
|
||||||
|
|
||||||
|
This is NOT the admin mutation endpoint (that's in ``admin.py`` and
|
||||||
|
gated by ``require_admin``). Here we only expose the effective boolean
|
||||||
|
state so the UI can hide things like the People view, Tags view, or
|
||||||
|
text-search affordances when the underlying pipeline stage is off.
|
||||||
|
"""
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
|
||||||
|
from app.dependencies import get_current_user
|
||||||
|
from app.models.user import User
|
||||||
|
from app.services.feature_flags import ALL_FLAGS, is_enabled
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("")
|
||||||
|
async def get_enabled_features(_: User = Depends(get_current_user)):
|
||||||
|
"""Return ``{flag_name: bool}`` for every known flag, reflecting
|
||||||
|
the currently effective value (admin override or YAML default)."""
|
||||||
|
return {name: is_enabled(name) for name in ALL_FLAGS}
|
||||||
303
backend/app/routers/upload.py
Normal file
303
backend/app/routers/upload.py
Normal file
@@ -0,0 +1,303 @@
|
|||||||
|
"""
|
||||||
|
Upload router — lets users drop files (or whole folders) from their
|
||||||
|
desktop into a destination Folder, preserving any sub-folder structure
|
||||||
|
they bring with them.
|
||||||
|
|
||||||
|
Each POST handles one file. The frontend fans out many parallel requests
|
||||||
|
per drop, giving it per-file progress without the server having to
|
||||||
|
invent a chunking protocol. For folder uploads, the browser passes
|
||||||
|
`webkitRelativePath` under the `relative_path` field; any leading
|
||||||
|
sub-directories there are materialised on disk (and as Folder rows)
|
||||||
|
under the destination.
|
||||||
|
|
||||||
|
Uploaded files are placed under the destination folder on the owner's
|
||||||
|
media mount, indexed immediately (Photo row created), and queued for
|
||||||
|
the same thumb + metadata pipeline that the scanner uses. An optional
|
||||||
|
`heap_id` also drops them into a heap in the same request.
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
||||||
|
from sqlalchemy import insert, select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.database import get_db
|
||||||
|
from app.dependencies import get_current_user
|
||||||
|
from app.models import Folder, Heap, Photo, SourceRoot
|
||||||
|
from app.models.heaps import heap_photos
|
||||||
|
from app.models.user import User
|
||||||
|
from app.services.date_guess import has_date_warning
|
||||||
|
from app.tasks.scan import SUPPORTED_EXTENSIONS, get_media_type
|
||||||
|
from app.tasks.thumbs import generate_thumbnails
|
||||||
|
from app.services.metadata import extract_metadata
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
MAX_UPLOAD_BYTES = 500 * 1024 * 1024 # 500 MB per file cap.
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_segment(segment: str) -> str:
|
||||||
|
"""Reject path segments that would escape the destination directory."""
|
||||||
|
segment = segment.strip()
|
||||||
|
if not segment or segment in ('.', '..') or '/' in segment or '\\' in segment:
|
||||||
|
raise HTTPException(status_code=400, detail=f"Invalid path segment: {segment!r}")
|
||||||
|
return segment
|
||||||
|
|
||||||
|
|
||||||
|
def _sanitize_relative_path(rel: Optional[str]) -> list[str]:
|
||||||
|
"""Split `relative_path` into safe segments (dirs + filename).
|
||||||
|
|
||||||
|
Empty or missing → []. Any absolute path, backslash, or `..` segment
|
||||||
|
raises 400 — we never want an upload to escape the destination.
|
||||||
|
"""
|
||||||
|
if not rel:
|
||||||
|
return []
|
||||||
|
# Normalise backslashes to forward slashes; browsers on Windows send
|
||||||
|
# webkitRelativePath with forward slashes anyway, but defend in depth.
|
||||||
|
rel = rel.replace('\\', '/').strip('/')
|
||||||
|
if not rel:
|
||||||
|
return []
|
||||||
|
segs = [_validate_segment(s) for s in rel.split('/') if s]
|
||||||
|
return segs
|
||||||
|
|
||||||
|
|
||||||
|
async def _resolve_destination(
|
||||||
|
folder_id: str,
|
||||||
|
user: User,
|
||||||
|
db: AsyncSession,
|
||||||
|
) -> Folder:
|
||||||
|
"""Resolve `folder_id` to a concrete Folder row the user owns.
|
||||||
|
|
||||||
|
Accepts both Folder ids and SourceRoot ids (for source roots, we
|
||||||
|
return the Folder row at the mount path — the scanner creates one
|
||||||
|
for every source root it walks). Raises 404 if neither matches.
|
||||||
|
"""
|
||||||
|
folder = (await db.execute(
|
||||||
|
select(Folder).where(Folder.id == folder_id, Folder.user_id == user.id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if folder is not None:
|
||||||
|
return folder
|
||||||
|
|
||||||
|
sr = (await db.execute(
|
||||||
|
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == user.id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if sr is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Destination folder not found")
|
||||||
|
|
||||||
|
root_folder = (await db.execute(
|
||||||
|
select(Folder).where(
|
||||||
|
Folder.source_root_id == sr.id,
|
||||||
|
Folder.user_id == user.id,
|
||||||
|
Folder.path == os.path.normpath(sr.path),
|
||||||
|
)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if root_folder is None:
|
||||||
|
# First-time source root with no walk yet — create the row now so
|
||||||
|
# uploads work even before the initial scan has run.
|
||||||
|
root_folder = Folder(
|
||||||
|
name=sr.name or os.path.basename(sr.path),
|
||||||
|
path=os.path.normpath(sr.path),
|
||||||
|
source_root_id=sr.id,
|
||||||
|
user_id=user.id,
|
||||||
|
)
|
||||||
|
os.makedirs(root_folder.path, exist_ok=True)
|
||||||
|
db.add(root_folder)
|
||||||
|
await db.flush()
|
||||||
|
return root_folder
|
||||||
|
|
||||||
|
|
||||||
|
async def _ensure_subfolder(
|
||||||
|
parent: Folder,
|
||||||
|
name: str,
|
||||||
|
user: User,
|
||||||
|
db: AsyncSession,
|
||||||
|
) -> Folder:
|
||||||
|
"""Return (or create) a Folder row named `name` under `parent`.
|
||||||
|
|
||||||
|
Also mkdirs the directory on disk. Idempotent — safe to call for a
|
||||||
|
path segment that already exists as a Folder row or directory.
|
||||||
|
"""
|
||||||
|
child_path = os.path.normpath(os.path.join(parent.path, name))
|
||||||
|
|
||||||
|
existing = (await db.execute(
|
||||||
|
select(Folder).where(
|
||||||
|
Folder.path == child_path,
|
||||||
|
Folder.user_id == user.id,
|
||||||
|
)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
os.makedirs(child_path, exist_ok=True)
|
||||||
|
return existing
|
||||||
|
|
||||||
|
os.makedirs(child_path, exist_ok=True)
|
||||||
|
child = Folder(
|
||||||
|
name=name,
|
||||||
|
path=child_path,
|
||||||
|
parent_id=parent.id,
|
||||||
|
source_root_id=parent.source_root_id,
|
||||||
|
user_id=user.id,
|
||||||
|
is_hidden=parent.is_hidden,
|
||||||
|
)
|
||||||
|
db.add(child)
|
||||||
|
await db.flush()
|
||||||
|
return child
|
||||||
|
|
||||||
|
|
||||||
|
def _unique_path(target_dir: str, filename: str) -> tuple[str, str]:
|
||||||
|
"""Return a (filepath, filename) that doesn't collide with an
|
||||||
|
existing file on disk. Suffixes " (2)", " (3)", ... until a free
|
||||||
|
slot is found. Prevents upload-over-existing and keeps the user's
|
||||||
|
original file intact.
|
||||||
|
"""
|
||||||
|
base, ext = os.path.splitext(filename)
|
||||||
|
candidate = os.path.join(target_dir, filename)
|
||||||
|
n = 2
|
||||||
|
while os.path.exists(candidate):
|
||||||
|
new_name = f"{base} ({n}){ext}"
|
||||||
|
candidate = os.path.join(target_dir, new_name)
|
||||||
|
n += 1
|
||||||
|
return candidate, os.path.basename(candidate)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("")
|
||||||
|
async def upload_file(
|
||||||
|
file: UploadFile = File(...),
|
||||||
|
destination_folder_id: str = Form(...),
|
||||||
|
relative_path: Optional[str] = Form(None),
|
||||||
|
heap_id: Optional[str] = Form(None),
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
"""Upload a single file into a destination folder (and optionally a
|
||||||
|
heap). For folder uploads, `relative_path` carries the sub-folder
|
||||||
|
chain from the browser's `webkitRelativePath`, and we materialise
|
||||||
|
it under the destination on disk + as Folder rows.
|
||||||
|
|
||||||
|
Returns the created photo's id on success. 4xx on unsupported file
|
||||||
|
type, bad path, missing destination, or too-large file.
|
||||||
|
"""
|
||||||
|
# --- validate inputs -------------------------------------------------
|
||||||
|
raw_name = file.filename or ''
|
||||||
|
if not raw_name:
|
||||||
|
raise HTTPException(status_code=400, detail="Missing filename")
|
||||||
|
|
||||||
|
# Prefer the leaf of relative_path when present (it contains the
|
||||||
|
# original filename as the browser saw it inside the picked folder).
|
||||||
|
segs = _sanitize_relative_path(relative_path)
|
||||||
|
if segs:
|
||||||
|
leaf = segs[-1]
|
||||||
|
subdirs = segs[:-1]
|
||||||
|
else:
|
||||||
|
leaf = _validate_segment(os.path.basename(raw_name))
|
||||||
|
subdirs = []
|
||||||
|
|
||||||
|
ext = Path(leaf).suffix.lower()
|
||||||
|
if ext not in SUPPORTED_EXTENSIONS:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Unsupported file type: {ext or '(none)'}",
|
||||||
|
)
|
||||||
|
|
||||||
|
dest_folder = await _resolve_destination(destination_folder_id, current_user, db)
|
||||||
|
|
||||||
|
target_folder = dest_folder
|
||||||
|
for seg in subdirs:
|
||||||
|
target_folder = await _ensure_subfolder(target_folder, seg, current_user, db)
|
||||||
|
|
||||||
|
target_dir = target_folder.path
|
||||||
|
os.makedirs(target_dir, exist_ok=True)
|
||||||
|
filepath, final_name = _unique_path(target_dir, leaf)
|
||||||
|
|
||||||
|
# --- stream to disk, hash as we go ----------------------------------
|
||||||
|
hasher = hashlib.sha256()
|
||||||
|
total = 0
|
||||||
|
try:
|
||||||
|
with open(filepath, 'wb') as out:
|
||||||
|
while True:
|
||||||
|
chunk = await file.read(1024 * 1024)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
total += len(chunk)
|
||||||
|
if total > MAX_UPLOAD_BYTES:
|
||||||
|
out.close()
|
||||||
|
os.unlink(filepath)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=413,
|
||||||
|
detail=f"File exceeds {MAX_UPLOAD_BYTES // (1024*1024)}MB limit",
|
||||||
|
)
|
||||||
|
hasher.update(chunk)
|
||||||
|
out.write(chunk)
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Upload write failed for {filepath}: {e}")
|
||||||
|
if os.path.exists(filepath):
|
||||||
|
try:
|
||||||
|
os.unlink(filepath)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
raise HTTPException(status_code=500, detail=f"Upload failed: {e}")
|
||||||
|
|
||||||
|
file_hash = hasher.hexdigest()
|
||||||
|
|
||||||
|
# --- validate heap before committing the DB row ---------------------
|
||||||
|
if heap_id:
|
||||||
|
heap = (await db.execute(
|
||||||
|
select(Heap).where(Heap.id == heap_id, Heap.user_id == current_user.id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if heap is None:
|
||||||
|
# Destination heap vanished — still keep the file + photo row,
|
||||||
|
# but tell the caller so the UI can surface the mismatch.
|
||||||
|
heap_id = None
|
||||||
|
|
||||||
|
# --- create Photo row ------------------------------------------------
|
||||||
|
mtime_dt = datetime.fromtimestamp(os.stat(filepath).st_mtime)
|
||||||
|
photo = Photo(
|
||||||
|
filepath=filepath,
|
||||||
|
filename=final_name,
|
||||||
|
folder_id=target_folder.id,
|
||||||
|
user_id=current_user.id,
|
||||||
|
file_hash=file_hash,
|
||||||
|
media_type=get_media_type(filepath),
|
||||||
|
original_format=Path(filepath).suffix.upper()[1:],
|
||||||
|
file_size=total,
|
||||||
|
taken_at=mtime_dt,
|
||||||
|
taken_at_source='filesystem',
|
||||||
|
has_date_warning=has_date_warning(filepath, mtime_dt),
|
||||||
|
is_hidden=bool(target_folder.is_hidden),
|
||||||
|
processing_status='pending',
|
||||||
|
)
|
||||||
|
db.add(photo)
|
||||||
|
await db.flush()
|
||||||
|
|
||||||
|
if heap_id:
|
||||||
|
await db.execute(
|
||||||
|
insert(heap_photos),
|
||||||
|
[{"heap_id": heap_id, "photo_id": photo.id}],
|
||||||
|
)
|
||||||
|
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
# Queue the same background work the scanner does so thumbnails +
|
||||||
|
# EXIF show up without the user having to trigger a rescan.
|
||||||
|
try:
|
||||||
|
generate_thumbnails.delay(photo.id)
|
||||||
|
extract_metadata.delay(photo.id)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to queue post-upload tasks for {photo.id}: {e}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"photo_id": photo.id,
|
||||||
|
"filename": final_name,
|
||||||
|
"folder_id": target_folder.id,
|
||||||
|
"folder_path": target_folder.path,
|
||||||
|
"heap_id": heap_id,
|
||||||
|
}
|
||||||
@@ -221,6 +221,13 @@ async def incremental_regroup(
|
|||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
since = datetime.now(timezone.utc) - timedelta(hours=1)
|
since = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||||
|
|
||||||
|
# Photo.added_at is stored as TIMESTAMP WITHOUT TIME ZONE, so
|
||||||
|
# asyncpg rejects aware datetimes with "can't subtract offset-naive
|
||||||
|
# and offset-aware". Normalise: if `since` has a tzinfo, convert
|
||||||
|
# it to UTC and drop the tzinfo so the bind parameter is naive.
|
||||||
|
if since.tzinfo is not None:
|
||||||
|
since = since.astimezone(timezone.utc).replace(tzinfo=None)
|
||||||
|
|
||||||
# Get newly added photos (the "new" set).
|
# Get newly added photos (the "new" set).
|
||||||
new_rows = (
|
new_rows = (
|
||||||
await session.execute(
|
await session.execute(
|
||||||
@@ -349,6 +356,14 @@ async def _clip_neighbor_scan(
|
|||||||
for photo_id, vector in target_embeddings:
|
for photo_id, vector in target_embeddings:
|
||||||
# pgvector cosine distance: <=> operator
|
# pgvector cosine distance: <=> operator
|
||||||
# Find top 20 nearest neighbors within threshold.
|
# Find top 20 nearest neighbors within threshold.
|
||||||
|
# Serialize the vector as "[a,b,c,...]" — pgvector's text
|
||||||
|
# format uses commas; numpy's default str() joins with spaces
|
||||||
|
# which Postgres rejects with "invalid input syntax for vector".
|
||||||
|
if hasattr(vector, 'tolist'):
|
||||||
|
vec_seq = vector.tolist()
|
||||||
|
else:
|
||||||
|
vec_seq = list(vector)
|
||||||
|
vec_text = '[' + ','.join(f'{float(x):.8f}' for x in vec_seq) + ']'
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
text("""
|
text("""
|
||||||
SELECT e.photo_id, (e.vector <=> :vec) AS distance
|
SELECT e.photo_id, (e.vector <=> :vec) AS distance
|
||||||
@@ -356,14 +371,14 @@ async def _clip_neighbor_scan(
|
|||||||
JOIN photos p ON p.id = e.photo_id
|
JOIN photos p ON p.id = e.photo_id
|
||||||
WHERE e.model = :model
|
WHERE e.model = :model
|
||||||
AND e.photo_id != :pid
|
AND e.photo_id != :pid
|
||||||
AND p.is_discarded = false
|
AND p.is_trashed = false
|
||||||
AND p.is_hidden = false
|
AND p.is_hidden = false
|
||||||
AND (e.vector <=> :vec) < :threshold
|
AND (e.vector <=> :vec) < :threshold
|
||||||
ORDER BY e.vector <=> :vec
|
ORDER BY e.vector <=> :vec
|
||||||
LIMIT 20
|
LIMIT 20
|
||||||
"""),
|
"""),
|
||||||
{
|
{
|
||||||
'vec': str(vector),
|
'vec': vec_text,
|
||||||
'pid': photo_id,
|
'pid': photo_id,
|
||||||
'model': embedder_model,
|
'model': embedder_model,
|
||||||
'threshold': threshold,
|
'threshold': threshold,
|
||||||
|
|||||||
301
backend/app/services/feature_flags.py
Normal file
301
backend/app/services/feature_flags.py
Normal file
@@ -0,0 +1,301 @@
|
|||||||
|
"""
|
||||||
|
Runtime feature flags for expensive pipeline stages.
|
||||||
|
|
||||||
|
The YAML config (``mulita.yml``) ships reasonable defaults. Admins can
|
||||||
|
toggle these at runtime from the Settings → AI Features tab without
|
||||||
|
rebuilding the image or editing the bind-mounted YAML; the overrides
|
||||||
|
live in Redis so both the FastAPI backend and the Celery workers see
|
||||||
|
the same value within ~1s of the write.
|
||||||
|
|
||||||
|
The key namespace is:
|
||||||
|
|
||||||
|
mulita:flags:<name> → "true" | "false"
|
||||||
|
|
||||||
|
An unset key means "fall back to the YAML default" — so an admin who
|
||||||
|
has never touched the tab sees exactly the config-file behaviour.
|
||||||
|
|
||||||
|
Only bool flags live here. Thresholds, batch sizes, model names etc.
|
||||||
|
stay in the YAML file because flipping them safely requires restarting
|
||||||
|
the vision workers (model reload, ONNX session re-init); that's not
|
||||||
|
something a single admin click should do.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# Feature identifiers. The public name is what the admin UI sends; the
|
||||||
|
# ``yaml_default`` getter returns the value the YAML would have set.
|
||||||
|
# Keep these in sync with the VisionSettings fields in ``config.py``.
|
||||||
|
FLAG_VISION_ENABLED = 'vision.enabled'
|
||||||
|
FLAG_OCR_ENABLED = 'vision.ocr.enabled'
|
||||||
|
FLAG_DETECTOR_ENABLED = 'vision.detector.enabled'
|
||||||
|
FLAG_FACES_ENABLED = 'vision.faces.enabled'
|
||||||
|
FLAG_CLASSIFIER_ENABLED = 'vision.classifier.enabled'
|
||||||
|
|
||||||
|
ALL_FLAGS = (
|
||||||
|
FLAG_VISION_ENABLED,
|
||||||
|
FLAG_OCR_ENABLED,
|
||||||
|
FLAG_DETECTOR_ENABLED,
|
||||||
|
FLAG_FACES_ENABLED,
|
||||||
|
FLAG_CLASSIFIER_ENABLED,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_REDIS: Optional[redis.Redis] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _redis() -> Optional[redis.Redis]:
|
||||||
|
"""Lazy Redis client. Returns None if the broker is unreachable so
|
||||||
|
callers can fall back to YAML defaults instead of crashing."""
|
||||||
|
global _REDIS
|
||||||
|
if _REDIS is None:
|
||||||
|
try:
|
||||||
|
_REDIS = redis.Redis.from_url(
|
||||||
|
settings.celery_broker_url, decode_responses=True
|
||||||
|
)
|
||||||
|
_REDIS.ping()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"feature_flags: Redis unavailable, using YAML defaults ({e})")
|
||||||
|
_REDIS = None
|
||||||
|
return _REDIS
|
||||||
|
|
||||||
|
|
||||||
|
def _yaml_default(name: str) -> bool:
|
||||||
|
"""Return the YAML-configured default for a flag. Used when Redis
|
||||||
|
has no value set (fresh install or admin never touched the tab)."""
|
||||||
|
v = settings.vision
|
||||||
|
if name == FLAG_VISION_ENABLED:
|
||||||
|
return bool(v.enabled)
|
||||||
|
if name == FLAG_OCR_ENABLED:
|
||||||
|
return bool(v.ocr.enabled)
|
||||||
|
if name == FLAG_DETECTOR_ENABLED:
|
||||||
|
return bool(v.detector.enabled)
|
||||||
|
if name == FLAG_FACES_ENABLED:
|
||||||
|
return bool(v.faces.enabled)
|
||||||
|
if name == FLAG_CLASSIFIER_ENABLED:
|
||||||
|
return bool(v.classifier.enabled)
|
||||||
|
raise ValueError(f"Unknown feature flag: {name!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def _redis_key(name: str) -> str:
|
||||||
|
return f"mulita:flags:{name}"
|
||||||
|
|
||||||
|
|
||||||
|
def is_enabled(name: str) -> bool:
|
||||||
|
"""Return True if feature ``name`` is currently enabled.
|
||||||
|
|
||||||
|
Order of precedence:
|
||||||
|
1. Redis override (set by PATCH /admin/feature-flags)
|
||||||
|
2. YAML default
|
||||||
|
|
||||||
|
Reads are cheap (~ms) and we intentionally do NOT add a local
|
||||||
|
process cache — the whole point of runtime flags is that a toggle
|
||||||
|
takes effect on the next task without a worker restart.
|
||||||
|
"""
|
||||||
|
r = _redis()
|
||||||
|
if r is not None:
|
||||||
|
try:
|
||||||
|
raw = r.get(_redis_key(name))
|
||||||
|
if raw is not None:
|
||||||
|
return raw.lower() == 'true'
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"feature_flags: Redis read failed for {name} ({e})")
|
||||||
|
return _yaml_default(name)
|
||||||
|
|
||||||
|
|
||||||
|
def set_flag(name: str, value: bool) -> None:
|
||||||
|
"""Persist a flag override to Redis. No-op if Redis is unreachable
|
||||||
|
(we don't silently pretend to have written; raise so the admin
|
||||||
|
request returns a 500 instead of misleading success)."""
|
||||||
|
if name not in ALL_FLAGS:
|
||||||
|
raise ValueError(f"Unknown feature flag: {name!r}")
|
||||||
|
r = _redis()
|
||||||
|
if r is None:
|
||||||
|
raise RuntimeError("Redis unavailable; cannot update feature flags")
|
||||||
|
r.set(_redis_key(name), 'true' if value else 'false')
|
||||||
|
_apply_worker_side_effects(name)
|
||||||
|
|
||||||
|
|
||||||
|
def reset_flag(name: str) -> None:
|
||||||
|
"""Delete the Redis override so the flag falls back to its YAML
|
||||||
|
default. Useful if an admin wants a clean slate without guessing
|
||||||
|
what the config defaults are."""
|
||||||
|
if name not in ALL_FLAGS:
|
||||||
|
raise ValueError(f"Unknown feature flag: {name!r}")
|
||||||
|
r = _redis()
|
||||||
|
if r is None:
|
||||||
|
raise RuntimeError("Redis unavailable; cannot reset feature flags")
|
||||||
|
r.delete(_redis_key(name))
|
||||||
|
_apply_worker_side_effects(name)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Worker-level side effects: when the admin flips a flag we don't just want
|
||||||
|
# gating at task-start (which still executes the message, it just returns
|
||||||
|
# 'skipped'). We also want queued work gone and the vision worker genuinely
|
||||||
|
# idle when the master switch is off.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_VISION_QUEUE = 'vision'
|
||||||
|
|
||||||
|
# Flag → celery task name(s) whose queued messages should be dropped when
|
||||||
|
# the flag goes off. Keeps the queue from replaying yesterday's work the
|
||||||
|
# moment someone re-enables the stage.
|
||||||
|
_TASKS_BY_FLAG: dict[str, tuple[str, ...]] = {
|
||||||
|
FLAG_VISION_ENABLED: (
|
||||||
|
'embed_photo', 'ocr_photo', 'detect_objects', 'extract_faces',
|
||||||
|
'classify_content', 'vision_fanout', 'recluster_faces',
|
||||||
|
),
|
||||||
|
FLAG_OCR_ENABLED: ('ocr_photo',),
|
||||||
|
FLAG_DETECTOR_ENABLED: ('detect_objects',),
|
||||||
|
FLAG_FACES_ENABLED: ('extract_faces', 'recluster_faces'),
|
||||||
|
FLAG_CLASSIFIER_ENABLED: ('classify_content',),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_worker_side_effects(name: str) -> None:
|
||||||
|
"""Bring the live workers in line with the new flag value.
|
||||||
|
|
||||||
|
For the master ``vision.enabled`` flag we go beyond task gating and
|
||||||
|
actually stop consumption from the ``vision`` queue — flipping it
|
||||||
|
off puts the vision worker to sleep (no CPU, no model memory
|
||||||
|
churn) until it's flipped back on. For per-feature flags, the
|
||||||
|
running tasks already skip via ``is_enabled``; we just purge any
|
||||||
|
messages already sitting in the queue so the admin doesn't pay for
|
||||||
|
a backlog on re-enable.
|
||||||
|
|
||||||
|
All operations are best-effort — if control messaging or a Redis
|
||||||
|
op fails, we log and return; the flag state itself is already
|
||||||
|
persisted so the gating path continues to work.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Lazy import: avoids a circular dependency between the services
|
||||||
|
# module (imported from tasks.vision) and the celery app config.
|
||||||
|
from app.tasks.celery import celery_app
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"feature_flags: celery app unavailable for side effects ({e})")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
if name == FLAG_VISION_ENABLED:
|
||||||
|
if is_enabled(FLAG_VISION_ENABLED):
|
||||||
|
# Re-attach the vision consumer so workers pick up tasks
|
||||||
|
# again. broadcast=True ensures every running worker
|
||||||
|
# receives the command.
|
||||||
|
celery_app.control.add_consumer(_VISION_QUEUE, reply=False)
|
||||||
|
logger.info("feature_flags: vision re-enabled; consumer added")
|
||||||
|
else:
|
||||||
|
celery_app.control.cancel_consumer(_VISION_QUEUE, reply=False)
|
||||||
|
_purge_queue(_VISION_QUEUE)
|
||||||
|
logger.info(
|
||||||
|
"feature_flags: vision disabled; consumer cancelled "
|
||||||
|
"and queue purged"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Per-feature flag going off → drop pending tasks of its types.
|
||||||
|
if not is_enabled(name):
|
||||||
|
targets = _TASKS_BY_FLAG.get(name, ())
|
||||||
|
if targets:
|
||||||
|
removed = _purge_queue_by_task_names(_VISION_QUEUE, targets)
|
||||||
|
logger.info(
|
||||||
|
f"feature_flags: {name} disabled; removed {removed} "
|
||||||
|
f"pending messages from {_VISION_QUEUE}"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"feature_flags: worker side effects failed for {name}: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def _purge_queue(queue: str) -> int:
|
||||||
|
"""Drop every pending message from ``queue``. Returns the count
|
||||||
|
deleted. Celery's control.purge() purges the default queue only,
|
||||||
|
so we delete the Redis key directly (the broker's queue list)."""
|
||||||
|
r = _redis()
|
||||||
|
if r is None:
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
removed = r.delete(queue)
|
||||||
|
return int(removed or 0)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"feature_flags: purge {queue} failed: {e}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _purge_queue_by_task_names(queue: str, task_names: tuple[str, ...]) -> int:
|
||||||
|
"""Walk ``queue`` and drop any message whose Celery task name is in
|
||||||
|
``task_names``. Other messages are preserved (pushed back in order)
|
||||||
|
so we don't flush embed tasks when the admin disabled only OCR.
|
||||||
|
|
||||||
|
Celery stores each message as a JSON blob in a Redis list; the
|
||||||
|
task name lives at ``headers.task``.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
r = _redis()
|
||||||
|
if r is None:
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
# Snapshot the queue, then rebuild it without the filtered names.
|
||||||
|
# Done inside a Redis transaction so a concurrent enqueue doesn't
|
||||||
|
# race with us (worst case it gets re-delivered after we release,
|
||||||
|
# which is the normal enqueue path anyway).
|
||||||
|
pipe = r.pipeline()
|
||||||
|
pipe.lrange(queue, 0, -1)
|
||||||
|
pipe.delete(queue)
|
||||||
|
raw_items, _ = pipe.execute()
|
||||||
|
kept: list[bytes | str] = []
|
||||||
|
removed = 0
|
||||||
|
for raw in raw_items or []:
|
||||||
|
try:
|
||||||
|
# Messages can be bytes or str depending on decode_responses.
|
||||||
|
payload = raw.decode() if isinstance(raw, bytes) else raw
|
||||||
|
msg = json.loads(payload)
|
||||||
|
task = (
|
||||||
|
msg.get('headers', {}).get('task')
|
||||||
|
or msg.get('task')
|
||||||
|
)
|
||||||
|
if task in task_names:
|
||||||
|
removed += 1
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
# Unparseable message — keep it, better to leak than
|
||||||
|
# to silently drop a message we can't identify.
|
||||||
|
pass
|
||||||
|
kept.append(raw)
|
||||||
|
if kept:
|
||||||
|
r.rpush(queue, *kept)
|
||||||
|
return removed
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"feature_flags: selective purge failed on {queue}: {e}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot() -> dict[str, dict[str, object]]:
|
||||||
|
"""Return every flag's current effective value, YAML default, and
|
||||||
|
whether it's overridden. Powers the admin UI tab.
|
||||||
|
"""
|
||||||
|
r = _redis()
|
||||||
|
out: dict[str, dict[str, object]] = {}
|
||||||
|
for name in ALL_FLAGS:
|
||||||
|
default = _yaml_default(name)
|
||||||
|
override = None
|
||||||
|
if r is not None:
|
||||||
|
try:
|
||||||
|
raw = r.get(_redis_key(name))
|
||||||
|
if raw is not None:
|
||||||
|
override = raw.lower() == 'true'
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
out[name] = {
|
||||||
|
'effective': override if override is not None else default,
|
||||||
|
'default': default,
|
||||||
|
'overridden': override is not None,
|
||||||
|
}
|
||||||
|
return out
|
||||||
@@ -313,6 +313,24 @@ async def _generate_thumbnails_async(photo_id: str, task):
|
|||||||
logger.error(f"Unsupported media type: {photo.media_type}")
|
logger.error(f"Unsupported media type: {photo.media_type}")
|
||||||
image = create_placeholder_thumbnail(photo.media_type)
|
image = create_placeholder_thumbnail(photo.media_type)
|
||||||
|
|
||||||
|
# Fallback: some files wear a RAW/HEIC extension but are actually
|
||||||
|
# plain JPEGs — e.g. iPhones that write ProRAW-style .DNG for
|
||||||
|
# images where no RAW sensor data was captured, or re-exports
|
||||||
|
# that kept the original suffix. Pillow can open them directly,
|
||||||
|
# so before giving up, try reading the file as a standard image.
|
||||||
|
if not image and photo.media_type in ('raw', 'heic'):
|
||||||
|
try:
|
||||||
|
image = process_standard_image(photo.filepath)
|
||||||
|
if image is not None:
|
||||||
|
logger.info(
|
||||||
|
f"{photo.filepath}: {photo.media_type} decode failed "
|
||||||
|
f"but file opens as a standard image — using fallback"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(
|
||||||
|
f"Standard-image fallback failed for {photo.filepath}: {e}"
|
||||||
|
)
|
||||||
|
|
||||||
if not image:
|
if not image:
|
||||||
raise Exception("Failed to process image")
|
raise Exception("Failed to process image")
|
||||||
|
|
||||||
@@ -493,7 +511,16 @@ async def _backfill_phashes_async():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@shared_task(name='regroup_duplicates')
|
@shared_task(
|
||||||
|
name='regroup_duplicates',
|
||||||
|
# Full regroup scales with O(N²) on phash plus one pgvector query per
|
||||||
|
# embedded photo. On a 16k-photo library that's comfortably past the
|
||||||
|
# default 5-minute soft limit — bump to 2h / 2h30m. (Passing None here
|
||||||
|
# does NOT disable limits; Celery falls back to the worker default
|
||||||
|
# of 300s/600s. An explicit number overrides.)
|
||||||
|
soft_time_limit=7200,
|
||||||
|
time_limit=9000,
|
||||||
|
)
|
||||||
def regroup_duplicates_task():
|
def regroup_duplicates_task():
|
||||||
"""Full recompute of duplicate groups (pHash + CLIP similarity).
|
"""Full recompute of duplicate groups (pHash + CLIP similarity).
|
||||||
|
|
||||||
@@ -502,7 +529,15 @@ def regroup_duplicates_task():
|
|||||||
return asyncio.run(regroup_duplicates())
|
return asyncio.run(regroup_duplicates())
|
||||||
|
|
||||||
|
|
||||||
@shared_task(name='incremental_regroup_duplicates')
|
@shared_task(
|
||||||
|
name='incremental_regroup_duplicates',
|
||||||
|
# O(new × N); still cheaper than a full regroup but can easily exceed
|
||||||
|
# the 5-minute default after a big batch import. Same caveat as
|
||||||
|
# regroup_duplicates above — None would just re-inherit the worker
|
||||||
|
# default, so we pass explicit values.
|
||||||
|
soft_time_limit=3600,
|
||||||
|
time_limit=4200,
|
||||||
|
)
|
||||||
def incremental_regroup_duplicates_task(since_iso: str | None = None):
|
def incremental_regroup_duplicates_task(since_iso: str | None = None):
|
||||||
"""Incremental duplicate detection for newly added photos.
|
"""Incremental duplicate detection for newly added photos.
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,14 @@ from PIL import Image
|
|||||||
|
|
||||||
from app.models.embeddings import Embedding
|
from app.models.embeddings import Embedding
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
from app.services.feature_flags import (
|
||||||
|
is_enabled,
|
||||||
|
FLAG_VISION_ENABLED,
|
||||||
|
FLAG_OCR_ENABLED,
|
||||||
|
FLAG_DETECTOR_ENABLED,
|
||||||
|
FLAG_FACES_ENABLED,
|
||||||
|
FLAG_CLASSIFIER_ENABLED,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -83,7 +91,7 @@ def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None:
|
|||||||
@shared_task(name='embed_photo', queue='vision', bind=True, max_retries=3)
|
@shared_task(name='embed_photo', queue='vision', bind=True, max_retries=3)
|
||||||
def embed_photo(self, photo_id: str):
|
def embed_photo(self, photo_id: str):
|
||||||
"""Generate CLIP embedding for a photo and store in pgvector."""
|
"""Generate CLIP embedding for a photo and store in pgvector."""
|
||||||
if not settings.vision.enabled:
|
if not is_enabled(FLAG_VISION_ENABLED):
|
||||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||||
|
|
||||||
image = _load_thumb(photo_id, "medium") # 640px
|
image = _load_thumb(photo_id, "medium") # 640px
|
||||||
@@ -128,18 +136,18 @@ def embed_photo(self, photo_id: str):
|
|||||||
@shared_task(name='vision_fanout', queue='vision')
|
@shared_task(name='vision_fanout', queue='vision')
|
||||||
def vision_fanout(photo_id: str):
|
def vision_fanout(photo_id: str):
|
||||||
"""Dispatch all enabled vision tasks for a photo."""
|
"""Dispatch all enabled vision tasks for a photo."""
|
||||||
if not settings.vision.enabled:
|
if not is_enabled(FLAG_VISION_ENABLED):
|
||||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||||
|
|
||||||
embed_photo.delay(photo_id)
|
embed_photo.delay(photo_id)
|
||||||
|
|
||||||
if settings.vision.ocr.enabled:
|
if is_enabled(FLAG_OCR_ENABLED):
|
||||||
ocr_photo.delay(photo_id)
|
ocr_photo.delay(photo_id)
|
||||||
if settings.vision.detector.enabled:
|
if is_enabled(FLAG_DETECTOR_ENABLED):
|
||||||
detect_objects.delay(photo_id)
|
detect_objects.delay(photo_id)
|
||||||
if settings.vision.faces.enabled:
|
if is_enabled(FLAG_FACES_ENABLED):
|
||||||
extract_faces.delay(photo_id)
|
extract_faces.delay(photo_id)
|
||||||
if settings.vision.classifier.enabled:
|
if is_enabled(FLAG_CLASSIFIER_ENABLED):
|
||||||
classify_content.delay(photo_id)
|
classify_content.delay(photo_id)
|
||||||
|
|
||||||
return {'status': 'dispatched', 'photo_id': photo_id}
|
return {'status': 'dispatched', 'photo_id': photo_id}
|
||||||
@@ -148,7 +156,7 @@ def vision_fanout(photo_id: str):
|
|||||||
@shared_task(name='ocr_photo', queue='vision', bind=True, max_retries=3)
|
@shared_task(name='ocr_photo', queue='vision', bind=True, max_retries=3)
|
||||||
def ocr_photo(self, photo_id: str):
|
def ocr_photo(self, photo_id: str):
|
||||||
"""Run OCR on a photo and store text regions."""
|
"""Run OCR on a photo and store text regions."""
|
||||||
if not settings.vision.enabled or not settings.vision.ocr.enabled:
|
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_OCR_ENABLED):
|
||||||
return {'status': 'skipped', 'reason': 'OCR disabled'}
|
return {'status': 'skipped', 'reason': 'OCR disabled'}
|
||||||
|
|
||||||
image = _load_thumb(photo_id, "large") # 1280px for better OCR accuracy
|
image = _load_thumb(photo_id, "large") # 1280px for better OCR accuracy
|
||||||
@@ -195,7 +203,7 @@ def ocr_photo(self, photo_id: str):
|
|||||||
def detect_objects(self, photo_id: str):
|
def detect_objects(self, photo_id: str):
|
||||||
"""Detect objects in a photo, create Tag(kind=object) rows, and
|
"""Detect objects in a photo, create Tag(kind=object) rows, and
|
||||||
link via photo_tags with confidence/bbox/source."""
|
link via photo_tags with confidence/bbox/source."""
|
||||||
if not settings.vision.enabled or not settings.vision.detector.enabled:
|
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_DETECTOR_ENABLED):
|
||||||
return {'status': 'skipped', 'reason': 'detection disabled'}
|
return {'status': 'skipped', 'reason': 'detection disabled'}
|
||||||
|
|
||||||
image = _load_thumb(photo_id, "medium") # 640px
|
image = _load_thumb(photo_id, "medium") # 640px
|
||||||
@@ -279,7 +287,7 @@ def detect_objects(self, photo_id: str):
|
|||||||
def classify_content(self, photo_id: str):
|
def classify_content(self, photo_id: str):
|
||||||
"""Classify image content type (screenshot, document, artwork, etc.)
|
"""Classify image content type (screenshot, document, artwork, etc.)
|
||||||
using CLIP zero-shot classification. Writes Tag(kind=content_type)."""
|
using CLIP zero-shot classification. Writes Tag(kind=content_type)."""
|
||||||
if not settings.vision.enabled or not settings.vision.classifier.enabled:
|
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_CLASSIFIER_ENABLED):
|
||||||
return {'status': 'skipped', 'reason': 'classifier disabled'}
|
return {'status': 'skipped', 'reason': 'classifier disabled'}
|
||||||
|
|
||||||
image = _load_thumb(photo_id, "medium")
|
image = _load_thumb(photo_id, "medium")
|
||||||
@@ -394,7 +402,7 @@ def extract_faces(self, photo_id: str):
|
|||||||
"""Detect faces and store recognition embeddings using InsightFace
|
"""Detect faces and store recognition embeddings using InsightFace
|
||||||
(RetinaFace + ArcFace). No YOLO workaround needed — RetinaFace has
|
(RetinaFace + ArcFace). No YOLO workaround needed — RetinaFace has
|
||||||
strong human-vs-non-human precision on its own."""
|
strong human-vs-non-human precision on its own."""
|
||||||
if not settings.vision.enabled or not settings.vision.faces.enabled:
|
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_FACES_ENABLED):
|
||||||
return {'status': 'skipped', 'reason': 'faces disabled'}
|
return {'status': 'skipped', 'reason': 'faces disabled'}
|
||||||
|
|
||||||
image = _load_original(photo_id)
|
image = _load_original(photo_id)
|
||||||
@@ -478,7 +486,7 @@ def recluster_faces(self):
|
|||||||
logger.info("Vision worker not ready yet — retrying in 30s")
|
logger.info("Vision worker not ready yet — retrying in 30s")
|
||||||
raise self.retry(countdown=30)
|
raise self.retry(countdown=30)
|
||||||
|
|
||||||
if not settings.vision.enabled or not settings.vision.faces.enabled:
|
if not is_enabled(FLAG_VISION_ENABLED) or not is_enabled(FLAG_FACES_ENABLED):
|
||||||
return {'status': 'skipped', 'reason': 'faces disabled'}
|
return {'status': 'skipped', 'reason': 'faces disabled'}
|
||||||
|
|
||||||
from app.models import Photo
|
from app.models import Photo
|
||||||
@@ -603,7 +611,7 @@ def backfill_vision(self, task: str | None = None, limit: int | None = None):
|
|||||||
embed_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
embed_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||||
|
|
||||||
ocr_ids = []
|
ocr_ids = []
|
||||||
if task in ('ocr', None) and settings.vision.ocr.enabled:
|
if task in ('ocr', None) and is_enabled(FLAG_OCR_ENABLED):
|
||||||
sql = f"""
|
sql = f"""
|
||||||
SELECT p.id FROM photos p
|
SELECT p.id FROM photos p
|
||||||
LEFT JOIN ocr_text o ON o.photo_id = p.id
|
LEFT JOIN ocr_text o ON o.photo_id = p.id
|
||||||
@@ -613,7 +621,7 @@ def backfill_vision(self, task: str | None = None, limit: int | None = None):
|
|||||||
ocr_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
ocr_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||||
|
|
||||||
detect_ids = []
|
detect_ids = []
|
||||||
if task in ('detect', None) and settings.vision.detector.enabled:
|
if task in ('detect', None) and is_enabled(FLAG_DETECTOR_ENABLED):
|
||||||
sql = f"""
|
sql = f"""
|
||||||
SELECT p.id FROM photos p
|
SELECT p.id FROM photos p
|
||||||
WHERE p.processing_status = 'completed'
|
WHERE p.processing_status = 'completed'
|
||||||
@@ -626,7 +634,7 @@ def backfill_vision(self, task: str | None = None, limit: int | None = None):
|
|||||||
detect_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
detect_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||||
|
|
||||||
face_ids = []
|
face_ids = []
|
||||||
if task in ('faces', None) and settings.vision.faces.enabled:
|
if task in ('faces', None) and is_enabled(FLAG_FACES_ENABLED):
|
||||||
sql = f"""
|
sql = f"""
|
||||||
SELECT p.id FROM photos p
|
SELECT p.id FROM photos p
|
||||||
LEFT JOIN face_embeddings fe ON fe.photo_id = p.id
|
LEFT JOIN face_embeddings fe ON fe.photo_id = p.id
|
||||||
@@ -636,7 +644,7 @@ def backfill_vision(self, task: str | None = None, limit: int | None = None):
|
|||||||
face_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
face_ids = [r[0] for r in session.execute(sa_text(sql), params).fetchall()]
|
||||||
|
|
||||||
classify_ids = []
|
classify_ids = []
|
||||||
if task in ('classify', None) and settings.vision.classifier.enabled:
|
if task in ('classify', None) and is_enabled(FLAG_CLASSIFIER_ENABLED):
|
||||||
sql = f"""
|
sql = f"""
|
||||||
SELECT p.id FROM photos p
|
SELECT p.id FROM photos p
|
||||||
WHERE p.processing_status = 'completed'
|
WHERE p.processing_status = 'completed'
|
||||||
|
|||||||
@@ -18,7 +18,12 @@ flower==2.0.1
|
|||||||
|
|
||||||
# Image processing
|
# Image processing
|
||||||
# pyvips==2.2.1 # Optional - having compatibility issues, using Pillow as fallback
|
# pyvips==2.2.1 # Optional - having compatibility issues, using Pillow as fallback
|
||||||
# rawpy==0.19.0 # Optional - numpy compatibility issues, using Pillow as fallback
|
rawpy==0.26.1 # RAW decoder (CR2/NEF/ARW/DNG/…). cp312 wheels
|
||||||
|
# ship with libraw bundled; the older 0.19 pin
|
||||||
|
# had numpy 2.x incompatibilities — 0.26 is fine
|
||||||
|
# with our numpy 1.26. iPhone ProRAW-style DNGs
|
||||||
|
# that aren't real RAW still fail here; thumbs.py
|
||||||
|
# falls back to opening them as JPEG in that case.
|
||||||
pillow==10.2.0
|
pillow==10.2.0
|
||||||
pillow-heif==0.15.0
|
pillow-heif==0.15.0
|
||||||
imagehash==4.3.1 # perceptual hash for duplicate detection
|
imagehash==4.3.1 # perceptual hash for duplicate detection
|
||||||
|
|||||||
@@ -15,15 +15,23 @@ import {
|
|||||||
Activity,
|
Activity,
|
||||||
FolderSearch,
|
FolderSearch,
|
||||||
Shield,
|
Shield,
|
||||||
|
Brain,
|
||||||
|
ScanText,
|
||||||
|
UserSquare2,
|
||||||
|
Boxes,
|
||||||
|
Tags as TagsIcon,
|
||||||
|
RotateCcw,
|
||||||
} from 'lucide-react'
|
} from 'lucide-react'
|
||||||
import clsx from 'clsx'
|
import clsx from 'clsx'
|
||||||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||||
import {
|
import {
|
||||||
library,
|
library,
|
||||||
|
admin as adminApi,
|
||||||
type MediaType,
|
type MediaType,
|
||||||
type PipelineStage,
|
type PipelineStage,
|
||||||
type ScanStatus,
|
type ScanStatus,
|
||||||
type WorkerStatus,
|
type WorkerStatus,
|
||||||
|
type FeatureFlagSnapshot,
|
||||||
} from '../../services/api'
|
} from '../../services/api'
|
||||||
import { toast } from '../ToastContainer'
|
import { toast } from '../ToastContainer'
|
||||||
import { useAuth } from '../../contexts/AuthContext'
|
import { useAuth } from '../../contexts/AuthContext'
|
||||||
@@ -42,13 +50,16 @@ const SETTINGS_SCAN_STATUS_KEY = ['settings', 'scan-status'] as const
|
|||||||
// the grid renders from. Imported via the canonical hook key.
|
// the grid renders from. Imported via the canonical hook key.
|
||||||
import { DUPLICATE_GROUPS_QUERY_KEY } from '../../hooks/useDuplicateGroupsQuery'
|
import { DUPLICATE_GROUPS_QUERY_KEY } from '../../hooks/useDuplicateGroupsQuery'
|
||||||
|
|
||||||
type SettingsTab = 'library' | 'users'
|
type SettingsTab = 'library' | 'ai' | 'users'
|
||||||
|
|
||||||
const TABS: { id: SettingsTab; label: string; adminOnly?: boolean }[] = [
|
const TABS: { id: SettingsTab; label: string; adminOnly?: boolean }[] = [
|
||||||
{ id: 'library', label: 'Library Management' },
|
{ id: 'library', label: 'Library Management' },
|
||||||
|
{ id: 'ai', label: 'AI Features', adminOnly: true },
|
||||||
{ id: 'users', label: 'Users', adminOnly: true },
|
{ id: 'users', label: 'Users', adminOnly: true },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
const SETTINGS_FEATURE_FLAGS_KEY = ['settings', 'feature-flags'] as const
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Full-page settings view with tabbed navigation. Replaces the old
|
* Full-page settings view with tabbed navigation. Replaces the old
|
||||||
* modal dialog — renders as a top-level section in the main content
|
* modal dialog — renders as a top-level section in the main content
|
||||||
@@ -842,6 +853,13 @@ export function SettingsPage() {
|
|||||||
</Section>
|
</Section>
|
||||||
</>)}
|
</>)}
|
||||||
|
|
||||||
|
{activeTab === 'ai' && isAdmin && (
|
||||||
|
<AiFeaturesTab
|
||||||
|
busy={busy}
|
||||||
|
runAction={runAction}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
{activeTab === 'users' && isAdmin && (
|
{activeTab === 'users' && isAdmin && (
|
||||||
<Section
|
<Section
|
||||||
icon={<Shield className="h-4 w-4" />}
|
icon={<Shield className="h-4 w-4" />}
|
||||||
@@ -1073,3 +1091,267 @@ function ActionButton({
|
|||||||
</button>
|
</button>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// AI Features admin tab
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
interface AiFeaturesTabProps {
|
||||||
|
busy: Record<string, boolean>
|
||||||
|
runAction: <T>(
|
||||||
|
key: string,
|
||||||
|
fn: () => Promise<T>,
|
||||||
|
successTitle: string,
|
||||||
|
describe?: (result: T) => string | undefined,
|
||||||
|
) => Promise<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flags are keyed by the backend's canonical name ("vision.enabled",
|
||||||
|
// "vision.ocr.enabled", ...). The metadata here just adds presentation
|
||||||
|
// (label, short description, icon) so the tab layout stays data-driven.
|
||||||
|
const FLAG_META: Array<{
|
||||||
|
id: string
|
||||||
|
label: string
|
||||||
|
description: string
|
||||||
|
icon: React.ReactNode
|
||||||
|
// Optional "run this backfill" hook — lets the user kick off a stage's
|
||||||
|
// backfill right from the toggle row without hopping to a separate UI.
|
||||||
|
backfillTask?: 'embed' | 'ocr' | 'detect' | 'faces' | 'classify'
|
||||||
|
}> = [
|
||||||
|
{
|
||||||
|
id: 'vision.enabled',
|
||||||
|
label: 'Vision pipeline (master switch)',
|
||||||
|
description:
|
||||||
|
'When off, every AI stage below is skipped — including newly uploaded photos. ' +
|
||||||
|
'Existing results stay intact.',
|
||||||
|
icon: <Sparkles className="h-3.5 w-3.5" />,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'vision.ocr.enabled',
|
||||||
|
label: 'Text recognition (OCR)',
|
||||||
|
description: 'Extract printed / handwritten text from photos so it becomes searchable.',
|
||||||
|
icon: <ScanText className="h-3.5 w-3.5" />,
|
||||||
|
backfillTask: 'ocr',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'vision.detector.enabled',
|
||||||
|
label: 'Object detection',
|
||||||
|
description: 'Tag photos with detected objects (person, car, dog, …) via YOLOv8n.',
|
||||||
|
icon: <Boxes className="h-3.5 w-3.5" />,
|
||||||
|
backfillTask: 'detect',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'vision.faces.enabled',
|
||||||
|
label: 'Face recognition',
|
||||||
|
description:
|
||||||
|
'Find and cluster faces across the library (RetinaFace + ArcFace). ' +
|
||||||
|
'Expensive on big libraries — disable if you don\'t need the People view.',
|
||||||
|
icon: <UserSquare2 className="h-3.5 w-3.5" />,
|
||||||
|
backfillTask: 'faces',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'vision.classifier.enabled',
|
||||||
|
label: 'Content classification',
|
||||||
|
description: 'Zero-shot CLIP tags for scenes / activities (beach, wedding, …).',
|
||||||
|
icon: <TagsIcon className="h-3.5 w-3.5" />,
|
||||||
|
backfillTask: 'classify',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
function AiFeaturesTab({ busy, runAction }: AiFeaturesTabProps) {
|
||||||
|
const queryClient = useQueryClient()
|
||||||
|
const flagsQuery = useQuery<{ flags: FeatureFlagSnapshot }>({
|
||||||
|
queryKey: SETTINGS_FEATURE_FLAGS_KEY,
|
||||||
|
queryFn: adminApi.listFeatureFlags,
|
||||||
|
staleTime: 5_000,
|
||||||
|
})
|
||||||
|
|
||||||
|
const flags = flagsQuery.data?.flags ?? {}
|
||||||
|
const masterOff = flags['vision.enabled'] && !flags['vision.enabled'].effective
|
||||||
|
|
||||||
|
const applyFlag = async (name: string, value: boolean | null) => {
|
||||||
|
await runAction(
|
||||||
|
`flag:${name}`,
|
||||||
|
() => adminApi.setFeatureFlag(name, value),
|
||||||
|
value === null ? 'Override cleared' : `Feature ${value ? 'enabled' : 'disabled'}`,
|
||||||
|
)
|
||||||
|
queryClient.invalidateQueries({ queryKey: SETTINGS_FEATURE_FLAGS_KEY })
|
||||||
|
// Non-admin feature map drives sidebar gating — invalidate so the
|
||||||
|
// People / Tags entries appear / disappear immediately without a
|
||||||
|
// page reload.
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['features'] })
|
||||||
|
}
|
||||||
|
|
||||||
|
type BackfillTask = 'embed' | 'ocr' | 'detect' | 'faces' | 'classify' | null
|
||||||
|
const runBackfill = (task: BackfillTask) =>
|
||||||
|
runAction(
|
||||||
|
`ai-backfill:${task ?? 'all'}`,
|
||||||
|
() => adminApi.triggerAiBackfill({ task }),
|
||||||
|
task ? `Backfill queued for ${task}` : 'Full backfill queued',
|
||||||
|
(r) => `Celery task ${r.task_id}`,
|
||||||
|
)
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Section icon={<Brain className="h-4 w-4" />} title="AI feature flags">
|
||||||
|
<p className="text-xs text-text-muted">
|
||||||
|
Toggle each stage at runtime. Changes are observed by Celery
|
||||||
|
workers on the next task — no restart needed. "Default" means
|
||||||
|
the flag hasn\'t been overridden and is tracking the YAML config;
|
||||||
|
an overridden flag is pinned to the value shown until cleared.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{flagsQuery.isLoading && (
|
||||||
|
<div className="mt-3 flex items-center gap-2 text-xs text-text-muted">
|
||||||
|
<Loader2 className="h-3 w-3 animate-spin" />
|
||||||
|
Loading feature flags…
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{flagsQuery.error && (
|
||||||
|
<ErrorBanner
|
||||||
|
title="Could not load feature flags"
|
||||||
|
detail={String((flagsQuery.error as Error).message || flagsQuery.error)}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!flagsQuery.isLoading && !flagsQuery.error && (
|
||||||
|
<div className="mt-3 space-y-2">
|
||||||
|
{FLAG_META.map((meta) => {
|
||||||
|
const state = flags[meta.id]
|
||||||
|
if (!state) return null
|
||||||
|
const busyKey = `flag:${meta.id}`
|
||||||
|
const isBusy = !!busy[busyKey]
|
||||||
|
const isMaster = meta.id === 'vision.enabled'
|
||||||
|
const dimmed = !isMaster && masterOff
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={meta.id}
|
||||||
|
className={clsx(
|
||||||
|
'rounded border border-border bg-surface p-3 text-xs',
|
||||||
|
dimmed && 'opacity-60',
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<div className="flex items-start justify-between gap-3">
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<div className="flex items-center gap-1.5 text-text">
|
||||||
|
{meta.icon}
|
||||||
|
<span className="font-medium">{meta.label}</span>
|
||||||
|
{state.overridden && (
|
||||||
|
<span className="rounded bg-primary/20 px-1 py-0.5 text-[9px] font-semibold uppercase tracking-wide text-primary">
|
||||||
|
overridden
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<p className="mt-1 text-[11px] text-text-muted">{meta.description}</p>
|
||||||
|
<p className="mt-1 text-[10px] text-text-faint">
|
||||||
|
Default: {state.default ? 'on' : 'off'} · Currently:{' '}
|
||||||
|
<span className={state.effective ? 'text-pick' : 'text-reject'}>
|
||||||
|
{state.effective ? 'on' : 'off'}
|
||||||
|
</span>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="flex shrink-0 items-center gap-2">
|
||||||
|
<button
|
||||||
|
role="switch"
|
||||||
|
aria-checked={state.effective}
|
||||||
|
onClick={() => applyFlag(meta.id, !state.effective)}
|
||||||
|
disabled={isBusy || (dimmed && !isMaster)}
|
||||||
|
className={clsx(
|
||||||
|
'relative inline-flex h-6 w-11 shrink-0 items-center rounded-full transition-colors',
|
||||||
|
state.effective ? 'bg-primary' : 'bg-surface-2 border border-border',
|
||||||
|
(isBusy || (dimmed && !isMaster)) && 'cursor-not-allowed opacity-50',
|
||||||
|
)}
|
||||||
|
title={state.effective ? 'Click to disable' : 'Click to enable'}
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
aria-hidden="true"
|
||||||
|
className={clsx(
|
||||||
|
'inline-block h-5 w-5 transform rounded-full bg-white shadow transition-transform',
|
||||||
|
state.effective ? 'translate-x-[22px]' : 'translate-x-0.5',
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
</button>
|
||||||
|
{state.overridden && (
|
||||||
|
<button
|
||||||
|
onClick={() => applyFlag(meta.id, null)}
|
||||||
|
disabled={isBusy}
|
||||||
|
className="rounded border border-border p-1 text-text-muted hover:bg-surface-2 hover:text-text disabled:opacity-50"
|
||||||
|
title="Reset to YAML default"
|
||||||
|
aria-label="Reset override"
|
||||||
|
>
|
||||||
|
<RotateCcw className="h-3 w-3" />
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{meta.backfillTask && state.effective && !masterOff && (
|
||||||
|
<div className="mt-2">
|
||||||
|
<ActionButton
|
||||||
|
loading={!!busy[`ai-backfill:${meta.backfillTask}`]}
|
||||||
|
onClick={() => runBackfill(meta.backfillTask!)}
|
||||||
|
>
|
||||||
|
<RefreshCw className="h-3.5 w-3.5" />
|
||||||
|
Run {meta.backfillTask} backfill
|
||||||
|
</ActionButton>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</Section>
|
||||||
|
|
||||||
|
<Section icon={<Cpu className="h-4 w-4" />} title="Manual pipeline triggers">
|
||||||
|
<p className="text-xs text-text-muted">
|
||||||
|
Run a full pass across the enabled stages, recompute face
|
||||||
|
clusters, or force a fresh filesystem scan. All three are safe
|
||||||
|
to run repeatedly — the backfill only touches photos that
|
||||||
|
don\'t yet have a given output, and the rescan skips files
|
||||||
|
that are already indexed.
|
||||||
|
</p>
|
||||||
|
<div className="mt-3 flex flex-wrap gap-2">
|
||||||
|
<ActionButton
|
||||||
|
loading={!!busy['ai-backfill:all']}
|
||||||
|
onClick={() => runBackfill(null)}
|
||||||
|
disabled={masterOff}
|
||||||
|
>
|
||||||
|
<Sparkles className="h-4 w-4" />
|
||||||
|
Run full vision backfill
|
||||||
|
</ActionButton>
|
||||||
|
<ActionButton
|
||||||
|
loading={!!busy['recluster']}
|
||||||
|
onClick={() =>
|
||||||
|
runAction(
|
||||||
|
'recluster',
|
||||||
|
() => adminApi.triggerFaceRecluster(),
|
||||||
|
'Face recluster queued',
|
||||||
|
(r) => `Celery task ${r.task_id}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
disabled={masterOff || !flags['vision.faces.enabled']?.effective}
|
||||||
|
>
|
||||||
|
<UserSquare2 className="h-4 w-4" />
|
||||||
|
Recluster faces
|
||||||
|
</ActionButton>
|
||||||
|
<ActionButton
|
||||||
|
loading={!!busy['rescan-full']}
|
||||||
|
onClick={() =>
|
||||||
|
runAction(
|
||||||
|
'rescan-full',
|
||||||
|
() => adminApi.triggerFullRescan(),
|
||||||
|
'Rescan queued',
|
||||||
|
(r) => `Celery task ${r.task_id}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<RefreshCw className="h-4 w-4" />
|
||||||
|
Rescan all source roots
|
||||||
|
</ActionButton>
|
||||||
|
</div>
|
||||||
|
</Section>
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,12 +11,13 @@ import {
|
|||||||
Copy,
|
Copy,
|
||||||
Trash2,
|
Trash2,
|
||||||
Users,
|
Users,
|
||||||
|
Download as DownloadIcon,
|
||||||
} from 'lucide-react'
|
} from 'lucide-react'
|
||||||
import clsx from 'clsx'
|
import clsx from 'clsx'
|
||||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||||
import { useHeapsQuery, HEAPS_QUERY_KEY } from '../../hooks/useHeapsQuery'
|
import { useHeapsQuery, HEAPS_QUERY_KEY } from '../../hooks/useHeapsQuery'
|
||||||
import { useSharedHeapsQuery } from '../../hooks/useSharingQueries'
|
import { useSharedHeapsQuery } from '../../hooks/useSharingQueries'
|
||||||
import { heaps as heapsApi, type Heap } from '../../services/api'
|
import { heaps as heapsApi, downloads, type Heap } from '../../services/api'
|
||||||
import { useFilterStore } from '../../store/filterStore'
|
import { useFilterStore } from '../../store/filterStore'
|
||||||
import { toast } from '../ToastContainer'
|
import { toast } from '../ToastContainer'
|
||||||
import { PHOTO_DRAG_MIME } from '../timeline/PhotoThumbnail'
|
import { PHOTO_DRAG_MIME } from '../timeline/PhotoThumbnail'
|
||||||
@@ -448,6 +449,14 @@ export function HeapsPanel() {
|
|||||||
setSharingHeap(heap)
|
setSharingHeap(heap)
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
<MenuItem
|
||||||
|
icon={<DownloadIcon className="h-3.5 w-3.5" />}
|
||||||
|
label="Download as zip"
|
||||||
|
onClick={() => {
|
||||||
|
setOpenMenuId(null)
|
||||||
|
downloads.trigger(downloads.heapUrl(heap.id))
|
||||||
|
}}
|
||||||
|
/>
|
||||||
<div className="my-1 h-px bg-border" />
|
<div className="my-1 h-px bg-border" />
|
||||||
<MenuItem
|
<MenuItem
|
||||||
icon={<Trash2 className="h-3.5 w-3.5" />}
|
icon={<Trash2 className="h-3.5 w-3.5" />}
|
||||||
|
|||||||
@@ -24,9 +24,11 @@ import {
|
|||||||
LogOut,
|
LogOut,
|
||||||
Shield,
|
Shield,
|
||||||
Clock,
|
Clock,
|
||||||
|
Upload as UploadIcon,
|
||||||
|
Download as DownloadIcon,
|
||||||
} from 'lucide-react'
|
} from 'lucide-react'
|
||||||
import clsx from 'clsx'
|
import clsx from 'clsx'
|
||||||
import { sourceFolders, photos as photosApi, type FolderTreeNode } from '../../services/api'
|
import { sourceFolders, photos as photosApi, downloads, type FolderTreeNode } from '../../services/api'
|
||||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||||
import { toast } from '../ToastContainer'
|
import { toast } from '../ToastContainer'
|
||||||
import { useFilterStore } from '../../store/filterStore'
|
import { useFilterStore } from '../../store/filterStore'
|
||||||
@@ -45,8 +47,10 @@ import { registerUndoable } from '../../store/undoStore'
|
|||||||
import type { Photo } from '../../types/photo'
|
import type { Photo } from '../../types/photo'
|
||||||
import { DeleteFolderDialog } from '../dialogs/DeleteFolderDialog'
|
import { DeleteFolderDialog } from '../dialogs/DeleteFolderDialog'
|
||||||
import { ShareDialog } from '../sharing/ShareDialog'
|
import { ShareDialog } from '../sharing/ShareDialog'
|
||||||
|
import { UploadModal } from '../upload/UploadModal'
|
||||||
import { useSharedFoldersQuery } from '../../hooks/useSharingQueries'
|
import { useSharedFoldersQuery } from '../../hooks/useSharingQueries'
|
||||||
import { useAuth } from '../../contexts/AuthContext'
|
import { useAuth } from '../../contexts/AuthContext'
|
||||||
|
import { useFeaturesQuery } from '../../hooks/useFeaturesQuery'
|
||||||
|
|
||||||
interface TreeItem {
|
interface TreeItem {
|
||||||
id: string
|
id: string
|
||||||
@@ -78,6 +82,15 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
|||||||
const { data: allTags = [] } = useTagsQuery()
|
const { data: allTags = [] } = useTagsQuery()
|
||||||
const { data: faceClusters = [] } = useTagsQuery('face_cluster')
|
const { data: faceClusters = [] } = useTagsQuery('face_cluster')
|
||||||
const { data: stats } = useLibraryStatsQuery()
|
const { data: stats } = useLibraryStatsQuery()
|
||||||
|
const { data: featuresMap } = useFeaturesQuery()
|
||||||
|
const visionOn = featuresMap ? featuresMap['vision.enabled'] !== false : true
|
||||||
|
const facesOn = visionOn && (featuresMap ? featuresMap['vision.faces.enabled'] !== false : true)
|
||||||
|
const tagsOn =
|
||||||
|
visionOn &&
|
||||||
|
(featuresMap
|
||||||
|
? featuresMap['vision.detector.enabled'] !== false ||
|
||||||
|
featuresMap['vision.classifier.enabled'] !== false
|
||||||
|
: true)
|
||||||
const [dropTargetId, setDropTargetId] = useState<string | null>(null)
|
const [dropTargetId, setDropTargetId] = useState<string | null>(null)
|
||||||
|
|
||||||
// Per-folder kebab menu open state. Stores the tree-item id ("folder-..."
|
// Per-folder kebab menu open state. Stores the tree-item id ("folder-..."
|
||||||
@@ -117,6 +130,14 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
|||||||
id: string
|
id: string
|
||||||
name: string
|
name: string
|
||||||
} | null>(null)
|
} | null>(null)
|
||||||
|
// Upload modal state. `uploadTarget` stores the pre-selected
|
||||||
|
// destination Folder/SourceRoot id so "Upload here…" on a folder row
|
||||||
|
// drops files straight into that folder; null means the general
|
||||||
|
// Library-level button (defaults to the first source root).
|
||||||
|
const [uploadTarget, setUploadTarget] = useState<{ open: boolean; folderId: string | null }>({
|
||||||
|
open: false,
|
||||||
|
folderId: null,
|
||||||
|
})
|
||||||
const { data: sharedFolders = [] } = useSharedFoldersQuery()
|
const { data: sharedFolders = [] } = useSharedFoldersQuery()
|
||||||
|
|
||||||
// Bulk discard mutation for the drag-onto-Discarded interaction.
|
// Bulk discard mutation for the drag-onto-Discarded interaction.
|
||||||
@@ -414,8 +435,8 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
|||||||
children: [
|
children: [
|
||||||
{ id: 'all-photos', label: 'All Photos', icon: <Image className="h-4 w-4" />, count: stats?.all_photos ?? 0 },
|
{ id: 'all-photos', label: 'All Photos', icon: <Image className="h-4 w-4" />, count: stats?.all_photos ?? 0 },
|
||||||
{ id: 'rated', label: 'Rated', icon: <Star className="h-4 w-4" />, count: stats?.rated ?? 0 },
|
{ id: 'rated', label: 'Rated', icon: <Star className="h-4 w-4" />, count: stats?.rated ?? 0 },
|
||||||
{ id: 'tags', label: 'Tags', icon: <TagIcon className="h-4 w-4" />, count: tagsTotalCount },
|
...(tagsOn ? [{ id: 'tags', label: 'Tags', icon: <TagIcon className="h-4 w-4" />, count: tagsTotalCount }] : []),
|
||||||
{ id: 'people', label: 'People', icon: <Users className="h-4 w-4" />, count: peopleTotalCount },
|
...(facesOn ? [{ id: 'people', label: 'People', icon: <Users className="h-4 w-4" />, count: peopleTotalCount }] : []),
|
||||||
{ id: 'colors', label: 'Colors', icon: <Palette className="h-4 w-4" />, count: stats?.colored ?? 0 },
|
{ id: 'colors', label: 'Colors', icon: <Palette className="h-4 w-4" />, count: stats?.colored ?? 0 },
|
||||||
{ id: 'map', label: 'Map', icon: <MapPin className="h-4 w-4" />, count: stats?.with_gps ?? 0 },
|
{ id: 'map', label: 'Map', icon: <MapPin className="h-4 w-4" />, count: stats?.with_gps ?? 0 },
|
||||||
{ id: 'memories', label: 'Memories', icon: <Clock className="h-4 w-4" /> },
|
{ id: 'memories', label: 'Memories', icon: <Clock className="h-4 w-4" /> },
|
||||||
@@ -670,6 +691,14 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
|||||||
onClick={(e) => e.stopPropagation()}
|
onClick={(e) => e.stopPropagation()}
|
||||||
className="absolute right-0 top-full z-30 mt-1 min-w-[180px] overflow-hidden rounded-lg border border-border bg-surface py-1 text-sm shadow-xl"
|
className="absolute right-0 top-full z-30 mt-1 min-w-[180px] overflow-hidden rounded-lg border border-border bg-surface py-1 text-sm shadow-xl"
|
||||||
>
|
>
|
||||||
|
<FolderMenuItem
|
||||||
|
icon={<UploadIcon className="h-3.5 w-3.5" />}
|
||||||
|
label="Upload here…"
|
||||||
|
onClick={() => {
|
||||||
|
setOpenMenuId(null)
|
||||||
|
setUploadTarget({ open: true, folderId })
|
||||||
|
}}
|
||||||
|
/>
|
||||||
<FolderMenuItem
|
<FolderMenuItem
|
||||||
icon={<FolderPlus className="h-3.5 w-3.5" />}
|
icon={<FolderPlus className="h-3.5 w-3.5" />}
|
||||||
label="New sub-folder"
|
label="New sub-folder"
|
||||||
@@ -718,6 +747,14 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
|||||||
setSharingFolder({ id: folderId, name: item.label })
|
setSharingFolder({ id: folderId, name: item.label })
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
<FolderMenuItem
|
||||||
|
icon={<DownloadIcon className="h-3.5 w-3.5" />}
|
||||||
|
label="Download as zip"
|
||||||
|
onClick={() => {
|
||||||
|
setOpenMenuId(null)
|
||||||
|
downloads.trigger(downloads.folderUrl(folderId))
|
||||||
|
}}
|
||||||
|
/>
|
||||||
<div className="my-1 h-px bg-border" />
|
<div className="my-1 h-px bg-border" />
|
||||||
<FolderMenuItem
|
<FolderMenuItem
|
||||||
icon={<Trash2 className="h-3.5 w-3.5" />}
|
icon={<Trash2 className="h-3.5 w-3.5" />}
|
||||||
@@ -801,14 +838,24 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
|||||||
<h2 className="text-[11px] font-semibold uppercase tracking-[0.14em] text-text-muted">
|
<h2 className="text-[11px] font-semibold uppercase tracking-[0.14em] text-text-muted">
|
||||||
Library
|
Library
|
||||||
</h2>
|
</h2>
|
||||||
<button
|
<div className="flex items-center gap-1">
|
||||||
onClick={onCollapse}
|
<button
|
||||||
className="rounded p-0.5 text-text-muted hover:bg-surface-2 hover:text-text"
|
onClick={() => setUploadTarget({ open: true, folderId: null })}
|
||||||
title="Collapse panel (Tab)"
|
className="rounded p-0.5 text-text-muted hover:bg-surface-2 hover:text-text"
|
||||||
aria-label="Collapse panel"
|
title="Upload photos"
|
||||||
>
|
aria-label="Upload photos"
|
||||||
<PanelLeftClose className="h-3.5 w-3.5" />
|
>
|
||||||
</button>
|
<UploadIcon className="h-3.5 w-3.5" />
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={onCollapse}
|
||||||
|
className="rounded p-0.5 text-text-muted hover:bg-surface-2 hover:text-text"
|
||||||
|
title="Collapse panel (Tab)"
|
||||||
|
aria-label="Collapse panel"
|
||||||
|
>
|
||||||
|
<PanelLeftClose className="h-3.5 w-3.5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{/* Active heap card — pinned just below the Library header so
|
{/* Active heap card — pinned just below the Library header so
|
||||||
* toasts (bottom-left fixed) can't cover it. Returns null when
|
* toasts (bottom-left fixed) can't cover it. Returns null when
|
||||||
@@ -922,6 +969,11 @@ export function LeftSidebar({ onCollapse }: LeftSidebarProps) {
|
|||||||
targetName={sharingFolder?.name ?? ''}
|
targetName={sharingFolder?.name ?? ''}
|
||||||
onClose={() => setSharingFolder(null)}
|
onClose={() => setSharingFolder(null)}
|
||||||
/>
|
/>
|
||||||
|
<UploadModal
|
||||||
|
isOpen={uploadTarget.open}
|
||||||
|
initialFolderId={uploadTarget.folderId}
|
||||||
|
onClose={() => setUploadTarget({ open: false, folderId: null })}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
608
frontend/src/components/upload/UploadModal.tsx
Normal file
608
frontend/src/components/upload/UploadModal.tsx
Normal file
@@ -0,0 +1,608 @@
|
|||||||
|
import { useEffect, useMemo, useRef, useState } from 'react'
|
||||||
|
import {
|
||||||
|
ChevronDown,
|
||||||
|
ChevronRight,
|
||||||
|
Folder as FolderIcon,
|
||||||
|
Upload as UploadIcon,
|
||||||
|
X,
|
||||||
|
FileImage,
|
||||||
|
CheckCircle2,
|
||||||
|
AlertCircle,
|
||||||
|
} from 'lucide-react'
|
||||||
|
import clsx from 'clsx'
|
||||||
|
import { useQueryClient } from '@tanstack/react-query'
|
||||||
|
import { uploads, type FolderTreeNode } from '../../services/api'
|
||||||
|
import { useFolderTreeQuery } from '../../hooks/useFolderTreeQuery'
|
||||||
|
import { useHeapsQuery, HEAPS_QUERY_KEY } from '../../hooks/useHeapsQuery'
|
||||||
|
import { FOLDER_TREE_QUERY_KEY } from '../../hooks/useFolderTreeQuery'
|
||||||
|
import { LIBRARY_STATS_QUERY_KEY } from '../../hooks/useLibraryStatsQuery'
|
||||||
|
import { toast } from '../ToastContainer'
|
||||||
|
|
||||||
|
interface UploadModalProps {
|
||||||
|
isOpen: boolean
|
||||||
|
onClose: () => void
|
||||||
|
/** Optional pre-selected destination. Accepts a Folder id or SourceRoot
|
||||||
|
* id — the backend resolves source roots to their root Folder row. */
|
||||||
|
initialFolderId?: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
interface QueuedFile {
|
||||||
|
/** Stable key; the browser may give us multiple files with the same
|
||||||
|
* name from different subfolders, so we key on index + path. */
|
||||||
|
key: string
|
||||||
|
file: File
|
||||||
|
relativePath: string
|
||||||
|
status: 'pending' | 'uploading' | 'done' | 'error'
|
||||||
|
progress: number
|
||||||
|
error?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
const SUPPORTED_EXTENSIONS = new Set([
|
||||||
|
'.jpg', '.jpeg', '.png', '.tiff', '.tif', '.webp', '.bmp',
|
||||||
|
'.cr2', '.cr3', '.nef', '.arw', '.raf', '.dng', '.orf', '.rw2', '.pef', '.srw',
|
||||||
|
'.heic', '.heif',
|
||||||
|
'.mp4', '.mov', '.avi', '.mkv', '.mts', '.m2ts', '.3gp', '.wmv', '.flv',
|
||||||
|
])
|
||||||
|
|
||||||
|
function extOf(name: string): string {
|
||||||
|
const i = name.lastIndexOf('.')
|
||||||
|
return i === -1 ? '' : name.slice(i).toLowerCase()
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSupported(name: string): boolean {
|
||||||
|
return SUPPORTED_EXTENSIONS.has(extOf(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
const MAX_PARALLEL = 4
|
||||||
|
|
||||||
|
/** Walk the tree to collect the ids on the path from root to `targetId`,
|
||||||
|
* excluding the target itself — used to expand ancestor rows so a
|
||||||
|
* pre-selected destination is visible. */
|
||||||
|
function collectAncestors(tree: FolderTreeNode[], targetId: string): string[] {
|
||||||
|
const path: string[] = []
|
||||||
|
const walk = (nodes: FolderTreeNode[], chain: string[]): boolean => {
|
||||||
|
for (const n of nodes) {
|
||||||
|
if (n.id === targetId) {
|
||||||
|
path.push(...chain)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if (n.children && walk(n.children, [...chain, n.id])) return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
walk(tree, [])
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload from desktop. Supports:
|
||||||
|
* - Dropping files or folders onto the drop zone
|
||||||
|
* - Picking files with "Select files"
|
||||||
|
* - Picking a whole folder with "Select folder" (webkitdirectory);
|
||||||
|
* every file's webkitRelativePath is sent to the backend so sub-
|
||||||
|
* folder structure is preserved under the chosen destination.
|
||||||
|
*
|
||||||
|
* Destination is a Folder row (or a source root, which the backend
|
||||||
|
* resolves to its root folder). An optional heap can also be chosen —
|
||||||
|
* uploaded photos are added to that heap in the same request.
|
||||||
|
*/
|
||||||
|
export function UploadModal({ isOpen, onClose, initialFolderId }: UploadModalProps) {
|
||||||
|
const queryClient = useQueryClient()
|
||||||
|
const { data: folderTree } = useFolderTreeQuery()
|
||||||
|
const { data: allHeaps = [] } = useHeapsQuery()
|
||||||
|
|
||||||
|
const [queue, setQueue] = useState<QueuedFile[]>([])
|
||||||
|
const [destFolderId, setDestFolderId] = useState<string | null>(null)
|
||||||
|
const [destHeapId, setDestHeapId] = useState<string | null>(null)
|
||||||
|
const [isUploading, setIsUploading] = useState(false)
|
||||||
|
const [dragOver, setDragOver] = useState(false)
|
||||||
|
const [expandedFolders, setExpandedFolders] = useState<Set<string>>(new Set())
|
||||||
|
|
||||||
|
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||||
|
const dirInputRef = useRef<HTMLInputElement>(null)
|
||||||
|
const abortRef = useRef<AbortController | null>(null)
|
||||||
|
|
||||||
|
// Default destination: caller-provided initialFolderId wins; otherwise
|
||||||
|
// first source root in the tree. Re-runs when the modal is re-opened
|
||||||
|
// with a different initial target so the right row is highlighted.
|
||||||
|
useEffect(() => {
|
||||||
|
if (!isOpen) return
|
||||||
|
if (initialFolderId) {
|
||||||
|
setDestFolderId(initialFolderId)
|
||||||
|
// Expand every ancestor of the pre-selected folder so the row is
|
||||||
|
// actually visible in the tree.
|
||||||
|
if (folderTree) {
|
||||||
|
const ancestors = collectAncestors(folderTree, initialFolderId)
|
||||||
|
setExpandedFolders((prev) => new Set([...prev, ...ancestors]))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (!destFolderId && folderTree && folderTree.length > 0) {
|
||||||
|
setDestFolderId(folderTree[0].id)
|
||||||
|
setExpandedFolders(new Set([folderTree[0].id]))
|
||||||
|
}
|
||||||
|
}, [isOpen, initialFolderId, folderTree, destFolderId])
|
||||||
|
|
||||||
|
// Esc closes (unless mid-upload — don't orphan in-flight requests).
|
||||||
|
useEffect(() => {
|
||||||
|
if (!isOpen) return
|
||||||
|
const handler = (e: KeyboardEvent) => {
|
||||||
|
if (e.key === 'Escape' && !isUploading) onClose()
|
||||||
|
}
|
||||||
|
window.addEventListener('keydown', handler)
|
||||||
|
return () => window.removeEventListener('keydown', handler)
|
||||||
|
}, [isOpen, isUploading, onClose])
|
||||||
|
|
||||||
|
// Reset transient state on open so a previous session's queue doesn't
|
||||||
|
// bleed into a fresh one.
|
||||||
|
useEffect(() => {
|
||||||
|
if (isOpen) {
|
||||||
|
setQueue([])
|
||||||
|
setIsUploading(false)
|
||||||
|
}
|
||||||
|
}, [isOpen])
|
||||||
|
|
||||||
|
const addFiles = (incoming: File[], relPathFn?: (f: File) => string) => {
|
||||||
|
const next: QueuedFile[] = []
|
||||||
|
let skipped = 0
|
||||||
|
for (const file of incoming) {
|
||||||
|
const relPath = (relPathFn?.(file) ?? '').replace(/\\/g, '/').replace(/^\/+/, '')
|
||||||
|
const filename = relPath || file.name
|
||||||
|
if (!isSupported(filename)) {
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
next.push({
|
||||||
|
key: `${relPath || file.name}::${file.size}::${file.lastModified}::${next.length}`,
|
||||||
|
file,
|
||||||
|
relativePath: relPath,
|
||||||
|
status: 'pending',
|
||||||
|
progress: 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (skipped > 0) {
|
||||||
|
toast.info?.(`Skipped ${skipped} unsupported file${skipped === 1 ? '' : 's'}`)
|
||||||
|
}
|
||||||
|
setQueue((prev) => [...prev, ...next])
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleFilePick = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||||
|
const files = Array.from(e.target.files ?? [])
|
||||||
|
addFiles(files, (f) => f.name) // no relative path for single files
|
||||||
|
e.target.value = '' // allow re-picking the same file
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleDirPick = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||||
|
const files = Array.from(e.target.files ?? [])
|
||||||
|
addFiles(files, (f) => (f as File & { webkitRelativePath?: string }).webkitRelativePath || f.name)
|
||||||
|
e.target.value = ''
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drag-and-drop handler. We walk the DataTransferItemList recursively
|
||||||
|
// with webkitGetAsEntry so dropped folders contribute every nested
|
||||||
|
// file, with relative paths reconstructed from the entry chain.
|
||||||
|
const handleDrop = async (e: React.DragEvent) => {
|
||||||
|
e.preventDefault()
|
||||||
|
setDragOver(false)
|
||||||
|
const items = Array.from(e.dataTransfer.items)
|
||||||
|
const collected: { file: File; relativePath: string }[] = []
|
||||||
|
|
||||||
|
const walkEntry = (entry: any, pathPrefix: string): Promise<void> => {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (!entry) return resolve()
|
||||||
|
if (entry.isFile) {
|
||||||
|
entry.file((f: File) => {
|
||||||
|
collected.push({
|
||||||
|
file: f,
|
||||||
|
relativePath: pathPrefix ? `${pathPrefix}/${entry.name}` : '',
|
||||||
|
})
|
||||||
|
resolve()
|
||||||
|
}, () => resolve())
|
||||||
|
} else if (entry.isDirectory) {
|
||||||
|
const reader = entry.createReader()
|
||||||
|
const readBatch = () => {
|
||||||
|
reader.readEntries(async (entries: any[]) => {
|
||||||
|
if (!entries.length) return resolve()
|
||||||
|
const childPrefix = pathPrefix ? `${pathPrefix}/${entry.name}` : entry.name
|
||||||
|
await Promise.all(entries.map((c) => walkEntry(c, childPrefix)))
|
||||||
|
// readEntries only returns a batch at a time; loop until empty.
|
||||||
|
readBatch()
|
||||||
|
}, () => resolve())
|
||||||
|
}
|
||||||
|
readBatch()
|
||||||
|
} else {
|
||||||
|
resolve()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
await Promise.all(
|
||||||
|
items.map((it) => {
|
||||||
|
const entry = (it as DataTransferItem & { webkitGetAsEntry?: () => any }).webkitGetAsEntry?.()
|
||||||
|
return walkEntry(entry, '')
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
if (collected.length === 0) {
|
||||||
|
// Fallback for browsers without webkitGetAsEntry — use plain files.
|
||||||
|
const files = Array.from(e.dataTransfer.files)
|
||||||
|
addFiles(files, (f) => f.name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const files = collected.map((c) => c.file)
|
||||||
|
const pathMap = new Map<File, string>(collected.map((c) => [c.file, c.relativePath]))
|
||||||
|
addFiles(files, (f) => pathMap.get(f) || f.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
const removeFromQueue = (key: string) => {
|
||||||
|
setQueue((prev) => prev.filter((q) => q.key !== key))
|
||||||
|
}
|
||||||
|
|
||||||
|
const startUpload = async () => {
|
||||||
|
if (!destFolderId || queue.length === 0) return
|
||||||
|
setIsUploading(true)
|
||||||
|
const ctrl = new AbortController()
|
||||||
|
abortRef.current = ctrl
|
||||||
|
|
||||||
|
// Simple worker-pool: up to MAX_PARALLEL concurrent uploads.
|
||||||
|
const pending = queue.filter((q) => q.status === 'pending' || q.status === 'error')
|
||||||
|
let cursor = 0
|
||||||
|
let successCount = 0
|
||||||
|
let failCount = 0
|
||||||
|
|
||||||
|
const uploadOne = async (item: QueuedFile) => {
|
||||||
|
setQueue((prev) =>
|
||||||
|
prev.map((q) => (q.key === item.key ? { ...q, status: 'uploading', progress: 0, error: undefined } : q))
|
||||||
|
)
|
||||||
|
try {
|
||||||
|
await uploads.uploadFile(item.file, destFolderId, {
|
||||||
|
relativePath: item.relativePath || undefined,
|
||||||
|
heapId: destHeapId,
|
||||||
|
signal: ctrl.signal,
|
||||||
|
onProgress: (loaded, total) => {
|
||||||
|
const pct = total > 0 ? loaded / total : 0
|
||||||
|
setQueue((prev) =>
|
||||||
|
prev.map((q) => (q.key === item.key ? { ...q, progress: pct } : q))
|
||||||
|
)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
successCount++
|
||||||
|
setQueue((prev) =>
|
||||||
|
prev.map((q) => (q.key === item.key ? { ...q, status: 'done', progress: 1 } : q))
|
||||||
|
)
|
||||||
|
} catch (err: any) {
|
||||||
|
failCount++
|
||||||
|
const msg = err?.response?.data?.detail || err?.message || 'Upload failed'
|
||||||
|
setQueue((prev) =>
|
||||||
|
prev.map((q) => (q.key === item.key ? { ...q, status: 'error', error: msg } : q))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const workers: Promise<void>[] = []
|
||||||
|
for (let i = 0; i < Math.min(MAX_PARALLEL, pending.length); i++) {
|
||||||
|
workers.push(
|
||||||
|
(async () => {
|
||||||
|
while (cursor < pending.length && !ctrl.signal.aborted) {
|
||||||
|
const idx = cursor++
|
||||||
|
await uploadOne(pending[idx])
|
||||||
|
}
|
||||||
|
})()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
await Promise.all(workers)
|
||||||
|
|
||||||
|
setIsUploading(false)
|
||||||
|
abortRef.current = null
|
||||||
|
|
||||||
|
// Refresh everything affected by new photos.
|
||||||
|
queryClient.invalidateQueries({ queryKey: FOLDER_TREE_QUERY_KEY })
|
||||||
|
queryClient.invalidateQueries({ queryKey: HEAPS_QUERY_KEY })
|
||||||
|
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||||
|
|
||||||
|
if (successCount > 0) {
|
||||||
|
toast.success(
|
||||||
|
`Uploaded ${successCount} file${successCount === 1 ? '' : 's'}`,
|
||||||
|
failCount > 0 ? `${failCount} failed — see list` : undefined
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (successCount === 0 && failCount > 0) {
|
||||||
|
toast.error('Upload failed', `${failCount} file${failCount === 1 ? '' : 's'} errored`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const cancelUpload = () => {
|
||||||
|
abortRef.current?.abort()
|
||||||
|
}
|
||||||
|
|
||||||
|
const totalBytes = useMemo(() => queue.reduce((s, q) => s + q.file.size, 0), [queue])
|
||||||
|
const uploadedBytes = useMemo(
|
||||||
|
() => queue.reduce((s, q) => s + q.file.size * (q.status === 'done' ? 1 : q.progress), 0),
|
||||||
|
[queue]
|
||||||
|
)
|
||||||
|
const overallPct = totalBytes > 0 ? Math.round((uploadedBytes / totalBytes) * 100) : 0
|
||||||
|
|
||||||
|
if (!isOpen) return null
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="fixed inset-0 z-50">
|
||||||
|
<div
|
||||||
|
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
|
||||||
|
onClick={!isUploading ? onClose : undefined}
|
||||||
|
/>
|
||||||
|
<div className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2">
|
||||||
|
<div className="relative z-10 flex max-h-[85vh] w-[760px] flex-col rounded-lg border border-border bg-surface shadow-2xl">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex items-center justify-between border-b border-border px-5 py-3">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<UploadIcon className="h-4 w-4 text-text-muted" />
|
||||||
|
<h2 className="text-base font-semibold text-text">Upload photos</h2>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={onClose}
|
||||||
|
disabled={isUploading}
|
||||||
|
className="rounded p-1 text-text-muted hover:bg-surface-2 hover:text-text disabled:opacity-40"
|
||||||
|
aria-label="Close"
|
||||||
|
>
|
||||||
|
<X className="h-4 w-4" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Body */}
|
||||||
|
<div className="flex min-h-0 flex-1 gap-4 overflow-hidden p-5">
|
||||||
|
{/* Left: destination picker */}
|
||||||
|
<div className="flex w-64 flex-col gap-4 overflow-hidden">
|
||||||
|
<div className="flex flex-col gap-2 overflow-hidden">
|
||||||
|
<label className="text-xs font-medium uppercase tracking-wide text-text-muted">
|
||||||
|
Destination folder
|
||||||
|
</label>
|
||||||
|
<div className="flex-1 overflow-auto rounded border border-border bg-bg p-1 text-sm">
|
||||||
|
{folderTree && folderTree.length > 0 ? (
|
||||||
|
folderTree.map((n) => (
|
||||||
|
<FolderTreeRow
|
||||||
|
key={n.id}
|
||||||
|
node={n}
|
||||||
|
depth={0}
|
||||||
|
selectedId={destFolderId}
|
||||||
|
onSelect={setDestFolderId}
|
||||||
|
expanded={expandedFolders}
|
||||||
|
onToggle={(id) =>
|
||||||
|
setExpandedFolders((prev) => {
|
||||||
|
const next = new Set(prev)
|
||||||
|
if (next.has(id)) next.delete(id)
|
||||||
|
else next.add(id)
|
||||||
|
return next
|
||||||
|
})
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
))
|
||||||
|
) : (
|
||||||
|
<div className="p-3 text-xs text-text-muted">No folders yet.</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex flex-col gap-2">
|
||||||
|
<label className="text-xs font-medium uppercase tracking-wide text-text-muted">
|
||||||
|
Also add to heap (optional)
|
||||||
|
</label>
|
||||||
|
<select
|
||||||
|
value={destHeapId ?? ''}
|
||||||
|
onChange={(e) => setDestHeapId(e.target.value || null)}
|
||||||
|
className="rounded border border-border bg-bg px-2 py-1.5 text-sm text-text"
|
||||||
|
>
|
||||||
|
<option value="">— none —</option>
|
||||||
|
{allHeaps.map((h) => (
|
||||||
|
<option key={h.id} value={h.id}>
|
||||||
|
{h.name}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Right: drop zone + queue */}
|
||||||
|
<div className="flex min-w-0 flex-1 flex-col gap-3 overflow-hidden">
|
||||||
|
<div
|
||||||
|
onDragOver={(e) => {
|
||||||
|
e.preventDefault()
|
||||||
|
setDragOver(true)
|
||||||
|
}}
|
||||||
|
onDragLeave={() => setDragOver(false)}
|
||||||
|
onDrop={handleDrop}
|
||||||
|
className={clsx(
|
||||||
|
'flex flex-col items-center justify-center rounded border-2 border-dashed px-4 py-6 text-center transition-colors',
|
||||||
|
dragOver
|
||||||
|
? 'border-primary bg-primary/10'
|
||||||
|
: 'border-border bg-bg'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<UploadIcon className="mb-2 h-6 w-6 text-text-muted" />
|
||||||
|
<div className="text-sm text-text">
|
||||||
|
Drop files or folders here
|
||||||
|
</div>
|
||||||
|
<div className="mt-1 text-xs text-text-muted">
|
||||||
|
Folder structure is preserved under the destination.
|
||||||
|
</div>
|
||||||
|
<div className="mt-3 flex gap-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => fileInputRef.current?.click()}
|
||||||
|
className="rounded border border-border px-3 py-1 text-xs text-text hover:bg-surface-2"
|
||||||
|
>
|
||||||
|
Select files
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => dirInputRef.current?.click()}
|
||||||
|
className="rounded border border-border px-3 py-1 text-xs text-text hover:bg-surface-2"
|
||||||
|
>
|
||||||
|
Select folder
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
ref={fileInputRef}
|
||||||
|
type="file"
|
||||||
|
multiple
|
||||||
|
accept="image/*,video/*,.heic,.heif,.cr2,.cr3,.nef,.arw,.raf,.dng,.orf,.rw2,.pef,.srw"
|
||||||
|
className="hidden"
|
||||||
|
onChange={handleFilePick}
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
ref={dirInputRef}
|
||||||
|
type="file"
|
||||||
|
multiple
|
||||||
|
// @ts-expect-error — non-standard but supported in Chromium/WebKit
|
||||||
|
webkitdirectory=""
|
||||||
|
directory=""
|
||||||
|
className="hidden"
|
||||||
|
onChange={handleDirPick}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Queue */}
|
||||||
|
<div className="min-h-0 flex-1 overflow-auto rounded border border-border bg-bg">
|
||||||
|
{queue.length === 0 ? (
|
||||||
|
<div className="flex h-full items-center justify-center text-xs text-text-muted">
|
||||||
|
No files added yet.
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<ul className="divide-y divide-border">
|
||||||
|
{queue.map((item) => (
|
||||||
|
<li key={item.key} className="flex items-center gap-2 px-3 py-2 text-sm">
|
||||||
|
<FileImage className="h-4 w-4 shrink-0 text-text-muted" />
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<div className="truncate text-text">
|
||||||
|
{item.relativePath || item.file.name}
|
||||||
|
</div>
|
||||||
|
{item.status === 'uploading' && (
|
||||||
|
<div className="mt-1 h-1 w-full overflow-hidden rounded bg-surface-2">
|
||||||
|
<div
|
||||||
|
className="h-full bg-primary transition-all"
|
||||||
|
style={{ width: `${Math.round(item.progress * 100)}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{item.status === 'error' && item.error && (
|
||||||
|
<div className="mt-0.5 truncate text-xs text-reject">{item.error}</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<div className="shrink-0">
|
||||||
|
{item.status === 'done' && <CheckCircle2 className="h-4 w-4 text-green-500" />}
|
||||||
|
{item.status === 'error' && <AlertCircle className="h-4 w-4 text-reject" />}
|
||||||
|
{item.status !== 'done' && !isUploading && (
|
||||||
|
<button
|
||||||
|
onClick={() => removeFromQueue(item.key)}
|
||||||
|
className="rounded p-0.5 text-text-muted hover:bg-surface-2 hover:text-text"
|
||||||
|
aria-label="Remove"
|
||||||
|
>
|
||||||
|
<X className="h-3.5 w-3.5" />
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{queue.length > 0 && (
|
||||||
|
<div className="text-xs text-text-muted">
|
||||||
|
{queue.length} file{queue.length === 1 ? '' : 's'} •{' '}
|
||||||
|
{(totalBytes / 1024 / 1024).toFixed(1)} MB
|
||||||
|
{isUploading && ` • ${overallPct}% uploaded`}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Footer */}
|
||||||
|
<div className="flex items-center justify-end gap-2 border-t border-border px-5 py-3">
|
||||||
|
<button
|
||||||
|
onClick={isUploading ? cancelUpload : onClose}
|
||||||
|
className="rounded border border-border px-3 py-1.5 text-sm text-text hover:bg-surface-2"
|
||||||
|
>
|
||||||
|
{isUploading ? 'Cancel' : 'Close'}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={startUpload}
|
||||||
|
disabled={isUploading || queue.length === 0 || !destFolderId}
|
||||||
|
className="flex items-center gap-1.5 rounded bg-primary px-3 py-1.5 text-sm font-medium text-white hover:bg-primary/80 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
|
>
|
||||||
|
<UploadIcon className="h-3.5 w-3.5" />
|
||||||
|
{isUploading ? 'Uploading…' : `Upload ${queue.length || ''}`.trim()}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FolderTreeRowProps {
|
||||||
|
node: FolderTreeNode
|
||||||
|
depth: number
|
||||||
|
selectedId: string | null
|
||||||
|
onSelect: (id: string) => void
|
||||||
|
expanded: Set<string>
|
||||||
|
onToggle: (id: string) => void
|
||||||
|
}
|
||||||
|
|
||||||
|
function FolderTreeRow({
|
||||||
|
node,
|
||||||
|
depth,
|
||||||
|
selectedId,
|
||||||
|
onSelect,
|
||||||
|
expanded,
|
||||||
|
onToggle,
|
||||||
|
}: FolderTreeRowProps) {
|
||||||
|
const isExpanded = expanded.has(node.id)
|
||||||
|
const hasChildren = node.children && node.children.length > 0
|
||||||
|
const isSelected = selectedId === node.id
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<div
|
||||||
|
className={clsx(
|
||||||
|
'flex cursor-pointer items-center gap-1 rounded px-1 py-1 text-sm',
|
||||||
|
isSelected ? 'bg-primary/20 text-text' : 'text-text-muted hover:bg-surface-2 hover:text-text'
|
||||||
|
)}
|
||||||
|
style={{ paddingLeft: `${depth * 12 + 4}px` }}
|
||||||
|
onClick={() => onSelect(node.id)}
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation()
|
||||||
|
if (hasChildren) onToggle(node.id)
|
||||||
|
}}
|
||||||
|
className="flex h-4 w-4 items-center justify-center"
|
||||||
|
aria-label={isExpanded ? 'Collapse' : 'Expand'}
|
||||||
|
>
|
||||||
|
{hasChildren ? (
|
||||||
|
isExpanded ? (
|
||||||
|
<ChevronDown className="h-3 w-3" />
|
||||||
|
) : (
|
||||||
|
<ChevronRight className="h-3 w-3" />
|
||||||
|
)
|
||||||
|
) : null}
|
||||||
|
</button>
|
||||||
|
<FolderIcon className="h-3.5 w-3.5 shrink-0" />
|
||||||
|
<span className="truncate">{node.name}</span>
|
||||||
|
</div>
|
||||||
|
{isExpanded &&
|
||||||
|
node.children?.map((c) => (
|
||||||
|
<FolderTreeRow
|
||||||
|
key={c.id}
|
||||||
|
node={c}
|
||||||
|
depth={depth + 1}
|
||||||
|
selectedId={selectedId}
|
||||||
|
onSelect={onSelect}
|
||||||
|
expanded={expanded}
|
||||||
|
onToggle={onToggle}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
}
|
||||||
39
frontend/src/hooks/useFeaturesQuery.ts
Normal file
39
frontend/src/hooks/useFeaturesQuery.ts
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||||
|
import { features, type FeaturesMap } from '../services/api'
|
||||||
|
|
||||||
|
export const FEATURES_QUERY_KEY = ['features'] as const
|
||||||
|
|
||||||
|
/** Read the effective feature-flag state (admin override or YAML
|
||||||
|
* default). Powers conditional rendering of pipeline-dependent UI —
|
||||||
|
* People view, Tags view, OCR snippets, etc. */
|
||||||
|
export function useFeaturesQuery() {
|
||||||
|
return useQuery<FeaturesMap>({
|
||||||
|
queryKey: FEATURES_QUERY_KEY,
|
||||||
|
queryFn: features.list,
|
||||||
|
// Re-read every minute so admin toggles reflect without a page
|
||||||
|
// reload. The admin tab also invalidates this key on write so the
|
||||||
|
// refresh can be immediate for the admin who just flipped it.
|
||||||
|
staleTime: 60_000,
|
||||||
|
refetchInterval: 60_000,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useIsFeatureEnabled(
|
||||||
|
name:
|
||||||
|
| 'vision.enabled'
|
||||||
|
| 'vision.ocr.enabled'
|
||||||
|
| 'vision.detector.enabled'
|
||||||
|
| 'vision.faces.enabled'
|
||||||
|
| 'vision.classifier.enabled',
|
||||||
|
): boolean {
|
||||||
|
const { data } = useFeaturesQuery()
|
||||||
|
// Default to enabled while loading so we don't flash "feature off"
|
||||||
|
// during a first-paint fetch. The backend is the source of truth;
|
||||||
|
// any gated UI that slipped through just returns empty data anyway.
|
||||||
|
if (!data) return true
|
||||||
|
return !!data[name]
|
||||||
|
}
|
||||||
|
|
||||||
|
export function invalidateFeaturesQuery(queryClient: ReturnType<typeof useQueryClient>) {
|
||||||
|
queryClient.invalidateQueries({ queryKey: FEATURES_QUERY_KEY })
|
||||||
|
}
|
||||||
@@ -713,6 +713,72 @@ export const heaps = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Upload API — single file per request so the browser can fan out many
|
||||||
|
// POSTs in parallel with per-file progress. For folder uploads the
|
||||||
|
// caller passes each File's webkitRelativePath so the backend can
|
||||||
|
// materialise the folder structure under the destination.
|
||||||
|
export const uploads = {
|
||||||
|
uploadFile: async (
|
||||||
|
file: File,
|
||||||
|
destinationFolderId: string,
|
||||||
|
opts: {
|
||||||
|
relativePath?: string
|
||||||
|
heapId?: string | null
|
||||||
|
onProgress?: (loadedBytes: number, totalBytes: number) => void
|
||||||
|
signal?: AbortSignal
|
||||||
|
} = {}
|
||||||
|
) => {
|
||||||
|
const form = new FormData()
|
||||||
|
form.append('file', file)
|
||||||
|
form.append('destination_folder_id', destinationFolderId)
|
||||||
|
if (opts.relativePath) form.append('relative_path', opts.relativePath)
|
||||||
|
if (opts.heapId) form.append('heap_id', opts.heapId)
|
||||||
|
|
||||||
|
const response = await api.post('/upload', form, {
|
||||||
|
headers: { 'Content-Type': 'multipart/form-data' },
|
||||||
|
signal: opts.signal,
|
||||||
|
onUploadProgress: (evt) => {
|
||||||
|
if (opts.onProgress && evt.total) opts.onProgress(evt.loaded, evt.total)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return response.data as {
|
||||||
|
photo_id: string
|
||||||
|
filename: string
|
||||||
|
folder_id: string
|
||||||
|
folder_path: string
|
||||||
|
heap_id: string | null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download helpers — build a URL the browser can pull directly via an
|
||||||
|
// <a href>. The backend accepts `?token=` so an <a> works without a
|
||||||
|
// custom fetch + save-blob dance; the Authorization header is not
|
||||||
|
// settable on a plain link click.
|
||||||
|
export const downloads = {
|
||||||
|
folderUrl: (folderId: string): string => {
|
||||||
|
const token = localStorage.getItem('access_token') || ''
|
||||||
|
return `${API_BASE_URL}/download/folders/${folderId}?token=${encodeURIComponent(token)}`
|
||||||
|
},
|
||||||
|
heapUrl: (heapId: string): string => {
|
||||||
|
const token = localStorage.getItem('access_token') || ''
|
||||||
|
return `${API_BASE_URL}/download/heaps/${heapId}?token=${encodeURIComponent(token)}`
|
||||||
|
},
|
||||||
|
trigger: (url: string) => {
|
||||||
|
// Kicking off a download via a transient <a> click keeps the
|
||||||
|
// browser in charge of the file dialog + progress indicator. We
|
||||||
|
// use target=_blank so the current SPA route isn't replaced if
|
||||||
|
// the server returns an error mid-stream.
|
||||||
|
const a = document.createElement('a')
|
||||||
|
a.href = url
|
||||||
|
a.rel = 'noopener'
|
||||||
|
a.target = '_blank'
|
||||||
|
document.body.appendChild(a)
|
||||||
|
a.click()
|
||||||
|
document.body.removeChild(a)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
// Sharing API
|
// Sharing API
|
||||||
|
|
||||||
export interface SharedHeap {
|
export interface SharedHeap {
|
||||||
@@ -928,6 +994,61 @@ export const admin = {
|
|||||||
const response = await api.delete(`/admin/users/${userId}`)
|
const response = await api.delete(`/admin/users/${userId}`)
|
||||||
return response.data
|
return response.data
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// --- AI / vision feature flags + manual pipeline triggers -----------
|
||||||
|
|
||||||
|
listFeatureFlags: async (): Promise<{ flags: FeatureFlagSnapshot }> => {
|
||||||
|
const response = await api.get('/admin/feature-flags')
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
|
||||||
|
/** Set ``value`` to toggle; pass ``null`` to clear the override and
|
||||||
|
* fall back to the YAML default. */
|
||||||
|
setFeatureFlag: async (
|
||||||
|
name: string,
|
||||||
|
value: boolean | null,
|
||||||
|
): Promise<{ flags: FeatureFlagSnapshot }> => {
|
||||||
|
const response = await api.patch(`/admin/feature-flags/${encodeURIComponent(name)}`, { value })
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
|
||||||
|
triggerAiBackfill: async (body: {
|
||||||
|
task?: 'embed' | 'ocr' | 'detect' | 'faces' | 'classify' | null
|
||||||
|
limit?: number | null
|
||||||
|
}): Promise<{ status: string; task_id: string }> => {
|
||||||
|
const response = await api.post('/admin/ai/backfill', body)
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
|
||||||
|
triggerFaceRecluster: async (): Promise<{ status: string; task_id: string }> => {
|
||||||
|
const response = await api.post('/admin/ai/recluster-faces')
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
|
||||||
|
triggerFullRescan: async (): Promise<{ status: string; task_id: string }> => {
|
||||||
|
const response = await api.post('/admin/ai/rescan')
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FeatureFlagState {
|
||||||
|
effective: boolean
|
||||||
|
default: boolean
|
||||||
|
overridden: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export type FeatureFlagSnapshot = Record<string, FeatureFlagState>
|
||||||
|
|
||||||
|
export type FeaturesMap = Record<string, boolean>
|
||||||
|
|
||||||
|
// Public read of effective feature flags. Available to any signed-in
|
||||||
|
// user so the frontend can hide sections that depend on a disabled
|
||||||
|
// pipeline stage (e.g. People when faces are off).
|
||||||
|
export const features = {
|
||||||
|
list: async (): Promise<FeaturesMap> => {
|
||||||
|
const response = await api.get('/features')
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
export default api
|
export default api
|
||||||
Reference in New Issue
Block a user