Files
mule-image/backend/app/routers/download.py
root 5c531f11da feat: runtime feature flags, upload/download, RAW decoding
Adds Redis-backed feature flags for vision stages with admin UI toggles
and manual backfill trigger, photo upload and download routers with
frontend upload modal, and rawpy-based RAW decoding with JPEG fallback
for misnamed DNGs. Fixes pgvector serialization, is_trashed filter, and
naive-datetime bind in incremental duplicate regrouping; bumps Celery
time limits on regroup tasks beyond the 5-minute default.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-14 21:31:52 +02:00

248 lines
8.8 KiB
Python

"""
Download router — streams a .zip of every photo in a folder (recursively)
or a heap back to the browser.
Auth: both endpoints accept the regular Authorization header *or* a
``?token=JWT`` query string, mirroring the media endpoints. That lets the
frontend trigger a download with a plain ``<a href>`` (which can't set a
header), keeping the client side a one-liner.
Implementation: we build the zip into a ``NamedTemporaryFile`` and then
stream its bytes back, deleting the temp file on the way out. Stored
(uncompressed) mode because photos and videos are already compressed —
deflating them again just burns CPU for a fraction of a percent. For
very large libraries the temp-file route is mildly wasteful vs. a true
streaming zip (zipstream-ng etc), but it avoids a new dependency and
handles arbitrary folder sizes without blowing out RAM.
"""
import logging
import os
import re
import tempfile
import zipfile
from typing import List
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import StreamingResponse
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.dependencies import get_current_user_media
from app.models import Folder, Heap, Photo, SourceRoot
from app.models.heaps import heap_photos
from app.models.user import User
logger = logging.getLogger(__name__)
router = APIRouter()
def _safe_filename(name: str) -> str:
"""Strip characters that Content-Disposition or Windows filesystems
would choke on. Keeps the download's filename readable without
needing any escaping on the client side."""
cleaned = re.sub(r'[\\/:*?"<>|\r\n\t]', '_', name).strip().strip('.')
return cleaned or 'download'
async def _collect_folder_photos(
folder_id: str,
user: User,
db: AsyncSession,
) -> tuple[str, str, List[Photo]]:
"""Resolve a folder or source-root id → (base_path, display_name,
photos). ``base_path`` is the prefix we strip off each photo's
filepath when naming zip entries, so the archive mirrors the user's
on-disk structure under that folder.
"""
folder = (await db.execute(
select(Folder).where(Folder.id == folder_id, Folder.user_id == user.id)
)).scalar_one_or_none()
base_path: str
display_name: str
if folder is not None:
base_path = os.path.normpath(folder.path)
display_name = folder.name or os.path.basename(base_path)
else:
sr = (await db.execute(
select(SourceRoot).where(
SourceRoot.id == folder_id,
SourceRoot.user_id == user.id,
)
)).scalar_one_or_none()
if sr is None:
raise HTTPException(status_code=404, detail="Folder not found")
base_path = os.path.normpath(sr.path)
display_name = sr.name or os.path.basename(base_path)
# Every photo whose filepath is at or below the base path — matches
# the same prefix convention folders.py uses for recursive deletes.
descendant_prefix = base_path.rstrip(os.sep) + os.sep
result = await db.execute(
select(Photo).where(
Photo.user_id == user.id,
Photo.is_discarded == False, # noqa: E712
(Photo.filepath == base_path) | (Photo.filepath.like(descendant_prefix + '%')),
)
)
photos = list(result.scalars().all())
return base_path, display_name, photos
def _build_zip(
photos: List[Photo],
arcname_fn,
) -> tempfile.NamedTemporaryFile:
"""Write ``photos`` into a fresh ZIP_STORED temp file.
``arcname_fn(photo, used_names)`` returns the entry name to use for
the given photo; the caller supplies it because folder downloads
want path-preserving names while heap downloads flatten to bare
filenames (with a collision suffix).
"""
tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.zip')
try:
used: set[str] = set()
with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_STORED, allowZip64=True) as zf:
for p in photos:
if not p.filepath or not os.path.exists(p.filepath):
# Silent skip: the scanner may have indexed files
# that have since been moved / unlinked by a shell.
continue
name = arcname_fn(p, used)
used.add(name)
try:
zf.write(p.filepath, name)
except OSError as e:
logger.warning(f"Skipping {p.filepath} in zip: {e}")
tmp.close()
return tmp
except Exception:
tmp.close()
try:
os.unlink(tmp.name)
except OSError:
pass
raise
def _stream_and_cleanup(path: str):
"""Yield the temp zip in 1 MiB chunks and unlink it when the
iterator is exhausted (or GC'd, if the client disconnects early)."""
try:
with open(path, 'rb') as f:
while True:
chunk = f.read(1024 * 1024)
if not chunk:
break
yield chunk
finally:
try:
os.unlink(path)
except OSError as e:
logger.debug(f"Temp zip cleanup failed for {path}: {e}")
def _dedupe(name: str, used: set[str]) -> str:
"""Return ``name`` (or ``name (2)``, ``name (3)`` ...) such that the
result doesn't collide with anything in ``used``. Needed for heap
downloads where two members can have identical filenames from
different folders."""
if name not in used:
return name
stem, ext = os.path.splitext(name)
n = 2
while True:
cand = f"{stem} ({n}){ext}"
if cand not in used:
return cand
n += 1
@router.get("/folders/{folder_id}")
async def download_folder(
folder_id: str,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user_media),
):
"""Zip every (non-discarded) photo under a folder/source-root and
stream it back. Entries preserve the folder structure relative to
the downloaded root so the resulting archive is a faithful snapshot.
"""
base_path, display_name, photos = await _collect_folder_photos(
folder_id, current_user, db
)
if not photos:
raise HTTPException(status_code=404, detail="No photos to download")
def arcname(p: Photo, _used: set[str]) -> str:
# Relative path from the download root, falling back to the
# bare filename if the photo somehow lives outside base_path.
abs_path = os.path.normpath(p.filepath)
if abs_path.startswith(base_path + os.sep):
rel = abs_path[len(base_path) + 1:]
elif abs_path == base_path:
rel = os.path.basename(abs_path)
else:
rel = p.filename or os.path.basename(abs_path)
# Nest everything under display_name so users see one top-level
# folder inside the zip rather than loose files.
return os.path.join(_safe_filename(display_name), rel)
tmp = _build_zip(photos, arcname)
filename = _safe_filename(display_name) + '.zip'
return StreamingResponse(
_stream_and_cleanup(tmp.name),
media_type='application/zip',
headers={
'Content-Disposition': f'attachment; filename="{filename}"',
'Content-Length': str(os.path.getsize(tmp.name)),
},
)
@router.get("/heaps/{heap_id}")
async def download_heap(
heap_id: str,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user_media),
):
"""Zip every photo in a heap. Heaps are flat collections, so entries
use the original filename (with a ``(2)`` collision suffix when
two members share a name)."""
heap = (await db.execute(
select(Heap).where(Heap.id == heap_id, Heap.user_id == current_user.id)
)).scalar_one_or_none()
if heap is None:
raise HTTPException(status_code=404, detail="Heap not found")
result = await db.execute(
select(Photo)
.join(heap_photos, heap_photos.c.photo_id == Photo.id)
.where(
heap_photos.c.heap_id == heap_id,
Photo.is_discarded == False, # noqa: E712
)
)
photos = list(result.scalars().all())
if not photos:
raise HTTPException(status_code=404, detail="Heap is empty")
def arcname(p: Photo, used: set[str]) -> str:
bare = p.filename or os.path.basename(p.filepath or 'photo')
entry = os.path.join(_safe_filename(heap.name), _dedupe(bare, used))
return entry
tmp = _build_zip(photos, arcname)
filename = _safe_filename(heap.name) + '.zip'
return StreamingResponse(
_stream_and_cleanup(tmp.name),
media_type='application/zip',
headers={
'Content-Disposition': f'attachment; filename="{filename}"',
'Content-Length': str(os.path.getsize(tmp.name)),
},
)