feat(thumbs): proxy Nextcloud previews instead of duplicating the cache

mule-image was generating and storing three WebP sizes per photo in
/data/thumbs while Nextcloud already keeps its own previews for the
same source files. Frontend thumbnail requests now proxy NC's
/index.php/core/preview keyed by the photo's Nextcloud fileid,
authenticated with the owner's encrypted app password.

- new column photos.nextcloud_fileid (alembic 0018) plus an index
- get_preview_async + fetch_fileid helpers in nextcloud_dav.py
- thumb route proxies NC primary, falls back to /data/thumbs (legacy
  rows / NC unreachable) so a single-file revert restores the old path
- extract_metadata caches the fileid on first run for new photos
- generate_thumbnails now writes only medium since the vision worker
  still loads it from disk; small + large drop out of the worker path
- backend/scripts/backfill_nextcloud_fileid.py for one-shot population
  of existing rows: docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid

X-Mule-Thumb-Source response header marks each request 'nextcloud' or
'disk' for observability while the rollout settles.
This commit is contained in:
Claudio
2026-05-11 11:34:58 +02:00
parent 9e9b1ba224
commit 576b0c236d
8 changed files with 378 additions and 16 deletions

View File

@@ -0,0 +1,121 @@
"""Backfill Photo.nextcloud_fileid for photos under Nextcloud-rooted paths.
The Phase-1 thumbnail proxy reads `Photo.nextcloud_fileid` to know which
file to ask Nextcloud's /core/preview endpoint about. New photos pick it
up at scan time; this script catches up the existing library.
Run inside the backend container, e.g.:
pct exec 120 -- docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid
Idempotent: skips rows that already have nextcloud_fileid set, and any
row whose path isn't under the Nextcloud bind mount. One PROPFIND per
photo. At ~50ms each that's ~18 minutes for a 22k-row library — run
during off-hours.
"""
from __future__ import annotations
import asyncio
import logging
from typing import Optional
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import AsyncSessionLocal
from app.models import Photo
from app.models.user import User
from app.services.nextcloud_dav import fetch_fileid, is_nextcloud_path
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
logger = logging.getLogger("backfill_nextcloud_fileid")
BATCH = 500
async def _user_cache(session: AsyncSession) -> dict[str, User]:
"""One SELECT per script run instead of per photo."""
result = await session.execute(select(User))
return {u.id: u for u in result.scalars().all()}
async def run() -> None:
async with AsyncSessionLocal() as session:
users = await _user_cache(session)
total = await session.scalar(
select(func.count(Photo.id)).where(Photo.nextcloud_fileid.is_(None))
)
logger.info("photos with NULL nextcloud_fileid: %s", total)
done = 0
skipped_no_user = 0
skipped_not_nc = 0
skipped_no_creds = 0
filled = 0
not_found = 0
offset = 0
while True:
result = await session.execute(
select(Photo)
.where(Photo.nextcloud_fileid.is_(None))
.order_by(Photo.id)
.offset(offset)
.limit(BATCH)
)
rows = list(result.scalars().all())
if not rows:
break
for photo in rows:
done += 1
if not photo.user_id:
skipped_no_user += 1
continue
owner = users.get(photo.user_id)
if owner is None:
skipped_no_user += 1
continue
if not photo.filepath or not is_nextcloud_path(photo.filepath):
skipped_not_nc += 1
continue
if not owner.nextcloud_app_password_enc:
skipped_no_creds += 1
continue
fid: Optional[int] = None
try:
fid = fetch_fileid(owner, photo.filepath)
except Exception as e:
logger.warning(
"PROPFIND failed for photo %s (%s): %s",
photo.id, photo.filepath, e,
)
if fid is None:
not_found += 1
continue
photo.nextcloud_fileid = fid
filled += 1
await session.commit()
offset += BATCH
logger.info(
"progress: scanned=%s filled=%s not_found=%s "
"skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s",
done, filled, not_found,
skipped_no_user, skipped_not_nc, skipped_no_creds,
total,
)
logger.info(
"done: scanned=%s filled=%s not_found=%s "
"skipped(no_user=%s not_nc=%s no_creds=%s)",
done, filled, not_found,
skipped_no_user, skipped_not_nc, skipped_no_creds,
)
if __name__ == "__main__":
asyncio.run(run())