feat(thumbs): proxy Nextcloud previews instead of duplicating the cache
mule-image was generating and storing three WebP sizes per photo in /data/thumbs while Nextcloud already keeps its own previews for the same source files. Frontend thumbnail requests now proxy NC's /index.php/core/preview keyed by the photo's Nextcloud fileid, authenticated with the owner's encrypted app password. - new column photos.nextcloud_fileid (alembic 0018) plus an index - get_preview_async + fetch_fileid helpers in nextcloud_dav.py - thumb route proxies NC primary, falls back to /data/thumbs (legacy rows / NC unreachable) so a single-file revert restores the old path - extract_metadata caches the fileid on first run for new photos - generate_thumbnails now writes only medium since the vision worker still loads it from disk; small + large drop out of the worker path - backend/scripts/backfill_nextcloud_fileid.py for one-shot population of existing rows: docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid X-Mule-Thumb-Source response header marks each request 'nextcloud' or 'disk' for observability while the rollout settles.
This commit is contained in:
121
backend/scripts/backfill_nextcloud_fileid.py
Normal file
121
backend/scripts/backfill_nextcloud_fileid.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""Backfill Photo.nextcloud_fileid for photos under Nextcloud-rooted paths.
|
||||
|
||||
The Phase-1 thumbnail proxy reads `Photo.nextcloud_fileid` to know which
|
||||
file to ask Nextcloud's /core/preview endpoint about. New photos pick it
|
||||
up at scan time; this script catches up the existing library.
|
||||
|
||||
Run inside the backend container, e.g.:
|
||||
|
||||
pct exec 120 -- docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid
|
||||
|
||||
Idempotent: skips rows that already have nextcloud_fileid set, and any
|
||||
row whose path isn't under the Nextcloud bind mount. One PROPFIND per
|
||||
photo. At ~50ms each that's ~18 minutes for a 22k-row library — run
|
||||
during off-hours.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo
|
||||
from app.models.user import User
|
||||
from app.services.nextcloud_dav import fetch_fileid, is_nextcloud_path
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("backfill_nextcloud_fileid")
|
||||
|
||||
BATCH = 500
|
||||
|
||||
|
||||
async def _user_cache(session: AsyncSession) -> dict[str, User]:
|
||||
"""One SELECT per script run instead of per photo."""
|
||||
result = await session.execute(select(User))
|
||||
return {u.id: u for u in result.scalars().all()}
|
||||
|
||||
|
||||
async def run() -> None:
|
||||
async with AsyncSessionLocal() as session:
|
||||
users = await _user_cache(session)
|
||||
total = await session.scalar(
|
||||
select(func.count(Photo.id)).where(Photo.nextcloud_fileid.is_(None))
|
||||
)
|
||||
logger.info("photos with NULL nextcloud_fileid: %s", total)
|
||||
|
||||
done = 0
|
||||
skipped_no_user = 0
|
||||
skipped_not_nc = 0
|
||||
skipped_no_creds = 0
|
||||
filled = 0
|
||||
not_found = 0
|
||||
|
||||
offset = 0
|
||||
while True:
|
||||
result = await session.execute(
|
||||
select(Photo)
|
||||
.where(Photo.nextcloud_fileid.is_(None))
|
||||
.order_by(Photo.id)
|
||||
.offset(offset)
|
||||
.limit(BATCH)
|
||||
)
|
||||
rows = list(result.scalars().all())
|
||||
if not rows:
|
||||
break
|
||||
|
||||
for photo in rows:
|
||||
done += 1
|
||||
if not photo.user_id:
|
||||
skipped_no_user += 1
|
||||
continue
|
||||
owner = users.get(photo.user_id)
|
||||
if owner is None:
|
||||
skipped_no_user += 1
|
||||
continue
|
||||
if not photo.filepath or not is_nextcloud_path(photo.filepath):
|
||||
skipped_not_nc += 1
|
||||
continue
|
||||
if not owner.nextcloud_app_password_enc:
|
||||
skipped_no_creds += 1
|
||||
continue
|
||||
fid: Optional[int] = None
|
||||
try:
|
||||
fid = fetch_fileid(owner, photo.filepath)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"PROPFIND failed for photo %s (%s): %s",
|
||||
photo.id, photo.filepath, e,
|
||||
)
|
||||
if fid is None:
|
||||
not_found += 1
|
||||
continue
|
||||
photo.nextcloud_fileid = fid
|
||||
filled += 1
|
||||
|
||||
await session.commit()
|
||||
offset += BATCH
|
||||
logger.info(
|
||||
"progress: scanned=%s filled=%s not_found=%s "
|
||||
"skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s",
|
||||
done, filled, not_found,
|
||||
skipped_no_user, skipped_not_nc, skipped_no_creds,
|
||||
total,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"done: scanned=%s filled=%s not_found=%s "
|
||||
"skipped(no_user=%s not_nc=%s no_creds=%s)",
|
||||
done, filled, not_found,
|
||||
skipped_no_user, skipped_not_nc, skipped_no_creds,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run())
|
||||
Reference in New Issue
Block a user