576b0c236d
mule-image was generating and storing three WebP sizes per photo in /data/thumbs while Nextcloud already keeps its own previews for the same source files. Frontend thumbnail requests now proxy NC's /index.php/core/preview keyed by the photo's Nextcloud fileid, authenticated with the owner's encrypted app password. - new column photos.nextcloud_fileid (alembic 0018) plus an index - get_preview_async + fetch_fileid helpers in nextcloud_dav.py - thumb route proxies NC primary, falls back to /data/thumbs (legacy rows / NC unreachable) so a single-file revert restores the old path - extract_metadata caches the fileid on first run for new photos - generate_thumbnails now writes only medium since the vision worker still loads it from disk; small + large drop out of the worker path - backend/scripts/backfill_nextcloud_fileid.py for one-shot population of existing rows: docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid X-Mule-Thumb-Source response header marks each request 'nextcloud' or 'disk' for observability while the rollout settles.
122 lines
3.9 KiB
Python
122 lines
3.9 KiB
Python
"""Backfill Photo.nextcloud_fileid for photos under Nextcloud-rooted paths.
|
|
|
|
The Phase-1 thumbnail proxy reads `Photo.nextcloud_fileid` to know which
|
|
file to ask Nextcloud's /core/preview endpoint about. New photos pick it
|
|
up at scan time; this script catches up the existing library.
|
|
|
|
Run inside the backend container, e.g.:
|
|
|
|
pct exec 120 -- docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid
|
|
|
|
Idempotent: skips rows that already have nextcloud_fileid set, and any
|
|
row whose path isn't under the Nextcloud bind mount. One PROPFIND per
|
|
photo. At ~50ms each that's ~18 minutes for a 22k-row library — run
|
|
during off-hours.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import Optional
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import Photo
|
|
from app.models.user import User
|
|
from app.services.nextcloud_dav import fetch_fileid, is_nextcloud_path
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
)
|
|
logger = logging.getLogger("backfill_nextcloud_fileid")
|
|
|
|
BATCH = 500
|
|
|
|
|
|
async def _user_cache(session: AsyncSession) -> dict[str, User]:
|
|
"""One SELECT per script run instead of per photo."""
|
|
result = await session.execute(select(User))
|
|
return {u.id: u for u in result.scalars().all()}
|
|
|
|
|
|
async def run() -> None:
|
|
async with AsyncSessionLocal() as session:
|
|
users = await _user_cache(session)
|
|
total = await session.scalar(
|
|
select(func.count(Photo.id)).where(Photo.nextcloud_fileid.is_(None))
|
|
)
|
|
logger.info("photos with NULL nextcloud_fileid: %s", total)
|
|
|
|
done = 0
|
|
skipped_no_user = 0
|
|
skipped_not_nc = 0
|
|
skipped_no_creds = 0
|
|
filled = 0
|
|
not_found = 0
|
|
|
|
offset = 0
|
|
while True:
|
|
result = await session.execute(
|
|
select(Photo)
|
|
.where(Photo.nextcloud_fileid.is_(None))
|
|
.order_by(Photo.id)
|
|
.offset(offset)
|
|
.limit(BATCH)
|
|
)
|
|
rows = list(result.scalars().all())
|
|
if not rows:
|
|
break
|
|
|
|
for photo in rows:
|
|
done += 1
|
|
if not photo.user_id:
|
|
skipped_no_user += 1
|
|
continue
|
|
owner = users.get(photo.user_id)
|
|
if owner is None:
|
|
skipped_no_user += 1
|
|
continue
|
|
if not photo.filepath or not is_nextcloud_path(photo.filepath):
|
|
skipped_not_nc += 1
|
|
continue
|
|
if not owner.nextcloud_app_password_enc:
|
|
skipped_no_creds += 1
|
|
continue
|
|
fid: Optional[int] = None
|
|
try:
|
|
fid = fetch_fileid(owner, photo.filepath)
|
|
except Exception as e:
|
|
logger.warning(
|
|
"PROPFIND failed for photo %s (%s): %s",
|
|
photo.id, photo.filepath, e,
|
|
)
|
|
if fid is None:
|
|
not_found += 1
|
|
continue
|
|
photo.nextcloud_fileid = fid
|
|
filled += 1
|
|
|
|
await session.commit()
|
|
offset += BATCH
|
|
logger.info(
|
|
"progress: scanned=%s filled=%s not_found=%s "
|
|
"skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s",
|
|
done, filled, not_found,
|
|
skipped_no_user, skipped_not_nc, skipped_no_creds,
|
|
total,
|
|
)
|
|
|
|
logger.info(
|
|
"done: scanned=%s filled=%s not_found=%s "
|
|
"skipped(no_user=%s not_nc=%s no_creds=%s)",
|
|
done, filled, not_found,
|
|
skipped_no_user, skipped_not_nc, skipped_no_creds,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(run())
|