"""Backfill Photo.nextcloud_fileid for photos under Nextcloud-rooted paths. The Phase-1 thumbnail proxy reads `Photo.nextcloud_fileid` to know which file to ask Nextcloud's /core/preview endpoint about. New photos pick it up at scan time; this script catches up the existing library. Run inside the backend container, e.g.: pct exec 120 -- docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid Idempotent: skips rows that already have nextcloud_fileid set, and any row whose path isn't under the Nextcloud bind mount. One PROPFIND per photo. At ~50ms each that's ~18 minutes for a 22k-row library — run during off-hours. """ from __future__ import annotations import asyncio import logging from typing import Optional from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from app.database import AsyncSessionLocal from app.models import Photo from app.models.user import User from app.services.nextcloud_dav import fetch_fileid, is_nextcloud_path logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s", ) logger = logging.getLogger("backfill_nextcloud_fileid") BATCH = 500 async def _user_cache(session: AsyncSession) -> dict[str, User]: """One SELECT per script run instead of per photo.""" result = await session.execute(select(User)) return {u.id: u for u in result.scalars().all()} async def run() -> None: async with AsyncSessionLocal() as session: users = await _user_cache(session) total = await session.scalar( select(func.count(Photo.id)).where(Photo.nextcloud_fileid.is_(None)) ) logger.info("photos with NULL nextcloud_fileid: %s", total) done = 0 skipped_no_user = 0 skipped_not_nc = 0 skipped_no_creds = 0 filled = 0 not_found = 0 # Keep selecting the next batch of NULL-fileid rows until the # set is empty. NO offset() — each batch's writes shrink the # `WHERE nextcloud_fileid IS NULL` set, so an offset would skip # over the rows that were just filled in by the previous batch. # Rows we couldn't resolve (skipped or not_found) stay in the # set; we track them in a "stuck ids" set so the loop terminates # instead of spinning on them forever. stuck: set[str] = set() while True: stmt = ( select(Photo) .where(Photo.nextcloud_fileid.is_(None)) .order_by(Photo.id) .limit(BATCH) ) if stuck: stmt = stmt.where(Photo.id.notin_(stuck)) result = await session.execute(stmt) rows = list(result.scalars().all()) if not rows: break batch_started_filled = filled for photo in rows: done += 1 if not photo.user_id: skipped_no_user += 1 stuck.add(photo.id) continue owner = users.get(photo.user_id) if owner is None: skipped_no_user += 1 stuck.add(photo.id) continue if not photo.filepath or not is_nextcloud_path(photo.filepath): skipped_not_nc += 1 stuck.add(photo.id) continue if not owner.nextcloud_app_password_enc: skipped_no_creds += 1 stuck.add(photo.id) continue fid: Optional[int] = None try: fid = fetch_fileid(owner, photo.filepath) except Exception as e: logger.warning( "PROPFIND failed for photo %s (%s): %s", photo.id, photo.filepath, e, ) if fid is None: not_found += 1 stuck.add(photo.id) continue photo.nextcloud_fileid = fid filled += 1 await session.commit() # Safety: if a whole batch produced no new fills, every row # in it is already in `stuck` — break to avoid an infinite # loop on the same set. if filled == batch_started_filled and len(rows) < BATCH: break logger.info( "progress: scanned=%s filled=%s not_found=%s " "skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s", done, filled, not_found, skipped_no_user, skipped_not_nc, skipped_no_creds, total, ) logger.info( "done: scanned=%s filled=%s not_found=%s " "skipped(no_user=%s not_nc=%s no_creds=%s)", done, filled, not_found, skipped_no_user, skipped_not_nc, skipped_no_creds, ) if __name__ == "__main__": asyncio.run(run())