28738acb56
The offset+limit loop walked the IS NULL set, but every batch's writes shrank that set, so batch N+1 with offset=N*BATCH skipped over the rows just filled. A 17k library backfilled only 9k before the loop walked off the (now-shorter) NULL set. Replace with a tail-recursive pattern: keep selecting LIMIT BATCH on the NULL set, tracking rows that won't ever resolve in a `stuck` set so the loop terminates instead of spinning on them.
141 lines
4.9 KiB
Python
141 lines
4.9 KiB
Python
"""Backfill Photo.nextcloud_fileid for photos under Nextcloud-rooted paths.
|
|
|
|
The Phase-1 thumbnail proxy reads `Photo.nextcloud_fileid` to know which
|
|
file to ask Nextcloud's /core/preview endpoint about. New photos pick it
|
|
up at scan time; this script catches up the existing library.
|
|
|
|
Run inside the backend container, e.g.:
|
|
|
|
pct exec 120 -- docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid
|
|
|
|
Idempotent: skips rows that already have nextcloud_fileid set, and any
|
|
row whose path isn't under the Nextcloud bind mount. One PROPFIND per
|
|
photo. At ~50ms each that's ~18 minutes for a 22k-row library — run
|
|
during off-hours.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import Optional
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import Photo
|
|
from app.models.user import User
|
|
from app.services.nextcloud_dav import fetch_fileid, is_nextcloud_path
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
)
|
|
logger = logging.getLogger("backfill_nextcloud_fileid")
|
|
|
|
BATCH = 500
|
|
|
|
|
|
async def _user_cache(session: AsyncSession) -> dict[str, User]:
|
|
"""One SELECT per script run instead of per photo."""
|
|
result = await session.execute(select(User))
|
|
return {u.id: u for u in result.scalars().all()}
|
|
|
|
|
|
async def run() -> None:
|
|
async with AsyncSessionLocal() as session:
|
|
users = await _user_cache(session)
|
|
total = await session.scalar(
|
|
select(func.count(Photo.id)).where(Photo.nextcloud_fileid.is_(None))
|
|
)
|
|
logger.info("photos with NULL nextcloud_fileid: %s", total)
|
|
|
|
done = 0
|
|
skipped_no_user = 0
|
|
skipped_not_nc = 0
|
|
skipped_no_creds = 0
|
|
filled = 0
|
|
not_found = 0
|
|
|
|
# Keep selecting the next batch of NULL-fileid rows until the
|
|
# set is empty. NO offset() — each batch's writes shrink the
|
|
# `WHERE nextcloud_fileid IS NULL` set, so an offset would skip
|
|
# over the rows that were just filled in by the previous batch.
|
|
# Rows we couldn't resolve (skipped or not_found) stay in the
|
|
# set; we track them in a "stuck ids" set so the loop terminates
|
|
# instead of spinning on them forever.
|
|
stuck: set[str] = set()
|
|
while True:
|
|
stmt = (
|
|
select(Photo)
|
|
.where(Photo.nextcloud_fileid.is_(None))
|
|
.order_by(Photo.id)
|
|
.limit(BATCH)
|
|
)
|
|
if stuck:
|
|
stmt = stmt.where(Photo.id.notin_(stuck))
|
|
result = await session.execute(stmt)
|
|
rows = list(result.scalars().all())
|
|
if not rows:
|
|
break
|
|
|
|
batch_started_filled = filled
|
|
for photo in rows:
|
|
done += 1
|
|
if not photo.user_id:
|
|
skipped_no_user += 1
|
|
stuck.add(photo.id)
|
|
continue
|
|
owner = users.get(photo.user_id)
|
|
if owner is None:
|
|
skipped_no_user += 1
|
|
stuck.add(photo.id)
|
|
continue
|
|
if not photo.filepath or not is_nextcloud_path(photo.filepath):
|
|
skipped_not_nc += 1
|
|
stuck.add(photo.id)
|
|
continue
|
|
if not owner.nextcloud_app_password_enc:
|
|
skipped_no_creds += 1
|
|
stuck.add(photo.id)
|
|
continue
|
|
fid: Optional[int] = None
|
|
try:
|
|
fid = fetch_fileid(owner, photo.filepath)
|
|
except Exception as e:
|
|
logger.warning(
|
|
"PROPFIND failed for photo %s (%s): %s",
|
|
photo.id, photo.filepath, e,
|
|
)
|
|
if fid is None:
|
|
not_found += 1
|
|
stuck.add(photo.id)
|
|
continue
|
|
photo.nextcloud_fileid = fid
|
|
filled += 1
|
|
|
|
await session.commit()
|
|
# Safety: if a whole batch produced no new fills, every row
|
|
# in it is already in `stuck` — break to avoid an infinite
|
|
# loop on the same set.
|
|
if filled == batch_started_filled and len(rows) < BATCH:
|
|
break
|
|
logger.info(
|
|
"progress: scanned=%s filled=%s not_found=%s "
|
|
"skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s",
|
|
done, filled, not_found,
|
|
skipped_no_user, skipped_not_nc, skipped_no_creds,
|
|
total,
|
|
)
|
|
|
|
logger.info(
|
|
"done: scanned=%s filled=%s not_found=%s "
|
|
"skipped(no_user=%s not_nc=%s no_creds=%s)",
|
|
done, filled, not_found,
|
|
skipped_no_user, skipped_not_nc, skipped_no_creds,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(run())
|