diff --git a/backend/scripts/backfill_nextcloud_fileid.py b/backend/scripts/backfill_nextcloud_fileid.py index 24d5d0b..6daec3b 100644 --- a/backend/scripts/backfill_nextcloud_fileid.py +++ b/backend/scripts/backfill_nextcloud_fileid.py @@ -57,33 +57,47 @@ async def run() -> None: filled = 0 not_found = 0 - offset = 0 + # Keep selecting the next batch of NULL-fileid rows until the + # set is empty. NO offset() — each batch's writes shrink the + # `WHERE nextcloud_fileid IS NULL` set, so an offset would skip + # over the rows that were just filled in by the previous batch. + # Rows we couldn't resolve (skipped or not_found) stay in the + # set; we track them in a "stuck ids" set so the loop terminates + # instead of spinning on them forever. + stuck: set[str] = set() while True: - result = await session.execute( + stmt = ( select(Photo) .where(Photo.nextcloud_fileid.is_(None)) .order_by(Photo.id) - .offset(offset) .limit(BATCH) ) + if stuck: + stmt = stmt.where(Photo.id.notin_(stuck)) + result = await session.execute(stmt) rows = list(result.scalars().all()) if not rows: break + batch_started_filled = filled for photo in rows: done += 1 if not photo.user_id: skipped_no_user += 1 + stuck.add(photo.id) continue owner = users.get(photo.user_id) if owner is None: skipped_no_user += 1 + stuck.add(photo.id) continue if not photo.filepath or not is_nextcloud_path(photo.filepath): skipped_not_nc += 1 + stuck.add(photo.id) continue if not owner.nextcloud_app_password_enc: skipped_no_creds += 1 + stuck.add(photo.id) continue fid: Optional[int] = None try: @@ -95,12 +109,17 @@ async def run() -> None: ) if fid is None: not_found += 1 + stuck.add(photo.id) continue photo.nextcloud_fileid = fid filled += 1 await session.commit() - offset += BATCH + # Safety: if a whole batch produced no new fills, every row + # in it is already in `stuck` — break to avoid an infinite + # loop on the same set. + if filled == batch_started_filled and len(rows) < BATCH: + break logger.info( "progress: scanned=%s filled=%s not_found=%s " "skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s",