fix(backfill): drop offset-based pagination — it skipped filled rows

The offset+limit loop walked the IS NULL set, but every batch's writes
shrank that set, so batch N+1 with offset=N*BATCH skipped over the rows
just filled. A 17k library backfilled only 9k before the loop walked
off the (now-shorter) NULL set.

Replace with a tail-recursive pattern: keep selecting LIMIT BATCH on
the NULL set, tracking rows that won't ever resolve in a `stuck` set so
the loop terminates instead of spinning on them.
This commit is contained in:
Claudio
2026-05-11 11:43:03 +02:00
parent 576b0c236d
commit 28738acb56

View File

@@ -57,33 +57,47 @@ async def run() -> None:
filled = 0
not_found = 0
offset = 0
# Keep selecting the next batch of NULL-fileid rows until the
# set is empty. NO offset() — each batch's writes shrink the
# `WHERE nextcloud_fileid IS NULL` set, so an offset would skip
# over the rows that were just filled in by the previous batch.
# Rows we couldn't resolve (skipped or not_found) stay in the
# set; we track them in a "stuck ids" set so the loop terminates
# instead of spinning on them forever.
stuck: set[str] = set()
while True:
result = await session.execute(
stmt = (
select(Photo)
.where(Photo.nextcloud_fileid.is_(None))
.order_by(Photo.id)
.offset(offset)
.limit(BATCH)
)
if stuck:
stmt = stmt.where(Photo.id.notin_(stuck))
result = await session.execute(stmt)
rows = list(result.scalars().all())
if not rows:
break
batch_started_filled = filled
for photo in rows:
done += 1
if not photo.user_id:
skipped_no_user += 1
stuck.add(photo.id)
continue
owner = users.get(photo.user_id)
if owner is None:
skipped_no_user += 1
stuck.add(photo.id)
continue
if not photo.filepath or not is_nextcloud_path(photo.filepath):
skipped_not_nc += 1
stuck.add(photo.id)
continue
if not owner.nextcloud_app_password_enc:
skipped_no_creds += 1
stuck.add(photo.id)
continue
fid: Optional[int] = None
try:
@@ -95,12 +109,17 @@ async def run() -> None:
)
if fid is None:
not_found += 1
stuck.add(photo.id)
continue
photo.nextcloud_fileid = fid
filled += 1
await session.commit()
offset += BATCH
# Safety: if a whole batch produced no new fills, every row
# in it is already in `stuck` — break to avoid an infinite
# loop on the same set.
if filled == batch_started_filled and len(rows) < BATCH:
break
logger.info(
"progress: scanned=%s filled=%s not_found=%s "
"skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s",