From 28738acb56f63b088d16ab29201ec30c2403ec34 Mon Sep 17 00:00:00 2001
From: Claudio <claudio@hubris.network>
Date: Mon, 11 May 2026 11:43:03 +0200
Subject: [PATCH] =?UTF-8?q?fix(backfill):=20drop=20offset-based=20paginati?=
 =?UTF-8?q?on=20=E2=80=94=20it=20skipped=20filled=20rows?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The offset+limit loop walked the IS NULL set, but every batch's writes
shrank that set, so batch N+1 with offset=N*BATCH skipped over the rows
just filled. A 17k library backfilled only 9k before the loop walked
off the (now-shorter) NULL set.

Replace with a tail-recursive pattern: keep selecting LIMIT BATCH on
the NULL set, tracking rows that won't ever resolve in a `stuck` set so
the loop terminates instead of spinning on them.
---
 backend/scripts/backfill_nextcloud_fileid.py | 27 +++++++++++++++++---
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/backend/scripts/backfill_nextcloud_fileid.py b/backend/scripts/backfill_nextcloud_fileid.py
index 24d5d0b..6daec3b 100644
--- a/backend/scripts/backfill_nextcloud_fileid.py
+++ b/backend/scripts/backfill_nextcloud_fileid.py
@@ -57,33 +57,47 @@ async def run() -> None:
         filled = 0
         not_found = 0
 
-        offset = 0
+        # Keep selecting the next batch of NULL-fileid rows until the
+        # set is empty. NO offset() — each batch's writes shrink the
+        # `WHERE nextcloud_fileid IS NULL` set, so an offset would skip
+        # over the rows that were just filled in by the previous batch.
+        # Rows we couldn't resolve (skipped or not_found) stay in the
+        # set; we track them in a "stuck ids" set so the loop terminates
+        # instead of spinning on them forever.
+        stuck: set[str] = set()
         while True:
-            result = await session.execute(
+            stmt = (
                 select(Photo)
                 .where(Photo.nextcloud_fileid.is_(None))
                 .order_by(Photo.id)
-                .offset(offset)
                 .limit(BATCH)
             )
+            if stuck:
+                stmt = stmt.where(Photo.id.notin_(stuck))
+            result = await session.execute(stmt)
             rows = list(result.scalars().all())
             if not rows:
                 break
 
+            batch_started_filled = filled
             for photo in rows:
                 done += 1
                 if not photo.user_id:
                     skipped_no_user += 1
+                    stuck.add(photo.id)
                     continue
                 owner = users.get(photo.user_id)
                 if owner is None:
                     skipped_no_user += 1
+                    stuck.add(photo.id)
                     continue
                 if not photo.filepath or not is_nextcloud_path(photo.filepath):
                     skipped_not_nc += 1
+                    stuck.add(photo.id)
                     continue
                 if not owner.nextcloud_app_password_enc:
                     skipped_no_creds += 1
+                    stuck.add(photo.id)
                     continue
                 fid: Optional[int] = None
                 try:
@@ -95,12 +109,17 @@ async def run() -> None:
                     )
                 if fid is None:
                     not_found += 1
+                    stuck.add(photo.id)
                     continue
                 photo.nextcloud_fileid = fid
                 filled += 1
 
             await session.commit()
-            offset += BATCH
+            # Safety: if a whole batch produced no new fills, every row
+            # in it is already in `stuck` — break to avoid an infinite
+            # loop on the same set.
+            if filled == batch_started_filled and len(rows) < BATCH:
+                break
             logger.info(
                 "progress: scanned=%s filled=%s not_found=%s "
                 "skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s",