feat: split celery workers, fix asyncpg-in-fork, add pipeline progress UI
Three overlapping fixes so the ingestion pipeline actually runs and the
user can see what it's doing:
Pipeline recovery
- app/database.py: use NullPool when MULITA_CELERY_WORKER=1 so each
Celery task opens a fresh asyncpg connection on its own event loop.
Fixes "another operation in progress" and "Future attached to a
different loop" errors that were dropping ~every thumbnail +
extract_metadata task on the floor.
- app/tasks/thumbs.py: initialize photo=None before the try and rollback
on error so a transport failure in the initial SELECT doesn't raise
UnboundLocalError in the except block and leak rows stuck in 'pending'.
- app/services/vision/bootstrap_models.py: on missing model files,
invoke export_models automatically instead of just warning. First
boot of a fresh install now self-heals.
- app/services/vision/export_models.py: shutil.move instead of
Path.rename so the YOLO export survives the /app → /data/models
cross-volume hop.
- requirements.txt: add ultralytics so export works in a stock image.
Worker topology
- docker-compose.yml: replace the single worker with worker-light
(default/high/low queues, c=2, IO-bound) and worker-vision (vision
queue, c=5, OMP_NUM_THREADS=1 to avoid oversubscription on 6 cores).
Vision is pinned to ≤5 parallel inferences so ONNX doesn't each
spawn an all-cores intra-op pool.
- .env / .env.example: CELERYD_CONCURRENCY replaced with
CELERY_LIGHT_CONCURRENCY + CELERY_VISION_CONCURRENCY.
- Backfill queries in thumbs / scan / vision now ORDER BY taken_at
DESC NULLS LAST so newest photos finish first — the library fills
in top-down in the UI instead of arbitrary insertion order.
Settings visibility
- routers/library.py: new GET /maintenance/pipeline-stats returning
done/total per stage (thumbnails, exif, gps, phash, embeddings,
tags, ocr, faces, face clusters, duplicate groups). Worker-status
now also reports the `vision` queue depth, which was missing.
- services/api.ts: PipelineStats / PipelineStage / ScanStatus types
and the matching client call.
- components/dialogs/SettingsDialog.tsx:
- new Pipeline Progress card with one progress bar per stage
- inline scan banner (processed/total/current folder) inside the
Library section while a scan is running
- Tasks/min throughput computed by diffing worker processed counters
between polls
- Workers section calls out the vision queue and documents the
CELERY_LIGHT/VISION_CONCURRENCY + docker compose up -d scale path
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -484,11 +484,18 @@ def backfill_gps():
|
||||
|
||||
async def _backfill_gps_async():
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Newest-first so the most recent photos get their GPS + EXIF
|
||||
# written before the worker climbs back through the archive.
|
||||
result = await session.execute(
|
||||
select(Photo.id).where(
|
||||
select(Photo.id)
|
||||
.where(
|
||||
Photo.latitude.is_(None),
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
)
|
||||
photo_ids = [row[0] for row in result.all()]
|
||||
|
||||
|
||||
@@ -237,6 +237,9 @@ def generate_thumbnails(self, photo_id: str):
|
||||
async def _generate_thumbnails_async(photo_id: str, task):
|
||||
"""Async implementation of thumbnail generation"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Declared up front so the except block below can safely check it
|
||||
# even if the initial SELECT raises (e.g. asyncpg transport error).
|
||||
photo: Optional[Photo] = None
|
||||
try:
|
||||
# Get photo from database
|
||||
result = await session.execute(
|
||||
@@ -330,13 +333,25 @@ async def _generate_thumbnails_async(photo_id: str, task):
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating thumbnails for {photo_id}: {e}")
|
||||
|
||||
# Update error status
|
||||
if photo:
|
||||
photo.processing_status = 'failed'
|
||||
photo.processing_error = str(e)
|
||||
await session.commit()
|
||||
|
||||
|
||||
# Update error status. If the session is in a bad state (e.g.
|
||||
# the original failure was a transport error) rollback first so
|
||||
# the status write has a clean transaction to commit into.
|
||||
try:
|
||||
await session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if photo is not None:
|
||||
try:
|
||||
photo.processing_status = 'failed'
|
||||
photo.processing_error = str(e)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
f"Could not mark photo {photo_id} as failed"
|
||||
)
|
||||
|
||||
return {'status': 'error', 'message': str(e)}
|
||||
|
||||
@shared_task(name='regenerate_all_thumbnails')
|
||||
@@ -345,12 +360,24 @@ def regenerate_all_thumbnails():
|
||||
return asyncio.run(_regenerate_all_thumbnails_async())
|
||||
|
||||
async def _regenerate_all_thumbnails_async():
|
||||
"""Async implementation of regenerating all thumbnails"""
|
||||
"""Async implementation of regenerating all thumbnails.
|
||||
|
||||
Queue order matters on first-boot and recovery runs: we dispatch
|
||||
newest-first (by EXIF taken_at, fallback added_at) so the user's
|
||||
most recent photos become fully-indexed before the 2012 archive even
|
||||
starts. Picking up the library in pipeline order means the grid,
|
||||
timeline and All Photos view populate top-down instead of the worker
|
||||
chewing through random insertion-order rows while the UI still
|
||||
shows grey placeholders.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Get all photos that need thumbnails
|
||||
# Get all photos that need thumbnails, newest first.
|
||||
result = await session.execute(
|
||||
select(Photo).where(
|
||||
Photo.processing_status.in_(['pending', 'failed'])
|
||||
select(Photo)
|
||||
.where(Photo.processing_status.in_(['pending', 'failed']))
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
@@ -389,10 +416,16 @@ async def _backfill_phashes_async():
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
while True:
|
||||
# Newest-first so the recent end of the library gets phashes
|
||||
# (and therefore duplicate detection) ahead of the archive.
|
||||
result = await session.execute(
|
||||
select(Photo)
|
||||
.where(Photo.phash.is_(None))
|
||||
.where(Photo.processing_status == 'completed')
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
.limit(BATCH)
|
||||
)
|
||||
batch = result.scalars().all()
|
||||
|
||||
@@ -446,12 +446,18 @@ def backfill_vision(task: str | None = None, limit: int | None = None):
|
||||
"""Queue vision tasks for photos that haven't been processed yet.
|
||||
Uses a sync DB connection to avoid asyncpg conflicts in Celery."""
|
||||
model_name = settings.vision.embedder.name
|
||||
# Newest-first ordering — matches regenerate_all_thumbnails so the
|
||||
# whole ingestion pipeline sweeps the library top-down and the user
|
||||
# sees recent photos fully-indexed long before the backlog drains.
|
||||
# `taken_at` is the canonical capture timestamp (from EXIF, falls
|
||||
# back to filesystem mtime in scan); `added_at` is the tie-breaker
|
||||
# when taken_at is null.
|
||||
sql = """
|
||||
SELECT p.id FROM photos p
|
||||
LEFT JOIN embeddings e ON e.photo_id = p.id AND e.model = :model
|
||||
WHERE e.photo_id IS NULL
|
||||
AND p.processing_status = 'completed'
|
||||
ORDER BY p.added_at DESC
|
||||
ORDER BY p.taken_at DESC NULLS LAST, p.added_at DESC NULLS LAST
|
||||
"""
|
||||
if limit:
|
||||
sql += f" LIMIT {limit}"
|
||||
|
||||
Reference in New Issue
Block a user