feat: split celery workers, fix asyncpg-in-fork, add pipeline progress UI
Three overlapping fixes so the ingestion pipeline actually runs and the
user can see what it's doing:
Pipeline recovery
- app/database.py: use NullPool when MULITA_CELERY_WORKER=1 so each
Celery task opens a fresh asyncpg connection on its own event loop.
Fixes "another operation in progress" and "Future attached to a
different loop" errors that were dropping ~every thumbnail +
extract_metadata task on the floor.
- app/tasks/thumbs.py: initialize photo=None before the try and rollback
on error so a transport failure in the initial SELECT doesn't raise
UnboundLocalError in the except block and leak rows stuck in 'pending'.
- app/services/vision/bootstrap_models.py: on missing model files,
invoke export_models automatically instead of just warning. First
boot of a fresh install now self-heals.
- app/services/vision/export_models.py: shutil.move instead of
Path.rename so the YOLO export survives the /app → /data/models
cross-volume hop.
- requirements.txt: add ultralytics so export works in a stock image.
Worker topology
- docker-compose.yml: replace the single worker with worker-light
(default/high/low queues, c=2, IO-bound) and worker-vision (vision
queue, c=5, OMP_NUM_THREADS=1 to avoid oversubscription on 6 cores).
Vision is pinned to ≤5 parallel inferences so ONNX doesn't each
spawn an all-cores intra-op pool.
- .env / .env.example: CELERYD_CONCURRENCY replaced with
CELERY_LIGHT_CONCURRENCY + CELERY_VISION_CONCURRENCY.
- Backfill queries in thumbs / scan / vision now ORDER BY taken_at
DESC NULLS LAST so newest photos finish first — the library fills
in top-down in the UI instead of arbitrary insertion order.
Settings visibility
- routers/library.py: new GET /maintenance/pipeline-stats returning
done/total per stage (thumbnails, exif, gps, phash, embeddings,
tags, ocr, faces, face clusters, duplicate groups). Worker-status
now also reports the `vision` queue depth, which was missing.
- services/api.ts: PipelineStats / PipelineStage / ScanStatus types
and the matching client call.
- components/dialogs/SettingsDialog.tsx:
- new Pipeline Progress card with one progress bar per stage
- inline scan banner (processed/total/current folder) inside the
Library section while a scan is running
- Tasks/min throughput computed by diffing worker processed counters
between polls
- Workers section calls out the vision queue and documents the
CELERY_LIGHT/VISION_CONCURRENCY + docker compose up -d scale path
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -378,7 +378,11 @@ async def get_worker_status(db: AsyncSession = Depends(get_db)):
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
r.ping()
|
||||
broker_ok = True
|
||||
for q in ('default', 'high', 'low'):
|
||||
# `vision` is the big one — embed / classify / detect / ocr /
|
||||
# extract_faces all land here, so it's where backlogs actually
|
||||
# pile up. Leaving it off the dashboard made it look like the
|
||||
# queue was always empty while the worker was clearly busy.
|
||||
for q in ('default', 'high', 'low', 'vision'):
|
||||
try:
|
||||
queue_depths[q] = int(r.llen(q) or 0)
|
||||
except Exception:
|
||||
@@ -445,6 +449,206 @@ async def get_worker_status(db: AsyncSession = Depends(get_db)):
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/pipeline-stats")
|
||||
async def get_pipeline_stats(db: AsyncSession = Depends(get_db)):
|
||||
"""Per-stage progress across the ingestion pipeline.
|
||||
|
||||
Returns a `{stage_key: {done, total, label}}` map so the Settings
|
||||
panel can render one progress bar per stage. `total` is the number
|
||||
of non-discarded photos the stage is *expected* to run on — which is
|
||||
every non-discarded photo for most stages, or a narrower subset when
|
||||
a stage is image-only (e.g. embeddings don't run on videos).
|
||||
|
||||
Keep the shape flat + serialisable; the frontend turns it straight
|
||||
into a list of rows without needing to know about the models.
|
||||
"""
|
||||
from app.config import settings as _settings
|
||||
from app.models import Embedding, FaceEmbedding, OCRText
|
||||
from app.models.tags import photo_tags # association Table, not a model
|
||||
|
||||
not_discarded = Photo.is_discarded.is_(False)
|
||||
|
||||
async def scalar_count(query):
|
||||
return (await db.execute(query)).scalar() or 0
|
||||
|
||||
# Total non-discarded photos — the denominator for most stages.
|
||||
total_photos = await scalar_count(
|
||||
select(func.count(Photo.id)).where(not_discarded)
|
||||
)
|
||||
|
||||
# Image-only denominator (embeddings, tags, faces, OCR, phash). We
|
||||
# exclude videos because those stages either don't apply or run off
|
||||
# the extracted video frame which is treated separately.
|
||||
total_images = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.media_type != 'video'
|
||||
)
|
||||
)
|
||||
|
||||
completed = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.processing_status == 'completed'
|
||||
)
|
||||
)
|
||||
with_exif = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.exif_json.is_not(None)
|
||||
)
|
||||
)
|
||||
with_gps = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded,
|
||||
Photo.latitude.is_not(None),
|
||||
Photo.longitude.is_not(None),
|
||||
)
|
||||
)
|
||||
with_phash = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.phash.is_not(None)
|
||||
)
|
||||
)
|
||||
|
||||
# Embeddings: count distinct photos that have a row for the currently
|
||||
# configured embedder model. A photo can have multiple model rows
|
||||
# (historical re-embeds) so COUNT(DISTINCT) is the right thing here.
|
||||
embedder_model = _settings.vision.embedder.name
|
||||
embeddings_done = await scalar_count(
|
||||
select(func.count(func.distinct(Embedding.photo_id)))
|
||||
.select_from(Embedding)
|
||||
.join(Photo, Photo.id == Embedding.photo_id)
|
||||
.where(not_discarded, Embedding.model == embedder_model)
|
||||
)
|
||||
|
||||
tagged_photos = await scalar_count(
|
||||
select(func.count(func.distinct(photo_tags.c.photo_id)))
|
||||
.select_from(photo_tags)
|
||||
.join(Photo, Photo.id == photo_tags.c.photo_id)
|
||||
.where(not_discarded)
|
||||
)
|
||||
ocr_done = await scalar_count(
|
||||
select(func.count(func.distinct(OCRText.photo_id)))
|
||||
.select_from(OCRText)
|
||||
.join(Photo, Photo.id == OCRText.photo_id)
|
||||
.where(not_discarded)
|
||||
)
|
||||
|
||||
# Faces: photos that have at least one face_embeddings row. A photo
|
||||
# with no faces legitimately finishes face extraction with zero rows,
|
||||
# so this undercounts by exactly "images with no visible people". We
|
||||
# surface the photo-with-faces count rather than "images scanned for
|
||||
# faces" because the latter isn't tracked anywhere.
|
||||
faces_photos = await scalar_count(
|
||||
select(func.count(func.distinct(FaceEmbedding.photo_id)))
|
||||
.select_from(FaceEmbedding)
|
||||
.join(Photo, Photo.id == FaceEmbedding.photo_id)
|
||||
.where(not_discarded)
|
||||
)
|
||||
face_rows = await scalar_count(select(func.count(FaceEmbedding.id)))
|
||||
face_clusters = await scalar_count(
|
||||
select(func.count(func.distinct(FaceEmbedding.cluster_id)))
|
||||
.where(FaceEmbedding.cluster_id.is_not(None))
|
||||
)
|
||||
|
||||
duplicate_groups = await scalar_count(
|
||||
select(func.count(func.distinct(Photo.duplicate_group_id))).where(
|
||||
not_discarded, Photo.duplicate_group_id.is_not(None)
|
||||
)
|
||||
)
|
||||
duplicate_members = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.duplicate_group_id.is_not(None)
|
||||
)
|
||||
)
|
||||
|
||||
# Ordered list so the frontend renders stages in pipeline order
|
||||
# without needing to know the sequence itself.
|
||||
stages = [
|
||||
{
|
||||
"key": "thumbnails",
|
||||
"label": "Thumbnails & pHash",
|
||||
"done": completed,
|
||||
"total": total_photos,
|
||||
"hint": "Generated on scan. Unlocks every downstream stage.",
|
||||
},
|
||||
{
|
||||
"key": "exif",
|
||||
"label": "EXIF metadata",
|
||||
"done": with_exif,
|
||||
"total": total_photos,
|
||||
"hint": "Camera, lens, capture time. Required for GPS + taken_at.",
|
||||
},
|
||||
{
|
||||
"key": "gps",
|
||||
"label": "GPS coordinates",
|
||||
"done": with_gps,
|
||||
"total": total_photos,
|
||||
"hint": "Subset of EXIF. Drives the map view; many photos legitimately have none.",
|
||||
"partial": True, # not every photo is expected to have GPS
|
||||
},
|
||||
{
|
||||
"key": "phash",
|
||||
"label": "Perceptual hashes",
|
||||
"done": with_phash,
|
||||
"total": total_images,
|
||||
"hint": "Feeds duplicate detection.",
|
||||
},
|
||||
{
|
||||
"key": "embeddings",
|
||||
"label": f"Embeddings ({embedder_model})",
|
||||
"done": embeddings_done,
|
||||
"total": total_images,
|
||||
"hint": "Semantic search + content classification.",
|
||||
},
|
||||
{
|
||||
"key": "tags",
|
||||
"label": "Object tags (YOLO)",
|
||||
"done": tagged_photos,
|
||||
"total": total_images,
|
||||
"hint": "Auto-generated object labels. Not every photo has a detectable object.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "ocr",
|
||||
"label": "OCR text",
|
||||
"done": ocr_done,
|
||||
"total": total_images,
|
||||
"hint": "Extracted text from screenshots / documents. Many photos have none.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "faces",
|
||||
"label": "Face detection",
|
||||
"done": faces_photos,
|
||||
"total": total_images,
|
||||
"hint": f"{face_rows} face rows detected across {faces_photos} photos.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "face_clusters",
|
||||
"label": "Face clusters",
|
||||
"done": face_clusters,
|
||||
"total": face_clusters, # no meaningful "total" — it's just the current count
|
||||
"hint": "Built by recluster_faces. Run it after backfill to populate the People view.",
|
||||
"standalone": True,
|
||||
},
|
||||
{
|
||||
"key": "duplicates",
|
||||
"label": "Duplicate groups",
|
||||
"done": duplicate_groups,
|
||||
"total": duplicate_groups, # same — current count, not a progress ratio
|
||||
"hint": f"{duplicate_members} photos in {duplicate_groups} groups. Run regroup_duplicates after new imports.",
|
||||
"standalone": True,
|
||||
},
|
||||
]
|
||||
|
||||
return {
|
||||
"total_photos": total_photos,
|
||||
"total_images": total_images,
|
||||
"embedder_model": embedder_model,
|
||||
"stages": stages,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/missing-stats")
|
||||
async def get_missing_stats():
|
||||
"""Count photos whose files no longer exist on disk under a mounted
|
||||
|
||||
Reference in New Issue
Block a user