feat: worker diagnostics in settings panel

Adds /api/v1/library/maintenance/worker-status (Celery inspect + queue
depths + recent failed photos) and a Workers section in the Settings
dialog so users can debug stuck queues and task failures without
tailing container logs. Auto-polls every 5s while open.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

backend/app/routers/library.py

@@ -252,6 +252,144 @@ async def regenerate_thumbnails(
     }
 
 
+@router.get("/maintenance/worker-status")
+async def get_worker_status(db: AsyncSession = Depends(get_db)):
+    """Diagnostics for the Celery worker fleet + recent task failures.
+
+    Surfaced in the Settings panel so the user can spot a stuck queue or
+    a worker that's gone away without tailing container logs. Returns:
+
+      - workers: list of {name, status, active, concurrency, queues}
+        derived from celery_app.control.inspect(). `status` is 'online'
+        when ping succeeds, 'unreachable' otherwise. Empty list means no
+        workers are responding at all (broker down, container crashed,
+        wrong queue routing, etc.).
+      - queues: per-queue depth read from Redis (LLEN of each queue key
+        used by celery.kombu). Mirrors what tasks are waiting to be
+        picked up.
+      - failures: aggregate count of photos with processing_status='failed'
+        plus the most recent N error messages so the user can see *why*
+        things failed without opening the DB.
+      - broker_ok: bool — could we even reach Redis?
+    """
+    from app.tasks.celery import celery_app
+    from app.config import settings
+    import redis as _redis
+
+    # ----- Celery inspect (workers + active tasks) -------------------------
+    workers: list[dict] = []
+    inspect_error: Optional[str] = None
+    try:
+        inspect = celery_app.control.inspect(timeout=1.0)
+        ping = inspect.ping() or {}
+        active = inspect.active() or {}
+        reserved = inspect.reserved() or {}
+        scheduled = inspect.scheduled() or {}
+        stats = inspect.stats() or {}
+        active_queues = inspect.active_queues() or {}
+
+        worker_names = set(ping) | set(active) | set(stats)
+        for name in sorted(worker_names):
+            wstats = stats.get(name) or {}
+            pool = wstats.get('pool') or {}
+            workers.append({
+                "name": name,
+                "status": "online" if name in ping else "unreachable",
+                "active": len(active.get(name, []) or []),
+                "reserved": len(reserved.get(name, []) or []),
+                "scheduled": len(scheduled.get(name, []) or []),
+                "concurrency": pool.get('max-concurrency'),
+                "processed": (wstats.get('total') or {}),
+                "queues": [q.get('name') for q in (active_queues.get(name) or [])],
+                "active_tasks": [
+                    {
+                        "id": t.get('id'),
+                        "name": t.get('name'),
+                        "args": t.get('args'),
+                        "time_start": t.get('time_start'),
+                    }
+                    for t in (active.get(name) or [])[:10]
+                ],
+            })
+    except Exception as e:
+        inspect_error = str(e)
+        logger.warning(f"Celery inspect failed: {e}")
+
+    # ----- Broker / queue depth --------------------------------------------
+    broker_ok = False
+    queue_depths: dict[str, int] = {}
+    broker_error: Optional[str] = None
+    try:
+        r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
+        r.ping()
+        broker_ok = True
+        for q in ('default', 'high', 'low'):
+            try:
+                queue_depths[q] = int(r.llen(q) or 0)
+            except Exception:
+                queue_depths[q] = 0
+    except Exception as e:
+        broker_error = str(e)
+        logger.warning(f"Redis broker unreachable: {e}")
+
+    # ----- Recent task failures from the photos table ----------------------
+    failed_total = (
+        await db.execute(
+            select(func.count(Photo.id)).where(Photo.processing_status == 'failed')
+        )
+    ).scalar() or 0
+
+    recent_failed_rows = (
+        await db.execute(
+            select(
+                Photo.id,
+                Photo.filename,
+                Photo.media_type,
+                Photo.processing_error,
+                Photo.updated_at,
+            )
+            .where(Photo.processing_status == 'failed')
+            .order_by(Photo.updated_at.desc().nullslast())
+            .limit(20)
+        )
+    ).all()
+
+    recent_failures = [
+        {
+            "photo_id": row[0],
+            "filename": row[1],
+            "media_type": row[2],
+            "error": (row[3] or '')[:500],
+            "updated_at": row[4].isoformat() if row[4] else None,
+        }
+        for row in recent_failed_rows
+    ]
+
+    # ----- Most recent scan errors (Redis list) ----------------------------
+    scan_errors: list[str] = []
+    try:
+        if broker_ok:
+            r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
+            raw = r.lrange('scan:errors', 0, 19) or []
+            scan_errors = [e.decode(errors='replace') for e in raw]
+    except Exception as e:
+        logger.debug(f"Could not read scan:errors: {e}")
+
+    return {
+        "broker_ok": broker_ok,
+        "broker_error": broker_error,
+        "inspect_error": inspect_error,
+        "workers": workers,
+        "worker_count": len(workers),
+        "queues": queue_depths,
+        "failures": {
+            "total": failed_total,
+            "recent": recent_failures,
+        },
+        "scan_errors": scan_errors,
+    }
+
+
 @router.post("/maintenance/cleanup")
 async def run_data_integrity_cleanup():
     """Re-run the source-roots / folders / photos data-integrity cleanup