feat: worker diagnostics in settings panel

Adds /api/v1/library/maintenance/worker-status (Celery inspect + queue
depths + recent failed photos) and a Workers section in the Settings
dialog so users can debug stuck queues and task failures without
tailing container logs. Auto-polls every 5s while open.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

backend/app/routers/library.py

@@ -252,6 +252,144 @@ async def regenerate_thumbnails(
     }
 
 
+@router.get("/maintenance/worker-status")
+async def get_worker_status(db: AsyncSession = Depends(get_db)):
+    """Diagnostics for the Celery worker fleet + recent task failures.
+
+    Surfaced in the Settings panel so the user can spot a stuck queue or
+    a worker that's gone away without tailing container logs. Returns:
+
+      - workers: list of {name, status, active, concurrency, queues}
+        derived from celery_app.control.inspect(). `status` is 'online'
+        when ping succeeds, 'unreachable' otherwise. Empty list means no
+        workers are responding at all (broker down, container crashed,
+        wrong queue routing, etc.).
+      - queues: per-queue depth read from Redis (LLEN of each queue key
+        used by celery.kombu). Mirrors what tasks are waiting to be
+        picked up.
+      - failures: aggregate count of photos with processing_status='failed'
+        plus the most recent N error messages so the user can see *why*
+        things failed without opening the DB.
+      - broker_ok: bool — could we even reach Redis?
+    """
+    from app.tasks.celery import celery_app
+    from app.config import settings
+    import redis as _redis
+
+    # ----- Celery inspect (workers + active tasks) -------------------------
+    workers: list[dict] = []
+    inspect_error: Optional[str] = None
+    try:
+        inspect = celery_app.control.inspect(timeout=1.0)
+        ping = inspect.ping() or {}
+        active = inspect.active() or {}
+        reserved = inspect.reserved() or {}
+        scheduled = inspect.scheduled() or {}
+        stats = inspect.stats() or {}
+        active_queues = inspect.active_queues() or {}
+
+        worker_names = set(ping) | set(active) | set(stats)
+        for name in sorted(worker_names):
+            wstats = stats.get(name) or {}
+            pool = wstats.get('pool') or {}
+            workers.append({
+                "name": name,
+                "status": "online" if name in ping else "unreachable",
+                "active": len(active.get(name, []) or []),
+                "reserved": len(reserved.get(name, []) or []),
+                "scheduled": len(scheduled.get(name, []) or []),
+                "concurrency": pool.get('max-concurrency'),
+                "processed": (wstats.get('total') or {}),
+                "queues": [q.get('name') for q in (active_queues.get(name) or [])],
+                "active_tasks": [
+                    {
+                        "id": t.get('id'),
+                        "name": t.get('name'),
+                        "args": t.get('args'),
+                        "time_start": t.get('time_start'),
+                    }
+                    for t in (active.get(name) or [])[:10]
+                ],
+            })
+    except Exception as e:
+        inspect_error = str(e)
+        logger.warning(f"Celery inspect failed: {e}")
+
+    # ----- Broker / queue depth --------------------------------------------
+    broker_ok = False
+    queue_depths: dict[str, int] = {}
+    broker_error: Optional[str] = None
+    try:
+        r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
+        r.ping()
+        broker_ok = True
+        for q in ('default', 'high', 'low'):
+            try:
+                queue_depths[q] = int(r.llen(q) or 0)
+            except Exception:
+                queue_depths[q] = 0
+    except Exception as e:
+        broker_error = str(e)
+        logger.warning(f"Redis broker unreachable: {e}")
+
+    # ----- Recent task failures from the photos table ----------------------
+    failed_total = (
+        await db.execute(
+            select(func.count(Photo.id)).where(Photo.processing_status == 'failed')
+        )
+    ).scalar() or 0
+
+    recent_failed_rows = (
+        await db.execute(
+            select(
+                Photo.id,
+                Photo.filename,
+                Photo.media_type,
+                Photo.processing_error,
+                Photo.updated_at,
+            )
+            .where(Photo.processing_status == 'failed')
+            .order_by(Photo.updated_at.desc().nullslast())
+            .limit(20)
+        )
+    ).all()
+
+    recent_failures = [
+        {
+            "photo_id": row[0],
+            "filename": row[1],
+            "media_type": row[2],
+            "error": (row[3] or '')[:500],
+            "updated_at": row[4].isoformat() if row[4] else None,
+        }
+        for row in recent_failed_rows
+    ]
+
+    # ----- Most recent scan errors (Redis list) ----------------------------
+    scan_errors: list[str] = []
+    try:
+        if broker_ok:
+            r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
+            raw = r.lrange('scan:errors', 0, 19) or []
+            scan_errors = [e.decode(errors='replace') for e in raw]
+    except Exception as e:
+        logger.debug(f"Could not read scan:errors: {e}")
+
+    return {
+        "broker_ok": broker_ok,
+        "broker_error": broker_error,
+        "inspect_error": inspect_error,
+        "workers": workers,
+        "worker_count": len(workers),
+        "queues": queue_depths,
+        "failures": {
+            "total": failed_total,
+            "recent": recent_failures,
+        },
+        "scan_errors": scan_errors,
+    }
+
+
 @router.post("/maintenance/cleanup")
 async def run_data_integrity_cleanup():
     """Re-run the source-roots / folders / photos data-integrity cleanup

frontend/src/components/dialogs/SettingsDialog.tsx

@@ -8,6 +8,9 @@ import {
   AlertTriangle,
   Database,
   Loader2,
+  Cpu,
+  AlertCircle,
+  CheckCircle2,
 } from 'lucide-react'
 import clsx from 'clsx'
 import {
@@ -15,6 +18,7 @@ import {
   type ThumbnailStats,
   type LibraryStats,
   type MediaType,
+  type WorkerStatus,
 } from '../../services/api'
 import { toast } from '../ToastContainer'
 
@@ -37,7 +41,10 @@ interface SettingsDialogProps {
 export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
   const [thumbStats, setThumbStats] = useState<ThumbnailStats | null>(null)
   const [libStats, setLibStats] = useState<LibraryStats | null>(null)
+  const [workerStatus, setWorkerStatus] = useState<WorkerStatus | null>(null)
   const [loadingStats, setLoadingStats] = useState(false)
+  const [loadingWorkers, setLoadingWorkers] = useState(false)
+  const [showAllErrors, setShowAllErrors] = useState(false)
   // One key per action so each button has its own spinner without
   // blocking the others.
   const [busy, setBusy] = useState<Record<string, boolean>>({})
@@ -59,16 +66,38 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
     }
   }, [])
 
-  // Esc closes; load stats when opened.
+  const refreshWorkers = useCallback(async () => {
+    setLoadingWorkers(true)
+    try {
+      const ws = await library.maintenance.workerStatus()
+      setWorkerStatus(ws)
+    } catch (e) {
+      console.error('Failed to load worker status', e)
+      toast.error('Could not load worker status')
+    } finally {
+      setLoadingWorkers(false)
+    }
+  }, [])
+
+  // Esc closes; load stats when opened. Workers section auto-polls
+  // every 5s while the dialog is open so the user sees live worker
+  // activity without manually hammering the refresh button.
   useEffect(() => {
     if (!isOpen) return
     refreshStats()
+    refreshWorkers()
     const handler = (e: KeyboardEvent) => {
       if (e.key === 'Escape') onClose()
     }
     window.addEventListener('keydown', handler)
-    return () => window.removeEventListener('keydown', handler)
+    const poll = window.setInterval(() => {
-  }, [isOpen, onClose, refreshStats])
+      refreshWorkers()
+    }, 5000)
+    return () => {
+      window.removeEventListener('keydown', handler)
+      window.clearInterval(poll)
+    }
+  }, [isOpen, onClose, refreshStats, refreshWorkers])
 
   const runAction = useCallback(
     async <T,>(
@@ -254,6 +283,235 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
               </div>
             </Section>
 
+            {/* ----------------------------------------------------- */}
+            {/* Worker fleet diagnostics                               */}
+            {/* ----------------------------------------------------- */}
+            <Section
+              icon={<Cpu className="h-4 w-4" />}
+              title="Workers"
+              right={
+                <button
+                  onClick={refreshWorkers}
+                  disabled={loadingWorkers}
+                  className="flex items-center gap-1 rounded border border-border px-2 py-1 text-xs text-text-muted hover:bg-surface-2 disabled:opacity-50"
+                >
+                  {loadingWorkers ? (
+                    <Loader2 className="h-3 w-3 animate-spin" />
+                  ) : (
+                    <RefreshCw className="h-3 w-3" />
+                  )}
+                  Refresh
+                </button>
+              }
+            >
+              {/* Top-line health */}
+              <div className="grid grid-cols-3 gap-2 text-xs">
+                <Stat
+                  label="Workers"
+                  value={workerStatus?.worker_count}
+                  tone={
+                    workerStatus
+                      ? workerStatus.worker_count > 0
+                        ? 'ok'
+                        : 'warn'
+                      : 'muted'
+                  }
+                />
+                <Stat
+                  label="Broker"
+                  value={
+                    workerStatus
+                      ? workerStatus.broker_ok
+                        ? 'OK'
+                        : 'DOWN'
+                      : undefined
+                  }
+                  tone={
+                    workerStatus
+                      ? workerStatus.broker_ok
+                        ? 'ok'
+                        : 'warn'
+                      : 'muted'
+                  }
+                />
+                <Stat
+                  label="Failed tasks"
+                  value={workerStatus?.failures.total}
+                  tone={
+                    workerStatus && workerStatus.failures.total > 0
+                      ? 'warn'
+                      : 'muted'
+                  }
+                />
+              </div>
+
+              {/* Inline error banners for the obvious failure modes */}
+              {workerStatus?.broker_error && (
+                <ErrorBanner
+                  title="Cannot reach Redis broker"
+                  detail={workerStatus.broker_error}
+                />
+              )}
+              {workerStatus?.inspect_error && (
+                <ErrorBanner
+                  title="Celery inspect failed"
+                  detail={workerStatus.inspect_error}
+                />
+              )}
+              {workerStatus &&
+                workerStatus.broker_ok &&
+                workerStatus.worker_count === 0 && (
+                  <ErrorBanner
+                    title="No workers responding"
+                    detail="Broker is reachable but no celery worker pinged back. Check the mulita-worker container logs."
+                  />
+                )}
+
+              {/* Queue depth */}
+              {workerStatus && (
+                <div className="mt-3">
+                  <div className="mb-1 text-[10px] uppercase tracking-wide text-text-muted">
+                    Queue depth
+                  </div>
+                  <div className="grid grid-cols-3 gap-2 text-xs">
+                    {Object.entries(workerStatus.queues).map(([name, depth]) => (
+                      <div
+                        key={name}
+                        className="flex items-center justify-between rounded bg-surface px-2 py-1"
+                      >
+                        <span className="text-text-muted">{name}</span>
+                        <span
+                          className={clsx(
+                            'font-mono font-semibold',
+                            depth > 0 ? 'text-text' : 'text-text-muted'
+                          )}
+                        >
+                          {depth}
+                        </span>
+                      </div>
+                    ))}
+                  </div>
+                </div>
+              )}
+
+              {/* Per-worker breakdown */}
+              {workerStatus && workerStatus.workers.length > 0 && (
+                <div className="mt-3 space-y-2">
+                  <div className="text-[10px] uppercase tracking-wide text-text-muted">
+                    Worker fleet
+                  </div>
+                  {workerStatus.workers.map((w) => (
+                    <div
+                      key={w.name}
+                      className="rounded border border-border bg-surface p-2 text-xs"
+                    >
+                      <div className="flex items-center justify-between">
+                        <div className="flex items-center gap-2">
+                          {w.status === 'online' ? (
+                            <CheckCircle2 className="h-3.5 w-3.5 text-pick" />
+                          ) : (
+                            <AlertCircle className="h-3.5 w-3.5 text-reject" />
+                          )}
+                          <span className="font-mono text-text">{w.name}</span>
+                        </div>
+                        <span className="text-text-muted">
+                          {w.active}/{w.concurrency ?? '?'} active
+                        </span>
+                      </div>
+                      <div className="mt-1 flex flex-wrap gap-x-3 gap-y-0.5 text-[10px] text-text-muted">
+                        <span>reserved: {w.reserved}</span>
+                        <span>scheduled: {w.scheduled}</span>
+                        {w.queues.length > 0 && (
+                          <span>queues: {w.queues.join(', ')}</span>
+                        )}
+                      </div>
+                      {w.active_tasks.length > 0 && (
+                        <div className="mt-1.5 space-y-0.5 border-t border-border pt-1.5">
+                          {w.active_tasks.map((t) => (
+                            <div
+                              key={t.id}
+                              className="truncate font-mono text-[10px] text-text-muted"
+                              title={`${t.name} ${JSON.stringify(t.args)}`}
+                            >
+                              <span className="text-text">{t.name}</span>{' '}
+                              {Array.isArray(t.args)
+                                ? t.args.map((a) => String(a)).join(', ')
+                                : ''}
+                            </div>
+                          ))}
+                        </div>
+                      )}
+                    </div>
+                  ))}
+                </div>
+              )}
+
+              {/* Recent task failures */}
+              {workerStatus && workerStatus.failures.recent.length > 0 && (
+                <div className="mt-3">
+                  <div className="mb-1 flex items-center justify-between">
+                    <div className="text-[10px] uppercase tracking-wide text-text-muted">
+                      Recent failures ({workerStatus.failures.total})
+                    </div>
+                    {workerStatus.failures.recent.length > 5 && (
+                      <button
+                        onClick={() => setShowAllErrors((v) => !v)}
+                        className="text-[10px] text-text-muted hover:text-text"
+                      >
+                        {showAllErrors ? 'Show less' : 'Show all'}
+                      </button>
+                    )}
+                  </div>
+                  <div className="max-h-48 space-y-1 overflow-y-auto rounded border border-border bg-surface p-2">
+                    {(showAllErrors
+                      ? workerStatus.failures.recent
+                      : workerStatus.failures.recent.slice(0, 5)
+                    ).map((f) => (
+                      <div
+                        key={f.photo_id}
+                        className="border-b border-border/40 pb-1 last:border-b-0 last:pb-0"
+                      >
+                        <div className="flex items-center justify-between gap-2 text-[11px]">
+                          <span className="truncate font-mono text-text">
+                            {f.filename}
+                          </span>
+                          <span className="shrink-0 text-[10px] text-text-muted">
+                            {f.media_type}
+                          </span>
+                        </div>
+                        <div className="break-all font-mono text-[10px] text-reject">
+                          {f.error || '(no error message)'}
+                        </div>
+                      </div>
+                    ))}
+                  </div>
+                </div>
+              )}
+
+              {/* Recent scan errors (Redis list) */}
+              {workerStatus && workerStatus.scan_errors.length > 0 && (
+                <div className="mt-3">
+                  <div className="mb-1 text-[10px] uppercase tracking-wide text-text-muted">
+                    Scan errors
+                  </div>
+                  <div className="max-h-32 space-y-0.5 overflow-y-auto rounded border border-border bg-surface p-2 font-mono text-[10px] text-reject">
+                    {workerStatus.scan_errors.map((e, i) => (
+                      <div key={i} className="break-all">
+                        {e}
+                      </div>
+                    ))}
+                  </div>
+                </div>
+              )}
+
+              {!workerStatus && (
+                <div className="mt-3 flex items-center gap-2 text-xs text-text-muted">
+                  <Loader2 className="h-3 w-3 animate-spin" />
+                  Loading worker status…
+                </div>
+              )}
+            </Section>
+
             {/* ----------------------------------------------------- */}
             {/* Data integrity                                         */}
             {/* ----------------------------------------------------- */}
@@ -345,6 +603,20 @@ function Stat({
   )
 }
 
+function ErrorBanner({ title, detail }: { title: string; detail: string }) {
+  return (
+    <div className="mt-3 rounded border border-reject/40 bg-reject/10 p-2 text-xs">
+      <div className="flex items-center gap-1.5 font-medium text-reject">
+        <AlertTriangle className="h-3.5 w-3.5" />
+        {title}
+      </div>
+      <div className="mt-0.5 break-all font-mono text-[10px] text-reject/80">
+        {detail}
+      </div>
+    </div>
+  )
+}
+
 function ActionButton({
   loading,
   disabled,

frontend/src/services/api.ts

@@ -230,6 +230,45 @@ export interface RegenerateResult {
   }
 }
 
+export interface WorkerInfo {
+  name: string
+  status: 'online' | 'unreachable'
+  active: number
+  reserved: number
+  scheduled: number
+  concurrency: number | null
+  processed: Record<string, number>
+  queues: string[]
+  active_tasks: Array<{
+    id: string
+    name: string
+    args: unknown
+    time_start: number | null
+  }>
+}
+
+export interface WorkerFailure {
+  photo_id: string
+  filename: string
+  media_type: string
+  error: string
+  updated_at: string | null
+}
+
+export interface WorkerStatus {
+  broker_ok: boolean
+  broker_error: string | null
+  inspect_error: string | null
+  workers: WorkerInfo[]
+  worker_count: number
+  queues: Record<string, number>
+  failures: {
+    total: number
+    recent: WorkerFailure[]
+  }
+  scan_errors: string[]
+}
+
 export const library = {
   scan: async () => {
     const response = await api.post('/library/scan')
@@ -265,6 +304,14 @@ export const library = {
       return response.data
     },
 
+    /** Celery worker fleet diagnostics + recent task failures. Surfaced
+     *  in the Settings panel so users can debug stuck queues without
+     *  tailing container logs. */
+    workerStatus: async (): Promise<WorkerStatus> => {
+      const response = await api.get('/library/maintenance/worker-status')
+      return response.data
+    },
+
     /** Re-run the source-roots / folders / photos integrity cleanup that
      *  normally runs on backend startup. */
     cleanup: async (): Promise<{ status: string; message?: string }> => {