feat: worker diagnostics in settings panel
Adds /api/v1/library/maintenance/worker-status (Celery inspect + queue depths + recent failed photos) and a Workers section in the Settings dialog so users can debug stuck queues and task failures without tailing container logs. Auto-polls every 5s while open. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -252,6 +252,144 @@ async def regenerate_thumbnails(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/worker-status")
|
||||
async def get_worker_status(db: AsyncSession = Depends(get_db)):
|
||||
"""Diagnostics for the Celery worker fleet + recent task failures.
|
||||
|
||||
Surfaced in the Settings panel so the user can spot a stuck queue or
|
||||
a worker that's gone away without tailing container logs. Returns:
|
||||
|
||||
- workers: list of {name, status, active, concurrency, queues}
|
||||
derived from celery_app.control.inspect(). `status` is 'online'
|
||||
when ping succeeds, 'unreachable' otherwise. Empty list means no
|
||||
workers are responding at all (broker down, container crashed,
|
||||
wrong queue routing, etc.).
|
||||
- queues: per-queue depth read from Redis (LLEN of each queue key
|
||||
used by celery.kombu). Mirrors what tasks are waiting to be
|
||||
picked up.
|
||||
- failures: aggregate count of photos with processing_status='failed'
|
||||
plus the most recent N error messages so the user can see *why*
|
||||
things failed without opening the DB.
|
||||
- broker_ok: bool — could we even reach Redis?
|
||||
"""
|
||||
from app.tasks.celery import celery_app
|
||||
from app.config import settings
|
||||
import redis as _redis
|
||||
|
||||
# ----- Celery inspect (workers + active tasks) -------------------------
|
||||
workers: list[dict] = []
|
||||
inspect_error: Optional[str] = None
|
||||
try:
|
||||
inspect = celery_app.control.inspect(timeout=1.0)
|
||||
ping = inspect.ping() or {}
|
||||
active = inspect.active() or {}
|
||||
reserved = inspect.reserved() or {}
|
||||
scheduled = inspect.scheduled() or {}
|
||||
stats = inspect.stats() or {}
|
||||
active_queues = inspect.active_queues() or {}
|
||||
|
||||
worker_names = set(ping) | set(active) | set(stats)
|
||||
for name in sorted(worker_names):
|
||||
wstats = stats.get(name) or {}
|
||||
pool = wstats.get('pool') or {}
|
||||
workers.append({
|
||||
"name": name,
|
||||
"status": "online" if name in ping else "unreachable",
|
||||
"active": len(active.get(name, []) or []),
|
||||
"reserved": len(reserved.get(name, []) or []),
|
||||
"scheduled": len(scheduled.get(name, []) or []),
|
||||
"concurrency": pool.get('max-concurrency'),
|
||||
"processed": (wstats.get('total') or {}),
|
||||
"queues": [q.get('name') for q in (active_queues.get(name) or [])],
|
||||
"active_tasks": [
|
||||
{
|
||||
"id": t.get('id'),
|
||||
"name": t.get('name'),
|
||||
"args": t.get('args'),
|
||||
"time_start": t.get('time_start'),
|
||||
}
|
||||
for t in (active.get(name) or [])[:10]
|
||||
],
|
||||
})
|
||||
except Exception as e:
|
||||
inspect_error = str(e)
|
||||
logger.warning(f"Celery inspect failed: {e}")
|
||||
|
||||
# ----- Broker / queue depth --------------------------------------------
|
||||
broker_ok = False
|
||||
queue_depths: dict[str, int] = {}
|
||||
broker_error: Optional[str] = None
|
||||
try:
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
r.ping()
|
||||
broker_ok = True
|
||||
for q in ('default', 'high', 'low'):
|
||||
try:
|
||||
queue_depths[q] = int(r.llen(q) or 0)
|
||||
except Exception:
|
||||
queue_depths[q] = 0
|
||||
except Exception as e:
|
||||
broker_error = str(e)
|
||||
logger.warning(f"Redis broker unreachable: {e}")
|
||||
|
||||
# ----- Recent task failures from the photos table ----------------------
|
||||
failed_total = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(Photo.processing_status == 'failed')
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
recent_failed_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Photo.id,
|
||||
Photo.filename,
|
||||
Photo.media_type,
|
||||
Photo.processing_error,
|
||||
Photo.updated_at,
|
||||
)
|
||||
.where(Photo.processing_status == 'failed')
|
||||
.order_by(Photo.updated_at.desc().nullslast())
|
||||
.limit(20)
|
||||
)
|
||||
).all()
|
||||
|
||||
recent_failures = [
|
||||
{
|
||||
"photo_id": row[0],
|
||||
"filename": row[1],
|
||||
"media_type": row[2],
|
||||
"error": (row[3] or '')[:500],
|
||||
"updated_at": row[4].isoformat() if row[4] else None,
|
||||
}
|
||||
for row in recent_failed_rows
|
||||
]
|
||||
|
||||
# ----- Most recent scan errors (Redis list) ----------------------------
|
||||
scan_errors: list[str] = []
|
||||
try:
|
||||
if broker_ok:
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
raw = r.lrange('scan:errors', 0, 19) or []
|
||||
scan_errors = [e.decode(errors='replace') for e in raw]
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not read scan:errors: {e}")
|
||||
|
||||
return {
|
||||
"broker_ok": broker_ok,
|
||||
"broker_error": broker_error,
|
||||
"inspect_error": inspect_error,
|
||||
"workers": workers,
|
||||
"worker_count": len(workers),
|
||||
"queues": queue_depths,
|
||||
"failures": {
|
||||
"total": failed_total,
|
||||
"recent": recent_failures,
|
||||
},
|
||||
"scan_errors": scan_errors,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/maintenance/cleanup")
|
||||
async def run_data_integrity_cleanup():
|
||||
"""Re-run the source-roots / folders / photos data-integrity cleanup
|
||||
|
||||
@@ -8,6 +8,9 @@ import {
|
||||
AlertTriangle,
|
||||
Database,
|
||||
Loader2,
|
||||
Cpu,
|
||||
AlertCircle,
|
||||
CheckCircle2,
|
||||
} from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import {
|
||||
@@ -15,6 +18,7 @@ import {
|
||||
type ThumbnailStats,
|
||||
type LibraryStats,
|
||||
type MediaType,
|
||||
type WorkerStatus,
|
||||
} from '../../services/api'
|
||||
import { toast } from '../ToastContainer'
|
||||
|
||||
@@ -37,7 +41,10 @@ interface SettingsDialogProps {
|
||||
export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
const [thumbStats, setThumbStats] = useState<ThumbnailStats | null>(null)
|
||||
const [libStats, setLibStats] = useState<LibraryStats | null>(null)
|
||||
const [workerStatus, setWorkerStatus] = useState<WorkerStatus | null>(null)
|
||||
const [loadingStats, setLoadingStats] = useState(false)
|
||||
const [loadingWorkers, setLoadingWorkers] = useState(false)
|
||||
const [showAllErrors, setShowAllErrors] = useState(false)
|
||||
// One key per action so each button has its own spinner without
|
||||
// blocking the others.
|
||||
const [busy, setBusy] = useState<Record<string, boolean>>({})
|
||||
@@ -59,16 +66,38 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Esc closes; load stats when opened.
|
||||
const refreshWorkers = useCallback(async () => {
|
||||
setLoadingWorkers(true)
|
||||
try {
|
||||
const ws = await library.maintenance.workerStatus()
|
||||
setWorkerStatus(ws)
|
||||
} catch (e) {
|
||||
console.error('Failed to load worker status', e)
|
||||
toast.error('Could not load worker status')
|
||||
} finally {
|
||||
setLoadingWorkers(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Esc closes; load stats when opened. Workers section auto-polls
|
||||
// every 5s while the dialog is open so the user sees live worker
|
||||
// activity without manually hammering the refresh button.
|
||||
useEffect(() => {
|
||||
if (!isOpen) return
|
||||
refreshStats()
|
||||
refreshWorkers()
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') onClose()
|
||||
}
|
||||
window.addEventListener('keydown', handler)
|
||||
return () => window.removeEventListener('keydown', handler)
|
||||
}, [isOpen, onClose, refreshStats])
|
||||
const poll = window.setInterval(() => {
|
||||
refreshWorkers()
|
||||
}, 5000)
|
||||
return () => {
|
||||
window.removeEventListener('keydown', handler)
|
||||
window.clearInterval(poll)
|
||||
}
|
||||
}, [isOpen, onClose, refreshStats, refreshWorkers])
|
||||
|
||||
const runAction = useCallback(
|
||||
async <T,>(
|
||||
@@ -254,6 +283,235 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
</div>
|
||||
</Section>
|
||||
|
||||
{/* ----------------------------------------------------- */}
|
||||
{/* Worker fleet diagnostics */}
|
||||
{/* ----------------------------------------------------- */}
|
||||
<Section
|
||||
icon={<Cpu className="h-4 w-4" />}
|
||||
title="Workers"
|
||||
right={
|
||||
<button
|
||||
onClick={refreshWorkers}
|
||||
disabled={loadingWorkers}
|
||||
className="flex items-center gap-1 rounded border border-border px-2 py-1 text-xs text-text-muted hover:bg-surface-2 disabled:opacity-50"
|
||||
>
|
||||
{loadingWorkers ? (
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
) : (
|
||||
<RefreshCw className="h-3 w-3" />
|
||||
)}
|
||||
Refresh
|
||||
</button>
|
||||
}
|
||||
>
|
||||
{/* Top-line health */}
|
||||
<div className="grid grid-cols-3 gap-2 text-xs">
|
||||
<Stat
|
||||
label="Workers"
|
||||
value={workerStatus?.worker_count}
|
||||
tone={
|
||||
workerStatus
|
||||
? workerStatus.worker_count > 0
|
||||
? 'ok'
|
||||
: 'warn'
|
||||
: 'muted'
|
||||
}
|
||||
/>
|
||||
<Stat
|
||||
label="Broker"
|
||||
value={
|
||||
workerStatus
|
||||
? workerStatus.broker_ok
|
||||
? 'OK'
|
||||
: 'DOWN'
|
||||
: undefined
|
||||
}
|
||||
tone={
|
||||
workerStatus
|
||||
? workerStatus.broker_ok
|
||||
? 'ok'
|
||||
: 'warn'
|
||||
: 'muted'
|
||||
}
|
||||
/>
|
||||
<Stat
|
||||
label="Failed tasks"
|
||||
value={workerStatus?.failures.total}
|
||||
tone={
|
||||
workerStatus && workerStatus.failures.total > 0
|
||||
? 'warn'
|
||||
: 'muted'
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Inline error banners for the obvious failure modes */}
|
||||
{workerStatus?.broker_error && (
|
||||
<ErrorBanner
|
||||
title="Cannot reach Redis broker"
|
||||
detail={workerStatus.broker_error}
|
||||
/>
|
||||
)}
|
||||
{workerStatus?.inspect_error && (
|
||||
<ErrorBanner
|
||||
title="Celery inspect failed"
|
||||
detail={workerStatus.inspect_error}
|
||||
/>
|
||||
)}
|
||||
{workerStatus &&
|
||||
workerStatus.broker_ok &&
|
||||
workerStatus.worker_count === 0 && (
|
||||
<ErrorBanner
|
||||
title="No workers responding"
|
||||
detail="Broker is reachable but no celery worker pinged back. Check the mulita-worker container logs."
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Queue depth */}
|
||||
{workerStatus && (
|
||||
<div className="mt-3">
|
||||
<div className="mb-1 text-[10px] uppercase tracking-wide text-text-muted">
|
||||
Queue depth
|
||||
</div>
|
||||
<div className="grid grid-cols-3 gap-2 text-xs">
|
||||
{Object.entries(workerStatus.queues).map(([name, depth]) => (
|
||||
<div
|
||||
key={name}
|
||||
className="flex items-center justify-between rounded bg-surface px-2 py-1"
|
||||
>
|
||||
<span className="text-text-muted">{name}</span>
|
||||
<span
|
||||
className={clsx(
|
||||
'font-mono font-semibold',
|
||||
depth > 0 ? 'text-text' : 'text-text-muted'
|
||||
)}
|
||||
>
|
||||
{depth}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Per-worker breakdown */}
|
||||
{workerStatus && workerStatus.workers.length > 0 && (
|
||||
<div className="mt-3 space-y-2">
|
||||
<div className="text-[10px] uppercase tracking-wide text-text-muted">
|
||||
Worker fleet
|
||||
</div>
|
||||
{workerStatus.workers.map((w) => (
|
||||
<div
|
||||
key={w.name}
|
||||
className="rounded border border-border bg-surface p-2 text-xs"
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
{w.status === 'online' ? (
|
||||
<CheckCircle2 className="h-3.5 w-3.5 text-pick" />
|
||||
) : (
|
||||
<AlertCircle className="h-3.5 w-3.5 text-reject" />
|
||||
)}
|
||||
<span className="font-mono text-text">{w.name}</span>
|
||||
</div>
|
||||
<span className="text-text-muted">
|
||||
{w.active}/{w.concurrency ?? '?'} active
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-1 flex flex-wrap gap-x-3 gap-y-0.5 text-[10px] text-text-muted">
|
||||
<span>reserved: {w.reserved}</span>
|
||||
<span>scheduled: {w.scheduled}</span>
|
||||
{w.queues.length > 0 && (
|
||||
<span>queues: {w.queues.join(', ')}</span>
|
||||
)}
|
||||
</div>
|
||||
{w.active_tasks.length > 0 && (
|
||||
<div className="mt-1.5 space-y-0.5 border-t border-border pt-1.5">
|
||||
{w.active_tasks.map((t) => (
|
||||
<div
|
||||
key={t.id}
|
||||
className="truncate font-mono text-[10px] text-text-muted"
|
||||
title={`${t.name} ${JSON.stringify(t.args)}`}
|
||||
>
|
||||
<span className="text-text">{t.name}</span>{' '}
|
||||
{Array.isArray(t.args)
|
||||
? t.args.map((a) => String(a)).join(', ')
|
||||
: ''}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Recent task failures */}
|
||||
{workerStatus && workerStatus.failures.recent.length > 0 && (
|
||||
<div className="mt-3">
|
||||
<div className="mb-1 flex items-center justify-between">
|
||||
<div className="text-[10px] uppercase tracking-wide text-text-muted">
|
||||
Recent failures ({workerStatus.failures.total})
|
||||
</div>
|
||||
{workerStatus.failures.recent.length > 5 && (
|
||||
<button
|
||||
onClick={() => setShowAllErrors((v) => !v)}
|
||||
className="text-[10px] text-text-muted hover:text-text"
|
||||
>
|
||||
{showAllErrors ? 'Show less' : 'Show all'}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div className="max-h-48 space-y-1 overflow-y-auto rounded border border-border bg-surface p-2">
|
||||
{(showAllErrors
|
||||
? workerStatus.failures.recent
|
||||
: workerStatus.failures.recent.slice(0, 5)
|
||||
).map((f) => (
|
||||
<div
|
||||
key={f.photo_id}
|
||||
className="border-b border-border/40 pb-1 last:border-b-0 last:pb-0"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2 text-[11px]">
|
||||
<span className="truncate font-mono text-text">
|
||||
{f.filename}
|
||||
</span>
|
||||
<span className="shrink-0 text-[10px] text-text-muted">
|
||||
{f.media_type}
|
||||
</span>
|
||||
</div>
|
||||
<div className="break-all font-mono text-[10px] text-reject">
|
||||
{f.error || '(no error message)'}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Recent scan errors (Redis list) */}
|
||||
{workerStatus && workerStatus.scan_errors.length > 0 && (
|
||||
<div className="mt-3">
|
||||
<div className="mb-1 text-[10px] uppercase tracking-wide text-text-muted">
|
||||
Scan errors
|
||||
</div>
|
||||
<div className="max-h-32 space-y-0.5 overflow-y-auto rounded border border-border bg-surface p-2 font-mono text-[10px] text-reject">
|
||||
{workerStatus.scan_errors.map((e, i) => (
|
||||
<div key={i} className="break-all">
|
||||
{e}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!workerStatus && (
|
||||
<div className="mt-3 flex items-center gap-2 text-xs text-text-muted">
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
Loading worker status…
|
||||
</div>
|
||||
)}
|
||||
</Section>
|
||||
|
||||
{/* ----------------------------------------------------- */}
|
||||
{/* Data integrity */}
|
||||
{/* ----------------------------------------------------- */}
|
||||
@@ -345,6 +603,20 @@ function Stat({
|
||||
)
|
||||
}
|
||||
|
||||
function ErrorBanner({ title, detail }: { title: string; detail: string }) {
|
||||
return (
|
||||
<div className="mt-3 rounded border border-reject/40 bg-reject/10 p-2 text-xs">
|
||||
<div className="flex items-center gap-1.5 font-medium text-reject">
|
||||
<AlertTriangle className="h-3.5 w-3.5" />
|
||||
{title}
|
||||
</div>
|
||||
<div className="mt-0.5 break-all font-mono text-[10px] text-reject/80">
|
||||
{detail}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ActionButton({
|
||||
loading,
|
||||
disabled,
|
||||
|
||||
@@ -230,6 +230,45 @@ export interface RegenerateResult {
|
||||
}
|
||||
}
|
||||
|
||||
export interface WorkerInfo {
|
||||
name: string
|
||||
status: 'online' | 'unreachable'
|
||||
active: number
|
||||
reserved: number
|
||||
scheduled: number
|
||||
concurrency: number | null
|
||||
processed: Record<string, number>
|
||||
queues: string[]
|
||||
active_tasks: Array<{
|
||||
id: string
|
||||
name: string
|
||||
args: unknown
|
||||
time_start: number | null
|
||||
}>
|
||||
}
|
||||
|
||||
export interface WorkerFailure {
|
||||
photo_id: string
|
||||
filename: string
|
||||
media_type: string
|
||||
error: string
|
||||
updated_at: string | null
|
||||
}
|
||||
|
||||
export interface WorkerStatus {
|
||||
broker_ok: boolean
|
||||
broker_error: string | null
|
||||
inspect_error: string | null
|
||||
workers: WorkerInfo[]
|
||||
worker_count: number
|
||||
queues: Record<string, number>
|
||||
failures: {
|
||||
total: number
|
||||
recent: WorkerFailure[]
|
||||
}
|
||||
scan_errors: string[]
|
||||
}
|
||||
|
||||
export const library = {
|
||||
scan: async () => {
|
||||
const response = await api.post('/library/scan')
|
||||
@@ -265,6 +304,14 @@ export const library = {
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Celery worker fleet diagnostics + recent task failures. Surfaced
|
||||
* in the Settings panel so users can debug stuck queues without
|
||||
* tailing container logs. */
|
||||
workerStatus: async (): Promise<WorkerStatus> => {
|
||||
const response = await api.get('/library/maintenance/worker-status')
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Re-run the source-roots / folders / photos integrity cleanup that
|
||||
* normally runs on backend startup. */
|
||||
cleanup: async (): Promise<{ status: string; message?: string }> => {
|
||||
|
||||
Reference in New Issue
Block a user