feat: worker diagnostics in settings panel
Adds /api/v1/library/maintenance/worker-status (Celery inspect + queue depths + recent failed photos) and a Workers section in the Settings dialog so users can debug stuck queues and task failures without tailing container logs. Auto-polls every 5s while open. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -8,6 +8,9 @@ import {
|
||||
AlertTriangle,
|
||||
Database,
|
||||
Loader2,
|
||||
Cpu,
|
||||
AlertCircle,
|
||||
CheckCircle2,
|
||||
} from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import {
|
||||
@@ -15,6 +18,7 @@ import {
|
||||
type ThumbnailStats,
|
||||
type LibraryStats,
|
||||
type MediaType,
|
||||
type WorkerStatus,
|
||||
} from '../../services/api'
|
||||
import { toast } from '../ToastContainer'
|
||||
|
||||
@@ -37,7 +41,10 @@ interface SettingsDialogProps {
|
||||
export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
const [thumbStats, setThumbStats] = useState<ThumbnailStats | null>(null)
|
||||
const [libStats, setLibStats] = useState<LibraryStats | null>(null)
|
||||
const [workerStatus, setWorkerStatus] = useState<WorkerStatus | null>(null)
|
||||
const [loadingStats, setLoadingStats] = useState(false)
|
||||
const [loadingWorkers, setLoadingWorkers] = useState(false)
|
||||
const [showAllErrors, setShowAllErrors] = useState(false)
|
||||
// One key per action so each button has its own spinner without
|
||||
// blocking the others.
|
||||
const [busy, setBusy] = useState<Record<string, boolean>>({})
|
||||
@@ -59,16 +66,38 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Esc closes; load stats when opened.
|
||||
const refreshWorkers = useCallback(async () => {
|
||||
setLoadingWorkers(true)
|
||||
try {
|
||||
const ws = await library.maintenance.workerStatus()
|
||||
setWorkerStatus(ws)
|
||||
} catch (e) {
|
||||
console.error('Failed to load worker status', e)
|
||||
toast.error('Could not load worker status')
|
||||
} finally {
|
||||
setLoadingWorkers(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Esc closes; load stats when opened. Workers section auto-polls
|
||||
// every 5s while the dialog is open so the user sees live worker
|
||||
// activity without manually hammering the refresh button.
|
||||
useEffect(() => {
|
||||
if (!isOpen) return
|
||||
refreshStats()
|
||||
refreshWorkers()
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') onClose()
|
||||
}
|
||||
window.addEventListener('keydown', handler)
|
||||
return () => window.removeEventListener('keydown', handler)
|
||||
}, [isOpen, onClose, refreshStats])
|
||||
const poll = window.setInterval(() => {
|
||||
refreshWorkers()
|
||||
}, 5000)
|
||||
return () => {
|
||||
window.removeEventListener('keydown', handler)
|
||||
window.clearInterval(poll)
|
||||
}
|
||||
}, [isOpen, onClose, refreshStats, refreshWorkers])
|
||||
|
||||
const runAction = useCallback(
|
||||
async <T,>(
|
||||
@@ -254,6 +283,235 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
</div>
|
||||
</Section>
|
||||
|
||||
{/* ----------------------------------------------------- */}
|
||||
{/* Worker fleet diagnostics */}
|
||||
{/* ----------------------------------------------------- */}
|
||||
<Section
|
||||
icon={<Cpu className="h-4 w-4" />}
|
||||
title="Workers"
|
||||
right={
|
||||
<button
|
||||
onClick={refreshWorkers}
|
||||
disabled={loadingWorkers}
|
||||
className="flex items-center gap-1 rounded border border-border px-2 py-1 text-xs text-text-muted hover:bg-surface-2 disabled:opacity-50"
|
||||
>
|
||||
{loadingWorkers ? (
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
) : (
|
||||
<RefreshCw className="h-3 w-3" />
|
||||
)}
|
||||
Refresh
|
||||
</button>
|
||||
}
|
||||
>
|
||||
{/* Top-line health */}
|
||||
<div className="grid grid-cols-3 gap-2 text-xs">
|
||||
<Stat
|
||||
label="Workers"
|
||||
value={workerStatus?.worker_count}
|
||||
tone={
|
||||
workerStatus
|
||||
? workerStatus.worker_count > 0
|
||||
? 'ok'
|
||||
: 'warn'
|
||||
: 'muted'
|
||||
}
|
||||
/>
|
||||
<Stat
|
||||
label="Broker"
|
||||
value={
|
||||
workerStatus
|
||||
? workerStatus.broker_ok
|
||||
? 'OK'
|
||||
: 'DOWN'
|
||||
: undefined
|
||||
}
|
||||
tone={
|
||||
workerStatus
|
||||
? workerStatus.broker_ok
|
||||
? 'ok'
|
||||
: 'warn'
|
||||
: 'muted'
|
||||
}
|
||||
/>
|
||||
<Stat
|
||||
label="Failed tasks"
|
||||
value={workerStatus?.failures.total}
|
||||
tone={
|
||||
workerStatus && workerStatus.failures.total > 0
|
||||
? 'warn'
|
||||
: 'muted'
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Inline error banners for the obvious failure modes */}
|
||||
{workerStatus?.broker_error && (
|
||||
<ErrorBanner
|
||||
title="Cannot reach Redis broker"
|
||||
detail={workerStatus.broker_error}
|
||||
/>
|
||||
)}
|
||||
{workerStatus?.inspect_error && (
|
||||
<ErrorBanner
|
||||
title="Celery inspect failed"
|
||||
detail={workerStatus.inspect_error}
|
||||
/>
|
||||
)}
|
||||
{workerStatus &&
|
||||
workerStatus.broker_ok &&
|
||||
workerStatus.worker_count === 0 && (
|
||||
<ErrorBanner
|
||||
title="No workers responding"
|
||||
detail="Broker is reachable but no celery worker pinged back. Check the mulita-worker container logs."
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Queue depth */}
|
||||
{workerStatus && (
|
||||
<div className="mt-3">
|
||||
<div className="mb-1 text-[10px] uppercase tracking-wide text-text-muted">
|
||||
Queue depth
|
||||
</div>
|
||||
<div className="grid grid-cols-3 gap-2 text-xs">
|
||||
{Object.entries(workerStatus.queues).map(([name, depth]) => (
|
||||
<div
|
||||
key={name}
|
||||
className="flex items-center justify-between rounded bg-surface px-2 py-1"
|
||||
>
|
||||
<span className="text-text-muted">{name}</span>
|
||||
<span
|
||||
className={clsx(
|
||||
'font-mono font-semibold',
|
||||
depth > 0 ? 'text-text' : 'text-text-muted'
|
||||
)}
|
||||
>
|
||||
{depth}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Per-worker breakdown */}
|
||||
{workerStatus && workerStatus.workers.length > 0 && (
|
||||
<div className="mt-3 space-y-2">
|
||||
<div className="text-[10px] uppercase tracking-wide text-text-muted">
|
||||
Worker fleet
|
||||
</div>
|
||||
{workerStatus.workers.map((w) => (
|
||||
<div
|
||||
key={w.name}
|
||||
className="rounded border border-border bg-surface p-2 text-xs"
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
{w.status === 'online' ? (
|
||||
<CheckCircle2 className="h-3.5 w-3.5 text-pick" />
|
||||
) : (
|
||||
<AlertCircle className="h-3.5 w-3.5 text-reject" />
|
||||
)}
|
||||
<span className="font-mono text-text">{w.name}</span>
|
||||
</div>
|
||||
<span className="text-text-muted">
|
||||
{w.active}/{w.concurrency ?? '?'} active
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-1 flex flex-wrap gap-x-3 gap-y-0.5 text-[10px] text-text-muted">
|
||||
<span>reserved: {w.reserved}</span>
|
||||
<span>scheduled: {w.scheduled}</span>
|
||||
{w.queues.length > 0 && (
|
||||
<span>queues: {w.queues.join(', ')}</span>
|
||||
)}
|
||||
</div>
|
||||
{w.active_tasks.length > 0 && (
|
||||
<div className="mt-1.5 space-y-0.5 border-t border-border pt-1.5">
|
||||
{w.active_tasks.map((t) => (
|
||||
<div
|
||||
key={t.id}
|
||||
className="truncate font-mono text-[10px] text-text-muted"
|
||||
title={`${t.name} ${JSON.stringify(t.args)}`}
|
||||
>
|
||||
<span className="text-text">{t.name}</span>{' '}
|
||||
{Array.isArray(t.args)
|
||||
? t.args.map((a) => String(a)).join(', ')
|
||||
: ''}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Recent task failures */}
|
||||
{workerStatus && workerStatus.failures.recent.length > 0 && (
|
||||
<div className="mt-3">
|
||||
<div className="mb-1 flex items-center justify-between">
|
||||
<div className="text-[10px] uppercase tracking-wide text-text-muted">
|
||||
Recent failures ({workerStatus.failures.total})
|
||||
</div>
|
||||
{workerStatus.failures.recent.length > 5 && (
|
||||
<button
|
||||
onClick={() => setShowAllErrors((v) => !v)}
|
||||
className="text-[10px] text-text-muted hover:text-text"
|
||||
>
|
||||
{showAllErrors ? 'Show less' : 'Show all'}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div className="max-h-48 space-y-1 overflow-y-auto rounded border border-border bg-surface p-2">
|
||||
{(showAllErrors
|
||||
? workerStatus.failures.recent
|
||||
: workerStatus.failures.recent.slice(0, 5)
|
||||
).map((f) => (
|
||||
<div
|
||||
key={f.photo_id}
|
||||
className="border-b border-border/40 pb-1 last:border-b-0 last:pb-0"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2 text-[11px]">
|
||||
<span className="truncate font-mono text-text">
|
||||
{f.filename}
|
||||
</span>
|
||||
<span className="shrink-0 text-[10px] text-text-muted">
|
||||
{f.media_type}
|
||||
</span>
|
||||
</div>
|
||||
<div className="break-all font-mono text-[10px] text-reject">
|
||||
{f.error || '(no error message)'}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Recent scan errors (Redis list) */}
|
||||
{workerStatus && workerStatus.scan_errors.length > 0 && (
|
||||
<div className="mt-3">
|
||||
<div className="mb-1 text-[10px] uppercase tracking-wide text-text-muted">
|
||||
Scan errors
|
||||
</div>
|
||||
<div className="max-h-32 space-y-0.5 overflow-y-auto rounded border border-border bg-surface p-2 font-mono text-[10px] text-reject">
|
||||
{workerStatus.scan_errors.map((e, i) => (
|
||||
<div key={i} className="break-all">
|
||||
{e}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!workerStatus && (
|
||||
<div className="mt-3 flex items-center gap-2 text-xs text-text-muted">
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
Loading worker status…
|
||||
</div>
|
||||
)}
|
||||
</Section>
|
||||
|
||||
{/* ----------------------------------------------------- */}
|
||||
{/* Data integrity */}
|
||||
{/* ----------------------------------------------------- */}
|
||||
@@ -345,6 +603,20 @@ function Stat({
|
||||
)
|
||||
}
|
||||
|
||||
function ErrorBanner({ title, detail }: { title: string; detail: string }) {
|
||||
return (
|
||||
<div className="mt-3 rounded border border-reject/40 bg-reject/10 p-2 text-xs">
|
||||
<div className="flex items-center gap-1.5 font-medium text-reject">
|
||||
<AlertTriangle className="h-3.5 w-3.5" />
|
||||
{title}
|
||||
</div>
|
||||
<div className="mt-0.5 break-all font-mono text-[10px] text-reject/80">
|
||||
{detail}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ActionButton({
|
||||
loading,
|
||||
disabled,
|
||||
|
||||
@@ -230,6 +230,45 @@ export interface RegenerateResult {
|
||||
}
|
||||
}
|
||||
|
||||
export interface WorkerInfo {
|
||||
name: string
|
||||
status: 'online' | 'unreachable'
|
||||
active: number
|
||||
reserved: number
|
||||
scheduled: number
|
||||
concurrency: number | null
|
||||
processed: Record<string, number>
|
||||
queues: string[]
|
||||
active_tasks: Array<{
|
||||
id: string
|
||||
name: string
|
||||
args: unknown
|
||||
time_start: number | null
|
||||
}>
|
||||
}
|
||||
|
||||
export interface WorkerFailure {
|
||||
photo_id: string
|
||||
filename: string
|
||||
media_type: string
|
||||
error: string
|
||||
updated_at: string | null
|
||||
}
|
||||
|
||||
export interface WorkerStatus {
|
||||
broker_ok: boolean
|
||||
broker_error: string | null
|
||||
inspect_error: string | null
|
||||
workers: WorkerInfo[]
|
||||
worker_count: number
|
||||
queues: Record<string, number>
|
||||
failures: {
|
||||
total: number
|
||||
recent: WorkerFailure[]
|
||||
}
|
||||
scan_errors: string[]
|
||||
}
|
||||
|
||||
export const library = {
|
||||
scan: async () => {
|
||||
const response = await api.post('/library/scan')
|
||||
@@ -265,6 +304,14 @@ export const library = {
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Celery worker fleet diagnostics + recent task failures. Surfaced
|
||||
* in the Settings panel so users can debug stuck queues without
|
||||
* tailing container logs. */
|
||||
workerStatus: async (): Promise<WorkerStatus> => {
|
||||
const response = await api.get('/library/maintenance/worker-status')
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Re-run the source-roots / folders / photos integrity cleanup that
|
||||
* normally runs on backend startup. */
|
||||
cleanup: async (): Promise<{ status: string; message?: string }> => {
|
||||
|
||||
Reference in New Issue
Block a user