dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: split celery workers, fix asyncpg-in-fork, add pipeline progress UI

Browse Source 
Three overlapping fixes so the ingestion pipeline actually runs and the
user can see what it's doing:

Pipeline recovery
- app/database.py: use NullPool when MULITA_CELERY_WORKER=1 so each
  Celery task opens a fresh asyncpg connection on its own event loop.
  Fixes "another operation in progress" and "Future attached to a
  different loop" errors that were dropping ~every thumbnail +
  extract_metadata task on the floor.
- app/tasks/thumbs.py: initialize photo=None before the try and rollback
  on error so a transport failure in the initial SELECT doesn't raise
  UnboundLocalError in the except block and leak rows stuck in 'pending'.
- app/services/vision/bootstrap_models.py: on missing model files,
  invoke export_models automatically instead of just warning. First
  boot of a fresh install now self-heals.
- app/services/vision/export_models.py: shutil.move instead of
  Path.rename so the YOLO export survives the /app → /data/models
  cross-volume hop.
- requirements.txt: add ultralytics so export works in a stock image.

Worker topology
- docker-compose.yml: replace the single worker with worker-light
  (default/high/low queues, c=2, IO-bound) and worker-vision (vision
  queue, c=5, OMP_NUM_THREADS=1 to avoid oversubscription on 6 cores).
  Vision is pinned to ≤5 parallel inferences so ONNX doesn't each
  spawn an all-cores intra-op pool.
- .env / .env.example: CELERYD_CONCURRENCY replaced with
  CELERY_LIGHT_CONCURRENCY + CELERY_VISION_CONCURRENCY.
- Backfill queries in thumbs / scan / vision now ORDER BY taken_at
  DESC NULLS LAST so newest photos finish first — the library fills
  in top-down in the UI instead of arbitrary insertion order.

Settings visibility
- routers/library.py: new GET /maintenance/pipeline-stats returning
  done/total per stage (thumbnails, exif, gps, phash, embeddings,
  tags, ocr, faces, face clusters, duplicate groups). Worker-status
  now also reports the `vision` queue depth, which was missing.
- services/api.ts: PipelineStats / PipelineStage / ScanStatus types
  and the matching client call.
- components/dialogs/SettingsDialog.tsx:
  - new Pipeline Progress card with one progress bar per stage
  - inline scan banner (processed/total/current folder) inside the
    Library section while a scan is running
  - Tasks/min throughput computed by diffing worker processed counters
    between polls
  - Workers section calls out the vision queue and documents the
    CELERY_LIGHT/VISION_CONCURRENCY + docker compose up -d scale path

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-11 10:06:45 +02:00
parent afe420c620
commit 07b1e5e02a
13 changed files with 746 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

298
frontend/src/components/dialogs/SettingsDialog.tsx
View File

@@ -1,4 +1,4 @@
import { useEffect, useState, useCallback } from 'react'
import { useEffect, useState, useCallback, useRef } from 'react'
import {
X,
RefreshCw,
@@ -13,12 +13,17 @@ import {
CheckCircle2,
Copy,
Sparkles,
Activity,
FolderSearch,
} from 'lucide-react'
import clsx from 'clsx'
import { useQuery, useQueryClient } from '@tanstack/react-query'
import {
library,
type MediaType,
type PipelineStage,
type ScanStatus,
type WorkerStatus,
} from '../../services/api'
import { toast } from '../ToastContainer'
@@ -29,6 +34,8 @@ const SETTINGS_THUMB_STATS_KEY = ['settings', 'thumbnail-stats'] as const
const SETTINGS_LIB_STATS_KEY = ['settings', 'library-stats'] as const
const SETTINGS_WORKER_STATUS_KEY = ['settings', 'worker-status'] as const
const SETTINGS_MISSING_STATS_KEY = ['settings', 'missing-stats'] as const
const SETTINGS_PIPELINE_STATS_KEY = ['settings', 'pipeline-stats'] as const
const SETTINGS_SCAN_STATUS_KEY = ['settings', 'scan-status'] as const
// Shared with the DuplicatesView so a regroup invalidates the same cache
// the grid renders from. Imported via the canonical hook key.
import { DUPLICATE_GROUPS_QUERY_KEY } from '../../hooks/useDuplicateGroupsQuery'
@@ -91,6 +98,28 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
refetchInterval: isOpen ? 5000 : false,
staleTime: 0,
})
// Pipeline progress polls on the same 5s cadence as the worker status
// so both cards update together. Cheap query — ten COUNT(*)s on
// indexed columns.
const pipelineStatsQuery = useQuery({
queryKey: SETTINGS_PIPELINE_STATS_KEY,
queryFn: library.maintenance.pipelineStats,
enabled: isOpen,
refetchInterval: isOpen ? 5000 : false,
staleTime: 0,
})
// Scan status — polls fast (2s) so the progress bar feels live during
// a scan, and slow (15s) when idle to cut chatter. `isScanning` is
// read from the latest fetched value so the cadence flips on its own
// the moment a scan kicks off or finishes.
const scanStatusQuery = useQuery<ScanStatus>({
queryKey: SETTINGS_SCAN_STATUS_KEY,
queryFn: library.scanStatus,
enabled: isOpen,
refetchInterval: (q) =>
isOpen ? ((q.state.data as ScanStatus | undefined)?.is_scanning ? 2000 : 15000) : false,
staleTime: 0,
})
// Duplicates: shares the cache with DuplicatesView so a regroup
// triggered from Settings updates the grid view immediately.
const duplicatesQuery = useQuery({
@@ -104,6 +133,19 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
const libStats = libStatsQuery.data
const workerStatus = workerStatusQuery.data
const missingStats = missingStatsQuery.data
const pipelineStats = pipelineStatsQuery.data
const scanStatus = scanStatusQuery.data
// Throughput — tasks/minute across the fleet, derived by diffing the
// `total` counters on `workers[*].processed` between successive polls.
// We keep the previous sample in a ref so recomputation happens inside
// the existing 5s polling rhythm without introducing extra state.
// First sample returns null (we need two points for a rate).
const throughputSampleRef = useRef<{
totals: Record<string, number>
at: number
} | null>(null)
const throughput = computeThroughput(workerStatus, throughputSampleRef)
// "loading" in the UI sense = fetching AND no cached data yet. Background
// refetches on top of cached data shouldn't flip the refresh spinners.
const loadingStats =
@@ -116,11 +158,13 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
const refreshStats = useCallback(() => {
queryClient.invalidateQueries({ queryKey: SETTINGS_THUMB_STATS_KEY })
queryClient.invalidateQueries({ queryKey: SETTINGS_LIB_STATS_KEY })
queryClient.invalidateQueries({ queryKey: SETTINGS_PIPELINE_STATS_KEY })
queryClient.invalidateQueries({ queryKey: DUPLICATE_GROUPS_QUERY_KEY })
}, [queryClient])
const refreshWorkers = useCallback(() => {
queryClient.invalidateQueries({ queryKey: SETTINGS_WORKER_STATUS_KEY })
queryClient.invalidateQueries({ queryKey: SETTINGS_MISSING_STATS_KEY })
queryClient.invalidateQueries({ queryKey: SETTINGS_SCAN_STATUS_KEY })
}, [queryClient])
// Surface fetch errors once (React Query de-dupes retries but we still
@@ -246,6 +290,33 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
}
/>
</div>
{/* Inline scan progress. Rendered as a full-width sub-card
when a scan is active so the user sees the same info
they'd get from the floating widget without leaving
settings. Hidden when idle to keep the section tight. */}
{scanStatus?.is_scanning && (
<div className="mt-3 rounded border border-border bg-surface p-2">
<div className="mb-1 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-text-muted">
<FolderSearch className="h-3 w-3 animate-pulse" />
Scanning
</div>
<div className="mb-1 flex items-center justify-between text-xs">
<span className="truncate font-mono text-text-muted" title={scanStatus.current_folder ?? ''}>
{scanStatus.current_folder ?? '—'}
</span>
<span className="ml-2 shrink-0 font-mono text-text">
{scanStatus.processed_files.toLocaleString()} /{' '}
{scanStatus.total_files.toLocaleString()}
</span>
</div>
<ProgressBar
done={scanStatus.processed_files}
total={scanStatus.total_files || 1}
/>
</div>
)}
<div className="mt-3">
<ActionButton
loading={busy.scan}
@@ -263,6 +334,51 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
</div>
</Section>
{/* ----------------------------------------------------- */}
{/* Pipeline progress — per-stage done/total */}
{/* ----------------------------------------------------- */}
<Section
icon={<Activity className="h-4 w-4" />}
title="Pipeline progress"
right={
<button
onClick={() =>
queryClient.invalidateQueries({
queryKey: SETTINGS_PIPELINE_STATS_KEY,
})
}
disabled={pipelineStatsQuery.isFetching && !pipelineStats}
className="flex items-center gap-1 rounded border border-border px-2 py-1 text-xs text-text-muted hover:bg-surface-2 disabled:opacity-50"
>
{pipelineStatsQuery.isFetching && !pipelineStats ? (
<Loader2 className="h-3 w-3 animate-spin" />
) : (
<RefreshCw className="h-3 w-3" />
)}
Refresh
</button>
}
>
{pipelineStats ? (
<div className="space-y-2">
{pipelineStats.stages.map((stage) => (
<PipelineRow key={stage.key} stage={stage} />
))}
<p className="pt-1 text-[10px] text-text-muted">
Partial stages (marked *) only apply to a subset of
photos — e.g. not every photo has GPS, text, or detectable
objects. Face clusters and duplicate groups are output
counts rather than ratios.
</p>
</div>
) : (
<div className="flex items-center gap-2 text-xs text-text-muted">
<Loader2 className="h-3 w-3 animate-spin" />
Loading pipeline progress…
</div>
)}
</Section>
{/* ----------------------------------------------------- */}
{/* Duplicate detection */}
{/* ----------------------------------------------------- */}
@@ -430,7 +546,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
}
>
{/* Top-line health */}
<div className="grid grid-cols-3 gap-2 text-xs">
<div className="grid grid-cols-4 gap-2 text-xs">
<Stat
label="Workers"
value={workerStatus?.worker_count}
@@ -443,20 +559,25 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
}
/>
<Stat
label="Broker"
label="Concurrency"
value={
workerStatus
? workerStatus.broker_ok
? 'OK'
: 'DOWN'
workerStatus && workerStatus.workers.length > 0
? workerStatus.workers.reduce(
(acc, w) => acc + (w.concurrency ?? 0),
0
)
: undefined
}
/>
<Stat
label="Tasks/min"
value={
throughput === null
? '…'
: throughput.toFixed(0)
}
tone={
workerStatus
? workerStatus.broker_ok
? 'ok'
: 'warn'
: 'muted'
throughput !== null && throughput > 0 ? 'ok' : 'muted'
}
/>
<Stat
@@ -470,6 +591,34 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
/>
</div>
{/* How to scale — explicit, because this question comes up
every time a large import is running and the user wants
it to go faster. The ingestion pipeline runs on two
worker services; each has its own concurrency env var. */}
<p className="mt-3 text-[10px] leading-relaxed text-text-muted">
Two worker services share the load:{' '}
<code className="rounded bg-surface px-1">worker-light</code>{' '}
(scan, thumbnails, EXIF — set by{' '}
<code className="rounded bg-surface px-1">
CELERY_LIGHT_CONCURRENCY
</code>
) and{' '}
<code className="rounded bg-surface px-1">worker-vision</code>{' '}
(embed, detect, OCR, faces — set by{' '}
<code className="rounded bg-surface px-1">
CELERY_VISION_CONCURRENCY
</code>
) in{' '}
<code className="rounded bg-surface px-1">.env</code>. To
scale, bump those values then run{' '}
<code className="rounded bg-surface px-1">
docker compose up -d worker-light worker-vision
</code>
. Keep vision below your physical core count — each fork
loads ~2 GB of ONNX models. The vision queue is usually the
bottleneck; watch its depth below.
</p>
{/* Inline error banners for the obvious failure modes */}
{workerStatus?.broker_error && (
<ErrorBanner
@@ -492,17 +641,24 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
/>
)}
{/* Queue depth */}
{/* Queue depth. `vision` is called out with a highlight
because it's where the serious backlog lives — every
embed / tag / OCR / face task routes here. */}
{workerStatus && (
<div className="mt-3">
<div className="mb-1 text-[10px] uppercase tracking-wide text-text-muted">
Queue depth
</div>
<div className="grid grid-cols-3 gap-2 text-xs">
<div className="grid grid-cols-4 gap-2 text-xs">
{Object.entries(workerStatus.queues).map(([name, depth]) => (
<div
key={name}
className="flex items-center justify-between rounded bg-surface px-2 py-1"
className={clsx(
'flex items-center justify-between rounded px-2 py-1',
name === 'vision' && depth > 0
? 'bg-surface-2'
: 'bg-surface'
)}
>
<span className="text-text-muted">{name}</span>
<span
@@ -511,7 +667,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
depth > 0 ? 'text-text' : 'text-text-muted'
)}
>
{depth}
{depth.toLocaleString()}
</span>
</div>
))}
@@ -797,6 +953,116 @@ function ErrorBanner({ title, detail }: { title: string; detail: string }) {
)
}
/**
* Compute the total task-completion rate across the worker fleet by
* diffing the per-task `processed` counters between successive poll
* samples. Returns null on the first call (we need two samples for a
* rate) and 0 when nothing has moved since the last poll.
*
* The sample is kept in a ref — not state — because we don't want to
* re-render on every update; we want the number to settle into the
* existing render cycle that React Query already drives.
*/
function computeThroughput(
status: WorkerStatus | undefined,
ref: React.MutableRefObject<{
totals: Record<string, number>
at: number
} | null>
): number | null {
if (!status || status.workers.length === 0) {
return null
}
// Flatten processed counts across all workers into one dict keyed by
// task name so worker restarts (which reset individual counters) are
// absorbed by the total.
const totals: Record<string, number> = {}
for (const w of status.workers) {
for (const [task, count] of Object.entries(w.processed ?? {})) {
totals[task] = (totals[task] ?? 0) + (count ?? 0)
}
}
const now = Date.now()
const prev = ref.current
// Update the ref BEFORE returning so the next call has a baseline.
ref.current = { totals, at: now }
if (!prev) {
return null
}
const elapsedSec = (now - prev.at) / 1000
if (elapsedSec <= 0) {
return null
}
let delta = 0
for (const [task, count] of Object.entries(totals)) {
const before = prev.totals[task] ?? 0
// Guard against counter resets (worker restart) — negative diffs
// are clamped to zero rather than dragging the rate down.
delta += Math.max(0, count - before)
}
return (delta / elapsedSec) * 60
}
function PipelineRow({ stage }: { stage: PipelineStage }) {
const isStandalone = stage.standalone === true
const isComplete = !isStandalone && stage.total > 0 && stage.done >= stage.total
const isActive = !isStandalone && stage.done > 0 && stage.done < stage.total
return (
<div className="rounded bg-surface px-2 py-1.5" title={stage.hint}>
<div className="flex items-center justify-between gap-2 text-xs">
<div className="flex min-w-0 items-center gap-1.5">
<span className="truncate text-text">
{stage.label}
{stage.partial && <span className="text-text-muted"> *</span>}
</span>
</div>
<span
className={clsx(
'shrink-0 font-mono text-[11px]',
isComplete
? 'text-pick'
: isActive
? 'text-text'
: 'text-text-muted'
)}
>
{isStandalone
? stage.done.toLocaleString()
: `${stage.done.toLocaleString()} / ${stage.total.toLocaleString()}`}
</span>
</div>
{!isStandalone && (
<div className="mt-1">
<ProgressBar done={stage.done} total={stage.total || 1} />
</div>
)}
</div>
)
}
function ProgressBar({ done, total }: { done: number; total: number }) {
const pct = total > 0 ? Math.min(100, (done / total) * 100) : 0
const complete = total > 0 && done >= total
return (
<div className="h-1 w-full overflow-hidden rounded bg-border">
<div
className={clsx(
'h-full rounded transition-[width] duration-500',
complete ? 'bg-pick' : 'bg-text-muted'
)}
style={{ width: `${pct}%` }}
/>
</div>
)
}
function ActionButton({
loading,
disabled,

39
frontend/src/services/api.ts
View File

@@ -297,6 +297,37 @@ export interface WorkerStatus {
scan_errors: string[]
}
export interface PipelineStage {
key: string
label: string
done: number
total: number
hint: string
/** True when the stage legitimately runs on a subset of photos — e.g.
* GPS / tags / OCR / faces — so 100% coverage is never expected and
* the UI should not frame "missing" as a problem. */
partial?: boolean
/** True when the stage doesn't have a done/total progress semantic
* (e.g. face clusters, duplicate groups — those are output counts,
* not ratios). The UI renders a plain count instead of a bar. */
standalone?: boolean
}
export interface PipelineStats {
total_photos: number
total_images: number
embedder_model: string
stages: PipelineStage[]
}
export interface ScanStatus {
is_scanning: boolean
current_folder: string | null
processed_files: number
total_files: number
errors: string[]
}
export const library = {
scan: async () => {
const response = await api.post('/library/scan')
@@ -344,6 +375,14 @@ export const library = {
return response.data
},
/** Per-stage ingestion progress — thumbnails, EXIF, GPS, phash,
* embeddings, object tags, OCR, faces, face clusters, duplicate
* groups. Drives the Pipeline Progress card in Settings. */
pipelineStats: async (): Promise<PipelineStats> => {
const response = await api.get('/library/maintenance/pipeline-stats')
return response.data
},
/** Dry-run count of photo rows whose files are no longer on disk
* (under a mounted source root). */
missingStats: async (): Promise<MissingStats> => {