dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: drop AI/vision pipeline + plain Postgres + full-refresh script

Browse Source 
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
  - backend: app/services/vision/, app/tasks/vision.py,
    app/services/feature_flags.py, app/routers/features.py — all
    deleted; admin AI/feature-flag endpoints and the worker-vision
    bootstrap call gone. Photo.needs_review and its index dropped.
  - frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
    types, "Needs Review" sidebar entry + filter, needs_review filter
    URL param all gone.
  - infra: worker-vision compose service + models_data volume deleted;
    worker-light command no longer runs bootstrap_models; the db
    image switches from pgvector/pgvector:pg16 to postgres:16; backend
    Dockerfile drops the dedicated torch RUN layer; requirements.txt
    drops torch/torchvision/open-clip-torch/onnxruntime.

Alembic 0019_drop_ai_remnants:
  - drops photos.needs_review + ix_photos_needs_review
  - DROP EXTENSION IF EXISTS vector (must run before the image swap;
    the new postgres:16 doesn't ship pgvector)

New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
claudio
2026-05-14 00:20:38 +02:00
parent 6915c30911
commit a27267f7ad
39 changed files with 265 additions and 1573 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
frontend/src/components/dialogs/SettingsDialog.tsx
View File

@@ -32,8 +32,6 @@ import {
type PipelineStage,
type ScanStatus,
type WorkerStatus,
type FeatureFlagSnapshot,
type FeaturesMap,
type NextcloudSourceRoot,
} from '../../services/api'
import { toast } from '../ToastContainer'
@@ -57,14 +55,10 @@ const SETTINGS_SCAN_STATUS_KEY = ['settings', 'scan-status'] as const
// the grid renders from. Imported via the canonical hook key.
import { DUPLICATE_GROUPS_QUERY_KEY } from '../../hooks/useDuplicateGroupsQuery'
type SettingsTab = 'library' | 'ai' | 'users'
type SettingsTab = 'library' | 'users'
const TABS: { id: SettingsTab; label: string; adminOnly?: boolean }[] = [
{ id: 'library', label: 'Library Management' },
// AI is open to non-admins; per-user scope. Admin-only mutations
// (system-wide flag toggles, bulk backfill, rescan-all) are hidden
// inside the panel for non-admins.
{ id: 'ai', label: 'AI Features' },
{ id: 'users', label: 'Users', adminOnly: true },
]
@@ -869,14 +863,6 @@ export function SettingsPage() {
</Section>
</>)}
{activeTab === 'ai' && (
<AiFeaturesTab
busy={busy}
runAction={runAction}
isAdmin={isAdmin}
/>
)}
{activeTab === 'users' && isAdmin && (
<Section
icon={<Shield className="h-4 w-4" />}
@@ -1110,223 +1096,6 @@ function ActionButton({
}
// ---------------------------------------------------------------------------
// AI Features admin tab
// ---------------------------------------------------------------------------
interface AiFeaturesTabProps {
busy: Record<string, boolean>
runAction: <T>(
key: string,
fn: () => Promise<T>,
successTitle: string,
describe?: (result: T) => string | undefined,
) => Promise<void>
isAdmin: boolean
}
const FLAG_META: Array<{
id: string
label: string
description: string
icon: React.ReactNode
}> = [
{
id: 'vision.enabled',
label: 'Vision classifier',
description:
'Binary photo-vs-other classifier. Flags screenshots, documents, memes and ' +
'scans with "needs review" so they can be triaged.',
icon: <Sparkles className="h-3.5 w-3.5" />,
},
]
function AiFeaturesTab({ busy, runAction, isAdmin }: AiFeaturesTabProps) {
const queryClient = useQueryClient()
// Admins get the full snapshot (default + effective + overridden) so
// they can toggle. Non-admins only read effective values via the
// public /features endpoint — no leaked override metadata, no 403s.
const flagsQuery = useQuery<{ flags: FeatureFlagSnapshot }>({
queryKey: SETTINGS_FEATURE_FLAGS_KEY,
queryFn: adminApi.listFeatureFlags,
staleTime: 5_000,
enabled: isAdmin,
})
const featuresQuery = useQuery<FeaturesMap>({
queryKey: ['features'],
queryFn: featuresApi.list,
staleTime: 5_000,
enabled: !isAdmin,
})
const flags: FeatureFlagSnapshot = isAdmin
? (flagsQuery.data?.flags ?? {})
: Object.fromEntries(
Object.entries(featuresQuery.data ?? {}).map(([name, on]) => [
name,
{ effective: on, default: on, overridden: false },
]),
)
const masterOff = flags['vision.enabled'] && !flags['vision.enabled'].effective
const applyFlag = async (name: string, value: boolean | null) => {
await runAction(
`flag:${name}`,
() => adminApi.setFeatureFlag(name, value),
value === null ? 'Override cleared' : `Feature ${value ? 'enabled' : 'disabled'}`,
)
queryClient.invalidateQueries({ queryKey: SETTINGS_FEATURE_FLAGS_KEY })
// Non-admin feature map drives sidebar gating — invalidate so the
// People / Tags entries appear / disappear immediately without a
// page reload.
queryClient.invalidateQueries({ queryKey: ['features'] })
}
const runBackfill = () =>
runAction(
`ai-backfill:all`,
() => adminApi.triggerAiBackfill({}),
'Classifier backfill queued',
(r) => `Celery task ${r.task_id}`,
)
return (
<>
<Section icon={<Brain className="h-4 w-4" />} title="AI feature flags">
<p className="text-xs text-text-muted">
{isAdmin
? 'Toggle each stage at runtime. Changes are observed by Celery workers on the next task — no restart needed. "Default" means the flag hasn\'t been overridden and is tracking the YAML config; an overridden flag is pinned to the value shown until cleared.'
: 'Effective AI features for your library. Flags are system-wide; only an admin can toggle them.'}
</p>
{(isAdmin ? flagsQuery.isLoading : featuresQuery.isLoading) && (
<div className="mt-3 flex items-center gap-2 text-xs text-text-muted">
<Loader2 className="h-3 w-3 animate-spin" />
Loading feature flags…
</div>
)}
{(isAdmin ? flagsQuery.error : featuresQuery.error) && (
<ErrorBanner
title="Could not load feature flags"
detail={String(
((isAdmin ? flagsQuery.error : featuresQuery.error) as Error).message
|| (isAdmin ? flagsQuery.error : featuresQuery.error),
)}
/>
)}
{!(isAdmin ? flagsQuery.isLoading : featuresQuery.isLoading)
&& !(isAdmin ? flagsQuery.error : featuresQuery.error) && (
<div className="mt-3 space-y-2">
{FLAG_META.map((meta) => {
const state = flags[meta.id]
if (!state) return null
const busyKey = `flag:${meta.id}`
const isBusy = !!busy[busyKey]
const isMaster = meta.id === 'vision.enabled'
const dimmed = !isMaster && masterOff
return (
<div
key={meta.id}
className={cn(
'rounded border border-border bg-surface p-3 text-xs',
dimmed && 'opacity-60',
)}
>
<div className="flex items-start justify-between gap-3">
<div className="min-w-0 flex-1">
<div className="flex items-center gap-1.5 text-text">
{meta.icon}
<span className="font-medium">{meta.label}</span>
{state.overridden && (
<span className="rounded bg-primary/20 px-1 py-0.5 text-[9px] font-semibold uppercase tracking-wide text-primary">
overridden
</span>
)}
</div>
<p className="mt-1 text-[11px] text-text-muted">{meta.description}</p>
<p className="mt-1 text-[10px] text-text-faint">
Default: {state.default ? 'on' : 'off'} · Currently:{' '}
<span className={state.effective ? 'text-pick' : 'text-reject'}>
{state.effective ? 'on' : 'off'}
</span>
</p>
</div>
<div className="flex shrink-0 items-center gap-2">
<Switch
checked={state.effective}
onCheckedChange={(v) => applyFlag(meta.id, v)}
disabled={!isAdmin || isBusy || (dimmed && !isMaster)}
title={
!isAdmin
? 'Admin only — flags are system-wide'
: state.effective
? 'Click to disable'
: 'Click to enable'
}
/>
{isAdmin && state.overridden && (
<Button
variant="outline"
size="icon"
onClick={() => applyFlag(meta.id, null)}
disabled={isBusy}
className="h-6 w-6"
title="Reset to YAML default"
aria-label="Reset override"
>
<RotateCcw className="h-3 w-3" />
</Button>
)}
</div>
</div>
</div>
)
})}
</div>
)}
</Section>
{isAdmin && (
<Section icon={<Cpu className="h-4 w-4" />} title="Manual pipeline triggers">
<p className="text-xs text-text-muted">
Run the classifier over any photos that haven't been classified yet,
or force a fresh filesystem scan. Both are safe to run repeatedly.
These are bulk admin operations across every user; non-admins
should use "Re-scan source folders" in the Library tab to
re-scan their own libraries.
</p>
<div className="mt-3 flex flex-wrap gap-2">
<ActionButton
loading={!!busy['ai-backfill:all']}
onClick={() => runBackfill()}
disabled={masterOff}
>
<Sparkles className="h-4 w-4" />
Run classifier backfill
</ActionButton>
<ActionButton
loading={!!busy['rescan-full']}
onClick={() =>
runAction(
'rescan-full',
() => adminApi.triggerFullRescan(),
'Rescan queued',
(r) => `Celery task ${r.task_id}`,
)
}
>
<RefreshCw className="h-4 w-4" />
Rescan all source roots
</ActionButton>
</div>
</Section>
)}
</>
)
}
// ── Nextcloud integration card ─────────────────────────────────────────
//

65
frontend/src/components/filter/FilterBar.tsx
View File

@@ -81,8 +81,6 @@ export function FilterBar({
const sortBy = useFilterStore((s) => s.sortBy)
const sortOrder = useFilterStore((s) => s.sortOrder)
const tagIds = useFilterStore((s) => s.tagIds)
const needsReview = useFilterStore((s) => s.needsReview)
const setNeedsReview = useFilterStore((s) => s.setNeedsReview)
const dateFrom = useFilterStore((s) => s.dateFrom)
const dateTo = useFilterStore((s) => s.dateTo)
const setDateFrom = useFilterStore((s) => s.setDateFrom)
@@ -119,14 +117,12 @@ export function FilterBar({
const colorActive = colorLabel !== null
const colorValue = colorActive ? colorLabel : null
const flagActive = flag !== 'any' || needsReview
const flagValue = needsReview
? 'needs review'
: flag !== 'any'
? flag === 'date_warning'
? 'date issues'
: flag
: null
const flagActive = flag !== 'any'
const flagValue = flagActive
? flag === 'date_warning'
? 'date issues'
: flag
: null
const tagActive = tagIds.length > 0
const activeTagNames = allTags
@@ -304,19 +300,13 @@ export function FilterBar({
</FilterPill>
{/* Flag — hidden in the Discarded section, where the flag is
* pinned to "discarded" by the section preset. The Needs review
* option lives here too: it sets a different store field
* (`needsReview`) but is mutually exclusive with the other flag
* values from the user's perspective. */}
* pinned to "discarded" by the section preset. */}
{!hideFlagPill && (
<FilterPill
label="Flag"
value={flagValue}
isActive={flagActive}
onClear={() => {
setFlag('any')
setNeedsReview(false)
}}
onClear={() => setFlag('any')}
>
<MultiSelect
searchable={false}
@@ -324,37 +314,18 @@ export function FilterBar({
options={[
{ value: 'discarded', label: 'Discarded' },
{ value: 'date_warning', label: 'Date issues' },
{ value: 'needs_review', label: 'Needs review' },
]}
values={
needsReview
? ['needs_review']
: flag !== 'any'
? [flag]
: []
}
values={flag !== 'any' ? [flag] : []}
onChange={(next) => {
// Flag state is mutually exclusive in the store; treat
// the just-added value as the new single selection, or
// clear everything when the user unchecks the current.
const added = next.find(
(v) =>
v !==
(needsReview ? 'needs_review' : flag !== 'any' ? flag : '')
)
if (!added) {
setFlag('any')
setNeedsReview(false)
return
}
if (added === 'needs_review') {
setFlag('any')
setNeedsReview(true)
} else {
setFlag(added as 'discarded' | 'date_warning')
setNeedsReview(false)
}
}}
// Flag state is mutually exclusive in the store; treat
// the just-added value as the new single selection.
const added = next.find((v) => v !== (flag !== 'any' ? flag : ''))
if (!added) {
setFlag('any')
return
}
setFlag(added as 'discarded' | 'date_warning')
}}
/>
</FilterPill>
)}

7
frontend/src/components/layout/LeftSidebar.tsx
View File

@@ -62,7 +62,6 @@ import {
ContextMenuTrigger,
} from '@/components/ui/context-menu'
import { useAuth } from '../../contexts/AuthContext'
import { useFeaturesQuery } from '../../hooks/useFeaturesQuery'
import { useScanActivity } from '../../hooks/useScanActivity'
import { Input } from '@/components/ui/input'
import {
@@ -105,8 +104,6 @@ export function LeftSidebar() {
const currentSection = useFilterStore((s) => s.currentSection)
const { data: allTags = [] } = useTagsQuery()
const { data: stats } = useLibraryStatsQuery()
const { data: featuresMap } = useFeaturesQuery()
const visionOn = featuresMap ? featuresMap['vision.enabled'] !== false : true
const tagsOn = true
const [dropTargetId, setDropTargetId] = useState<string | null>(null)
@@ -289,9 +286,6 @@ export function LeftSidebar() {
case 'tags':
navigateToSection('tags', { groupBy: 'tag' })
break
case 'needs-review':
navigateToSection('needs-review', { needsReview: true })
break
case 'colors':
navigateToSection('colors', { groupBy: 'color' })
break
@@ -430,7 +424,6 @@ export function LeftSidebar() {
{ id: 'all-photos', label: 'All Photos', icon: <Image className="h-4 w-4" />, count: stats?.all_photos ?? 0 },
{ id: 'rated', label: 'Rated', icon: <Star className="h-4 w-4" />, count: stats?.rated ?? 0 },
...(tagsOn ? [{ id: 'tags', label: 'Tags', icon: <TagIcon className="h-4 w-4" />, count: tagsTotalCount }] : []),
...(visionOn ? [{ id: 'needs-review', label: 'Needs Review', icon: <Users className="h-4 w-4" />, count: stats?.needs_review ?? 0 }] : []),
{ id: 'colors', label: 'Colors', icon: <Palette className="h-4 w-4" />, count: stats?.colored ?? 0 },
{ id: 'map', label: 'Map', icon: <MapPin className="h-4 w-4" />, count: stats?.with_gps ?? 0 },
{ id: 'memories', label: 'Memories', icon: <Clock className="h-4 w-4" /> },