dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: drop AI/vision pipeline + plain Postgres + full-refresh script

Browse Source 
Removes the OpenCLIP-on-ONNX classifier and everything that fed or
consumed it:
  - backend: app/services/vision/, app/tasks/vision.py,
    app/services/feature_flags.py, app/routers/features.py — all
    deleted; admin AI/feature-flag endpoints and the worker-vision
    bootstrap call gone. Photo.needs_review and its index dropped.
  - frontend: AI Settings tab, useFeaturesQuery hook, FeatureFlag
    types, "Needs Review" sidebar entry + filter, needs_review filter
    URL param all gone.
  - infra: worker-vision compose service + models_data volume deleted;
    worker-light command no longer runs bootstrap_models; the db
    image switches from pgvector/pgvector:pg16 to postgres:16; backend
    Dockerfile drops the dedicated torch RUN layer; requirements.txt
    drops torch/torchvision/open-clip-torch/onnxruntime.

Alembic 0019_drop_ai_remnants:
  - drops photos.needs_review + ix_photos_needs_review
  - DROP EXTENSION IF EXISTS vector (must run before the image swap;
    the new postgres:16 doesn't ship pgvector)

New scripts/full_refresh.py: one-shot DB ↔ filesystem reconciliation.
Runs cleanup_data_integrity, scans every active SourceRoot inline
(no celery dependency so the worker can be stopped), hard-prunes
photo + folder rows for files that are gone, removes orphan
/data/thumbs/{user}/{photo}/ directories. New helper
prune_orphan_thumbnails in cleanup.py.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
claudio
2026-05-14 00:20:38 +02:00
parent 6915c30911
commit a27267f7ad
39 changed files with 265 additions and 1573 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
frontend/src/components/dialogs/SettingsDialog.tsx
View File

@@ -32,8 +32,6 @@ import {
type PipelineStage,
type ScanStatus,
type WorkerStatus,
type FeatureFlagSnapshot,
type FeaturesMap,
type NextcloudSourceRoot,
} from '../../services/api'
import { toast } from '../ToastContainer'
@@ -57,14 +55,10 @@ const SETTINGS_SCAN_STATUS_KEY = ['settings', 'scan-status'] as const
// the grid renders from. Imported via the canonical hook key.
import { DUPLICATE_GROUPS_QUERY_KEY } from '../../hooks/useDuplicateGroupsQuery'
type SettingsTab = 'library' | 'ai' | 'users'
type SettingsTab = 'library' | 'users'
const TABS: { id: SettingsTab; label: string; adminOnly?: boolean }[] = [
{ id: 'library', label: 'Library Management' },
// AI is open to non-admins; per-user scope. Admin-only mutations
// (system-wide flag toggles, bulk backfill, rescan-all) are hidden
// inside the panel for non-admins.
{ id: 'ai', label: 'AI Features' },
{ id: 'users', label: 'Users', adminOnly: true },
]
@@ -869,14 +863,6 @@ export function SettingsPage() {
</Section>
</>)}
{activeTab === 'ai' && (
<AiFeaturesTab
busy={busy}
runAction={runAction}
isAdmin={isAdmin}
/>
)}
{activeTab === 'users' && isAdmin && (
<Section
icon={<Shield className="h-4 w-4" />}
@@ -1110,223 +1096,6 @@ function ActionButton({
}
// ---------------------------------------------------------------------------
// AI Features admin tab
// ---------------------------------------------------------------------------
interface AiFeaturesTabProps {
busy: Record<string, boolean>
runAction: <T>(
key: string,
fn: () => Promise<T>,
successTitle: string,
describe?: (result: T) => string | undefined,
) => Promise<void>
isAdmin: boolean
}
const FLAG_META: Array<{
id: string
label: string
description: string
icon: React.ReactNode
}> = [
{
id: 'vision.enabled',
label: 'Vision classifier',
description:
'Binary photo-vs-other classifier. Flags screenshots, documents, memes and ' +
'scans with "needs review" so they can be triaged.',
icon: <Sparkles className="h-3.5 w-3.5" />,
},
]
function AiFeaturesTab({ busy, runAction, isAdmin }: AiFeaturesTabProps) {
const queryClient = useQueryClient()
// Admins get the full snapshot (default + effective + overridden) so
// they can toggle. Non-admins only read effective values via the
// public /features endpoint — no leaked override metadata, no 403s.
const flagsQuery = useQuery<{ flags: FeatureFlagSnapshot }>({
queryKey: SETTINGS_FEATURE_FLAGS_KEY,
queryFn: adminApi.listFeatureFlags,
staleTime: 5_000,
enabled: isAdmin,
})
const featuresQuery = useQuery<FeaturesMap>({
queryKey: ['features'],
queryFn: featuresApi.list,
staleTime: 5_000,
enabled: !isAdmin,
})
const flags: FeatureFlagSnapshot = isAdmin
? (flagsQuery.data?.flags ?? {})
: Object.fromEntries(
Object.entries(featuresQuery.data ?? {}).map(([name, on]) => [
name,
{ effective: on, default: on, overridden: false },
]),
)
const masterOff = flags['vision.enabled'] && !flags['vision.enabled'].effective
const applyFlag = async (name: string, value: boolean | null) => {
await runAction(
`flag:${name}`,
() => adminApi.setFeatureFlag(name, value),
value === null ? 'Override cleared' : `Feature ${value ? 'enabled' : 'disabled'}`,
)
queryClient.invalidateQueries({ queryKey: SETTINGS_FEATURE_FLAGS_KEY })
// Non-admin feature map drives sidebar gating — invalidate so the
// People / Tags entries appear / disappear immediately without a
// page reload.
queryClient.invalidateQueries({ queryKey: ['features'] })
}
const runBackfill = () =>
runAction(
`ai-backfill:all`,
() => adminApi.triggerAiBackfill({}),
'Classifier backfill queued',
(r) => `Celery task ${r.task_id}`,
)
return (
<>
<Section icon={<Brain className="h-4 w-4" />} title="AI feature flags">
<p className="text-xs text-text-muted">
{isAdmin
? 'Toggle each stage at runtime. Changes are observed by Celery workers on the next task — no restart needed. "Default" means the flag hasn\'t been overridden and is tracking the YAML config; an overridden flag is pinned to the value shown until cleared.'
: 'Effective AI features for your library. Flags are system-wide; only an admin can toggle them.'}
</p>
{(isAdmin ? flagsQuery.isLoading : featuresQuery.isLoading) && (
<div className="mt-3 flex items-center gap-2 text-xs text-text-muted">
<Loader2 className="h-3 w-3 animate-spin" />
Loading feature flags…
</div>
)}
{(isAdmin ? flagsQuery.error : featuresQuery.error) && (
<ErrorBanner
title="Could not load feature flags"
detail={String(
((isAdmin ? flagsQuery.error : featuresQuery.error) as Error).message
|| (isAdmin ? flagsQuery.error : featuresQuery.error),
)}
/>
)}
{!(isAdmin ? flagsQuery.isLoading : featuresQuery.isLoading)
&& !(isAdmin ? flagsQuery.error : featuresQuery.error) && (
<div className="mt-3 space-y-2">
{FLAG_META.map((meta) => {
const state = flags[meta.id]
if (!state) return null
const busyKey = `flag:${meta.id}`
const isBusy = !!busy[busyKey]
const isMaster = meta.id === 'vision.enabled'
const dimmed = !isMaster && masterOff
return (
<div
key={meta.id}
className={cn(
'rounded border border-border bg-surface p-3 text-xs',
dimmed && 'opacity-60',
)}
>
<div className="flex items-start justify-between gap-3">
<div className="min-w-0 flex-1">
<div className="flex items-center gap-1.5 text-text">
{meta.icon}
<span className="font-medium">{meta.label}</span>
{state.overridden && (
<span className="rounded bg-primary/20 px-1 py-0.5 text-[9px] font-semibold uppercase tracking-wide text-primary">
overridden
</span>
)}
</div>
<p className="mt-1 text-[11px] text-text-muted">{meta.description}</p>
<p className="mt-1 text-[10px] text-text-faint">
Default: {state.default ? 'on' : 'off'} · Currently:{' '}
<span className={state.effective ? 'text-pick' : 'text-reject'}>
{state.effective ? 'on' : 'off'}
</span>
</p>
</div>
<div className="flex shrink-0 items-center gap-2">
<Switch
checked={state.effective}
onCheckedChange={(v) => applyFlag(meta.id, v)}
disabled={!isAdmin || isBusy || (dimmed && !isMaster)}
title={
!isAdmin
? 'Admin only — flags are system-wide'
: state.effective
? 'Click to disable'
: 'Click to enable'
}
/>
{isAdmin && state.overridden && (
<Button
variant="outline"
size="icon"
onClick={() => applyFlag(meta.id, null)}
disabled={isBusy}
className="h-6 w-6"
title="Reset to YAML default"
aria-label="Reset override"
>
<RotateCcw className="h-3 w-3" />
</Button>
)}
</div>
</div>
</div>
)
})}
</div>
)}
</Section>
{isAdmin && (
<Section icon={<Cpu className="h-4 w-4" />} title="Manual pipeline triggers">
<p className="text-xs text-text-muted">
Run the classifier over any photos that haven't been classified yet,
or force a fresh filesystem scan. Both are safe to run repeatedly.
These are bulk admin operations across every user; non-admins
should use "Re-scan source folders" in the Library tab to
re-scan their own libraries.
</p>
<div className="mt-3 flex flex-wrap gap-2">
<ActionButton
loading={!!busy['ai-backfill:all']}
onClick={() => runBackfill()}
disabled={masterOff}
>
<Sparkles className="h-4 w-4" />
Run classifier backfill
</ActionButton>
<ActionButton
loading={!!busy['rescan-full']}
onClick={() =>
runAction(
'rescan-full',
() => adminApi.triggerFullRescan(),
'Rescan queued',
(r) => `Celery task ${r.task_id}`,
)
}
>
<RefreshCw className="h-4 w-4" />
Rescan all source roots
</ActionButton>
</div>
</Section>
)}
</>
)
}
// ── Nextcloud integration card ─────────────────────────────────────────
//

65
frontend/src/components/filter/FilterBar.tsx
View File

@@ -81,8 +81,6 @@ export function FilterBar({
const sortBy = useFilterStore((s) => s.sortBy)
const sortOrder = useFilterStore((s) => s.sortOrder)
const tagIds = useFilterStore((s) => s.tagIds)
const needsReview = useFilterStore((s) => s.needsReview)
const setNeedsReview = useFilterStore((s) => s.setNeedsReview)
const dateFrom = useFilterStore((s) => s.dateFrom)
const dateTo = useFilterStore((s) => s.dateTo)
const setDateFrom = useFilterStore((s) => s.setDateFrom)
@@ -119,14 +117,12 @@ export function FilterBar({
const colorActive = colorLabel !== null
const colorValue = colorActive ? colorLabel : null
const flagActive = flag !== 'any' || needsReview
const flagValue = needsReview
? 'needs review'
: flag !== 'any'
? flag === 'date_warning'
? 'date issues'
: flag
: null
const flagActive = flag !== 'any'
const flagValue = flagActive
? flag === 'date_warning'
? 'date issues'
: flag
: null
const tagActive = tagIds.length > 0
const activeTagNames = allTags
@@ -304,19 +300,13 @@ export function FilterBar({
</FilterPill>
{/* Flag — hidden in the Discarded section, where the flag is
* pinned to "discarded" by the section preset. The Needs review
* option lives here too: it sets a different store field
* (`needsReview`) but is mutually exclusive with the other flag
* values from the user's perspective. */}
* pinned to "discarded" by the section preset. */}
{!hideFlagPill && (
<FilterPill
label="Flag"
value={flagValue}
isActive={flagActive}
onClear={() => {
setFlag('any')
setNeedsReview(false)
}}
onClear={() => setFlag('any')}
>
<MultiSelect
searchable={false}
@@ -324,37 +314,18 @@ export function FilterBar({
options={[
{ value: 'discarded', label: 'Discarded' },
{ value: 'date_warning', label: 'Date issues' },
{ value: 'needs_review', label: 'Needs review' },
]}
values={
needsReview
? ['needs_review']
: flag !== 'any'
? [flag]
: []
}
values={flag !== 'any' ? [flag] : []}
onChange={(next) => {
// Flag state is mutually exclusive in the store; treat
// the just-added value as the new single selection, or
// clear everything when the user unchecks the current.
const added = next.find(
(v) =>
v !==
(needsReview ? 'needs_review' : flag !== 'any' ? flag : '')
)
if (!added) {
setFlag('any')
setNeedsReview(false)
return
}
if (added === 'needs_review') {
setFlag('any')
setNeedsReview(true)
} else {
setFlag(added as 'discarded' | 'date_warning')
setNeedsReview(false)
}
}}
// Flag state is mutually exclusive in the store; treat
// the just-added value as the new single selection.
const added = next.find((v) => v !== (flag !== 'any' ? flag : ''))
if (!added) {
setFlag('any')
return
}
setFlag(added as 'discarded' | 'date_warning')
}}
/>
</FilterPill>
)}

7
frontend/src/components/layout/LeftSidebar.tsx
View File

@@ -62,7 +62,6 @@ import {
ContextMenuTrigger,
} from '@/components/ui/context-menu'
import { useAuth } from '../../contexts/AuthContext'
import { useFeaturesQuery } from '../../hooks/useFeaturesQuery'
import { useScanActivity } from '../../hooks/useScanActivity'
import { Input } from '@/components/ui/input'
import {
@@ -105,8 +104,6 @@ export function LeftSidebar() {
const currentSection = useFilterStore((s) => s.currentSection)
const { data: allTags = [] } = useTagsQuery()
const { data: stats } = useLibraryStatsQuery()
const { data: featuresMap } = useFeaturesQuery()
const visionOn = featuresMap ? featuresMap['vision.enabled'] !== false : true
const tagsOn = true
const [dropTargetId, setDropTargetId] = useState<string | null>(null)
@@ -289,9 +286,6 @@ export function LeftSidebar() {
case 'tags':
navigateToSection('tags', { groupBy: 'tag' })
break
case 'needs-review':
navigateToSection('needs-review', { needsReview: true })
break
case 'colors':
navigateToSection('colors', { groupBy: 'color' })
break
@@ -430,7 +424,6 @@ export function LeftSidebar() {
{ id: 'all-photos', label: 'All Photos', icon: <Image className="h-4 w-4" />, count: stats?.all_photos ?? 0 },
{ id: 'rated', label: 'Rated', icon: <Star className="h-4 w-4" />, count: stats?.rated ?? 0 },
...(tagsOn ? [{ id: 'tags', label: 'Tags', icon: <TagIcon className="h-4 w-4" />, count: tagsTotalCount }] : []),
...(visionOn ? [{ id: 'needs-review', label: 'Needs Review', icon: <Users className="h-4 w-4" />, count: stats?.needs_review ?? 0 }] : []),
{ id: 'colors', label: 'Colors', icon: <Palette className="h-4 w-4" />, count: stats?.colored ?? 0 },
{ id: 'map', label: 'Map', icon: <MapPin className="h-4 w-4" />, count: stats?.with_gps ?? 0 },
{ id: 'memories', label: 'Memories', icon: <Clock className="h-4 w-4" /> },

2
frontend/src/hooks/useBulkPhotoMutations.ts
View File

@@ -18,7 +18,7 @@ import type { MemoriesResponse, MemoryPhoto } from '../services/api'
* per-photo cache so rating stars / color swatches flip instantly.
* - Rolls back the patch on error and surfaces a toast.
* - Invalidates the photo/library queries on success so server-side
* derived fields (needs_review, date_warning, etc.) reconcile.
* derived fields (date_warning, etc.) reconcile.
*
* Discard lives elsewhere — its two call sites have deliberately
* different semantics (keep-in-place for the X hotkey; strip-from-

32
frontend/src/hooks/useFeaturesQuery.ts
View File

@@ -1,32 +0,0 @@
import { useQuery, useQueryClient } from '@tanstack/react-query'
import { features, type FeaturesMap } from '../services/api'
export const FEATURES_QUERY_KEY = ['features'] as const
/** Read the effective feature-flag state (admin override or YAML
* default). Powers conditional rendering of pipeline-dependent UI —
* People view, Tags view, OCR snippets, etc. */
export function useFeaturesQuery() {
return useQuery<FeaturesMap>({
queryKey: FEATURES_QUERY_KEY,
queryFn: features.list,
// Re-read every minute so admin toggles reflect without a page
// reload. The admin tab also invalidates this key on write so the
// refresh can be immediate for the admin who just flipped it.
staleTime: 60_000,
refetchInterval: 60_000,
})
}
export function useIsFeatureEnabled(name: 'vision.enabled'): boolean {
const { data } = useFeaturesQuery()
// Default to enabled while loading so we don't flash "feature off"
// during a first-paint fetch. The backend is the source of truth;
// any gated UI that slipped through just returns empty data anyway.
if (!data) return true
return !!data[name]
}
export function invalidateFeaturesQuery(queryClient: ReturnType<typeof useQueryClient>) {
queryClient.invalidateQueries({ queryKey: FEATURES_QUERY_KEY })
}

2
frontend/src/hooks/useFilterUrlSync.ts
View File

@@ -86,7 +86,6 @@ function parseUrl(): HydratePayload {
}
if (sp.get('duplicates') === 'true') out.duplicates = true
if (sp.get('needs_review') === 'true') out.needsReview = true
const groupBy = sp.get('group')
if (groupBy === 'date' || groupBy === 'tag') out.groupBy = groupBy
@@ -120,7 +119,6 @@ function writeUrl(f: FilterState & { currentSection?: string }) {
if (f.folderId) sp.set('folder_id', f.folderId)
if (f.tagIds.length > 0) sp.set('tag_ids', f.tagIds.join(','))
if (f.duplicates) sp.set('duplicates', 'true')
if (f.needsReview) sp.set('needs_review', 'true')
if (f.groupBy !== 'date') sp.set('group', f.groupBy)
if (f.currentSection && f.currentSection !== 'all-photos')
sp.set('section', f.currentSection)

1
frontend/src/hooks/usePhotosQuery.ts
View File

@@ -65,7 +65,6 @@ export function usePhotosQuery() {
folderId: s.folderId,
tagIds: s.tagIds,
duplicates: s.duplicates,
needsReview: s.needsReview,
groupBy: s.groupBy,
sortBy: s.sortBy,
sortOrder: s.sortOrder,

50
frontend/src/services/api.ts
View File

@@ -611,7 +611,6 @@ export interface LibraryStats {
with_gps: number
duplicates: number
discarded: number
needs_review: number
total_photos: number
total_videos: number
total_size: number
@@ -1008,55 +1007,6 @@ export const admin = {
const response = await api.delete(`/admin/users/${userId}`)
return response.data
},
// --- AI / vision feature flags + manual pipeline triggers -----------
listFeatureFlags: async (): Promise<{ flags: FeatureFlagSnapshot }> => {
const response = await api.get('/admin/feature-flags')
return response.data
},
/** Set ``value`` to toggle; pass ``null`` to clear the override and
* fall back to the YAML default. */
setFeatureFlag: async (
name: string,
value: boolean | null,
): Promise<{ flags: FeatureFlagSnapshot }> => {
const response = await api.patch(`/admin/feature-flags/${encodeURIComponent(name)}`, { value })
return response.data
},
triggerAiBackfill: async (body: {
limit?: number | null
}): Promise<{ status: string; task_id: string }> => {
const response = await api.post('/admin/ai/backfill', body)
return response.data
},
triggerFullRescan: async (): Promise<{ status: string; task_id: string }> => {
const response = await api.post('/admin/ai/rescan')
return response.data
},
}
export interface FeatureFlagState {
effective: boolean
default: boolean
overridden: boolean
}
export type FeatureFlagSnapshot = Record<string, FeatureFlagState>
export type FeaturesMap = Record<string, boolean>
// Public read of effective feature flags. Available to any signed-in
// user so the frontend can hide sections that depend on a disabled
// pipeline stage (e.g. People when faces are off).
export const features = {
list: async (): Promise<FeaturesMap> => {
const response = await api.get('/features')
return response.data
},
}
// ── Nextcloud integration ──────────────────────────────────────────────

10
frontend/src/store/filterStore.ts
View File

@@ -30,8 +30,6 @@ export interface FilterState {
tagIds: string[]
/** When true, restrict to photos flagged as duplicates by the scanner. */
duplicates: boolean
/** When true, restrict to photos classified as 'other' (needs_review). */
needsReview: boolean
/** Visual grouping mode. 'date' groups by month when sortBy is a date
* field; 'tag' groups by photo tag membership. Independent of filters. */
groupBy: GroupBy
@@ -68,7 +66,6 @@ interface FilterStore extends FilterState {
setTagIds: (ids: string[]) => void
toggleTagId: (id: string) => void
setDuplicates: (v: boolean) => void
setNeedsReview: (v: boolean) => void
setGroupBy: (mode: GroupBy) => void
setSortBy: (field: SortField) => void
setSortOrder: (order: SortOrder) => void
@@ -102,7 +99,6 @@ export const INITIAL_FILTERS: FilterState = {
folderId: null,
tagIds: [],
duplicates: false,
needsReview: false,
groupBy: 'date',
sortBy: 'taken_at',
sortOrder: 'desc',
@@ -124,7 +120,6 @@ function snapshotFilters(s: FilterState): FilterState {
folderId: s.folderId,
tagIds: [...s.tagIds],
duplicates: s.duplicates,
needsReview: s.needsReview,
groupBy: s.groupBy,
sortBy: s.sortBy,
sortOrder: s.sortOrder,
@@ -159,7 +154,6 @@ export const useFilterStore = create<FilterStore>((set) => ({
: [...s.tagIds, id],
})),
setDuplicates: (duplicates) => set({ duplicates }),
setNeedsReview: (needsReview) => set({ needsReview }),
setGroupBy: (groupBy) => set({ groupBy }),
setSortBy: (sortBy) => set({ sortBy }),
setSortOrder: (sortOrder) => set({ sortOrder }),
@@ -223,7 +217,6 @@ export function filtersToParams(f: FilterState): Record<string, string | number>
if (f.folderId) params.folder_id = f.folderId
if (f.tagIds.length > 0) params.tag_ids = f.tagIds.join(',')
if (f.duplicates) params.is_duplicate = 'true'
if (f.needsReview) params.needs_review = 'true'
params.sort = f.sortBy
params.order = f.sortOrder
return params
@@ -242,7 +235,6 @@ export function hasActiveFilters(f: FilterState): boolean {
f.heapId !== null ||
f.folderId !== null ||
f.tagIds.length > 0 ||
f.duplicates ||
f.needsReview
f.duplicates
)
}

1
frontend/src/types/photo.ts
View File

@@ -18,7 +18,6 @@ export interface Photo {
user_notes?: string | null
is_discarded: boolean
is_duplicate: boolean
needs_review?: boolean
has_date_warning?: boolean
file_hash: string
folder_id: string | null