dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(nextcloud): hard-delete SourceRoot + reliable delete sync

Browse Source 
Two related fixes for the Nextcloud library lifecycle.

1. DELETE /api/v1/nextcloud/source-roots/{id} now actually deletes
   the SourceRoot, every Folder under it, and every Photo in those
   folders (Nextcloud files untouched). Was a soft-deactivate
   (is_active=false) that left the rows around forever, so re-adding
   the same path resurrected ghosts and prune-missing reported zero.
   Returns {deleted_photos, deleted_folders}; the Settings UI toasts
   the count and invalidates photos/folders/stats so cached lists
   don't show ghosts. photo_tags and heap_photos already cascade via
   ON DELETE CASCADE; FolderShare uses a stringly-typed folder_id
   with no FK so cleaned up explicitly.

2. The watcher (watch_folders task) was getting killed every five
   minutes by the global task_soft_time_limit=300 in app/tasks/celery.py
   despite passing soft_time_limit=None on the decorator (None falls
   back to the worker default in this Celery version). Override with
   soft_time_limit=0, time_limit=0 (= unlimited) so the watch loop
   actually stays alive. The 'Soft time limit (300s) exceeded' /
   'Worker exited prematurely' lines should stop in worker-watcher
   logs.

3. Added discard_missing_photos() in services/cleanup.py — a soft
   variant of prune_missing_photos that walks every present source
   root, checks os.path.exists for each non-discarded Photo, and
   flips is_discarded=true on the missing ones (UPDATE not DELETE).
   Wired as discard_missing_photos_beat in tasks/scan.py and
   scheduled every 30 min via celery beat. Beat runs in-process on
   worker-watcher (--beat flag in compose) — there's only ever one
   watcher and we don't need a separate container.

Hard delete remains manual via prune-missing for users who want to
review before committing. The beat catch-up only soft-discards (file
gone -> mule-image trash, restorable).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claudio
2026-05-10 22:19:58 +02:00
parent 99d504842e
commit 09a00f7419
7 changed files with 174 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
backend/app/routers/nextcloud.py
View File

@@ -30,12 +30,14 @@ from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.dependencies import get_current_user
from app.models.folders import SourceRoot, Folder
from app.models.photos import Photo
from app.models.sharing import FolderShare
from app.models.user import User
from app.services.nextcloud_dav import (
NEXTCLOUD_USERS_ROOT,
@@ -267,16 +269,25 @@ async def create_nextcloud_source_root(
}
@router.delete("/source-roots/{source_root_id}", status_code=204)
@router.delete("/source-roots/{source_root_id}")
async def delete_nextcloud_source_root(
source_root_id: str,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user),
):
"""Soft-delete: deactivate the SourceRoot and orphan its photo rows
(mark them is_discarded). The actual files in Nextcloud are NOT
touched — this is just unwiring the user's view of the library.
The user can re-add the same path later to restart scanning."""
"""Hard-delete: drop the SourceRoot row and cascade-delete every
Folder and Photo underneath it. The actual files in Nextcloud are
NOT touched — this is just unwiring the user's library in mule-image.
Implementation notes:
- photo_tags and heap_photos cascade via DB-level ON DELETE CASCADE,
so deleting Photo rows is enough to clean those up.
- FolderShare uses a stringly-typed folder_id (no FK), so we have
to clean those rows by hand for both the SourceRoot itself and
every Folder we're about to delete.
- Chunked at 500 to mirror prune_missing_photos so postgres doesn't
choke on a 21k-photo source root.
"""
sr = (await db.execute(
select(SourceRoot).where(
SourceRoot.id == source_root_id,
@@ -291,11 +302,49 @@ async def delete_nextcloud_source_root(
detail="This endpoint only manages Nextcloud-rooted SourceRoots",
)
# Soft-deactivate. Existing photos stay in the DB but won't appear
# in active queries (folders.py filters by is_active=true).
sr.is_active = False
folder_ids = (await db.execute(
select(Folder.id).where(Folder.source_root_id == sr.id)
)).scalars().all()
photo_ids: list[str] = []
if folder_ids:
photo_ids = (await db.execute(
select(Photo.id).where(Photo.folder_id.in_(folder_ids))
)).scalars().all()
CHUNK = 500
for i in range(0, len(photo_ids), CHUNK):
chunk = photo_ids[i:i + CHUNK]
await db.execute(delete(Photo).where(Photo.id.in_(chunk)))
# FolderShare rows: not a real FK, clean both 'source_root' and
# 'folder' typed shares pointing at anything we're tearing down.
await db.execute(
delete(FolderShare).where(
FolderShare.folder_id == sr.id,
FolderShare.folder_type == 'source_root',
)
)
if folder_ids:
await db.execute(
delete(FolderShare).where(
FolderShare.folder_id.in_(folder_ids),
FolderShare.folder_type == 'folder',
)
)
await db.execute(delete(Folder).where(Folder.id.in_(folder_ids)))
await db.delete(sr)
await db.commit()
return None
logger.info(
f"Deleted SourceRoot {sr.id} ({sr.name}): "
f"{len(photo_ids)} photos, {len(folder_ids)} folders"
)
return {
"deleted_photos": len(photo_ids),
"deleted_folders": len(folder_ids),
}
# ---------------------------------------------------------------------------

65
backend/app/services/cleanup.py
View File

@@ -299,6 +299,71 @@ async def prune_missing_photos(dry_run: bool = True) -> dict:
raise
async def discard_missing_photos() -> dict:
"""Soft variant of prune_missing_photos for the periodic beat
catch-up. Walks every active source root that is currently
`present` (not 'renamed' — the user-driven manual flow handles
those — and not 'unmounted'), and for each Photo whose file is
gone from disk, sets is_discarded=True so it shows up in the
in-app trash. Idempotent: skips photos that are already
discarded.
Hard-deletion stays manual via prune_missing_photos so users
can review the list before committing.
"""
async with AsyncSessionLocal() as session:
try:
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
present_sr_ids = {
sr.id for sr in sr_rows if _sr_state(sr.path) == 'present'
}
if not present_sr_ids:
return {"discarded": 0, "checked": 0}
folder_to_sr = {
fid: srid
for fid, _path, srid in (await session.execute(
select(Folder.id, Folder.path, Folder.source_root_id)
)).all()
}
photo_rows = (await session.execute(
select(Photo.id, Photo.filepath, Photo.folder_id)
.where(Photo.is_discarded.is_(False))
)).all()
missing_ids: list[str] = []
checked = 0
for pid, fp, folder_id in photo_rows:
sr_id = folder_to_sr.get(folder_id)
if sr_id not in present_sr_ids:
continue
checked += 1
if not os.path.exists(fp):
missing_ids.append(pid)
if missing_ids:
CHUNK = 500
now = datetime.utcnow()
for i in range(0, len(missing_ids), CHUNK):
await session.execute(
update(Photo)
.where(Photo.id.in_(missing_ids[i:i + CHUNK]))
.values(is_discarded=True, discarded_at=now)
)
await session.commit()
logger.info(
f"discard_missing_photos: discarded {len(missing_ids)} "
f"of {checked} photos under {len(present_sr_ids)} present source roots"
)
return {"discarded": len(missing_ids), "checked": checked}
except Exception as e:
logger.error(f"discard_missing_photos failed: {e}")
await session.rollback()
raise
async def cleanup_data_integrity() -> dict:
"""Top-level entry point. Runs the dedupe + count refresh in a single
transaction. Returns a small summary dict for logging."""

10
backend/app/tasks/celery.py
View File

@@ -54,12 +54,22 @@ celery_app.conf.update(
'backfill_gps': {'queue': 'low'},
# Dedicated watcher queue
'watch_folders': {'queue': 'watcher'},
'discard_missing_photos_beat': {'queue': 'low'},
},
task_default_queue='default',
task_default_exchange='default',
task_default_exchange_type='direct',
task_default_routing_key='default',
broker_connection_retry_on_startup=True,
# Periodic catch-up so external file deletions in Nextcloud get
# reflected even when the real-time watcher missed the event
# (worker restart window, mount transient, etc).
beat_schedule={
'discard-missing-photos-every-30min': {
'task': 'discard_missing_photos_beat',
'schedule': 30 * 60,
},
},
)

18
backend/app/tasks/scan.py
View File

@@ -474,7 +474,7 @@ WATCHER_LOCK_KEY = "mulita:watch_folders:lock"
WATCHER_LOCK_TTL = 60 # 1 min — renewed every event batch via wall-clock check
@shared_task(name='watch_folders', bind=True, soft_time_limit=None, time_limit=None)
@shared_task(name='watch_folders', bind=True, soft_time_limit=0, time_limit=0)
def watch_folders(self):
"""
Watch folders for changes using watchfiles. Long-running task that
@@ -615,4 +615,18 @@ async def _backfill_gps_async():
extract_metadata.delay(pid)
logger.info(f"backfill_gps: queued extract_metadata for {len(photo_ids)} photos")
return {'queued': len(photo_ids)}
return {'queued': len(photo_ids)}
@shared_task(name='discard_missing_photos_beat')
def discard_missing_photos_beat():
"""Periodic catch-up for filesystem deletions the watcher missed
(e.g. while the worker was restarting). Walks every active source
root that is currently mounted and present, and soft-discards any
Photo whose file is gone. Hard-deletion stays manual via
POST /api/v1/library/maintenance/prune-missing.
Wired to a 30-minute beat schedule in app/tasks/celery.py.
"""
from app.services.cleanup import discard_missing_photos
return asyncio.run(discard_missing_photos())

6
docker-compose.yml
View File

@@ -197,7 +197,11 @@ services:
dockerfile: Dockerfile
image: mule-image-worker
container_name: mulita-worker-watcher
command: sh -c "celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=1 -Q watcher -n watcher@%h"
# --beat runs the celery beat scheduler in-process alongside the
# watcher worker — there's only ever one watcher (Redis-locked
# singleton) and we don't need a separate container just to fire a
# 30-minute periodic task. Beat schedule lives in app/tasks/celery.py.
command: sh -c "celery -A app.tasks.celery worker --beat --loglevel=${LOG_LEVEL:-info} --concurrency=1 -Q watcher -n watcher@%h"
volumes:
- ./mulita.yml:/app/config/mulita.yml:ro
- ${PHOTO_DIRS:-./photos}:/photos:rw

13
frontend/src/components/dialogs/SettingsDialog.tsx
View File

@@ -1401,9 +1401,18 @@ function NextcloudIntegrationCard() {
const removeRoot = useMutation({
mutationFn: async (id: string) => nextcloudApi.deleteSourceRoot(id),
onSuccess: () => {
toast.success('Nextcloud library removed')
onSuccess: ({ deleted_photos, deleted_folders }) => {
const photos = `${deleted_photos} ${deleted_photos === 1 ? 'photo' : 'photos'}`
const folders = `${deleted_folders} ${deleted_folders === 1 ? 'folder' : 'folders'}`
toast.success(`Removed library — ${photos}, ${folders}. Files in Nextcloud are untouched.`)
queryClient.invalidateQueries({ queryKey: NEXTCLOUD_ROOTS_KEY })
// The list of photos/folders/heaps the rest of the app caches is
// now stale — invalidate everything photo-shaped so the user
// doesn't see ghosts of the removed library until next refresh.
queryClient.invalidateQueries({ queryKey: ['photos'] })
queryClient.invalidateQueries({ queryKey: ['folders'] })
queryClient.invalidateQueries({ queryKey: ['folders', 'tree'] })
queryClient.invalidateQueries({ queryKey: ['library', 'stats'] })
},
onError: (e) => {
const message = e instanceof Error ? e.message : String(e)

10
frontend/src/services/api.ts
View File

@@ -1139,8 +1139,14 @@ export const nextcloud = {
return response.data
},
deleteSourceRoot: async (id: string): Promise<void> => {
await api.delete(`/nextcloud/source-roots/${id}`)
deleteSourceRoot: async (
id: string,
): Promise<{ deleted_photos: number; deleted_folders: number }> => {
const response = await api.delete<{
deleted_photos: number
deleted_folders: number
}>(`/nextcloud/source-roots/${id}`)
return response.data
},
}