diff --git a/backend/app/routers/nextcloud.py b/backend/app/routers/nextcloud.py index 2115c8a..552b1be 100644 --- a/backend/app/routers/nextcloud.py +++ b/backend/app/routers/nextcloud.py @@ -30,12 +30,14 @@ from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel -from sqlalchemy import select +from sqlalchemy import delete, select from sqlalchemy.ext.asyncio import AsyncSession from app.database import get_db from app.dependencies import get_current_user from app.models.folders import SourceRoot, Folder +from app.models.photos import Photo +from app.models.sharing import FolderShare from app.models.user import User from app.services.nextcloud_dav import ( NEXTCLOUD_USERS_ROOT, @@ -267,16 +269,25 @@ async def create_nextcloud_source_root( } -@router.delete("/source-roots/{source_root_id}", status_code=204) +@router.delete("/source-roots/{source_root_id}") async def delete_nextcloud_source_root( source_root_id: str, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user), ): - """Soft-delete: deactivate the SourceRoot and orphan its photo rows - (mark them is_discarded). The actual files in Nextcloud are NOT - touched — this is just unwiring the user's view of the library. - The user can re-add the same path later to restart scanning.""" + """Hard-delete: drop the SourceRoot row and cascade-delete every + Folder and Photo underneath it. The actual files in Nextcloud are + NOT touched — this is just unwiring the user's library in mule-image. + + Implementation notes: + - photo_tags and heap_photos cascade via DB-level ON DELETE CASCADE, + so deleting Photo rows is enough to clean those up. + - FolderShare uses a stringly-typed folder_id (no FK), so we have + to clean those rows by hand for both the SourceRoot itself and + every Folder we're about to delete. + - Chunked at 500 to mirror prune_missing_photos so postgres doesn't + choke on a 21k-photo source root. + """ sr = (await db.execute( select(SourceRoot).where( SourceRoot.id == source_root_id, @@ -291,11 +302,49 @@ async def delete_nextcloud_source_root( detail="This endpoint only manages Nextcloud-rooted SourceRoots", ) - # Soft-deactivate. Existing photos stay in the DB but won't appear - # in active queries (folders.py filters by is_active=true). - sr.is_active = False + folder_ids = (await db.execute( + select(Folder.id).where(Folder.source_root_id == sr.id) + )).scalars().all() + + photo_ids: list[str] = [] + if folder_ids: + photo_ids = (await db.execute( + select(Photo.id).where(Photo.folder_id.in_(folder_ids)) + )).scalars().all() + + CHUNK = 500 + for i in range(0, len(photo_ids), CHUNK): + chunk = photo_ids[i:i + CHUNK] + await db.execute(delete(Photo).where(Photo.id.in_(chunk))) + + # FolderShare rows: not a real FK, clean both 'source_root' and + # 'folder' typed shares pointing at anything we're tearing down. + await db.execute( + delete(FolderShare).where( + FolderShare.folder_id == sr.id, + FolderShare.folder_type == 'source_root', + ) + ) + if folder_ids: + await db.execute( + delete(FolderShare).where( + FolderShare.folder_id.in_(folder_ids), + FolderShare.folder_type == 'folder', + ) + ) + await db.execute(delete(Folder).where(Folder.id.in_(folder_ids))) + + await db.delete(sr) await db.commit() - return None + + logger.info( + f"Deleted SourceRoot {sr.id} ({sr.name}): " + f"{len(photo_ids)} photos, {len(folder_ids)} folders" + ) + return { + "deleted_photos": len(photo_ids), + "deleted_folders": len(folder_ids), + } # --------------------------------------------------------------------------- diff --git a/backend/app/services/cleanup.py b/backend/app/services/cleanup.py index 60fb488..14c8852 100644 --- a/backend/app/services/cleanup.py +++ b/backend/app/services/cleanup.py @@ -299,6 +299,71 @@ async def prune_missing_photos(dry_run: bool = True) -> dict: raise +async def discard_missing_photos() -> dict: + """Soft variant of prune_missing_photos for the periodic beat + catch-up. Walks every active source root that is currently + `present` (not 'renamed' — the user-driven manual flow handles + those — and not 'unmounted'), and for each Photo whose file is + gone from disk, sets is_discarded=True so it shows up in the + in-app trash. Idempotent: skips photos that are already + discarded. + + Hard-deletion stays manual via prune_missing_photos so users + can review the list before committing. + """ + async with AsyncSessionLocal() as session: + try: + sr_rows = (await session.execute(select(SourceRoot))).scalars().all() + present_sr_ids = { + sr.id for sr in sr_rows if _sr_state(sr.path) == 'present' + } + if not present_sr_ids: + return {"discarded": 0, "checked": 0} + + folder_to_sr = { + fid: srid + for fid, _path, srid in (await session.execute( + select(Folder.id, Folder.path, Folder.source_root_id) + )).all() + } + + photo_rows = (await session.execute( + select(Photo.id, Photo.filepath, Photo.folder_id) + .where(Photo.is_discarded.is_(False)) + )).all() + + missing_ids: list[str] = [] + checked = 0 + for pid, fp, folder_id in photo_rows: + sr_id = folder_to_sr.get(folder_id) + if sr_id not in present_sr_ids: + continue + checked += 1 + if not os.path.exists(fp): + missing_ids.append(pid) + + if missing_ids: + CHUNK = 500 + now = datetime.utcnow() + for i in range(0, len(missing_ids), CHUNK): + await session.execute( + update(Photo) + .where(Photo.id.in_(missing_ids[i:i + CHUNK])) + .values(is_discarded=True, discarded_at=now) + ) + await session.commit() + logger.info( + f"discard_missing_photos: discarded {len(missing_ids)} " + f"of {checked} photos under {len(present_sr_ids)} present source roots" + ) + + return {"discarded": len(missing_ids), "checked": checked} + except Exception as e: + logger.error(f"discard_missing_photos failed: {e}") + await session.rollback() + raise + + async def cleanup_data_integrity() -> dict: """Top-level entry point. Runs the dedupe + count refresh in a single transaction. Returns a small summary dict for logging.""" diff --git a/backend/app/tasks/celery.py b/backend/app/tasks/celery.py index ad5118e..f4f7c80 100644 --- a/backend/app/tasks/celery.py +++ b/backend/app/tasks/celery.py @@ -54,12 +54,22 @@ celery_app.conf.update( 'backfill_gps': {'queue': 'low'}, # Dedicated watcher queue 'watch_folders': {'queue': 'watcher'}, + 'discard_missing_photos_beat': {'queue': 'low'}, }, task_default_queue='default', task_default_exchange='default', task_default_exchange_type='direct', task_default_routing_key='default', broker_connection_retry_on_startup=True, + # Periodic catch-up so external file deletions in Nextcloud get + # reflected even when the real-time watcher missed the event + # (worker restart window, mount transient, etc). + beat_schedule={ + 'discard-missing-photos-every-30min': { + 'task': 'discard_missing_photos_beat', + 'schedule': 30 * 60, + }, + }, ) diff --git a/backend/app/tasks/scan.py b/backend/app/tasks/scan.py index 0b82b8b..e7b974c 100644 --- a/backend/app/tasks/scan.py +++ b/backend/app/tasks/scan.py @@ -474,7 +474,7 @@ WATCHER_LOCK_KEY = "mulita:watch_folders:lock" WATCHER_LOCK_TTL = 60 # 1 min — renewed every event batch via wall-clock check -@shared_task(name='watch_folders', bind=True, soft_time_limit=None, time_limit=None) +@shared_task(name='watch_folders', bind=True, soft_time_limit=0, time_limit=0) def watch_folders(self): """ Watch folders for changes using watchfiles. Long-running task that @@ -615,4 +615,18 @@ async def _backfill_gps_async(): extract_metadata.delay(pid) logger.info(f"backfill_gps: queued extract_metadata for {len(photo_ids)} photos") - return {'queued': len(photo_ids)} \ No newline at end of file + return {'queued': len(photo_ids)} + + +@shared_task(name='discard_missing_photos_beat') +def discard_missing_photos_beat(): + """Periodic catch-up for filesystem deletions the watcher missed + (e.g. while the worker was restarting). Walks every active source + root that is currently mounted and present, and soft-discards any + Photo whose file is gone. Hard-deletion stays manual via + POST /api/v1/library/maintenance/prune-missing. + + Wired to a 30-minute beat schedule in app/tasks/celery.py. + """ + from app.services.cleanup import discard_missing_photos + return asyncio.run(discard_missing_photos()) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index e412235..0ccf5a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -197,7 +197,11 @@ services: dockerfile: Dockerfile image: mule-image-worker container_name: mulita-worker-watcher - command: sh -c "celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=1 -Q watcher -n watcher@%h" + # --beat runs the celery beat scheduler in-process alongside the + # watcher worker — there's only ever one watcher (Redis-locked + # singleton) and we don't need a separate container just to fire a + # 30-minute periodic task. Beat schedule lives in app/tasks/celery.py. + command: sh -c "celery -A app.tasks.celery worker --beat --loglevel=${LOG_LEVEL:-info} --concurrency=1 -Q watcher -n watcher@%h" volumes: - ./mulita.yml:/app/config/mulita.yml:ro - ${PHOTO_DIRS:-./photos}:/photos:rw diff --git a/frontend/src/components/dialogs/SettingsDialog.tsx b/frontend/src/components/dialogs/SettingsDialog.tsx index 46f4d65..1c7c40e 100644 --- a/frontend/src/components/dialogs/SettingsDialog.tsx +++ b/frontend/src/components/dialogs/SettingsDialog.tsx @@ -1401,9 +1401,18 @@ function NextcloudIntegrationCard() { const removeRoot = useMutation({ mutationFn: async (id: string) => nextcloudApi.deleteSourceRoot(id), - onSuccess: () => { - toast.success('Nextcloud library removed') + onSuccess: ({ deleted_photos, deleted_folders }) => { + const photos = `${deleted_photos} ${deleted_photos === 1 ? 'photo' : 'photos'}` + const folders = `${deleted_folders} ${deleted_folders === 1 ? 'folder' : 'folders'}` + toast.success(`Removed library — ${photos}, ${folders}. Files in Nextcloud are untouched.`) queryClient.invalidateQueries({ queryKey: NEXTCLOUD_ROOTS_KEY }) + // The list of photos/folders/heaps the rest of the app caches is + // now stale — invalidate everything photo-shaped so the user + // doesn't see ghosts of the removed library until next refresh. + queryClient.invalidateQueries({ queryKey: ['photos'] }) + queryClient.invalidateQueries({ queryKey: ['folders'] }) + queryClient.invalidateQueries({ queryKey: ['folders', 'tree'] }) + queryClient.invalidateQueries({ queryKey: ['library', 'stats'] }) }, onError: (e) => { const message = e instanceof Error ? e.message : String(e) diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 918dea8..a77ad68 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -1139,8 +1139,14 @@ export const nextcloud = { return response.data }, - deleteSourceRoot: async (id: string): Promise => { - await api.delete(`/nextcloud/source-roots/${id}`) + deleteSourceRoot: async ( + id: string, + ): Promise<{ deleted_photos: number; deleted_folders: number }> => { + const response = await api.delete<{ + deleted_photos: number + deleted_folders: number + }>(`/nextcloud/source-roots/${id}`) + return response.data }, }