dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(library): tighten duplicates scope, drop in-app upload, polish loaders

Browse Source 
- duplicates: restrict the /library/duplicates/groups query to photos whose
  folder path actually lives under an active SourceRoot in the user's
  settings. Nextcloud's "move to trash" flow was leaving .delete/purge-1
  Folder rows wired to the original source_root_id, leaking those entries
  into the Duplicates view as ghost paths that the user never opted into.
- discard: add a spinner to the "Delete N" and "Empty discard pile"
  buttons (and their confirm dialogs) while the destructive mutation is
  in flight, so the user gets immediate feedback for a slow operation.
- timeline: render a bottom-of-grid "Loading more photos…" indicator
  while usePhotosQuery's background cursor loop is still pulling pages.
  Backed by a tiny Zustand store the loop drives via a balanced
  start/stop (counter, not boolean, so rapid filter changes can't flip
  the flag false while a fresh loop is alive).
- remove client-side upload UI + /upload endpoint. Nextcloud is the
  authoritative ingress now; the duplicate path created confusion and
  the backend route is gone too.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
claudio
2026-05-13 20:44:36 +02:00
parent 99edd7d395
commit 5e2823ae94
11 changed files with 151 additions and 1059 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/app/main.py
View File

@@ -12,7 +12,7 @@ import os
from app.config import settings
from app.database import init_db
from app.routers import photos, folders, heaps, tags, discard, library, search, auth, admin, sharing, upload, download, features, nextcloud, nc_webhook
from app.routers import photos, folders, heaps, tags, discard, library, search, auth, admin, sharing, download, features, nextcloud, nc_webhook
from app.services.scanner import start_initial_scan, bootstrap_default_source_root
from app.services.cleanup import cleanup_data_integrity
from app.services.nextcloud_dav import init_preview_client, close_preview_client
@@ -116,7 +116,6 @@ app.include_router(tags.router, prefix="/api/v1/tags", tags=["tags"])
app.include_router(discard.router, prefix="/api/v1/discard", tags=["discard"])
app.include_router(library.router, prefix="/api/v1/library", tags=["library"])
app.include_router(search.router, prefix="/api/v1/photos/search", tags=["search"])
app.include_router(upload.router, prefix="/api/v1/upload", tags=["upload"])
app.include_router(download.router, prefix="/api/v1/download", tags=["download"])
app.include_router(features.router, prefix="/api/v1/features", tags=["features"])
app.include_router(nextcloud.router, prefix="/api/v1/nextcloud", tags=["nextcloud"])

33
backend/app/routers/library.py
View File

@@ -13,12 +13,12 @@ from typing import List, Optional
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel, Field
from sqlalchemy import select, func, update, true as sa_true
from sqlalchemy import select, func, update, or_, true as sa_true
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models import Photo
from app.models.folders import SourceRoot
from app.models.folders import SourceRoot, Folder
from app.models.user import User
from app.dependencies import get_current_user
@@ -730,6 +730,33 @@ async def get_duplicate_groups(
* "similar" — members differ at the byte level but match perceptually
"""
owner = _owner_filter(current_user, scope)
# Restrict to photos that actually live under an active SourceRoot
# in the user's settings. The folder→source_root link alone is not
# enough: Nextcloud's "move to trash" flow can leave a Folder row at
# a path like `…/files/.delete/purge-1` still wired to the original
# source_root_id, which then leaks its photos into the duplicates
# view even though that path is outside everything the user
# configured. Filtering on the folder's path being a descendant of
# the active root's path matches what the user actually expects to
# see in Settings → Source folders.
sr_path_query = select(SourceRoot.path).where(SourceRoot.is_active.is_(True))
if not (scope == "global" and current_user.role == "admin"):
sr_path_query = sr_path_query.where(SourceRoot.user_id == current_user.id)
active_root_paths = (await db.execute(sr_path_query)).scalars().all()
if not active_root_paths:
return {"groups": [], "total_groups": 0, "total_members": 0}
# Match the source root path itself OR a strict child (path + '/').
# `startswith` alone would accept `/files/Photos2` for a `/files/Photos`
# root.
folder_in_scope = or_(
*[
(Folder.path == p) | (Folder.path.like(p.rstrip('/') + '/%'))
for p in active_root_paths
]
)
rows = (
await db.execute(
select(
@@ -746,10 +773,12 @@ async def get_duplicate_groups(
Photo.media_type,
Photo.duplicate_group_id,
)
.join(Folder, Folder.id == Photo.folder_id)
.where(owner)
.where(Photo.duplicate_group_id.is_not(None))
.where(Photo.is_discarded.is_(False))
.where(Photo.is_hidden.is_(False))
.where(folder_in_scope)
.order_by(Photo.duplicate_group_id)
)
).all()

361
backend/app/routers/upload.py
View File

@@ -1,361 +0,0 @@
"""
Upload router — lets users drop files (or whole folders) from their
desktop into a destination Folder, preserving any sub-folder structure
they bring with them.
Each POST handles one file. The frontend fans out many parallel requests
per drop, giving it per-file progress without the server having to
invent a chunking protocol. For folder uploads, the browser passes
`webkitRelativePath` under the `relative_path` field; any leading
sub-directories there are materialised on disk (and as Folder rows)
under the destination.
Uploaded files are placed under the destination folder on the owner's
media mount, indexed immediately (Photo row created), and queued for
the same thumb + metadata pipeline that the scanner uses. An optional
`heap_id` also drops them into a heap in the same request.
"""
import hashlib
import logging
import os
import tempfile
from pathlib import Path
from datetime import datetime
from typing import Optional
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
from sqlalchemy import insert, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.dependencies import get_current_user
from app.models import Folder, Heap, Photo, SourceRoot
from app.models.heaps import heap_photos
from app.models.user import User
from app.services.date_guess import has_date_warning
from app.services.nextcloud_dav import (
ensure_parents_for_user,
is_nextcloud_path,
put_for_user,
)
from app.tasks.scan import SUPPORTED_EXTENSIONS, get_media_type
from app.tasks.thumbs import generate_thumbnails
from app.services.metadata import extract_metadata
logger = logging.getLogger(__name__)
router = APIRouter()
MAX_UPLOAD_BYTES = 500 * 1024 * 1024 # 500 MB per file cap.
def _validate_segment(segment: str) -> str:
"""Reject path segments that would escape the destination directory."""
segment = segment.strip()
if not segment or segment in ('.', '..') or '/' in segment or '\\' in segment:
raise HTTPException(status_code=400, detail=f"Invalid path segment: {segment!r}")
return segment
def _sanitize_relative_path(rel: Optional[str]) -> list[str]:
"""Split `relative_path` into safe segments (dirs + filename).
Empty or missing → []. Any absolute path, backslash, or `..` segment
raises 400 — we never want an upload to escape the destination.
"""
if not rel:
return []
# Normalise backslashes to forward slashes; browsers on Windows send
# webkitRelativePath with forward slashes anyway, but defend in depth.
rel = rel.replace('\\', '/').strip('/')
if not rel:
return []
segs = [_validate_segment(s) for s in rel.split('/') if s]
return segs
async def _resolve_destination(
folder_id: str,
user: User,
db: AsyncSession,
) -> Folder:
"""Resolve `folder_id` to a concrete Folder row the user owns.
Accepts both Folder ids and SourceRoot ids (for source roots, we
return the Folder row at the mount path — the scanner creates one
for every source root it walks). Raises 404 if neither matches.
"""
folder = (await db.execute(
select(Folder).where(Folder.id == folder_id, Folder.user_id == user.id)
)).scalar_one_or_none()
if folder is not None:
return folder
sr = (await db.execute(
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == user.id)
)).scalar_one_or_none()
if sr is None:
raise HTTPException(status_code=404, detail="Destination folder not found")
root_folder = (await db.execute(
select(Folder).where(
Folder.source_root_id == sr.id,
Folder.user_id == user.id,
Folder.path == os.path.normpath(sr.path),
)
)).scalar_one_or_none()
if root_folder is None:
# First-time source root with no walk yet — create the row now so
# uploads work even before the initial scan has run.
root_folder = Folder(
name=sr.name or os.path.basename(sr.path),
path=os.path.normpath(sr.path),
source_root_id=sr.id,
user_id=user.id,
)
os.makedirs(root_folder.path, exist_ok=True)
db.add(root_folder)
await db.flush()
return root_folder
async def _ensure_subfolder(
parent: Folder,
name: str,
user: User,
db: AsyncSession,
) -> Folder:
"""Return (or create) a Folder row named `name` under `parent`.
Also mkdirs the directory on disk. For Nextcloud-rooted paths the
directory is created via WebDAV MKCOL so Nextcloud's `oc_filecache`
knows about it; otherwise plain `os.makedirs`. Idempotent — safe
to call for a path segment that already exists as a Folder row or
directory.
"""
child_path = os.path.normpath(os.path.join(parent.path, name))
existing = (await db.execute(
select(Folder).where(
Folder.path == child_path,
Folder.user_id == user.id,
)
)).scalar_one_or_none()
if existing is not None:
# Materialise the directory if it wasn't already.
if is_nextcloud_path(child_path):
ensure_parents_for_user(user, child_path)
# Also ensure the leaf collection exists; ensure_parents
# only handles intermediate dirs.
from app.services.nextcloud_dav import mkcol_for_user
mkcol_for_user(user, child_path)
else:
os.makedirs(child_path, exist_ok=True)
return existing
if is_nextcloud_path(child_path):
ensure_parents_for_user(user, child_path)
from app.services.nextcloud_dav import mkcol_for_user
mkcol_for_user(user, child_path)
else:
os.makedirs(child_path, exist_ok=True)
child = Folder(
name=name,
path=child_path,
parent_id=parent.id,
source_root_id=parent.source_root_id,
user_id=user.id,
is_hidden=parent.is_hidden,
)
db.add(child)
await db.flush()
return child
def _unique_path(target_dir: str, filename: str) -> tuple[str, str]:
"""Return a (filepath, filename) that doesn't collide with an
existing file on disk. Suffixes " (2)", " (3)", ... until a free
slot is found. Prevents upload-over-existing and keeps the user's
original file intact.
"""
base, ext = os.path.splitext(filename)
candidate = os.path.join(target_dir, filename)
n = 2
while os.path.exists(candidate):
new_name = f"{base} ({n}){ext}"
candidate = os.path.join(target_dir, new_name)
n += 1
return candidate, os.path.basename(candidate)
@router.post("")
async def upload_file(
file: UploadFile = File(...),
destination_folder_id: str = Form(...),
relative_path: Optional[str] = Form(None),
heap_id: Optional[str] = Form(None),
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user),
):
"""Upload a single file into a destination folder (and optionally a
heap). For folder uploads, `relative_path` carries the sub-folder
chain from the browser's `webkitRelativePath`, and we materialise
it under the destination on disk + as Folder rows.
Returns the created photo's id on success. 4xx on unsupported file
type, bad path, missing destination, or too-large file.
"""
# --- validate inputs -------------------------------------------------
raw_name = file.filename or ''
if not raw_name:
raise HTTPException(status_code=400, detail="Missing filename")
# Prefer the leaf of relative_path when present (it contains the
# original filename as the browser saw it inside the picked folder).
segs = _sanitize_relative_path(relative_path)
if segs:
leaf = segs[-1]
subdirs = segs[:-1]
else:
leaf = _validate_segment(os.path.basename(raw_name))
subdirs = []
ext = Path(leaf).suffix.lower()
if ext not in SUPPORTED_EXTENSIONS:
raise HTTPException(
status_code=400,
detail=f"Unsupported file type: {ext or '(none)'}",
)
dest_folder = await _resolve_destination(destination_folder_id, current_user, db)
target_folder = dest_folder
for seg in subdirs:
target_folder = await _ensure_subfolder(target_folder, seg, current_user, db)
target_dir = target_folder.path
nc_managed = is_nextcloud_path(target_dir)
if not nc_managed:
os.makedirs(target_dir, exist_ok=True)
# else: target_dir was created via WebDAV MKCOL by _ensure_subfolder
filepath, final_name = _unique_path(target_dir, leaf)
# --- stream the upload to a tempfile, hashing as we go ---------------
# For Nextcloud-managed destinations we then PUT the tempfile to
# WebDAV so Nextcloud's oc_filecache + sharing/comment metadata stay
# consistent. For local destinations we just rename the tempfile
# into place. Either way the hash + size are computed once.
hasher = hashlib.sha256()
total = 0
tmp_dir = os.path.dirname(filepath) if not nc_managed else None
# NamedTemporaryFile in the same directory as filepath when local
# (so the final rename is atomic on the same filesystem). For NC,
# use the system tmpdir — we re-upload via HTTP either way.
tmp = tempfile.NamedTemporaryFile(
delete=False, dir=tmp_dir, suffix=".part"
)
tmp_path = tmp.name
try:
try:
while True:
chunk = await file.read(1024 * 1024)
if not chunk:
break
total += len(chunk)
if total > MAX_UPLOAD_BYTES:
raise HTTPException(
status_code=413,
detail=f"File exceeds {MAX_UPLOAD_BYTES // (1024*1024)}MB limit",
)
hasher.update(chunk)
tmp.write(chunk)
finally:
tmp.close()
if nc_managed:
# PUT to Nextcloud WebDAV. The PUT lands the bytes on
# `/mnt/library/homecloud/<nc_user>/files/<rel>` AND
# registers the file in oc_filecache, so the desktop sync
# client and Nextcloud's web UI both see it.
with open(tmp_path, "rb") as body:
put_for_user(current_user, filepath, body)
else:
os.replace(tmp_path, filepath)
tmp_path = None # consumed
except HTTPException:
raise
except Exception as e:
logger.error(f"Upload write failed for {filepath}: {e}")
raise HTTPException(status_code=500, detail=f"Upload failed: {e}")
finally:
if tmp_path and os.path.exists(tmp_path):
try:
os.unlink(tmp_path)
except OSError:
pass
if not os.path.exists(filepath):
# WebDAV wrote it; the bind mount should reflect it. If it
# doesn't, surface a clean error rather than building a Photo
# row that points at a missing file.
raise HTTPException(
status_code=502,
detail="Nextcloud accepted the upload but the file isn't visible on the mount yet.",
)
file_hash = hasher.hexdigest()
# --- validate heap before committing the DB row ---------------------
if heap_id:
heap = (await db.execute(
select(Heap).where(Heap.id == heap_id, Heap.user_id == current_user.id)
)).scalar_one_or_none()
if heap is None:
# Destination heap vanished — still keep the file + photo row,
# but tell the caller so the UI can surface the mismatch.
heap_id = None
# --- create Photo row ------------------------------------------------
mtime_dt = datetime.fromtimestamp(os.stat(filepath).st_mtime)
photo = Photo(
filepath=filepath,
filename=final_name,
folder_id=target_folder.id,
user_id=current_user.id,
file_hash=file_hash,
media_type=get_media_type(filepath),
original_format=Path(filepath).suffix.upper()[1:],
file_size=total,
taken_at=mtime_dt,
taken_at_source='filesystem',
has_date_warning=has_date_warning(filepath, mtime_dt),
is_hidden=bool(target_folder.is_hidden),
processing_status='pending',
)
db.add(photo)
await db.flush()
if heap_id:
await db.execute(
insert(heap_photos),
[{"heap_id": heap_id, "photo_id": photo.id}],
)
await db.commit()
# Queue the same background work the scanner does so thumbnails +
# EXIF show up without the user having to trigger a rescan.
try:
generate_thumbnails.delay(photo.id)
extract_metadata.delay(photo.id)
except Exception as e:
logger.warning(f"Failed to queue post-upload tasks for {photo.id}: {e}")
return {
"photo_id": photo.id,
"filename": final_name,
"folder_id": target_folder.id,
"folder_path": target_folder.path,
"heap_id": heap_id,
}