feat(thumbs): proxy Nextcloud previews instead of duplicating the cache
mule-image was generating and storing three WebP sizes per photo in /data/thumbs while Nextcloud already keeps its own previews for the same source files. Frontend thumbnail requests now proxy NC's /index.php/core/preview keyed by the photo's Nextcloud fileid, authenticated with the owner's encrypted app password. - new column photos.nextcloud_fileid (alembic 0018) plus an index - get_preview_async + fetch_fileid helpers in nextcloud_dav.py - thumb route proxies NC primary, falls back to /data/thumbs (legacy rows / NC unreachable) so a single-file revert restores the old path - extract_metadata caches the fileid on first run for new photos - generate_thumbnails now writes only medium since the vision worker still loads it from disk; small + large drop out of the worker path - backend/scripts/backfill_nextcloud_fileid.py for one-shot population of existing rows: docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid X-Mule-Thumb-Source response header marks each request 'nextcloud' or 'disk' for observability while the rollout settles.
This commit is contained in:
@@ -204,11 +204,39 @@ async def _extract_metadata_async(photo_id: str):
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
|
||||
|
||||
if not photo:
|
||||
logger.error(f"Photo not found: {photo_id}")
|
||||
return {'status': 'error', 'message': 'Photo not found'}
|
||||
|
||||
|
||||
# Cache Nextcloud's numeric fileid on the row so the thumbnail
|
||||
# handler can proxy /index.php/core/preview without doing a
|
||||
# PROPFIND per request. PROPFIND blocks for ~50ms; tolerable
|
||||
# because extract_metadata already does seconds of ExifTool
|
||||
# work. Failures are silent — the thumb handler falls back
|
||||
# to its on-disk path when the column is NULL.
|
||||
if photo.nextcloud_fileid is None and photo.user_id:
|
||||
from app.models.user import User
|
||||
from app.services.nextcloud_dav import (
|
||||
fetch_fileid, is_nextcloud_path,
|
||||
)
|
||||
if photo.filepath and is_nextcloud_path(photo.filepath):
|
||||
owner = (
|
||||
await session.execute(
|
||||
select(User).where(User.id == photo.user_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if owner is not None and owner.nextcloud_app_password_enc:
|
||||
try:
|
||||
fid = fetch_fileid(owner, photo.filepath)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"fileid lookup failed for %s: %s", photo_id, e
|
||||
)
|
||||
fid = None
|
||||
if fid is not None:
|
||||
photo.nextcloud_fileid = fid
|
||||
|
||||
# Check if file exists
|
||||
if not Path(photo.filepath).exists():
|
||||
logger.error(f"File not found: {photo.filepath}")
|
||||
|
||||
@@ -291,6 +291,90 @@ def ensure_parents_for_user(user: User, abs_path: str) -> None:
|
||||
mkcol_for_user(user, sub_abs)
|
||||
|
||||
|
||||
_FILEID_PROPFIND = (
|
||||
b'<?xml version="1.0"?>'
|
||||
b'<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns">'
|
||||
b'<d:prop><oc:fileid/></d:prop>'
|
||||
b'</d:propfind>'
|
||||
)
|
||||
|
||||
|
||||
def fetch_fileid(user: User, abs_path: str) -> Optional[int]:
|
||||
"""Look up Nextcloud's numeric fileid for the file at `abs_path`.
|
||||
|
||||
`abs_path` is the absolute filesystem path under the bind mount,
|
||||
e.g. `/nextcloud-users/admin/files/Photos/2024/01/foo.jpg`. Returns
|
||||
None when the file isn't under a Nextcloud-rooted tree, the user
|
||||
has no app password set, or Nextcloud returns 404 — callers should
|
||||
treat None as "skip this row" rather than an error.
|
||||
|
||||
Used by `scripts/backfill_nextcloud_fileid.py`. The hot path (the
|
||||
thumbnail handler) reads `Photo.nextcloud_fileid` directly so it
|
||||
doesn't round-trip to Nextcloud per request.
|
||||
"""
|
||||
if not is_nextcloud_path(abs_path):
|
||||
return None
|
||||
try:
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
except NextcloudCredentialsMissing:
|
||||
return None
|
||||
try:
|
||||
expected_user, rel = split_nextcloud_path(abs_path)
|
||||
except ValueError:
|
||||
return None
|
||||
if expected_user != nc_user:
|
||||
return None
|
||||
url = _dav_url(nc_user, rel)
|
||||
with _client((nc_user, app_pw)) as c:
|
||||
resp = c.request(
|
||||
"PROPFIND",
|
||||
url,
|
||||
headers={"Depth": "0", "Content-Type": "application/xml"},
|
||||
content=_FILEID_PROPFIND,
|
||||
)
|
||||
if resp.status_code == 404:
|
||||
return None
|
||||
if not resp.is_success:
|
||||
logger.warning(
|
||||
"Nextcloud PROPFIND %s returned %s", rel, resp.status_code
|
||||
)
|
||||
return None
|
||||
import re as _re
|
||||
m = _re.search(rb"<oc:fileid>(\d+)</oc:fileid>", resp.content)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
async def get_preview_async(
|
||||
user: User, fileid: int, x: int, y: int
|
||||
) -> httpx.Response:
|
||||
"""Fetch a Nextcloud preview for `fileid` sized up to (x, y).
|
||||
|
||||
Nextcloud's `/index.php/core/preview` endpoint returns a JPEG (or
|
||||
icon fallback) sized so the longest edge fits within the requested
|
||||
box. `a=true` preserves the source aspect ratio; `forceIcon=false`
|
||||
makes it 404 rather than returning a placeholder if no real preview
|
||||
can be produced.
|
||||
|
||||
Auth uses the user's encrypted app password — same path as every
|
||||
other mutation in this module. The caller streams the body back
|
||||
to the frontend; we don't buffer the bytes here.
|
||||
"""
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
url = f"{_base_url()}/index.php/core/preview"
|
||||
params = {
|
||||
"fileId": str(fileid),
|
||||
"x": str(x),
|
||||
"y": str(y),
|
||||
"a": "true",
|
||||
"forceIcon": "false",
|
||||
}
|
||||
client = _async_client((nc_user, app_pw))
|
||||
try:
|
||||
return await client.get(url, params=params)
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
def whoami_dir_exists(nc_username: str) -> bool:
|
||||
"""True iff the bind-mounted `<NEXTCLOUD_USERS_ROOT>/<user>/files`
|
||||
directory exists. Used by the UI to validate the override field
|
||||
|
||||
Reference in New Issue
Block a user