perf(thumbs): pool NC client, smaller grid thumbs, eager owner load
Five stacked optimisations for the thumbnail hot path so the timeline grid lands in fewer round trips and fewer bytes. 1. PhotoThumbnail: switch from 'medium' (640px) to 'small' (240px) for grid cells. 240px oversamples 150-200px logical cells on 2x retina and drops payload 5-8x. Lightbox and preview filmstrip keep 'large' and 'medium' respectively. 2. nextcloud_dav: pool the httpx client. A module-level AsyncClient with HTTP/2 + keepalive (max_connections=64, keepalive_expiry=120s) replaces the per-request constructor that paid a fresh TCP+TLS handshake on every preview fetch. Auth is per-user so it stays at the call site via auth=BasicAuth(...). Lifespan-managed: init in main.py's lifespan startup, aclose on shutdown. requirements.txt gains the http2 extra to pull in h2 (not currently installed). Same change applies to fetch_memories_info_async since it hits the same host. 3. PhotoThumbnail img: add decoding="async" so JPEG/WebP decode moves off the main thread, plus fetchPriority="low" so grid backfill doesn't fight UI fetches. 4. Eager-load Photo.user via joinedload from the thumb handler. _get_photo_with_share_fallback gains an options parameter so other callers stay zero-overhead; only the thumb handler asks for the owner join. Eliminates the second SELECT users per request. 5. Disk-fallback path picks up Cache-Control: private, max-age=86400 in both the FileResponse and X-Accel branches so re-renders match the NC primary path's caching behaviour. Net: a warm grid page should drop from ~200-400 ms median per thumb to well under 100 ms; payload drops ~5-8x; backend sustains higher concurrency with fewer sockets to Nextcloud and one fewer Postgres round-trip per request.
This commit is contained in:
@@ -15,6 +15,7 @@ from app.database import init_db
|
||||
from app.routers import photos, folders, heaps, tags, discard, library, search, auth, admin, sharing, upload, download, features, nextcloud, nc_webhook
|
||||
from app.services.scanner import start_initial_scan, bootstrap_default_source_root
|
||||
from app.services.cleanup import cleanup_data_integrity
|
||||
from app.services.nextcloud_dav import init_preview_client, close_preview_client
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
@@ -31,6 +32,11 @@ async def lifespan(app: FastAPI):
|
||||
# Initialize database
|
||||
await init_db()
|
||||
|
||||
# Pooled httpx client to Nextcloud — keepalive + HTTP/2 means every
|
||||
# thumbnail / Memories-info call after the first reuses one socket
|
||||
# instead of paying TCP+TLS handshake per request.
|
||||
await init_preview_client()
|
||||
|
||||
# First-boot convenience: if there are no source roots in the DB yet,
|
||||
# create one for the default /photos mount so the user sees their
|
||||
# library immediately without configuring anything in the UI.
|
||||
@@ -54,6 +60,7 @@ async def lifespan(app: FastAPI):
|
||||
yield
|
||||
|
||||
logger.info("Shutting down Mulita application...")
|
||||
await close_preview_client()
|
||||
|
||||
# Create FastAPI app
|
||||
app = FastAPI(
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
Photo model definition
|
||||
"""
|
||||
from sqlalchemy import Column, String, Integer, Float, Boolean, DateTime, ForeignKey, Text, Index
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
@@ -10,12 +11,17 @@ from app.database import Base
|
||||
|
||||
class Photo(Base):
|
||||
__tablename__ = 'photos'
|
||||
|
||||
|
||||
# Primary key
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
# Eager-loadable owner relationship. Used by the thumbnail handler so
|
||||
# one photo lookup also yields the NC creds we need to call the
|
||||
# preview endpoint, instead of issuing a second SELECT users WHERE
|
||||
# id=…. No FK change — user_id above already exists.
|
||||
user = relationship("User", lazy="select")
|
||||
|
||||
# File information
|
||||
filepath = Column(String, unique=True, nullable=False)
|
||||
|
||||
@@ -9,7 +9,7 @@ from fastapi.responses import FileResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, and_, or_, func, tuple_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlalchemy.orm import joinedload, selectinload
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
@@ -597,25 +597,33 @@ async def remove_photo_tag(
|
||||
|
||||
async def _get_photo_with_share_fallback(
|
||||
photo_id: str, user: User, db: AsyncSession,
|
||||
*, options=None,
|
||||
) -> Photo:
|
||||
"""Fetch a photo the user owns, or one they can access via a share.
|
||||
|
||||
The fast path (owned photo) does a single indexed query. The share
|
||||
fallback only runs when the first query returns nothing — this
|
||||
happens only for shared photos, not during normal browsing.
|
||||
|
||||
`options` is a list of SQLAlchemy loader options applied to both
|
||||
queries. Pass `[joinedload(Photo.user)]` from the thumb handler so
|
||||
the photo lookup also yields the owner row (NC creds) in one round
|
||||
trip instead of issuing a second SELECT users.
|
||||
"""
|
||||
# Fast path — owned photo (single indexed query, no extra joins).
|
||||
result = await db.execute(
|
||||
select(Photo).where(Photo.id == photo_id, Photo.user_id == user.id)
|
||||
)
|
||||
stmt = select(Photo).where(Photo.id == photo_id, Photo.user_id == user.id)
|
||||
if options:
|
||||
stmt = stmt.options(*options)
|
||||
result = await db.execute(stmt)
|
||||
photo = result.scalar_one_or_none()
|
||||
if photo:
|
||||
return photo
|
||||
# Slow path — check share access (only for shared photos).
|
||||
if await can_access_photo_via_share(photo_id, user, db):
|
||||
result = await db.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
)
|
||||
stmt = select(Photo).where(Photo.id == photo_id)
|
||||
if options:
|
||||
stmt = stmt.options(*options)
|
||||
result = await db.execute(stmt)
|
||||
photo = result.scalar_one_or_none()
|
||||
if photo:
|
||||
return photo
|
||||
@@ -653,13 +661,15 @@ async def get_thumbnail(
|
||||
if size not in _NC_PREVIEW_PX:
|
||||
raise HTTPException(status_code=400, detail="Invalid thumbnail size")
|
||||
|
||||
photo = await _get_photo_with_share_fallback(photo_id, current_user, db)
|
||||
# joinedload(Photo.user) folds the owner row into the same query so
|
||||
# we don't issue a follow-up SELECT users on every thumbnail.
|
||||
photo = await _get_photo_with_share_fallback(
|
||||
photo_id, current_user, db, options=[joinedload(Photo.user)],
|
||||
)
|
||||
|
||||
# ── Primary: proxy Nextcloud's preview endpoint ────────────────────
|
||||
if photo.nextcloud_fileid and photo.user_id:
|
||||
owner = (
|
||||
await db.execute(select(User).where(User.id == photo.user_id))
|
||||
).scalar_one_or_none()
|
||||
owner = photo.user
|
||||
if owner is not None:
|
||||
try:
|
||||
upstream = await get_preview_async(
|
||||
@@ -766,6 +776,10 @@ async def get_thumbnail(
|
||||
)
|
||||
|
||||
response.headers["X-Mule-Thumb-Source"] = "disk"
|
||||
# Same one-day cache as the NC primary path so re-renders / back-
|
||||
# button navigation hit the browser cache instead of round-tripping
|
||||
# the backend just to re-serve an unchanged WebP.
|
||||
response.headers["Cache-Control"] = "private, max-age=86400"
|
||||
# Check if we're behind Nginx
|
||||
if os.environ.get('USE_X_ACCEL_REDIRECT'):
|
||||
# Use Nginx X-Accel-Redirect for better performance
|
||||
@@ -774,7 +788,11 @@ async def get_thumbnail(
|
||||
return Response()
|
||||
else:
|
||||
# Direct file serving for development
|
||||
return FileResponse(thumb_path, media_type='image/webp')
|
||||
return FileResponse(
|
||||
thumb_path,
|
||||
media_type='image/webp',
|
||||
headers={"Cache-Control": "private, max-age=86400"},
|
||||
)
|
||||
|
||||
_INLINE_MEDIA_TYPES = {
|
||||
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
||||
|
||||
@@ -148,6 +148,47 @@ def _async_client(auth: tuple[str, str]) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(timeout=_TIMEOUT, auth=httpx.BasicAuth(*auth), follow_redirects=False)
|
||||
|
||||
|
||||
# Pooled async client for the read-heavy NC endpoints (preview proxy,
|
||||
# Memories info). Auth is per-user, so it's passed at call time via
|
||||
# `auth=BasicAuth(...)`; the pool itself is auth-less. Keepalive +
|
||||
# HTTP/2 cuts the TCP+TLS handshake from every thumbnail request and
|
||||
# multiplexes the dozens of concurrent grid fetches over one socket.
|
||||
_PREVIEW_LIMITS = httpx.Limits(
|
||||
max_connections=64, max_keepalive_connections=32, keepalive_expiry=120.0,
|
||||
)
|
||||
_preview_client: httpx.AsyncClient | None = None
|
||||
|
||||
|
||||
async def init_preview_client() -> None:
|
||||
"""Called from the FastAPI lifespan startup hook."""
|
||||
global _preview_client
|
||||
if _preview_client is None:
|
||||
_preview_client = httpx.AsyncClient(
|
||||
timeout=_TIMEOUT,
|
||||
limits=_PREVIEW_LIMITS,
|
||||
http2=True,
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
|
||||
async def close_preview_client() -> None:
|
||||
"""Called from the FastAPI lifespan shutdown hook."""
|
||||
global _preview_client
|
||||
if _preview_client is not None:
|
||||
await _preview_client.aclose()
|
||||
_preview_client = None
|
||||
|
||||
|
||||
def _shared_preview_client() -> httpx.AsyncClient:
|
||||
"""Return the pooled client. Falls back to a one-shot AsyncClient if
|
||||
init wasn't called (tests, scripts) — caller must aclose it."""
|
||||
if _preview_client is not None:
|
||||
return _preview_client
|
||||
return httpx.AsyncClient(
|
||||
timeout=_TIMEOUT, http2=True, follow_redirects=False,
|
||||
)
|
||||
|
||||
|
||||
def _raise_for_dav(resp: httpx.Response, action: str) -> None:
|
||||
"""Translate Nextcloud WebDAV errors into FastAPI HTTPExceptions
|
||||
the frontend can show. We surface Nextcloud's body verbatim when
|
||||
@@ -395,7 +436,7 @@ async def fetch_memories_info_async(
|
||||
url = (
|
||||
f"{_base_url()}/index.php/apps/memories/api/image/info/{int(fileid)}"
|
||||
)
|
||||
client = _async_client((nc_user, app_pw))
|
||||
client = _shared_preview_client()
|
||||
try:
|
||||
resp = await client.get(
|
||||
url,
|
||||
@@ -403,9 +444,13 @@ async def fetch_memories_info_async(
|
||||
"OCS-APIRequest": "true",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
auth=httpx.BasicAuth(nc_user, app_pw),
|
||||
)
|
||||
finally:
|
||||
await client.aclose()
|
||||
# Only close if we got a one-shot fallback client; the pooled
|
||||
# one is owned by the lifespan hook.
|
||||
if client is not _preview_client:
|
||||
await client.aclose()
|
||||
if resp.status_code == 404:
|
||||
return None
|
||||
if not resp.is_success:
|
||||
@@ -444,11 +489,16 @@ async def get_preview_async(
|
||||
"a": "true",
|
||||
"forceIcon": "false",
|
||||
}
|
||||
client = _async_client((nc_user, app_pw))
|
||||
client = _shared_preview_client()
|
||||
try:
|
||||
return await client.get(url, params=params)
|
||||
return await client.get(
|
||||
url, params=params, auth=httpx.BasicAuth(nc_user, app_pw),
|
||||
)
|
||||
finally:
|
||||
await client.aclose()
|
||||
# Only close if we got a one-shot fallback client; the pooled
|
||||
# one is owned by the lifespan hook.
|
||||
if client is not _preview_client:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
def whoami_dir_exists(nc_username: str) -> bool:
|
||||
|
||||
@@ -48,7 +48,7 @@ pyyaml==6.0.1
|
||||
pydantic==2.5.3
|
||||
pydantic-settings==2.1.0
|
||||
python-dotenv==1.0.0
|
||||
httpx==0.26.0
|
||||
httpx[http2]==0.26.0
|
||||
aiofiles==23.2.1
|
||||
|
||||
# Security and authentication
|
||||
|
||||
Reference in New Issue
Block a user