feat(playback): pre-transcode HEVC videos in background; veryfast preset

Long videos blocked /playback for the entire encode duration. The fix
is to populate the cache before the user clicks, not when they click.

Changes:
- Extract ffprobe + ffmpeg helpers to services/video.py so the request
  handler and the background task share one sync implementation. The
  endpoint wraps calls in asyncio.to_thread; celery just calls them.
- New tasks/video.py with pretranscode_video. Idempotent: skips when
  the cache is already current and skips passthrough-safe sources
  (h264 in mp4/m4v/webm). 30-min task time limit so the long-tail
  files (3GP archive, multi-minute 1080p clips) still complete.
- scan_folder now dispatches pretranscode_video alongside
  generate_thumbnails / extract_metadata for any new video row.
- POST /library/maintenance/backfill-video-cache enqueues every
  active video so the existing library catches up.
- libx264 preset bumped from fast to veryfast. ~2x throughput on this
  CPU-only box, output a few % larger but well within disk budget.
- /playback simplifies to: cache check, passthrough if h264 in
  web-safe container, else sync transcode (still there as fallback
  for races against the queued task).

Once the backfill task drains, /playback should be near-instant for
every video. Any video added afterwards is pre-transcoded at scan
time, so the user keeps that property going forward.
This commit is contained in:
Claudio
2026-05-12 00:03:15 +02:00
parent 2a5270d399
commit c4df92720b
6 changed files with 253 additions and 127 deletions

View File

@@ -18,6 +18,7 @@ celery_app = Celery(
include=[
'app.tasks.scan',
'app.tasks.thumbs',
'app.tasks.video',
'app.tasks.vision',
'app.services.metadata', # extract_metadata lives here
]
@@ -52,6 +53,10 @@ celery_app.conf.update(
'scan_folder': {'queue': 'low'},
'scan_all_source_roots': {'queue': 'low'},
'backfill_gps': {'queue': 'low'},
# Pre-transcode HEVC videos in the background so /playback is a
# cache hit on first user click. CPU-heavy but tolerant of the
# low-priority queue (it doesn't block any user-facing flow).
'pretranscode_video': {'queue': 'low'},
# `watch_folders` is retired (file events come from NC webhooks)
# but the task definition still exists as a no-op shim for any
# in-flight apply_async. Route it to the default queue so the

View File

@@ -20,6 +20,7 @@ from app.database import AsyncSessionLocal
from app.models import Photo, Folder, SourceRoot
from app.config import settings
from app.tasks.thumbs import generate_thumbnails
from app.tasks.video import pretranscode_video
from app.services.metadata import extract_metadata
from app.services.date_guess import has_date_warning
@@ -195,6 +196,7 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
# Defer task dispatch until AFTER commit so workers don't
# query for rows that aren't visible to other sessions yet.
pending_dispatch: list[str] = []
pending_video_pretranscode: list[tuple[str, str]] = []
for filename in batch:
filepath = os.path.join(root, filename)
@@ -316,6 +318,13 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
# below; otherwise the worker can race the writer
# and see "Photo not found".
pending_dispatch.append(photo.id)
if photo.media_type == 'video':
# Pre-transcode HEVC and other non-web-safe
# videos at scan time so the user doesn't
# pay the encode cost on first <video> click.
pending_video_pretranscode.append(
(photo.id, filepath)
)
processed_files += 1
progress_set(REDIS_KEY_PROCESSED, processed_files)
@@ -344,6 +353,8 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
for photo_id in pending_dispatch:
generate_thumbnails.delay(photo_id)
extract_metadata.delay(photo_id)
for vid_photo_id, vid_path in pending_video_pretranscode:
pretranscode_video.delay(vid_photo_id, vid_path)
# Update folder scan timestamp
folder.last_scanned = datetime.utcnow()

View File

@@ -0,0 +1,62 @@
"""Background pre-transcode for video photos so /playback is a cache
hit on first user click. Dispatched from scan_folder when a new video
row is created, and from the backfill admin endpoint for the existing
library."""
from __future__ import annotations
import logging
import os
from celery import shared_task
from app.services.video import (
PLAYBACK_OK_EXTS,
PLAYBACK_OK_VCODECS,
cache_path_for,
ffprobe_video_codec,
transcode_to_h264_mp4,
)
logger = logging.getLogger(__name__)
@shared_task(
name='pretranscode_video',
# Override the 10-minute global task_time_limit. Long videos (30+ min
# raw clips, the occasional .3gp from the 2016 archive) can legitimately
# take half an hour to transcode on this CPU-only box.
time_limit=1800,
soft_time_limit=1740,
)
def pretranscode_video(photo_id: str, src_path: str):
"""Idempotent: skip if cache exists and is newer than source, skip
if source is already a passthrough-safe codec/container.
Reports the outcome as a status string so the admin backfill can
summarise. The /playback endpoint also falls back to a sync
transcode if the cache miss races a queued task."""
if not os.path.exists(src_path):
logger.debug("pretranscode skip — source missing: %s", src_path)
return {'status': 'missing', 'photo_id': photo_id}
cache_path = cache_path_for(photo_id)
if cache_path.exists():
try:
if os.path.getmtime(src_path) <= os.path.getmtime(cache_path):
return {'status': 'cached', 'photo_id': photo_id}
cache_path.unlink()
except OSError:
pass
ext = os.path.splitext(src_path)[1].lower()
if ext in PLAYBACK_OK_EXTS:
codec = ffprobe_video_codec(src_path)
if codec in PLAYBACK_OK_VCODECS:
return {'status': 'passthrough', 'photo_id': photo_id, 'codec': codec}
ok = transcode_to_h264_mp4(src_path, str(cache_path))
return {
'status': 'transcoded' if ok else 'failed',
'photo_id': photo_id,
}