feat(playback): pre-transcode HEVC videos in background; veryfast preset
Long videos blocked /playback for the entire encode duration. The fix is to populate the cache before the user clicks, not when they click. Changes: - Extract ffprobe + ffmpeg helpers to services/video.py so the request handler and the background task share one sync implementation. The endpoint wraps calls in asyncio.to_thread; celery just calls them. - New tasks/video.py with pretranscode_video. Idempotent: skips when the cache is already current and skips passthrough-safe sources (h264 in mp4/m4v/webm). 30-min task time limit so the long-tail files (3GP archive, multi-minute 1080p clips) still complete. - scan_folder now dispatches pretranscode_video alongside generate_thumbnails / extract_metadata for any new video row. - POST /library/maintenance/backfill-video-cache enqueues every active video so the existing library catches up. - libx264 preset bumped from fast to veryfast. ~2x throughput on this CPU-only box, output a few % larger but well within disk budget. - /playback simplifies to: cache check, passthrough if h264 in web-safe container, else sync transcode (still there as fallback for races against the queued task). Once the backfill task drains, /playback should be near-instant for every video. Any video added afterwards is pre-transcoded at scan time, so the user keeps that property going forward.
This commit is contained in:
@@ -18,6 +18,7 @@ celery_app = Celery(
|
||||
include=[
|
||||
'app.tasks.scan',
|
||||
'app.tasks.thumbs',
|
||||
'app.tasks.video',
|
||||
'app.tasks.vision',
|
||||
'app.services.metadata', # extract_metadata lives here
|
||||
]
|
||||
@@ -52,6 +53,10 @@ celery_app.conf.update(
|
||||
'scan_folder': {'queue': 'low'},
|
||||
'scan_all_source_roots': {'queue': 'low'},
|
||||
'backfill_gps': {'queue': 'low'},
|
||||
# Pre-transcode HEVC videos in the background so /playback is a
|
||||
# cache hit on first user click. CPU-heavy but tolerant of the
|
||||
# low-priority queue (it doesn't block any user-facing flow).
|
||||
'pretranscode_video': {'queue': 'low'},
|
||||
# `watch_folders` is retired (file events come from NC webhooks)
|
||||
# but the task definition still exists as a no-op shim for any
|
||||
# in-flight apply_async. Route it to the default queue so the
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.database import AsyncSessionLocal
|
||||
from app.models import Photo, Folder, SourceRoot
|
||||
from app.config import settings
|
||||
from app.tasks.thumbs import generate_thumbnails
|
||||
from app.tasks.video import pretranscode_video
|
||||
from app.services.metadata import extract_metadata
|
||||
from app.services.date_guess import has_date_warning
|
||||
|
||||
@@ -195,6 +196,7 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
|
||||
# Defer task dispatch until AFTER commit so workers don't
|
||||
# query for rows that aren't visible to other sessions yet.
|
||||
pending_dispatch: list[str] = []
|
||||
pending_video_pretranscode: list[tuple[str, str]] = []
|
||||
|
||||
for filename in batch:
|
||||
filepath = os.path.join(root, filename)
|
||||
@@ -316,6 +318,13 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
|
||||
# below; otherwise the worker can race the writer
|
||||
# and see "Photo not found".
|
||||
pending_dispatch.append(photo.id)
|
||||
if photo.media_type == 'video':
|
||||
# Pre-transcode HEVC and other non-web-safe
|
||||
# videos at scan time so the user doesn't
|
||||
# pay the encode cost on first <video> click.
|
||||
pending_video_pretranscode.append(
|
||||
(photo.id, filepath)
|
||||
)
|
||||
|
||||
processed_files += 1
|
||||
progress_set(REDIS_KEY_PROCESSED, processed_files)
|
||||
@@ -344,6 +353,8 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
|
||||
for photo_id in pending_dispatch:
|
||||
generate_thumbnails.delay(photo_id)
|
||||
extract_metadata.delay(photo_id)
|
||||
for vid_photo_id, vid_path in pending_video_pretranscode:
|
||||
pretranscode_video.delay(vid_photo_id, vid_path)
|
||||
|
||||
# Update folder scan timestamp
|
||||
folder.last_scanned = datetime.utcnow()
|
||||
|
||||
62
backend/app/tasks/video.py
Normal file
62
backend/app/tasks/video.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""Background pre-transcode for video photos so /playback is a cache
|
||||
hit on first user click. Dispatched from scan_folder when a new video
|
||||
row is created, and from the backfill admin endpoint for the existing
|
||||
library."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from celery import shared_task
|
||||
|
||||
from app.services.video import (
|
||||
PLAYBACK_OK_EXTS,
|
||||
PLAYBACK_OK_VCODECS,
|
||||
cache_path_for,
|
||||
ffprobe_video_codec,
|
||||
transcode_to_h264_mp4,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task(
|
||||
name='pretranscode_video',
|
||||
# Override the 10-minute global task_time_limit. Long videos (30+ min
|
||||
# raw clips, the occasional .3gp from the 2016 archive) can legitimately
|
||||
# take half an hour to transcode on this CPU-only box.
|
||||
time_limit=1800,
|
||||
soft_time_limit=1740,
|
||||
)
|
||||
def pretranscode_video(photo_id: str, src_path: str):
|
||||
"""Idempotent: skip if cache exists and is newer than source, skip
|
||||
if source is already a passthrough-safe codec/container.
|
||||
|
||||
Reports the outcome as a status string so the admin backfill can
|
||||
summarise. The /playback endpoint also falls back to a sync
|
||||
transcode if the cache miss races a queued task."""
|
||||
if not os.path.exists(src_path):
|
||||
logger.debug("pretranscode skip — source missing: %s", src_path)
|
||||
return {'status': 'missing', 'photo_id': photo_id}
|
||||
|
||||
cache_path = cache_path_for(photo_id)
|
||||
if cache_path.exists():
|
||||
try:
|
||||
if os.path.getmtime(src_path) <= os.path.getmtime(cache_path):
|
||||
return {'status': 'cached', 'photo_id': photo_id}
|
||||
cache_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
ext = os.path.splitext(src_path)[1].lower()
|
||||
if ext in PLAYBACK_OK_EXTS:
|
||||
codec = ffprobe_video_codec(src_path)
|
||||
if codec in PLAYBACK_OK_VCODECS:
|
||||
return {'status': 'passthrough', 'photo_id': photo_id, 'codec': codec}
|
||||
|
||||
ok = transcode_to_h264_mp4(src_path, str(cache_path))
|
||||
return {
|
||||
'status': 'transcoded' if ok else 'failed',
|
||||
'photo_id': photo_id,
|
||||
}
|
||||
Reference in New Issue
Block a user