From 1b6ff4572648d874045775f2b6f1f6a8eb062474 Mon Sep 17 00:00:00 2001
From: Claudio <claudio@hubris.network>
Date: Mon, 11 May 2026 22:40:05 +0200
Subject: [PATCH] feat(playback): transcode HEVC .mov to H.264 MP4 on first hit

iPhone .mov files are HEVC Main 10 with codec_tag hvc1. Safari decodes
that fine; Chrome and Firefox refuse 10-bit HEVC entirely, which the
browser surfaces as "playback is not supported" against the existing
/original endpoint. Confirmed against the user's
26-05-01 13-13-26 0525.mov: codec_name=hevc, profile=Main 10,
audio=aac/48kHz.

New endpoint /photos/{id}/playback handles this transparently:
- check the on-disk cache at /data/video-cache/{id}.mp4 first; serve
  if newer than the source
- passthrough h264 in mp4/m4v/webm containers (ffprobe to confirm)
- otherwise transcode src -> H.264 8-bit MP4 with libx264 fast/CRF 23,
  audio re-encoded to AAC because the iPhone 16 ships APAC audio that
  no browser can decode; +faststart for progressive load
- atomic publish via tmp + os.replace so a failed run never leaves a
  half-written cache entry
- HTTP Range support so <video> can seek the result

The .mov container is excluded from the passthrough fast path because
Chrome/Firefox refuse to play even h264-in-mov reliably, so .mov always
goes through the cache (transcode-or-remux). /original is refactored
to share the new _serve_file_with_range helper.

Frontend getVideoSrc swaps from /original to /playback. /original
stays for downloads and any non-<video> fetches.

First-hit cost is ~9s wall for a 13s 1080p HEVC clip on this box
(software libx264, 4 cores). Long videos are still sync-in-request
because the browser's <video> can't deal with a 202 response; if that
becomes painful, lift the transcode into a celery task with a polling
endpoint.
---
 backend/app/routers/photos.py                 | 196 +++++++++++++++++-
 frontend/src/components/preview/previewSrc.ts |   5 +-
 frontend/src/services/api.ts                  |  10 +
 3 files changed, 204 insertions(+), 7 deletions(-)

diff --git a/backend/app/routers/photos.py b/backend/app/routers/photos.py
index 8320370..3d78d6b 100644
--- a/backend/app/routers/photos.py
+++ b/backend/app/routers/photos.py
@@ -10,9 +10,11 @@ from pydantic import BaseModel
 from sqlalchemy import select, and_, or_, func, tuple_
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import selectinload
+import asyncio
 import base64
 import json
 import os
+import subprocess
 import logging
 
 logger = logging.getLogger(__name__)
@@ -842,17 +844,52 @@ async def get_original(
 
     ext = Path(photo.filepath).suffix.lower()
     media_type = _INLINE_MEDIA_TYPES.get(ext, 'application/octet-stream')
-    file_size = os.path.getsize(photo.filepath)
+    return _serve_file_with_range(
+        photo.filepath,
+        request,
+        media_type,
+        download_filename=(
+            photo.filename if media_type == 'application/octet-stream' else None
+        ),
+    )
 
+
+# Cached H.264/MP4 transcodes for /playback. iPhone shoots HEVC Main 10 by
+# default and Chrome/Firefox can't decode 10-bit HEVC reliably, so anything
+# that isn't already h264 in a web-safe container gets transcoded once and
+# served from this directory thereafter.
+_VIDEO_CACHE_DIR = Path('/data/video-cache')
+_VIDEO_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+# Video codecs that play in <video> across the major browsers without
+# re-encoding. AV1 / VP9 are technically web-safe in webm but we only have
+# h264 source material in practice; expand if that changes.
+_PLAYBACK_OK_VCODECS = {'h264', 'avc1'}
+# Container extensions we trust to passthrough when the codec is OK. .mov
+# is excluded — Chrome/Firefox refuse to play even h264 inside a .mov
+# container reliably, so we always remux .mov to .mp4 via the cache path.
+_PLAYBACK_OK_EXTS = {'.mp4', '.m4v', '.webm'}
+
+
+def _serve_file_with_range(
+    path: str,
+    request: Request,
+    media_type: str,
+    download_filename: Optional[str] = None,
+) -> Response:
+    """Range-aware file streamer extracted so /original and /playback share
+    one implementation. Returns 200 (full body) when no Range header is
+    present, 206 with the requested slice otherwise. Browsers refuse to
+    seek long videos without Accept-Ranges + 206, so this is mandatory for
+    <video> playback rather than a nice-to-have."""
+    file_size = os.path.getsize(path)
     range_header = request.headers.get("range")
     parsed = _parse_range(range_header, file_size) if range_header else None
 
     if parsed is None:
-        # No Range header or malformed: full body, but advertise
-        # Accept-Ranges so the browser knows it can ask for one next.
         return FileResponse(
-            photo.filepath,
-            filename=photo.filename if media_type == 'application/octet-stream' else None,
+            path,
+            filename=download_filename,
             media_type=media_type,
             headers={"Accept-Ranges": "bytes"},
         )
@@ -861,7 +898,7 @@ async def get_original(
     length = end - start + 1
 
     def _iter_range():
-        with open(photo.filepath, 'rb') as f:
+        with open(path, 'rb') as f:
             f.seek(start)
             remaining = length
             while remaining > 0:
@@ -883,6 +920,153 @@ async def get_original(
     )
 
 
+async def _ffprobe_video_codec(path: str) -> Optional[str]:
+    """Return the video stream's codec_name (e.g. 'hevc', 'h264'), or None
+    on probe failure. Used to decide passthrough vs transcode in /playback."""
+    def _run():
+        return subprocess.run(
+            [
+                'ffprobe', '-v', 'error',
+                '-select_streams', 'v:0',
+                '-show_entries', 'stream=codec_name',
+                '-of', 'default=noprint_wrappers=1:nokey=1',
+                path,
+            ],
+            capture_output=True, text=True, timeout=10,
+        )
+    try:
+        result = await asyncio.to_thread(_run)
+    except (subprocess.TimeoutExpired, OSError) as e:
+        logger.warning(f"ffprobe failed for {path}: {e}")
+        return None
+    if result.returncode != 0:
+        logger.warning(
+            f"ffprobe returned {result.returncode} for {path}: "
+            f"{(result.stderr or '').strip()[:200]}"
+        )
+        return None
+    out = (result.stdout or '').strip().lower()
+    return out or None
+
+
+async def _transcode_to_h264_mp4(src: str, dst: str) -> bool:
+    """Transcode `src` to H.264 8-bit MP4 at `dst`. Returns True on success.
+
+    -pix_fmt yuv420p forces 8-bit output so 10-bit HEVC sources still play
+    on browsers that lack 10-bit decode. Audio is re-encoded to AAC because
+    the iPhone 16's APAC stream isn't decodable in browsers, and even when
+    the source audio is already AAC the cost of re-encoding is negligible
+    next to the video pass. +faststart moves the moov atom to the front
+    of the file so playback can start before the whole download completes.
+    Atomic via tmp file + os.replace so a failed run never leaves a
+    half-written .mp4 in the cache.
+    """
+    tmp = dst + '.tmp'
+    def _run():
+        return subprocess.run(
+            [
+                'ffmpeg', '-y', '-loglevel', 'error',
+                '-i', src,
+                '-map', '0:v:0',
+                '-map', '0:a:0?',
+                '-c:v', 'libx264',
+                '-preset', 'fast',
+                '-crf', '23',
+                '-pix_fmt', 'yuv420p',
+                '-c:a', 'aac',
+                '-b:a', '160k',
+                '-movflags', '+faststart',
+                tmp,
+            ],
+            capture_output=True, text=True, timeout=600,
+        )
+    try:
+        result = await asyncio.to_thread(_run)
+    except (subprocess.TimeoutExpired, OSError) as e:
+        logger.error(f"ffmpeg transcode failed for {src}: {e}")
+        try:
+            os.remove(tmp)
+        except OSError:
+            pass
+        return False
+    if result.returncode != 0:
+        logger.error(
+            f"ffmpeg returned {result.returncode} for {src}: "
+            f"{(result.stderr or '').strip()[:500]}"
+        )
+        try:
+            os.remove(tmp)
+        except OSError:
+            pass
+        return False
+    try:
+        os.replace(tmp, dst)
+    except OSError as e:
+        logger.error(f"failed to publish transcoded {dst}: {e}")
+        return False
+    return True
+
+
+@router.get("/{photo_id}/playback")
+async def get_playback(
+    photo_id: str,
+    request: Request,
+    db: AsyncSession = Depends(get_db),
+    current_user: User = Depends(get_current_user_media),
+):
+    """Serve a video in a format every browser can play.
+
+    iPhone .mov files are HEVC Main 10 (codec_tag hvc1), which Chrome and
+    Firefox cannot decode reliably — Safari is the only browser that
+    handles 10-bit HEVC out of the box. This endpoint transcodes to H.264
+    8-bit MP4 on first hit, caches under /data/video-cache/{id}.mp4, and
+    serves with HTTP Range like /original. Web-safe inputs (h264 in
+    mp4/webm) skip the transcode and stream the original.
+
+    First-hit latency for an HEVC video is dominated by ffmpeg — typically
+    a few seconds per video-second on CPU. Subsequent hits serve straight
+    from the cache."""
+    photo = await _get_photo_with_share_fallback(photo_id, current_user, db)
+    if not os.path.exists(photo.filepath):
+        raise HTTPException(status_code=404, detail="File not found")
+
+    src_path = photo.filepath
+    cache_path = _VIDEO_CACHE_DIR / f'{photo.id}.mp4'
+
+    # Source-newer-than-cache invalidates the cache. Covers in-place file
+    # replacement (re-upload via NC) without leaving stale playback bytes.
+    if cache_path.exists():
+        try:
+            if os.path.getmtime(src_path) > os.path.getmtime(cache_path):
+                cache_path.unlink()
+        except OSError:
+            pass
+
+    if cache_path.exists():
+        return _serve_file_with_range(str(cache_path), request, 'video/mp4')
+
+    # Passthrough fast path: source is already h264 in a web-safe container.
+    ext = Path(src_path).suffix.lower()
+    if ext in _PLAYBACK_OK_EXTS:
+        codec = await _ffprobe_video_codec(src_path)
+        if codec in _PLAYBACK_OK_VCODECS:
+            return _serve_file_with_range(
+                src_path,
+                request,
+                _INLINE_MEDIA_TYPES.get(ext, 'video/mp4'),
+            )
+
+    # Transcode-and-cache path. Sync inside the request because the
+    # browser's <video> can't deal with a 202 response — better to make
+    # the user wait once than to ship a UX that pretends the video isn't
+    # there. If this becomes a bottleneck for long videos, lift it into
+    # a celery task with an async polling endpoint.
+    ok = await _transcode_to_h264_mp4(src_path, str(cache_path))
+    if not ok:
+        raise HTTPException(status_code=500, detail="Video transcoding failed")
+    return _serve_file_with_range(str(cache_path), request, 'video/mp4')
+
+
 # Extensions that the browser can decode natively. Anything else (RAW, HEIC,
 # TIFF) needs the /proxy endpoint to convert to WebP for display.
 _WEB_SAFE_DISPLAY_EXTS = {'.jpg', '.jpeg', '.png', '.webp', '.gif'}
diff --git a/frontend/src/components/preview/previewSrc.ts b/frontend/src/components/preview/previewSrc.ts
index 8cd91f1..2468cb4 100644
--- a/frontend/src/components/preview/previewSrc.ts
+++ b/frontend/src/components/preview/previewSrc.ts
@@ -36,5 +36,8 @@ export function getPreviewFullResSrc(photo: Photo): string {
 }
 
 export function getVideoSrc(photo: Photo): string {
-  return photosApi.getOriginalUrl(photo.id)
+  // /playback transcodes HEVC (iPhone .mov) to H.264 on first hit and
+  // caches it. /original would just hand the browser raw HEVC, which
+  // Chrome and Firefox refuse to decode.
+  return photosApi.getPlaybackUrl(photo.id)
 }
diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts
index ee0f1eb..5226208 100644
--- a/frontend/src/services/api.ts
+++ b/frontend/src/services/api.ts
@@ -310,6 +310,16 @@ export const photos = {
     return `${API_BASE_URL}/photos/${photoId}/proxy${qs}`
   },
 
+  /** Browser-playable video URL. Backend passthroughs h264/mp4 sources and
+   *  transcodes everything else (notably iPhone HEVC .mov) to H.264 MP4
+   *  on first hit, cached thereafter. Use this for any <video src> in the
+   *  UI; /original stays for downloads. */
+  getPlaybackUrl: (photoId: string) => {
+    const token = localStorage.getItem('access_token')
+    const qs = token ? `?token=${encodeURIComponent(token)}` : ''
+    return `${API_BASE_URL}/photos/${photoId}/playback${qs}`
+  },
+
   /** "On this day" memories — photos taken on this date in previous years. */
   memories: async (): Promise<MemoriesResponse> => {
     const response = await api.get('/photos/memories')