From 7b153f0d285cc8ba15360b361530a6a3bf0b52ec Mon Sep 17 00:00:00 2001 From: Claudio Date: Sun, 10 May 2026 23:02:32 +0200 Subject: [PATCH] fix(metadata): drop EXIF:ModifyDate fallback, prefer SubSec, fall back to path The taken_at extractor walked four EXIF fields in order: DateTimeOriginal, CreateDate, MediaCreateDate, ModifyDate. The last one is set every time a file is re-saved (Lightroom export, EXIF strip, batch resize), so any photo whose original capture metadata was lost during editing ended up labeled 'exif' with the *edit* date instead of the shoot date. Changes: - SubSecDateTimeOriginal at the top of the list (sub-second precision, often carries OffsetTime). - QuickTime:CreateDate added next to MediaCreateDate. - ModifyDate dropped from the trusted list entirely. - When no trusted EXIF date is present, fall back to guess_date_from_path (already used for has_date_warning) and tag taken_at_source='path'. Better than filesystem mtime, which on Nextcloud-mounted libraries just reflects the upload time. - Skip the date-write block entirely if photo.taken_at_source == 'manual' so a rescan can't clobber a user correction. - parse_exif_datetime: handle the all-zero placeholder some cameras emit, accept tz-aware variants (%z), normalize to naive UTC. Frontend: new 'PATH' badge in TakenAtEditor with a tooltip explaining the date came from filename / folder rather than real EXIF. Backfill: new backfill_taken_at celery task and POST /api/v1/library/maintenance/backfill-taken-at endpoint that re-enqueues extract_metadata for every non-manual photo. ~21k tasks finish in ~15 min on the existing worker-light concurrency. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/routers/library.py | 13 ++ backend/app/services/metadata.py | 131 +++++++++++++++--- .../src/components/sidebar/TakenAtEditor.tsx | 14 +- 3 files changed, 134 insertions(+), 24 deletions(-) diff --git a/backend/app/routers/library.py b/backend/app/routers/library.py index 4923db7..976dc14 100644 --- a/backend/app/routers/library.py +++ b/backend/app/routers/library.py @@ -187,6 +187,19 @@ async def trigger_backfill_gps(current_user: User = Depends(get_current_user)): backfill_gps.delay() return {"status": "success", "message": "GPS backfill queued"} + +@router.post("/maintenance/backfill-taken-at") +async def trigger_backfill_taken_at(current_user: User = Depends(get_current_user)): + """Re-run extract_metadata on every non-manual photo to recompute + taken_at with the current EXIF-priority list and path-based fallback. + Useful after the date-extraction logic changes (e.g. dropping the + ModifyDate fallback). Manual edits are preserved.""" + from app.services.metadata import backfill_taken_at + + backfill_taken_at.delay() + return {"status": "success", "message": "taken_at backfill queued"} + + @router.get("/scan/status") async def get_scan_status(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)): """Get current scan status""" diff --git a/backend/app/services/metadata.py b/backend/app/services/metadata.py index 4f251f1..50130bb 100644 --- a/backend/app/services/metadata.py +++ b/backend/app/services/metadata.py @@ -20,26 +20,46 @@ from app.services.date_guess import has_date_warning logger = logging.getLogger(__name__) def parse_exif_datetime(date_str: str) -> Optional[datetime]: - """Parse EXIF datetime string to Python datetime""" + """Parse EXIF datetime string to Python datetime. + + Returns a tz-naive datetime — the photos.taken_at column is + `timestamp without time zone`. Tz-aware inputs (e.g. SubSec + fields with `+02:00` or QuickTime UTC `Z`) are converted to UTC + and stripped. Cameras that wrote the all-zero placeholder + return None. + """ if not date_str: return None - - # Common EXIF datetime formats + s = str(date_str).strip() + # All-zero placeholder some cameras emit when the clock isn't set. + if s.startswith("0000:00:00") or s.startswith("0000-00-00"): + return None + formats = [ "%Y:%m:%d %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y:%m:%d %H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f", - "%Y-%m-%dT%H:%M:%S%z" + # Tz-aware variants: SubSecDateTimeOriginal often looks like + # "2023:11:30 14:30:45.123+02:00", QuickTime CreateDate as + # "2023:11:30 14:30:45Z" or with offsets. + "%Y:%m:%d %H:%M:%S%z", + "%Y:%m:%d %H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S.%f%z", ] - + for fmt in formats: try: - return datetime.strptime(date_str, fmt) + dt = datetime.strptime(s, fmt) except ValueError: continue - + if dt.tzinfo is not None: + from datetime import timezone + dt = dt.astimezone(timezone.utc).replace(tzinfo=None) + return dt + return None _DMS_RE = re.compile( @@ -227,21 +247,51 @@ async def _extract_metadata_async(photo_id: str): # Store full metadata as JSON photo.exif_json = json.dumps(exif_data) - # Extract taken_at date - date_fields = [ - 'EXIF:DateTimeOriginal', - 'EXIF:CreateDate', - 'QuickTime:MediaCreateDate', - 'EXIF:ModifyDate' - ] + # Extract taken_at date — but only if the user hasn't + # explicitly set it via the UI. Manual edits are the + # source of truth and must survive any rescan. + if photo.taken_at_source != 'manual': + # Trusted EXIF fields, in order of preference. + # SubSecDateTimeOriginal includes sub-second + # precision and often a tz offset, so it's the + # most accurate when present. ModifyDate is NOT + # in this list — it's set every time the file + # is re-saved (Lightroom export, EXIF strip, + # batch resize) and routinely overwrote correct + # capture dates with edit-time dates. + date_fields = [ + 'EXIF:SubSecDateTimeOriginal', + 'EXIF:DateTimeOriginal', + 'EXIF:CreateDate', + 'QuickTime:MediaCreateDate', + 'QuickTime:CreateDate', + ] - for field in date_fields: - if field in exif_data: - taken_at = parse_exif_datetime(exif_data[field]) - if taken_at: - photo.taken_at = taken_at - photo.taken_at_source = 'exif' - break + new_taken_at = None + for field in date_fields: + if field in exif_data: + parsed = parse_exif_datetime(exif_data[field]) + if parsed: + new_taken_at = parsed + photo.taken_at = parsed + photo.taken_at_source = 'exif' + break + + # Fallback: if the file has no trusted EXIF date, + # try to extract one from the filename / folder + # path. The same date_guess module powers the + # has_date_warning flag — reusing it here means + # photos without EXIF (scanned prints, stripped + # JPEGs, re-saved exports) get a sensible date + # instead of falling back to filesystem mtime + # (which on Nextcloud-mounted files is just the + # upload time). + if new_taken_at is None: + from app.services.date_guess import guess_date_from_path + guess = guess_date_from_path(photo.filepath) + if guess is not None: + photo.taken_at = guess.date + photo.taken_at_source = 'path' # Re-run the path-vs-date heuristic now that we know # whether EXIF provided a real capture date. A true EXIF @@ -288,4 +338,41 @@ async def _extract_metadata_async(photo_id: str): except Exception as e: logger.error(f"Error extracting metadata for {photo_id}: {e}") - return {'status': 'error', 'message': str(e)} \ No newline at end of file + return {'status': 'error', 'message': str(e)} + + +@shared_task(name='backfill_taken_at') +def backfill_taken_at(): + """Re-enqueue extract_metadata for every non-manual photo. + + Used after fixing the date-extraction logic (removing ModifyDate + fallback, adding path-based fallback) to re-derive taken_at across + the whole library without touching photos the user has manually + corrected. Each enqueued task is fast (~90ms) and runs on the + default queue; ~21k photos finish in ~15 min on the existing + worker-light concurrency. + """ + return asyncio.run(_backfill_taken_at_async()) + + +async def _backfill_taken_at_async(): + from sqlalchemy import or_ + async with AsyncSessionLocal() as session: + result = await session.execute( + select(Photo.id).where( + # NULL taken_at_source predates the column default and + # should still be re-extracted; only 'manual' is sacred. + or_( + Photo.taken_at_source != 'manual', + Photo.taken_at_source.is_(None), + ), + Photo.is_discarded.is_(False), + ) + ) + photo_ids = [row[0] for row in result.all()] + + for pid in photo_ids: + extract_metadata.delay(pid) + + logger.info(f"backfill_taken_at: queued extract_metadata for {len(photo_ids)} photos") + return {'queued': len(photo_ids)} \ No newline at end of file diff --git a/frontend/src/components/sidebar/TakenAtEditor.tsx b/frontend/src/components/sidebar/TakenAtEditor.tsx index 5cd5dce..5272f8a 100644 --- a/frontend/src/components/sidebar/TakenAtEditor.tsx +++ b/frontend/src/components/sidebar/TakenAtEditor.tsx @@ -36,7 +36,17 @@ export function TakenAtEditor({ ? 'FILE' : source === 'manual' ? 'MANUAL' - : null + : source === 'path' + ? 'PATH' + : null + // Path-derived dates are guesses, so spell out where they came from + // in the tooltip. Other sources just need a short label. + const sourceTitle = + source === 'path' + ? 'Date inferred from filename or folder name (no trusted EXIF capture date)' + : sourceLabel + ? `Source: ${sourceLabel.toLowerCase()}` + : undefined const guess = useMemo( () => guessDateFromPath(photo.filepath), @@ -89,7 +99,7 @@ export function TakenAtEditor({ {sourceLabel && ( {sourceLabel}