dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(metadata): drop EXIF:ModifyDate fallback, prefer SubSec, fall back to path

Browse Source 
The taken_at extractor walked four EXIF fields in order: DateTimeOriginal,
CreateDate, MediaCreateDate, ModifyDate. The last one is set every time
a file is re-saved (Lightroom export, EXIF strip, batch resize), so any
photo whose original capture metadata was lost during editing ended up
labeled 'exif' with the *edit* date instead of the shoot date.

Changes:
  - SubSecDateTimeOriginal at the top of the list (sub-second precision,
    often carries OffsetTime).
  - QuickTime:CreateDate added next to MediaCreateDate.
  - ModifyDate dropped from the trusted list entirely.
  - When no trusted EXIF date is present, fall back to guess_date_from_path
    (already used for has_date_warning) and tag taken_at_source='path'.
    Better than filesystem mtime, which on Nextcloud-mounted libraries
    just reflects the upload time.
  - Skip the date-write block entirely if photo.taken_at_source == 'manual'
    so a rescan can't clobber a user correction.
  - parse_exif_datetime: handle the all-zero placeholder some cameras
    emit, accept tz-aware variants (%z), normalize to naive UTC.

Frontend: new 'PATH' badge in TakenAtEditor with a tooltip explaining
the date came from filename / folder rather than real EXIF.

Backfill: new backfill_taken_at celery task and
POST /api/v1/library/maintenance/backfill-taken-at endpoint that
re-enqueues extract_metadata for every non-manual photo. ~21k tasks
finish in ~15 min on the existing worker-light concurrency.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claudio
2026-05-10 23:02:32 +02:00
parent 76551d898b
commit 7b153f0d28
3 changed files with 134 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backend/app/routers/library.py
View File

@@ -187,6 +187,19 @@ async def trigger_backfill_gps(current_user: User = Depends(get_current_user)):
backfill_gps.delay()
return {"status": "success", "message": "GPS backfill queued"}
@router.post("/maintenance/backfill-taken-at")
async def trigger_backfill_taken_at(current_user: User = Depends(get_current_user)):
"""Re-run extract_metadata on every non-manual photo to recompute
taken_at with the current EXIF-priority list and path-based fallback.
Useful after the date-extraction logic changes (e.g. dropping the
ModifyDate fallback). Manual edits are preserved."""
from app.services.metadata import backfill_taken_at
backfill_taken_at.delay()
return {"status": "success", "message": "taken_at backfill queued"}
@router.get("/scan/status")
async def get_scan_status(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
"""Get current scan status"""

131
backend/app/services/metadata.py
View File

@@ -20,26 +20,46 @@ from app.services.date_guess import has_date_warning
logger = logging.getLogger(__name__)
def parse_exif_datetime(date_str: str) -> Optional[datetime]:
"""Parse EXIF datetime string to Python datetime"""
"""Parse EXIF datetime string to Python datetime.
Returns a tz-naive datetime — the photos.taken_at column is
`timestamp without time zone`. Tz-aware inputs (e.g. SubSec
fields with `+02:00` or QuickTime UTC `Z`) are converted to UTC
and stripped. Cameras that wrote the all-zero placeholder
return None.
"""
if not date_str:
return None
# Common EXIF datetime formats
s = str(date_str).strip()
# All-zero placeholder some cameras emit when the clock isn't set.
if s.startswith("0000:00:00") or s.startswith("0000-00-00"):
return None
formats = [
"%Y:%m:%d %H:%M:%S",
"%Y-%m-%d %H:%M:%S",
"%Y:%m:%d %H:%M:%S.%f",
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%dT%H:%M:%S.%f",
"%Y-%m-%dT%H:%M:%S%z"
# Tz-aware variants: SubSecDateTimeOriginal often looks like
# "2023:11:30 14:30:45.123+02:00", QuickTime CreateDate as
# "2023:11:30 14:30:45Z" or with offsets.
"%Y:%m:%d %H:%M:%S%z",
"%Y:%m:%d %H:%M:%S.%f%z",
"%Y-%m-%dT%H:%M:%S%z",
"%Y-%m-%dT%H:%M:%S.%f%z",
]
for fmt in formats:
try:
return datetime.strptime(date_str, fmt)
dt = datetime.strptime(s, fmt)
except ValueError:
continue
if dt.tzinfo is not None:
from datetime import timezone
dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
return dt
return None
_DMS_RE = re.compile(
@@ -227,21 +247,51 @@ async def _extract_metadata_async(photo_id: str):
# Store full metadata as JSON
photo.exif_json = json.dumps(exif_data)
# Extract taken_at date
date_fields = [
'EXIF:DateTimeOriginal',
'EXIF:CreateDate',
'QuickTime:MediaCreateDate',
'EXIF:ModifyDate'
]
# Extract taken_at date — but only if the user hasn't
# explicitly set it via the UI. Manual edits are the
# source of truth and must survive any rescan.
if photo.taken_at_source != 'manual':
# Trusted EXIF fields, in order of preference.
# SubSecDateTimeOriginal includes sub-second
# precision and often a tz offset, so it's the
# most accurate when present. ModifyDate is NOT
# in this list — it's set every time the file
# is re-saved (Lightroom export, EXIF strip,
# batch resize) and routinely overwrote correct
# capture dates with edit-time dates.
date_fields = [
'EXIF:SubSecDateTimeOriginal',
'EXIF:DateTimeOriginal',
'EXIF:CreateDate',
'QuickTime:MediaCreateDate',
'QuickTime:CreateDate',
]
for field in date_fields:
if field in exif_data:
taken_at = parse_exif_datetime(exif_data[field])
if taken_at:
photo.taken_at = taken_at
photo.taken_at_source = 'exif'
break
new_taken_at = None
for field in date_fields:
if field in exif_data:
parsed = parse_exif_datetime(exif_data[field])
if parsed:
new_taken_at = parsed
photo.taken_at = parsed
photo.taken_at_source = 'exif'
break
# Fallback: if the file has no trusted EXIF date,
# try to extract one from the filename / folder
# path. The same date_guess module powers the
# has_date_warning flag — reusing it here means
# photos without EXIF (scanned prints, stripped
# JPEGs, re-saved exports) get a sensible date
# instead of falling back to filesystem mtime
# (which on Nextcloud-mounted files is just the
# upload time).
if new_taken_at is None:
from app.services.date_guess import guess_date_from_path
guess = guess_date_from_path(photo.filepath)
if guess is not None:
photo.taken_at = guess.date
photo.taken_at_source = 'path'
# Re-run the path-vs-date heuristic now that we know
# whether EXIF provided a real capture date. A true EXIF
@@ -288,4 +338,41 @@ async def _extract_metadata_async(photo_id: str):
except Exception as e:
logger.error(f"Error extracting metadata for {photo_id}: {e}")
return {'status': 'error', 'message': str(e)}
return {'status': 'error', 'message': str(e)}
@shared_task(name='backfill_taken_at')
def backfill_taken_at():
"""Re-enqueue extract_metadata for every non-manual photo.
Used after fixing the date-extraction logic (removing ModifyDate
fallback, adding path-based fallback) to re-derive taken_at across
the whole library without touching photos the user has manually
corrected. Each enqueued task is fast (~90ms) and runs on the
default queue; ~21k photos finish in ~15 min on the existing
worker-light concurrency.
"""
return asyncio.run(_backfill_taken_at_async())
async def _backfill_taken_at_async():
from sqlalchemy import or_
async with AsyncSessionLocal() as session:
result = await session.execute(
select(Photo.id).where(
# NULL taken_at_source predates the column default and
# should still be re-extracted; only 'manual' is sacred.
or_(
Photo.taken_at_source != 'manual',
Photo.taken_at_source.is_(None),
),
Photo.is_discarded.is_(False),
)
)
photo_ids = [row[0] for row in result.all()]
for pid in photo_ids:
extract_metadata.delay(pid)
logger.info(f"backfill_taken_at: queued extract_metadata for {len(photo_ids)} photos")
return {'queued': len(photo_ids)}