dtoro/mule-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: structure

Browse Source
This commit is contained in:
authentik Default Admin
2026-04-06 23:30:19 +02:00
commit 46a0d7aba8
41 changed files with 3480 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
backend/Dockerfile Normal file
View File

@@ -0,0 +1,39 @@
FROM python:3.12-slim
# Install system dependencies
RUN apt-get update && apt-get install -y \
# Build dependencies
gcc \
g++ \
make \
# Image processing libraries
libvips42 \
libvips-dev \
# ExifTool for metadata extraction
libimage-exiftool-perl \
# FFmpeg for video processing
ffmpeg \
# Git for some Python packages
git \
# PostgreSQL client (for potential future use)
postgresql-client \
# Clean up
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy requirements first for better caching
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Copy application code
COPY . .
# Create necessary directories
RUN mkdir -p /data/thumbs /data/db /data/trash /app/config
# Expose port
EXPOSE 8000
# Run the application
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]

130
backend/app/config.py Normal file
View File

@@ -0,0 +1,130 @@
"""
Application configuration using Pydantic Settings
"""
from pydantic_settings import BaseSettings
from pydantic import BaseModel, Field
from typing import List, Optional
import os
import yaml
from pathlib import Path
class ThumbnailSettings(BaseModel):
"""Thumbnail generation settings"""
small: int = 240
medium: int = 640
large: int = 1280
quality: int = 85
format: str = "webp"
class ScannerSettings(BaseModel):
"""File scanner settings"""
watch: bool = True
initial_scan_on_start: bool = True
batch_size: int = 100
concurrent_workers: int = 4
class SourceRoot(BaseModel):
"""Source root directory configuration"""
name: str
path: str
class TrashSettings(BaseModel):
"""Trash settings"""
path: str = "/data/trash"
auto_empty_days: Optional[int] = 30
class PerformanceSettings(BaseModel):
"""Performance tuning settings"""
max_concurrent_thumbnails: int = 10
cache_ttl: int = 3600
db_pool_size: int = 20
db_pool_recycle: int = 3600
class MulitaConfig(BaseModel):
"""Main configuration from YAML file"""
source_roots: List[SourceRoot] = []
thumbnails: ThumbnailSettings = ThumbnailSettings()
scanner: ScannerSettings = ScannerSettings()
trash: TrashSettings = TrashSettings()
performance: PerformanceSettings = PerformanceSettings()
class Settings(BaseSettings):
"""Application settings"""
# Database
database_url: str = Field(
default="sqlite+aiosqlite:///data/db/mulita.db",
env="DATABASE_URL"
)
# Redis
redis_url: str = Field(
default="redis://localhost:6379",
env="REDIS_URL"
)
# Celery
celery_broker_url: str = Field(
default="redis://localhost:6379",
env="CELERY_BROKER_URL"
)
celery_result_backend: str = Field(
default="redis://localhost:6379",
env="CELERY_RESULT_BACKEND"
)
# Photo directories
photo_dirs: str = Field(
default="/photos",
env="PHOTO_DIRS"
)
# API settings
api_host: str = Field(default="0.0.0.0", env="API_HOST")
api_port: int = Field(default=8000, env="API_PORT")
# App configuration from YAML
_config: Optional[MulitaConfig] = None
@property
def config(self) -> MulitaConfig:
"""Load configuration from YAML file"""
if self._config is None:
config_path = Path("/app/config/mulita.yml")
if not config_path.exists():
config_path = Path("mulita.yml")
if config_path.exists():
with open(config_path, "r") as f:
config_data = yaml.safe_load(f)
self._config = MulitaConfig(**config_data)
else:
self._config = MulitaConfig()
return self._config
@property
def thumbnails(self) -> ThumbnailSettings:
return self.config.thumbnails
@property
def scanner(self) -> ScannerSettings:
return self.config.scanner
@property
def trash(self) -> TrashSettings:
return self.config.trash
@property
def performance(self) -> PerformanceSettings:
return self.config.performance
@property
def source_roots(self) -> List[SourceRoot]:
return self.config.source_roots
class Config:
env_file = ".env"
case_sensitive = False
# Global settings instance
settings = Settings()

82
backend/app/database.py Normal file
View File

@@ -0,0 +1,82 @@
"""
Database configuration and session management
"""
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
from sqlalchemy.orm import declarative_base
from sqlalchemy import event
import logging
import os
from pathlib import Path
from app.config import settings
logger = logging.getLogger(__name__)
# Create database directory if it doesn't exist
db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", ""))
db_path.parent.mkdir(parents=True, exist_ok=True)
# Create async engine
engine = create_async_engine(
settings.database_url,
echo=False, # Set to True for SQL debugging
pool_size=settings.performance.db_pool_size,
pool_recycle=settings.performance.db_pool_recycle,
connect_args={
"check_same_thread": False, # SQLite specific
"timeout": 30
} if "sqlite" in settings.database_url else {}
)
# Create async session factory
AsyncSessionLocal = async_sessionmaker(
engine,
class_=AsyncSession,
expire_on_commit=False
)
# Base class for models
Base = declarative_base()
async def get_db() -> AsyncSession:
"""Dependency to get database session"""
async with AsyncSessionLocal() as session:
try:
yield session
finally:
await session.close()
async def init_db():
"""Initialize database, create tables if they don't exist"""
async with engine.begin() as conn:
# Import all models to register them with Base
from app.models import Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto, Embedding
# Create all tables
await conn.run_sync(Base.metadata.create_all)
# Enable WAL mode for SQLite (better concurrency)
if "sqlite" in settings.database_url:
await conn.execute("PRAGMA journal_mode=WAL")
await conn.execute("PRAGMA synchronous=NORMAL")
await conn.execute("PRAGMA cache_size=10000")
await conn.execute("PRAGMA temp_store=MEMORY")
logger.info("Database initialized successfully")
async def create_fts_table():
"""Create Full-Text Search table for SQLite"""
if "sqlite" in settings.database_url:
async with engine.begin() as conn:
# Create FTS5 virtual table for full-text search
await conn.execute("""
CREATE VIRTUAL TABLE IF NOT EXISTS photos_fts USING fts5(
photo_id UNINDEXED,
filename,
user_title,
user_notes,
exif_text,
tokenize='unicode61'
)
""")
logger.info("FTS5 table created successfully")

82
backend/app/main.py Normal file
View File

@@ -0,0 +1,82 @@
"""
Mulita - Photo Management Application
Main FastAPI application entry point
"""
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
import logging
import os
from app.config import settings
from app.database import init_db
from app.routers import photos, folders, heaps, tags, trash, library
from app.services.scanner import start_initial_scan
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Manage application lifecycle"""
logger.info("Starting Mulita application...")
# Initialize database
await init_db()
# Start initial scan if configured
if settings.scanner.initial_scan_on_start:
logger.info("Starting initial library scan...")
await start_initial_scan()
yield
logger.info("Shutting down Mulita application...")
# Create FastAPI app
app = FastAPI(
title="Mulita Photo Management API",
description="Self-hosted photo management application inspired by Lightroom",
version="1.0.0",
lifespan=lifespan
)
# Configure CORS
app.add_middleware(
CORSMiddleware,
allow_origins=["http://localhost:3000", "http://localhost:5173"], # Frontend URLs
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Mount static files for serving thumbnails (with X-Accel-Redirect support)
if os.path.exists("/data/thumbs"):
app.mount("/thumbs", StaticFiles(directory="/data/thumbs"), name="thumbs")
# Include routers
app.include_router(photos.router, prefix="/api/v1/photos", tags=["photos"])
app.include_router(folders.router, prefix="/api/v1/folders", tags=["folders"])
app.include_router(heaps.router, prefix="/api/v1/heaps", tags=["heaps"])
app.include_router(tags.router, prefix="/api/v1/tags", tags=["tags"])
app.include_router(trash.router, prefix="/api/v1/trash", tags=["trash"])
app.include_router(library.router, prefix="/api/v1/library", tags=["library"])
@app.get("/")
async def root():
"""Root endpoint"""
return {
"name": "Mulita Photo Management API",
"version": "1.0.0",
"status": "running"
}
@app.get("/health")
async def health_check():
"""Health check endpoint for Docker"""
return {"status": "healthy"}

19
backend/app/models/__init__.py Normal file
View File

@@ -0,0 +1,19 @@
"""
Database models for Mulita
"""
from app.models.photos import Photo
from app.models.folders import Folder, SourceRoot
from app.models.tags import Tag, PhotoTag
from app.models.heaps import Heap, HeapPhoto
from app.models.embeddings import Embedding
__all__ = [
'Photo',
'Folder',
'SourceRoot',
'Tag',
'PhotoTag',
'Heap',
'HeapPhoto',
'Embedding'
]

17
backend/app/models/embeddings.py Normal file
View File

@@ -0,0 +1,17 @@
"""
Embedding model definition (placeholder for AI features)
"""
from sqlalchemy import Column, String, ForeignKey, LargeBinary
import uuid
from app.database import Base
class Embedding(Base):
"""
Placeholder table for future AI embeddings (CLIP, face recognition, etc.)
"""
__tablename__ = 'embeddings'
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True)
model = Column(String) # e.g., 'clip-vit-b32', 'face-recognition', etc.
vector = Column(LargeBinary) # raw float32 bytes for embedding vector

43
backend/app/models/folders.py Normal file
View File

@@ -0,0 +1,43 @@
"""
Folder and SourceRoot model definitions
"""
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey, Index
from sqlalchemy.sql import func
from sqlalchemy.orm import relationship
import uuid
from app.database import Base
class SourceRoot(Base):
__tablename__ = 'source_roots'
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, nullable=False)
path = Column(String, unique=True, nullable=False)
is_active = Column(Boolean, default=True)
added_at = Column(DateTime, server_default=func.now())
# Relationships
folders = relationship("Folder", back_populates="source_root")
class Folder(Base):
__tablename__ = 'folders'
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, nullable=False)
path = Column(String, unique=True, nullable=False)
parent_id = Column(String, ForeignKey('folders.id'))
source_root_id = Column(String, ForeignKey('source_roots.id'))
photo_count = Column(Integer, default=0)
last_scanned = Column(DateTime)
# Relationships
source_root = relationship("SourceRoot", back_populates="folders")
photos = relationship("Photo", backref="folder")
# Indexes
__table_args__ = (
Index('ix_folders_path', 'path'),
Index('ix_folders_parent_id', 'parent_id'),
Index('ix_folders_source_root_id', 'source_root_id'),
)

37
backend/app/models/heaps.py Normal file
View File

@@ -0,0 +1,37 @@
"""
Heap model definitions
"""
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey, Table, Index
from sqlalchemy.sql import func
from sqlalchemy.orm import relationship
import uuid
from app.database import Base
# Association table for many-to-many relationship with additional fields
heap_photos = Table(
'heap_photos',
Base.metadata,
Column('heap_id', String, ForeignKey('heaps.id', ondelete='CASCADE'), primary_key=True),
Column('photo_id', String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True),
Column('added_at', DateTime, server_default=func.now()),
Column('sort_order', Integer, default=0),
Index('ix_heap_photos_heap_id', 'heap_id'),
Index('ix_heap_photos_photo_id', 'photo_id'),
)
class Heap(Base):
__tablename__ = 'heaps'
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, nullable=False)
created_at = Column(DateTime, server_default=func.now())
updated_at = Column(DateTime, onupdate=func.now())
is_active = Column(Boolean, default=False) # For active heap feature
# Relationships
photos = relationship("Photo", secondary=heap_photos, backref="heaps")
class HeapPhoto:
"""Helper class for heap-photo associations (not a table model)"""
pass

75
backend/app/models/photos.py Normal file
View File

@@ -0,0 +1,75 @@
"""
Photo model definition
"""
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey, Text, Index
from sqlalchemy.sql import func
from datetime import datetime
import uuid
from app.database import Base
class Photo(Base):
__tablename__ = 'photos'
# Primary key
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
# File information
filepath = Column(String, unique=True, nullable=False)
filename = Column(String, nullable=False)
folder_id = Column(String, ForeignKey('folders.id'))
file_hash = Column(String, index=True) # SHA-256 hash for duplicate detection
# Media information
media_type = Column(String, nullable=False) # 'photo' | 'video' | 'raw' | 'heic'
original_format = Column(String) # 'CR3', 'NEF', 'HEIC', 'MP4', 'JPEG', etc.
width = Column(Integer)
height = Column(Integer)
file_size = Column(Integer)
# Timestamps
taken_at = Column(DateTime) # from EXIF DateTimeOriginal, fallback to file mtime
taken_at_source = Column(String) # 'exif' | 'filesystem' | 'manual'
added_at = Column(DateTime, server_default=func.now())
updated_at = Column(DateTime, onupdate=func.now())
# Trash status
is_trashed = Column(Boolean, default=False)
trashed_at = Column(DateTime)
# Thumbnail paths
thumb_small = Column(String) # path to 240px thumb
thumb_medium = Column(String) # path to 640px thumb
thumb_large = Column(String) # path to 1280px thumb
# Processing status
processing_status = Column(String, default='pending') # 'pending' | 'processing' | 'completed' | 'failed'
processing_error = Column(Text)
# Metadata
exif_json = Column(Text) # full EXIF/XMP blob as JSON
# User-editable fields
user_title = Column(String)
user_notes = Column(Text)
rating = Column(Integer, default=0) # 0-5 stars
color_label = Column(String) # 'red'|'orange'|'yellow'|'green'|'blue'|'purple'|NULL
is_picked = Column(Boolean, default=False)
is_rejected = Column(Boolean, default=False)
# Duplicate detection
is_duplicate = Column(Boolean, default=False)
# Live photo support
live_photo_video_id = Column(String, ForeignKey('photos.id'))
# Indexes for performance
__table_args__ = (
Index('ix_photos_taken_at', 'taken_at'),
Index('ix_photos_folder_id', 'folder_id'),
Index('ix_photos_is_trashed', 'is_trashed'),
Index('ix_photos_rating', 'rating'),
Index('ix_photos_color_label', 'color_label'),
Index('ix_photos_media_type', 'media_type'),
Index('ix_photos_processing_status', 'processing_status'),
)

32
backend/app/models/tags.py Normal file
View File

@@ -0,0 +1,32 @@
"""
Tag model definitions
"""
from sqlalchemy import Column, String, ForeignKey, Table, Index
from sqlalchemy.orm import relationship
import uuid
from app.database import Base
# Association table for many-to-many relationship
photo_tags = Table(
'photo_tags',
Base.metadata,
Column('photo_id', String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True),
Column('tag_id', String, ForeignKey('tags.id', ondelete='CASCADE'), primary_key=True),
Index('ix_photo_tags_photo_id', 'photo_id'),
Index('ix_photo_tags_tag_id', 'tag_id'),
)
class Tag(Base):
__tablename__ = 'tags'
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, unique=True, nullable=False, index=True)
color = Column(String) # Hex color code for UI display
# Relationships
photos = relationship("Photo", secondary=photo_tags, backref="tags")
class PhotoTag:
"""Helper class for photo-tag associations (not a table model)"""
pass

33
backend/app/routers/folders.py Normal file
View File

@@ -0,0 +1,33 @@
"""
Folders API router
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from typing import List
from app.database import get_db
from app.models import Folder, SourceRoot
router = APIRouter()
@router.get("")
async def get_folders(db: AsyncSession = Depends(get_db)):
"""Get folder tree"""
result = await db.execute(select(Folder))
folders = result.scalars().all()
return folders
@router.post("/{folder_id}/scan")
async def scan_folder(folder_id: str, db: AsyncSession = Depends(get_db)):
"""Trigger manual re-scan of folder"""
from app.tasks.scan import scan_folder as scan_task
result = await db.execute(select(Folder).where(Folder.id == folder_id))
folder = result.scalar_one_or_none()
if not folder:
raise HTTPException(status_code=404, detail="Folder not found")
scan_task.delay(folder.path)
return {"status": "success", "message": f"Scan queued for {folder.path}"}

27
backend/app/routers/heaps.py Normal file
View File

@@ -0,0 +1,27 @@
"""
Heaps API router
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models import Heap
router = APIRouter()
@router.get("")
async def list_heaps(db: AsyncSession = Depends(get_db)):
"""List all heaps"""
result = await db.execute(select(Heap))
heaps = result.scalars().all()
return heaps
@router.post("")
async def create_heap(name: str, db: AsyncSession = Depends(get_db)):
"""Create a new heap"""
heap = Heap(name=name)
db.add(heap)
await db.commit()
await db.refresh(heap)
return heap

61
backend/app/routers/library.py Normal file
View File

@@ -0,0 +1,61 @@
"""
Library API router for stats and scanning
"""
from fastapi import APIRouter, Depends
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models import Photo
router = APIRouter()
@router.get("/stats")
async def get_library_stats(db: AsyncSession = Depends(get_db)):
"""Get library statistics"""
# Count total photos
total_photos = await db.execute(
select(func.count(Photo.id)).where(Photo.media_type.in_(['photo', 'heic', 'raw']))
)
photo_count = total_photos.scalar()
# Count total videos
total_videos = await db.execute(
select(func.count(Photo.id)).where(Photo.media_type == 'video')
)
video_count = total_videos.scalar()
# Calculate total size
total_size = await db.execute(
select(func.sum(Photo.file_size))
)
size = total_size.scalar() or 0
return {
"total_photos": photo_count,
"total_videos": video_count,
"total_size": size,
"total_size_gb": round(size / (1024**3), 2) if size else 0
}
@router.post("/scan")
async def trigger_scan():
"""Trigger full library re-scan"""
from app.tasks.scan import scan_all_source_roots
scan_all_source_roots.delay()
return {"status": "success", "message": "Library scan started"}
@router.get("/scan/status")
async def get_scan_status():
"""Get current scan status"""
# This would connect to Celery to get task status
# For now, return a simple response
return {
"status": "idle",
"progress": 0,
"current_folder": None,
"queued": 0,
"done": 0
}

310
backend/app/routers/photos.py Normal file
View File

@@ -0,0 +1,310 @@
"""
Photos API router
"""
from typing import List, Optional, Dict, Any
from datetime import datetime
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from fastapi.responses import FileResponse
from sqlalchemy import select, and_, or_, func
from sqlalchemy.ext.asyncio import AsyncSession
import json
import os
from app.database import get_db
from app.models import Photo, Folder, Tag, PhotoTag
from app.schemas.photos import PhotoResponse, PhotoUpdate, PhotoListResponse, BulkAction
from app.config import settings
router = APIRouter()
@router.get("", response_model=PhotoListResponse)
async def list_photos(
q: Optional[str] = None,
date_from: Optional[datetime] = None,
date_to: Optional[datetime] = None,
folder_id: Optional[str] = None,
tag_ids: Optional[str] = None,
media_type: Optional[str] = None,
rating_min: Optional[int] = Query(None, ge=0, le=5),
rating_max: Optional[int] = Query(None, ge=0, le=5),
color_label: Optional[str] = None,
is_picked: Optional[bool] = None,
is_rejected: Optional[bool] = None,
is_trashed: Optional[bool] = False,
heap_id: Optional[str] = None,
sort: str = "taken_at",
order: str = "desc",
page: int = Query(1, ge=1),
per_page: int = Query(100, ge=1, le=500),
db: AsyncSession = Depends(get_db)
):
"""List photos with filters and pagination"""
# Build query
query = select(Photo)
# Apply filters
filters = []
# Text search (would use FTS5 in production)
if q:
search_pattern = f"%{q}%"
filters.append(
or_(
Photo.filename.ilike(search_pattern),
Photo.user_title.ilike(search_pattern),
Photo.user_notes.ilike(search_pattern),
Photo.exif_json.ilike(search_pattern)
)
)
# Date range
if date_from:
filters.append(Photo.taken_at >= date_from)
if date_to:
filters.append(Photo.taken_at <= date_to)
# Folder filter
if folder_id:
filters.append(Photo.folder_id == folder_id)
# Media type filter
if media_type:
types = media_type.split(',')
filters.append(Photo.media_type.in_(types))
# Rating filter
if rating_min is not None:
filters.append(Photo.rating >= rating_min)
if rating_max is not None:
filters.append(Photo.rating <= rating_max)
# Color label filter
if color_label:
if color_label == 'none':
filters.append(Photo.color_label.is_(None))
else:
filters.append(Photo.color_label == color_label)
# Flag filters
if is_picked is not None:
filters.append(Photo.is_picked == is_picked)
if is_rejected is not None:
filters.append(Photo.is_rejected == is_rejected)
# Trash filter
filters.append(Photo.is_trashed == is_trashed)
# Apply all filters
if filters:
query = query.where(and_(*filters))
# Apply sorting
sort_column = getattr(Photo, sort, Photo.taken_at)
if order == "desc":
query = query.order_by(sort_column.desc())
else:
query = query.order_by(sort_column.asc())
# Count total results
count_query = select(func.count()).select_from(query.subquery())
total_result = await db.execute(count_query)
total = total_result.scalar()
# Apply pagination
offset = (page - 1) * per_page
query = query.offset(offset).limit(per_page)
# Execute query
result = await db.execute(query)
photos = result.scalars().all()
# Convert to response
return PhotoListResponse(
photos=[PhotoResponse.from_orm(photo) for photo in photos],
total=total,
page=page,
per_page=per_page,
pages=(total + per_page - 1) // per_page
)
@router.get("/{photo_id}", response_model=PhotoResponse)
async def get_photo(
photo_id: str,
db: AsyncSession = Depends(get_db)
):
"""Get single photo with full EXIF and tags"""
result = await db.execute(
select(Photo).where(Photo.id == photo_id)
)
photo = result.scalar_one_or_none()
if not photo:
raise HTTPException(status_code=404, detail="Photo not found")
return PhotoResponse.from_orm(photo)
@router.get("/{photo_id}/thumb/{size}")
async def get_thumbnail(
photo_id: str,
size: str,
response: Response,
db: AsyncSession = Depends(get_db)
):
"""Serve thumbnail (with Nginx X-Accel-Redirect support)"""
if size not in ['small', 'medium', 'large']:
raise HTTPException(status_code=400, detail="Invalid thumbnail size")
result = await db.execute(
select(Photo).where(Photo.id == photo_id)
)
photo = result.scalar_one_or_none()
if not photo:
raise HTTPException(status_code=404, detail="Photo not found")
thumb_path = getattr(photo, f'thumb_{size}')
if not thumb_path or not os.path.exists(thumb_path):
raise HTTPException(status_code=404, detail="Thumbnail not found")
# Check if we're behind Nginx
if os.environ.get('USE_X_ACCEL_REDIRECT'):
# Use Nginx X-Accel-Redirect for better performance
response.headers['X-Accel-Redirect'] = f'/internal_thumbs/{photo_id}/{size}.webp'
response.headers['Content-Type'] = 'image/webp'
return Response()
else:
# Direct file serving for development
return FileResponse(thumb_path, media_type='image/webp')
@router.get("/{photo_id}/original")
async def get_original(
photo_id: str,
db: AsyncSession = Depends(get_db)
):
"""Serve original file for download"""
result = await db.execute(
select(Photo).where(Photo.id == photo_id)
)
photo = result.scalar_one_or_none()
if not photo:
raise HTTPException(status_code=404, detail="Photo not found")
if not os.path.exists(photo.filepath):
raise HTTPException(status_code=404, detail="File not found")
return FileResponse(
photo.filepath,
filename=photo.filename,
media_type='application/octet-stream'
)
@router.patch("/{photo_id}", response_model=PhotoResponse)
async def update_photo(
photo_id: str,
update: PhotoUpdate,
db: AsyncSession = Depends(get_db)
):
"""Update photo metadata"""
result = await db.execute(
select(Photo).where(Photo.id == photo_id)
)
photo = result.scalar_one_or_none()
if not photo:
raise HTTPException(status_code=404, detail="Photo not found")
# Apply updates
update_data = update.dict(exclude_unset=True)
for field, value in update_data.items():
setattr(photo, field, value)
await db.commit()
await db.refresh(photo)
return PhotoResponse.from_orm(photo)
@router.delete("/{photo_id}")
async def trash_photo(
photo_id: str,
db: AsyncSession = Depends(get_db)
):
"""Move photo to trash"""
result = await db.execute(
select(Photo).where(Photo.id == photo_id)
)
photo = result.scalar_one_or_none()
if not photo:
raise HTTPException(status_code=404, detail="Photo not found")
# Move file to trash directory
import shutil
trash_dir = f"{settings.trash.path}/{photo_id}"
os.makedirs(trash_dir, exist_ok=True)
trash_path = f"{trash_dir}/original{Path(photo.filepath).suffix}"
try:
shutil.move(photo.filepath, trash_path)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to move file: {e}")
# Update database
photo.is_trashed = True
photo.trashed_at = datetime.utcnow()
await db.commit()
return {"status": "success", "message": "Photo moved to trash"}
@router.post("/bulk")
async def bulk_action(
action: BulkAction,
db: AsyncSession = Depends(get_db)
):
"""Perform bulk actions on multiple photos"""
# Get photos
result = await db.execute(
select(Photo).where(Photo.id.in_(action.ids))
)
photos = result.scalars().all()
if not photos:
raise HTTPException(status_code=404, detail="No photos found")
# Perform action based on type
if action.action == 'trash':
for photo in photos:
photo.is_trashed = True
photo.trashed_at = datetime.utcnow()
elif action.action == 'restore':
for photo in photos:
photo.is_trashed = False
photo.trashed_at = None
elif action.action == 'set_rating':
for photo in photos:
photo.rating = action.value
elif action.action == 'set_color':
for photo in photos:
photo.color_label = action.value
elif action.action == 'pick':
for photo in photos:
photo.is_picked = True
photo.is_rejected = False
elif action.action == 'reject':
for photo in photos:
photo.is_rejected = True
photo.is_picked = False
else:
raise HTTPException(status_code=400, detail="Invalid action")
await db.commit()
return {
"status": "success",
"message": f"{action.action} applied to {len(photos)} photos"
}

27
backend/app/routers/tags.py Normal file
View File

@@ -0,0 +1,27 @@
"""
Tags API router
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models import Tag
router = APIRouter()
@router.get("")
async def list_tags(db: AsyncSession = Depends(get_db)):
"""List all tags with usage counts"""
result = await db.execute(select(Tag))
tags = result.scalars().all()
return tags
@router.post("")
async def create_tag(name: str, color: str = None, db: AsyncSession = Depends(get_db)):
"""Create a new tag"""
tag = Tag(name=name, color=color)
db.add(tag)
await db.commit()
await db.refresh(tag)
return tag

50
backend/app/routers/trash.py Normal file
View File

@@ -0,0 +1,50 @@
"""
Trash API router
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select, and_
from sqlalchemy.ext.asyncio import AsyncSession
from datetime import datetime
from app.database import get_db
from app.models import Photo
router = APIRouter()
@router.get("")
async def list_trashed(db: AsyncSession = Depends(get_db)):
"""List trashed photos"""
result = await db.execute(
select(Photo).where(Photo.is_trashed == True)
)
photos = result.scalars().all()
return photos
@router.post("/restore")
async def restore_photos(photo_ids: list[str], db: AsyncSession = Depends(get_db)):
"""Restore photos from trash"""
result = await db.execute(
select(Photo).where(and_(Photo.id.in_(photo_ids), Photo.is_trashed == True))
)
photos = result.scalars().all()
for photo in photos:
photo.is_trashed = False
photo.trashed_at = None
await db.commit()
return {"status": "success", "restored": len(photos)}
@router.delete("/empty")
async def empty_trash(db: AsyncSession = Depends(get_db)):
"""Permanently delete all trashed photos"""
result = await db.execute(
select(Photo).where(Photo.is_trashed == True)
)
photos = result.scalars().all()
for photo in photos:
await db.delete(photo)
await db.commit()
return {"status": "success", "deleted": len(photos)}

71
backend/app/schemas/photos.py Normal file
View File

@@ -0,0 +1,71 @@
"""
Pydantic schemas for photos
"""
from pydantic import BaseModel, Field
from typing import Optional, List, Dict, Any
from datetime import datetime
class PhotoBase(BaseModel):
"""Base photo schema"""
filename: str
media_type: str
original_format: Optional[str] = None
width: Optional[int] = None
height: Optional[int] = None
file_size: Optional[int] = None
taken_at: Optional[datetime] = None
taken_at_source: Optional[str] = None
user_title: Optional[str] = None
user_notes: Optional[str] = None
rating: int = 0
color_label: Optional[str] = None
is_picked: bool = False
is_rejected: bool = False
class PhotoResponse(PhotoBase):
"""Photo response schema"""
id: str
filepath: str
folder_id: Optional[str] = None
file_hash: Optional[str] = None
added_at: datetime
updated_at: Optional[datetime] = None
is_trashed: bool = False
trashed_at: Optional[datetime] = None
thumb_small: Optional[str] = None
thumb_medium: Optional[str] = None
thumb_large: Optional[str] = None
processing_status: str = 'pending'
processing_error: Optional[str] = None
exif_json: Optional[str] = None
is_duplicate: bool = False
live_photo_video_id: Optional[str] = None
tags: List[Dict[str, Any]] = []
class Config:
orm_mode = True
from_attributes = True
class PhotoUpdate(BaseModel):
"""Photo update schema"""
user_title: Optional[str] = None
user_notes: Optional[str] = None
rating: Optional[int] = Field(None, ge=0, le=5)
color_label: Optional[str] = None
is_picked: Optional[bool] = None
is_rejected: Optional[bool] = None
taken_at: Optional[datetime] = None
class PhotoListResponse(BaseModel):
"""Photo list response with pagination"""
photos: List[PhotoResponse]
total: int
page: int
per_page: int
pages: int
class BulkAction(BaseModel):
"""Bulk action on photos"""
ids: List[str]
action: str # 'trash', 'restore', 'delete_permanent', 'move', 'copy', 'add_tag', 'remove_tag', 'set_rating', 'set_color', 'pick', 'reject'
value: Optional[Any] = None # For actions that need a value (rating, color, tag_id, folder_id)

183
backend/app/services/metadata.py Normal file
View File

@@ -0,0 +1,183 @@
"""
Metadata extraction service using ExifTool
"""
import json
import logging
import asyncio
from datetime import datetime
from typing import Dict, Optional
import subprocess
from pathlib import Path
from celery import shared_task
from sqlalchemy import select
from app.database import AsyncSessionLocal
from app.models import Photo
logger = logging.getLogger(__name__)
def parse_exif_datetime(date_str: str) -> Optional[datetime]:
"""Parse EXIF datetime string to Python datetime"""
if not date_str:
return None
# Common EXIF datetime formats
formats = [
"%Y:%m:%d %H:%M:%S",
"%Y-%m-%d %H:%M:%S",
"%Y:%m:%d %H:%M:%S.%f",
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%dT%H:%M:%S.%f",
"%Y-%m-%dT%H:%M:%S%z"
]
for fmt in formats:
try:
return datetime.strptime(date_str, fmt)
except ValueError:
continue
return None
def extract_key_metadata(exif_data: Dict) -> Dict:
"""Extract key metadata fields for FTS indexing"""
key_fields = []
# Camera information
if 'Make' in exif_data:
key_fields.append(exif_data['Make'])
if 'Model' in exif_data:
key_fields.append(exif_data['Model'])
if 'LensModel' in exif_data:
key_fields.append(exif_data['LensModel'])
# Location information
if 'GPSLatitude' in exif_data and 'GPSLongitude' in exif_data:
key_fields.append(f"GPS: {exif_data['GPSLatitude']}, {exif_data['GPSLongitude']}")
# IPTC/XMP keywords
if 'Keywords' in exif_data:
if isinstance(exif_data['Keywords'], list):
key_fields.extend(exif_data['Keywords'])
else:
key_fields.append(exif_data['Keywords'])
# Copyright and creator
if 'Copyright' in exif_data:
key_fields.append(exif_data['Copyright'])
if 'Creator' in exif_data:
key_fields.append(exif_data['Creator'])
if 'Artist' in exif_data:
key_fields.append(exif_data['Artist'])
return {
'exif_text': ' '.join(key_fields),
'camera_make': exif_data.get('Make'),
'camera_model': exif_data.get('Model'),
'lens_model': exif_data.get('LensModel'),
'gps_latitude': exif_data.get('GPSLatitude'),
'gps_longitude': exif_data.get('GPSLongitude'),
}
@shared_task(name='extract_metadata')
def extract_metadata(photo_id: str):
"""Extract metadata from a photo using ExifTool"""
return asyncio.run(_extract_metadata_async(photo_id))
async def _extract_metadata_async(photo_id: str):
"""Async implementation of metadata extraction"""
async with AsyncSessionLocal() as session:
try:
# Get photo from database
result = await session.execute(
select(Photo).where(Photo.id == photo_id)
)
photo = result.scalar_one_or_none()
if not photo:
logger.error(f"Photo not found: {photo_id}")
return {'status': 'error', 'message': 'Photo not found'}
# Check if file exists
if not Path(photo.filepath).exists():
logger.error(f"File not found: {photo.filepath}")
return {'status': 'error', 'message': 'File not found'}
# Run ExifTool to extract metadata
cmd = [
'exiftool',
'-j', # JSON output
'-G', # Group names
'-s', # Short output format
'-All', # All metadata
photo.filepath
]
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=30
)
if result.returncode != 0:
logger.error(f"ExifTool error: {result.stderr}")
return {'status': 'error', 'message': result.stderr}
# Parse JSON output
metadata = json.loads(result.stdout)
if metadata and len(metadata) > 0:
exif_data = metadata[0]
# Store full metadata as JSON
photo.exif_json = json.dumps(exif_data)
# Extract taken_at date
date_fields = [
'EXIF:DateTimeOriginal',
'EXIF:CreateDate',
'QuickTime:MediaCreateDate',
'EXIF:ModifyDate'
]
for field in date_fields:
if field in exif_data:
taken_at = parse_exif_datetime(exif_data[field])
if taken_at:
photo.taken_at = taken_at
photo.taken_at_source = 'exif'
break
# Extract dimensions if not already set
if not photo.width:
photo.width = exif_data.get('EXIF:ImageWidth') or exif_data.get('File:ImageWidth')
if not photo.height:
photo.height = exif_data.get('EXIF:ImageHeight') or exif_data.get('File:ImageHeight')
# Extract and store key metadata for search
key_metadata = extract_key_metadata(exif_data)
# Update FTS table (would be done via trigger in production)
# For now, we'll store it in a comment
await session.commit()
logger.info(f"Metadata extracted for photo {photo_id}")
return {
'status': 'success',
'photo_id': photo_id,
'taken_at': photo.taken_at.isoformat() if photo.taken_at else None
}
except subprocess.TimeoutExpired:
logger.error(f"ExifTool timeout for {photo.filepath}")
return {'status': 'error', 'message': 'ExifTool timeout'}
except json.JSONDecodeError as e:
logger.error(f"Failed to parse ExifTool output: {e}")
return {'status': 'error', 'message': 'Invalid ExifTool output'}
except Exception as e:
logger.error(f"Error extracting metadata for {photo_id}: {e}")
return {'status': 'error', 'message': str(e)}

22
backend/app/services/scanner.py Normal file
View File

@@ -0,0 +1,22 @@
"""
Scanner service for initial library scan
"""
import logging
from app.tasks.scan import scan_all_source_roots, watch_folders
from app.config import settings
logger = logging.getLogger(__name__)
async def start_initial_scan():
"""Start the initial library scan"""
try:
# Queue scan of all source roots
scan_all_source_roots.delay()
# Start folder watcher if configured
if settings.scanner.watch:
watch_folders.delay()
logger.info("Initial scan queued successfully")
except Exception as e:
logger.error(f"Failed to start initial scan: {e}")

31
backend/app/tasks/celery.py Normal file
View File

@@ -0,0 +1,31 @@
"""
Celery configuration and app initialization
"""
from celery import Celery
from app.config import settings
# Create Celery app
celery_app = Celery(
'mulita',
broker=settings.celery_broker_url,
backend=settings.celery_result_backend,
include=['app.tasks.scan', 'app.tasks.thumbs']
)
# Configure Celery
celery_app.conf.update(
task_serializer='json',
accept_content=['json'],
result_serializer='json',
timezone='UTC',
enable_utc=True,
task_routes={
'app.tasks.thumbs.*': {'queue': 'high'},
'app.tasks.scan.*': {'queue': 'low'},
},
task_default_queue='default',
task_default_exchange='default',
task_default_exchange_type='direct',
task_default_routing_key='default',
broker_connection_retry_on_startup=True,
)

298
backend/app/tasks/scan.py Normal file
View File

@@ -0,0 +1,298 @@
"""
Celery tasks for scanning folders and indexing photos
"""
import os
import hashlib
import asyncio
from pathlib import Path
from datetime import datetime
import logging
import json
from typing import List, Dict, Optional
from celery import shared_task
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
import aiofiles
from app.database import AsyncSessionLocal
from app.models import Photo, Folder, SourceRoot
from app.config import settings
from app.tasks.thumbs import generate_thumbnails
from app.services.metadata import extract_metadata
logger = logging.getLogger(__name__)
# Supported file extensions
PHOTO_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.tiff', '.tif', '.webp', '.bmp'}
RAW_EXTENSIONS = {'.cr2', '.cr3', '.nef', '.arw', '.raf', '.dng', '.orf', '.rw2', '.pef', '.srw'}
HEIC_EXTENSIONS = {'.heic', '.heif'}
VIDEO_EXTENSIONS = {'.mp4', '.mov', '.avi', '.mkv', '.mts', '.m2ts', '.3gp', '.wmv', '.flv'}
SUPPORTED_EXTENSIONS = PHOTO_EXTENSIONS | RAW_EXTENSIONS | HEIC_EXTENSIONS | VIDEO_EXTENSIONS
def get_media_type(filepath: str) -> str:
"""Determine media type from file extension"""
ext = Path(filepath).suffix.lower()
if ext in PHOTO_EXTENSIONS:
return 'photo'
elif ext in RAW_EXTENSIONS:
return 'raw'
elif ext in HEIC_EXTENSIONS:
return 'heic'
elif ext in VIDEO_EXTENSIONS:
return 'video'
return 'unknown'
async def calculate_file_hash(filepath: str) -> str:
"""Calculate SHA-256 hash of a file"""
hash_sha256 = hashlib.sha256()
try:
async with aiofiles.open(filepath, 'rb') as f:
while chunk := await f.read(8192):
hash_sha256.update(chunk)
return hash_sha256.hexdigest()
except Exception as e:
logger.error(f"Error calculating hash for {filepath}: {e}")
return ""
@shared_task(bind=True, name='scan_folder')
def scan_folder(self, folder_path: str, source_root_id: Optional[str] = None):
"""
Scan a folder and index all photos/videos
"""
# Run async function in sync context
return asyncio.run(_scan_folder_async(folder_path, source_root_id, self))
async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], task):
"""Async implementation of folder scanning"""
logger.info(f"Starting scan of folder: {folder_path}")
async with AsyncSessionLocal() as session:
try:
# Get or create source root
if not source_root_id:
source_root = await get_or_create_source_root(session, folder_path)
source_root_id = source_root.id
# Walk the directory tree
total_files = 0
processed_files = 0
errors = []
for root, dirs, files in os.walk(folder_path):
# Get or create folder entry
folder = await get_or_create_folder(session, root, source_root_id)
# Filter supported files
supported_files = [f for f in files if Path(f).suffix.lower() in SUPPORTED_EXTENSIONS]
total_files += len(supported_files)
# Process files in batches
batch_size = settings.scanner.batch_size
for i in range(0, len(supported_files), batch_size):
batch = supported_files[i:i + batch_size]
for filename in batch:
filepath = os.path.join(root, filename)
try:
# Check if file already exists in database
existing = await session.execute(
select(Photo).where(Photo.filepath == filepath)
)
if existing.scalar_one_or_none():
logger.debug(f"File already indexed: {filepath}")
processed_files += 1
continue
# Get file stats
stat = os.stat(filepath)
# Calculate file hash for duplicate detection
file_hash = await calculate_file_hash(filepath)
# Check for duplicate by hash
duplicate = await session.execute(
select(Photo).where(Photo.file_hash == file_hash)
) if file_hash else None
# Create photo entry
photo = Photo(
filepath=filepath,
filename=filename,
folder_id=folder.id,
file_hash=file_hash,
media_type=get_media_type(filepath),
original_format=Path(filepath).suffix.upper()[1:],
file_size=stat.st_size,
taken_at=datetime.fromtimestamp(stat.st_mtime),
taken_at_source='filesystem',
is_duplicate=bool(duplicate.scalar_one_or_none() if duplicate else False),
processing_status='pending'
)
session.add(photo)
await session.flush() # Get the photo ID
# Queue thumbnail generation
generate_thumbnails.delay(photo.id)
# Queue metadata extraction
extract_metadata.delay(photo.id)
processed_files += 1
# Update progress
if processed_files % 10 == 0:
task.update_state(
state='PROGRESS',
meta={
'current': processed_files,
'total': total_files,
'folder': root
}
)
except Exception as e:
logger.error(f"Error processing file {filepath}: {e}")
errors.append({'file': filepath, 'error': str(e)})
continue
# Commit batch
await session.commit()
# Update folder scan timestamp
folder.last_scanned = datetime.utcnow()
folder.photo_count = processed_files
await session.commit()
logger.info(f"Scan complete. Processed {processed_files}/{total_files} files. Errors: {len(errors)}")
return {
'status': 'completed',
'processed': processed_files,
'total': total_files,
'errors': errors
}
except Exception as e:
logger.error(f"Scan failed: {e}")
await session.rollback()
raise
async def get_or_create_source_root(session: AsyncSession, path: str) -> SourceRoot:
"""Get or create a source root entry"""
from sqlalchemy import select
result = await session.execute(
select(SourceRoot).where(SourceRoot.path == path)
)
source_root = result.scalar_one_or_none()
if not source_root:
source_root = SourceRoot(
name=Path(path).name,
path=path
)
session.add(source_root)
await session.flush()
return source_root
async def get_or_create_folder(session: AsyncSession, path: str, source_root_id: str) -> Folder:
"""Get or create a folder entry"""
from sqlalchemy import select
result = await session.execute(
select(Folder).where(Folder.path == path)
)
folder = result.scalar_one_or_none()
if not folder:
parent_path = str(Path(path).parent)
parent = None
if parent_path != path: # Not root folder
parent_result = await session.execute(
select(Folder).where(Folder.path == parent_path)
)
parent = parent_result.scalar_one_or_none()
if parent:
parent_id = parent.id
else:
# Recursively create parent
parent = await get_or_create_folder(session, parent_path, source_root_id)
parent_id = parent.id
else:
parent_id = None
folder = Folder(
name=Path(path).name,
path=path,
parent_id=parent_id,
source_root_id=source_root_id
)
session.add(folder)
await session.flush()
return folder
@shared_task(name='scan_all_source_roots')
def scan_all_source_roots():
"""Scan all configured source roots"""
for source_root in settings.source_roots:
if os.path.exists(source_root.path):
scan_folder.delay(source_root.path)
else:
logger.warning(f"Source root path does not exist: {source_root.path}")
@shared_task(name='watch_folders')
def watch_folders():
"""
Watch folders for changes using watchfiles
This is a long-running task that monitors file system events
"""
from watchfiles import watch
paths = [sr.path for sr in settings.source_roots if os.path.exists(sr.path)]
if not paths:
logger.warning("No valid source roots to watch")
return
logger.info(f"Starting folder watcher for: {paths}")
for changes in watch(*paths):
for change_type, filepath in changes:
filepath = str(filepath)
# Check if it's a supported file type
if Path(filepath).suffix.lower() not in SUPPORTED_EXTENSIONS:
continue
if change_type == 'added' or change_type == 'modified':
# Queue scan for the parent folder
parent_dir = str(Path(filepath).parent)
scan_folder.delay(parent_dir)
logger.info(f"File {change_type}: {filepath}, queued scan for {parent_dir}")
elif change_type == 'deleted':
# Handle file deletion
asyncio.run(handle_file_deletion(filepath))
async def handle_file_deletion(filepath: str):
"""Handle deletion of a file from the filesystem"""
from sqlalchemy import select
async with AsyncSessionLocal() as session:
result = await session.execute(
select(Photo).where(Photo.filepath == filepath)
)
photo = result.scalar_one_or_none()
if photo:
# Mark as missing or delete from database
photo.is_trashed = True
photo.trashed_at = datetime.utcnow()
await session.commit()
logger.info(f"Marked photo as trashed: {filepath}")

277
backend/app/tasks/thumbs.py Normal file
View File

@@ -0,0 +1,277 @@
"""
Celery tasks for thumbnail generation
"""
import os
import asyncio
from pathlib import Path
import logging
from typing import Tuple, Optional
import json
from celery import shared_task
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
import pyvips
import rawpy
import imageio
from PIL import Image
from pillow_heif import register_heif_opener
import ffmpeg
from app.database import AsyncSessionLocal
from app.models import Photo
from app.config import settings
# Register HEIF opener with Pillow
register_heif_opener()
logger = logging.getLogger(__name__)
# Thumbnail sizes configuration
THUMB_SIZES = {
'small': settings.thumbnails.small,
'medium': settings.thumbnails.medium,
'large': settings.thumbnails.large
}
def get_thumb_path(photo_id: str, size: str) -> str:
"""Get the path for a thumbnail file"""
thumb_dir = f"/data/thumbs/{photo_id}"
os.makedirs(thumb_dir, exist_ok=True)
return f"{thumb_dir}/{size}.{settings.thumbnails.format}"
def process_standard_image(filepath: str) -> pyvips.Image:
"""Process standard image formats (JPEG, PNG, etc.)"""
return pyvips.Image.new_from_file(filepath, access='sequential')
def process_raw_image(filepath: str) -> pyvips.Image:
"""Process RAW image formats"""
try:
with rawpy.imread(filepath) as raw:
# Use half_size for faster processing
rgb = raw.postprocess(use_camera_wb=True, half_size=True)
# Convert numpy array to pyvips image
return pyvips.Image.new_from_array(rgb)
except Exception as e:
logger.error(f"Error processing RAW file {filepath}: {e}")
# Try to extract embedded JPEG preview
return extract_raw_preview(filepath)
def extract_raw_preview(filepath: str) -> Optional[pyvips.Image]:
"""Extract embedded JPEG preview from RAW file"""
try:
# Use exiftool to extract preview
import subprocess
import tempfile
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp:
cmd = ['exiftool', '-b', '-PreviewImage', filepath]
result = subprocess.run(cmd, capture_output=True)
if result.returncode == 0 and result.stdout:
tmp.write(result.stdout)
tmp.flush()
return pyvips.Image.new_from_file(tmp.name, access='sequential')
except Exception as e:
logger.error(f"Error extracting RAW preview from {filepath}: {e}")
return None
def process_heic_image(filepath: str) -> pyvips.Image:
"""Process HEIC/HEIF image formats"""
try:
# Use pillow-heif to open the image
img = Image.open(filepath)
# Convert to RGB if needed
if img.mode != 'RGB':
img = img.convert('RGB')
# Save to temp file and load with pyvips
import tempfile
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp:
img.save(tmp.name, 'JPEG')
return pyvips.Image.new_from_file(tmp.name, access='sequential')
except Exception as e:
logger.error(f"Error processing HEIC file {filepath}: {e}")
raise
def process_video_thumbnail(filepath: str) -> pyvips.Image:
"""Extract thumbnail from video file"""
try:
# Get video duration
probe = ffmpeg.probe(filepath)
duration = float(probe['streams'][0]['duration'])
# Extract frame at 10% of duration
timestamp = duration * 0.1
# Extract frame using ffmpeg
import tempfile
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp:
stream = ffmpeg.input(filepath, ss=timestamp)
stream = ffmpeg.output(stream, tmp.name, vframes=1, format='image2', vcodec='mjpeg')
ffmpeg.run(stream, capture_stdout=True, capture_stderr=True)
return pyvips.Image.new_from_file(tmp.name, access='sequential')
except Exception as e:
logger.error(f"Error extracting video thumbnail from {filepath}: {e}")
# Create a placeholder thumbnail
return create_placeholder_thumbnail('video')
def create_placeholder_thumbnail(media_type: str) -> pyvips.Image:
"""Create a placeholder thumbnail for failed processing"""
# Create a simple gray placeholder
placeholder = pyvips.Image.black(640, 480)
placeholder = placeholder + [128, 128, 128] # Make it gray
return placeholder
def auto_rotate_image(image: pyvips.Image) -> pyvips.Image:
"""Auto-rotate image based on EXIF orientation"""
try:
orientation = image.get('orientation')
rotation_map = {
3: 180,
6: 90,
8: 270
}
if orientation in rotation_map:
image = image.rot(rotation_map[orientation])
except:
pass # No orientation data available
return image
def generate_thumbnail(image: pyvips.Image, size: int, output_path: str):
"""Generate a thumbnail of the specified size"""
# Calculate scale to fit within size (longest edge)
width = image.width
height = image.height
if width > height:
scale = size / width
else:
scale = size / height
# Only downscale, never upscale
if scale < 1:
image = image.resize(scale)
# Save as WebP with specified quality
image.webpsave(
output_path,
Q=settings.thumbnails.quality,
effort=4 # Balance between speed and compression
)
@shared_task(bind=True, name='generate_thumbnails')
def generate_thumbnails(self, photo_id: str):
"""Generate thumbnails for a photo"""
return asyncio.run(_generate_thumbnails_async(photo_id, self))
async def _generate_thumbnails_async(photo_id: str, task):
"""Async implementation of thumbnail generation"""
async with AsyncSessionLocal() as session:
try:
# Get photo from database
result = await session.execute(
select(Photo).where(Photo.id == photo_id)
)
photo = result.scalar_one_or_none()
if not photo:
logger.error(f"Photo not found: {photo_id}")
return {'status': 'error', 'message': 'Photo not found'}
# Check if file exists
if not os.path.exists(photo.filepath):
logger.error(f"File not found: {photo.filepath}")
photo.processing_status = 'failed'
photo.processing_error = 'File not found'
await session.commit()
return {'status': 'error', 'message': 'File not found'}
# Update processing status
photo.processing_status = 'processing'
await session.commit()
# Load and process the image based on type
image = None
if photo.media_type == 'photo':
image = process_standard_image(photo.filepath)
elif photo.media_type == 'raw':
image = process_raw_image(photo.filepath)
elif photo.media_type == 'heic':
image = process_heic_image(photo.filepath)
elif photo.media_type == 'video':
image = process_video_thumbnail(photo.filepath)
else:
logger.error(f"Unsupported media type: {photo.media_type}")
image = create_placeholder_thumbnail(photo.media_type)
if not image:
raise Exception("Failed to process image")
# Auto-rotate based on EXIF
image = auto_rotate_image(image)
# Store original dimensions
photo.width = image.width
photo.height = image.height
# Generate thumbnails for each size
for size_name, size_value in THUMB_SIZES.items():
thumb_path = get_thumb_path(photo_id, size_name)
generate_thumbnail(image, size_value, thumb_path)
# Update database with thumbnail path
setattr(photo, f'thumb_{size_name}', thumb_path)
# Update progress
task.update_state(
state='PROGRESS',
meta={'current_size': size_name, 'photo_id': photo_id}
)
# Update processing status
photo.processing_status = 'completed'
photo.processing_error = None
await session.commit()
logger.info(f"Thumbnails generated for photo {photo_id}")
return {'status': 'success', 'photo_id': photo_id}
except Exception as e:
logger.error(f"Error generating thumbnails for {photo_id}: {e}")
# Update error status
if photo:
photo.processing_status = 'failed'
photo.processing_error = str(e)
await session.commit()
return {'status': 'error', 'message': str(e)}
@shared_task(name='regenerate_all_thumbnails')
def regenerate_all_thumbnails():
"""Regenerate thumbnails for all photos"""
return asyncio.run(_regenerate_all_thumbnails_async())
async def _regenerate_all_thumbnails_async():
"""Async implementation of regenerating all thumbnails"""
async with AsyncSessionLocal() as session:
# Get all photos that need thumbnails
result = await session.execute(
select(Photo).where(
Photo.processing_status.in_(['pending', 'failed'])
)
)
photos = result.scalars().all()
logger.info(f"Regenerating thumbnails for {len(photos)} photos")
for photo in photos:
generate_thumbnails.delay(photo.id)
return {'status': 'queued', 'count': len(photos)}

50
backend/requirements.txt Normal file
View File

@@ -0,0 +1,50 @@
# Core dependencies
fastapi==0.109.0
uvicorn[standard]==0.27.0
python-multipart==0.0.6
# Database
sqlalchemy[asyncio]==2.0.25
aiosqlite==0.19.0
alembic==1.13.1
# Redis and Celery
redis==5.0.1
celery==5.3.6
flower==2.0.1
# Image processing
pyvips==2.2.2
rawpy==0.19.0
pillow==10.2.0
pillow-heif==0.15.0
imageio==2.33.1
imageio-ffmpeg==0.4.9
# Video processing
ffmpeg-python==0.2.0
# Metadata extraction
pyexiftool==0.5.6
# File watching
watchfiles==0.21.0
# Utilities
pyyaml==6.0.1
pydantic==2.5.3
pydantic-settings==2.1.0
python-dotenv==1.0.0
httpx==0.26.0
aiofiles==23.2.1
# Hashing and security
hashlib
python-jose[cryptography]==3.3.0
passlib[bcrypt]==1.7.4
# Development
pytest==7.4.4
pytest-asyncio==0.23.3
black==23.12.1
ruff==0.1.11