""" Celery configuration and app initialization """ import logging import os from celery import Celery from celery.signals import worker_process_init from app.config import settings logger = logging.getLogger(__name__) # Create Celery app celery_app = Celery( 'mulita', broker=settings.celery_broker_url, backend=settings.celery_result_backend, include=[ 'app.tasks.scan', 'app.tasks.thumbs', 'app.tasks.vision', 'app.services.metadata', # extract_metadata lives here ] ) # Configure Celery celery_app.conf.update( task_serializer='json', accept_content=['json'], result_serializer='json', timezone='UTC', enable_utc=True, # Robust acknowledgment: keep message in broker until task succeeds. task_acks_late=True, task_reject_on_worker_lost=True, # Global time limits — individual tasks can override via decorator. task_soft_time_limit=300, # 5 min — raises SoftTimeLimitExceeded task_time_limit=600, # 10 min — SIGKILL # Explicit routes for every task name. Wildcard patterns don't match # short names produced by @shared_task(name='...'). task_routes={ # Vision queue — GPU/CPU-bound inference 'embed_photo': {'queue': 'vision'}, 'ocr_photo': {'queue': 'vision'}, 'detect_objects': {'queue': 'vision'}, 'extract_faces': {'queue': 'vision'}, 'classify_content': {'queue': 'vision'}, 'vision_fanout': {'queue': 'vision'}, 'recluster_faces': {'queue': 'vision'}, # High-priority queue — thumbnails & duplicates 'generate_thumbnails': {'queue': 'high'}, 'regenerate_all_thumbnails': {'queue': 'high'}, 'backfill_phashes': {'queue': 'high'}, 'regroup_duplicates': {'queue': 'high'}, 'incremental_regroup_duplicates': {'queue': 'high'}, # Low-priority queue — scans 'scan_folder': {'queue': 'low'}, 'scan_all_source_roots': {'queue': 'low'}, 'backfill_gps': {'queue': 'low'}, # Dedicated watcher queue 'watch_folders': {'queue': 'watcher'}, }, task_default_queue='default', task_default_exchange='default', task_default_exchange_type='direct', task_default_routing_key='default', broker_connection_retry_on_startup=True, ) @worker_process_init.connect def _warmup_vision_models(**kwargs): """Pre-load vision models in the worker process so the first task doesn't pay cold-start latency. Only runs on the vision queue.""" # The worker name contains the queue — only warm up vision workers. worker_queues = os.environ.get("CELERY_QUEUES", "") if "vision" not in worker_queues: # Heuristic: check the celery command line for -Q vision import sys if "vision" not in " ".join(sys.argv): return try: from app.services.vision.registry import registry registry.warmup() except Exception: logger.exception("Vision model warmup failed")