09a00f7419
Two related fixes for the Nextcloud library lifecycle.
1. DELETE /api/v1/nextcloud/source-roots/{id} now actually deletes
the SourceRoot, every Folder under it, and every Photo in those
folders (Nextcloud files untouched). Was a soft-deactivate
(is_active=false) that left the rows around forever, so re-adding
the same path resurrected ghosts and prune-missing reported zero.
Returns {deleted_photos, deleted_folders}; the Settings UI toasts
the count and invalidates photos/folders/stats so cached lists
don't show ghosts. photo_tags and heap_photos already cascade via
ON DELETE CASCADE; FolderShare uses a stringly-typed folder_id
with no FK so cleaned up explicitly.
2. The watcher (watch_folders task) was getting killed every five
minutes by the global task_soft_time_limit=300 in app/tasks/celery.py
despite passing soft_time_limit=None on the decorator (None falls
back to the worker default in this Celery version). Override with
soft_time_limit=0, time_limit=0 (= unlimited) so the watch loop
actually stays alive. The 'Soft time limit (300s) exceeded' /
'Worker exited prematurely' lines should stop in worker-watcher
logs.
3. Added discard_missing_photos() in services/cleanup.py — a soft
variant of prune_missing_photos that walks every present source
root, checks os.path.exists for each non-discarded Photo, and
flips is_discarded=true on the missing ones (UPDATE not DELETE).
Wired as discard_missing_photos_beat in tasks/scan.py and
scheduled every 30 min via celery beat. Beat runs in-process on
worker-watcher (--beat flag in compose) — there's only ever one
watcher and we don't need a separate container.
Hard delete remains manual via prune-missing for users who want to
review before committing. The beat catch-up only soft-discards (file
gone -> mule-image trash, restorable).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
375 lines
16 KiB
YAML
375 lines
16 KiB
YAML
services:
|
||
frontend:
|
||
build:
|
||
context: ./frontend
|
||
dockerfile: Dockerfile
|
||
container_name: mulita-frontend
|
||
ports:
|
||
# Host port is configurable via FRONTEND_PORT in .env so multiple
|
||
# instances / other services on the same host don't collide.
|
||
- "${FRONTEND_PORT:-3000}:80"
|
||
depends_on:
|
||
- backend
|
||
networks:
|
||
- mulita-network
|
||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||
# docker DNS forwards the lookup to the host's resolver, which
|
||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||
# traefik exposure list, so TLS handshakes against it die with
|
||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||
extra_hosts:
|
||
- "cloud.hubris.network:192.168.8.175"
|
||
restart: unless-stopped
|
||
|
||
backend:
|
||
build:
|
||
context: ./backend
|
||
dockerfile: Dockerfile
|
||
container_name: mulita-backend
|
||
ports:
|
||
# Direct backend access on the host is rarely needed (the frontend
|
||
# talks to it through the nginx /api proxy on the same network),
|
||
# but it's exposed for debugging / curl. Override with BACKEND_PORT.
|
||
- "${BACKEND_PORT:-8001}:8000"
|
||
volumes:
|
||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||
# The single host → container mount for your photo library. Set
|
||
# PHOTO_DIRS in .env to your library root. Mounted :rw because file
|
||
# operations (rename, move, empty discard pile) need to mutate the
|
||
# filesystem; flip to :ro for a strict read-only library and the
|
||
# write endpoints will return EROFS.
|
||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||
# Optional Nextcloud integration: mount the homecloud data dir so
|
||
# users can register subfolders of their Nextcloud `files/` tree
|
||
# as per-user SourceRoots. Reads use this path directly; mutations
|
||
# (upload, delete, rename, move) dispatch via WebDAV against
|
||
# NEXTCLOUD_BASE_URL so Nextcloud's oc_filecache stays in sync.
|
||
# Leave NEXTCLOUD_USERS_HOST_PATH unset (or pointing at a no-op
|
||
# path) to disable.
|
||
- ${NEXTCLOUD_USERS_HOST_PATH:-./photos}:/nextcloud-users:rw
|
||
- thumbs_data:/data/thumbs
|
||
- proxies_data:/data/proxies
|
||
- db_data:/data/db # retained so the docker-compose.sqlite.yml override has somewhere to put mulita.db
|
||
# Run Alembic migrations before starting uvicorn. On a fresh Postgres
|
||
# the empty 0001 baseline is a no-op stamp; create_all in init_db then
|
||
# builds the schema.
|
||
# init_db creates all tables from models (idempotent create_all),
|
||
# then Alembic runs migrations for existing installs. On fresh DBs
|
||
# create_all already built the full schema, so bootstrap.py stamps
|
||
# alembic head to skip redundant ALTER statements.
|
||
command: sh -c "python -c 'import asyncio; from app.database import init_db; asyncio.run(init_db())' && python bootstrap.py && uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers 2 --proxy-headers"
|
||
environment:
|
||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||
- REDIS_URL=redis://redis:6379
|
||
- CELERY_BROKER_URL=redis://redis:6379
|
||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||
- PHOTO_DIRS=/photos
|
||
- ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-*}
|
||
- SECRET_KEY=${SECRET_KEY:-mulita-dev-secret-change-me}
|
||
- ACCESS_TOKEN_EXPIRE_MINUTES=${ACCESS_TOKEN_EXPIRE_MINUTES:-60}
|
||
- REFRESH_TOKEN_EXPIRE_DAYS=${REFRESH_TOKEN_EXPIRE_DAYS:-30}
|
||
# Authentik / OIDC single sign-on. Leave OIDC_ENABLED=false to
|
||
# hide the SSO button and stick with username/password. When
|
||
# enabled, set OIDC_ISSUER to the Authentik provider URL (the one
|
||
# that serves /.well-known/openid-configuration), and paste the
|
||
# client id/secret from the Authentik application. OIDC_REDIRECT_URI
|
||
# must match the one registered on the Authentik side exactly —
|
||
# e.g. https://photovault.example.com/api/v1/auth/oidc/callback.
|
||
- OIDC_ENABLED=${OIDC_ENABLED:-false}
|
||
- OIDC_ISSUER=${OIDC_ISSUER:-}
|
||
- OIDC_CLIENT_ID=${OIDC_CLIENT_ID:-}
|
||
- OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET:-}
|
||
- OIDC_REDIRECT_URI=${OIDC_REDIRECT_URI:-}
|
||
- OIDC_SCOPES=${OIDC_SCOPES:-openid profile email}
|
||
- OIDC_PROVIDER_LABEL=${OIDC_PROVIDER_LABEL:-Authentik}
|
||
- OIDC_ALLOW_SIGNUP=${OIDC_ALLOW_SIGNUP:-true}
|
||
- OIDC_ADMIN_GROUPS=${OIDC_ADMIN_GROUPS:-}
|
||
- OIDC_LINK_BY_USERNAME=${OIDC_LINK_BY_USERNAME:-false}
|
||
- SESSION_SECRET=${SESSION_SECRET:-}
|
||
# Nextcloud integration. NEXTCLOUD_USERS_ROOT is the in-container
|
||
# path that NEXTCLOUD_USERS_HOST_PATH binds to. NEXTCLOUD_BASE_URL
|
||
# is the public-facing Nextcloud URL used for outgoing WebDAV
|
||
# calls (must be reachable from the backend container; e.g.
|
||
# https://cloud.example.com or http://nextcloud:80 if you put it
|
||
# on the same docker network). Leave NEXTCLOUD_BASE_URL unset to
|
||
# keep the integration off — the router endpoints stay registered
|
||
# but mutating endpoints fail with a clear error.
|
||
- NEXTCLOUD_USERS_ROOT=${NEXTCLOUD_USERS_ROOT:-/nextcloud-users}
|
||
- NEXTCLOUD_BASE_URL=${NEXTCLOUD_BASE_URL:-}
|
||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||
- TZ=${TZ:-UTC}
|
||
depends_on:
|
||
redis:
|
||
condition: service_started
|
||
db:
|
||
condition: service_healthy
|
||
networks:
|
||
- mulita-network
|
||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||
# docker DNS forwards the lookup to the host's resolver, which
|
||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||
# traefik exposure list, so TLS handshakes against it die with
|
||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||
extra_hosts:
|
||
- "cloud.hubris.network:192.168.8.175"
|
||
restart: unless-stopped
|
||
|
||
# ── Celery workers ─────────────────────────────────────────────────────
|
||
#
|
||
# The ingestion pipeline is split across two worker services so CPU-heavy
|
||
# vision tasks (embed / detect / OCR / faces / classify) cannot starve
|
||
# the fast IO-bound tasks (scan / thumbnails / EXIF / phash / duplicates).
|
||
#
|
||
# worker-light listens on default,high,low — IO-bound, cheap
|
||
# worker-vision listens on vision — CPU-bound, loads ONNX
|
||
#
|
||
# Both share the same image, photo volume, and model cache, so there's
|
||
# no disk duplication and model weights are loaded lazily only by
|
||
# worker-vision. Each service has its own concurrency knob; both
|
||
# workers ship their heartbeat to the same Redis broker so the
|
||
# Settings > Workers panel lists them side-by-side.
|
||
#
|
||
# Sizing defaults target a 6-core / 16 GB host:
|
||
# CELERY_LIGHT_CONCURRENCY=2 (enough for parallel thumbnail + EXIF)
|
||
# CELERY_VISION_CONCURRENCY=5 (5 × ~2GB ONNX = ~10GB RAM, 5/6 cores)
|
||
# Raise these in .env and run `docker compose up -d worker-light worker-vision`
|
||
# to scale. Keep light under ~4 and vision under your physical core
|
||
# count; more just thrashes.
|
||
worker-light:
|
||
build:
|
||
context: ./backend
|
||
dockerfile: Dockerfile
|
||
image: mule-image-worker
|
||
container_name: mulita-worker-light
|
||
command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_LIGHT_CONCURRENCY:-2} -Q default,high,low -n light@%h"
|
||
volumes:
|
||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||
- ${NEXTCLOUD_USERS_HOST_PATH:-./photos}:/nextcloud-users:rw
|
||
- thumbs_data:/data/thumbs
|
||
- proxies_data:/data/proxies
|
||
- db_data:/data/db
|
||
- models_data:/data/models
|
||
environment:
|
||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||
- REDIS_URL=redis://redis:6379
|
||
- CELERY_BROKER_URL=redis://redis:6379
|
||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||
- PHOTO_DIRS=/photos
|
||
- NEXTCLOUD_USERS_ROOT=${NEXTCLOUD_USERS_ROOT:-/nextcloud-users}
|
||
- NEXTCLOUD_BASE_URL=${NEXTCLOUD_BASE_URL:-}
|
||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||
- TZ=${TZ:-UTC}
|
||
# NullPool — see app/database.py for rationale.
|
||
- MULITA_CELERY_WORKER=1
|
||
depends_on:
|
||
redis:
|
||
condition: service_started
|
||
backend:
|
||
condition: service_started
|
||
db:
|
||
condition: service_healthy
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "celery -A app.tasks.celery inspect ping -d light@$$HOSTNAME 2>/dev/null | grep -q OK"]
|
||
interval: 30s
|
||
timeout: 10s
|
||
retries: 3
|
||
start_period: 120s
|
||
networks:
|
||
- mulita-network
|
||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||
# docker DNS forwards the lookup to the host's resolver, which
|
||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||
# traefik exposure list, so TLS handshakes against it die with
|
||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||
extra_hosts:
|
||
- "cloud.hubris.network:192.168.8.175"
|
||
restart: unless-stopped
|
||
|
||
# Dedicated watcher worker — runs the long-lived watch_folders task
|
||
# on its own queue so it never blocks scan/thumbnail workers.
|
||
worker-watcher:
|
||
build:
|
||
context: ./backend
|
||
dockerfile: Dockerfile
|
||
image: mule-image-worker
|
||
container_name: mulita-worker-watcher
|
||
# --beat runs the celery beat scheduler in-process alongside the
|
||
# watcher worker — there's only ever one watcher (Redis-locked
|
||
# singleton) and we don't need a separate container just to fire a
|
||
# 30-minute periodic task. Beat schedule lives in app/tasks/celery.py.
|
||
command: sh -c "celery -A app.tasks.celery worker --beat --loglevel=${LOG_LEVEL:-info} --concurrency=1 -Q watcher -n watcher@%h"
|
||
volumes:
|
||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||
- ${NEXTCLOUD_USERS_HOST_PATH:-./photos}:/nextcloud-users:rw
|
||
- db_data:/data/db
|
||
environment:
|
||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||
- REDIS_URL=redis://redis:6379
|
||
- CELERY_BROKER_URL=redis://redis:6379
|
||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||
- PHOTO_DIRS=/photos
|
||
- NEXTCLOUD_USERS_ROOT=${NEXTCLOUD_USERS_ROOT:-/nextcloud-users}
|
||
- NEXTCLOUD_BASE_URL=${NEXTCLOUD_BASE_URL:-}
|
||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||
- TZ=${TZ:-UTC}
|
||
- MULITA_CELERY_WORKER=1
|
||
depends_on:
|
||
redis:
|
||
condition: service_started
|
||
db:
|
||
condition: service_healthy
|
||
networks:
|
||
- mulita-network
|
||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||
# docker DNS forwards the lookup to the host's resolver, which
|
||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||
# traefik exposure list, so TLS handshakes against it die with
|
||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||
extra_hosts:
|
||
- "cloud.hubris.network:192.168.8.175"
|
||
restart: unless-stopped
|
||
|
||
worker-vision:
|
||
build:
|
||
context: ./backend
|
||
dockerfile: Dockerfile
|
||
image: mule-image-worker
|
||
container_name: mulita-worker-vision
|
||
command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_VISION_CONCURRENCY:-5} -Q vision -n vision@%h"
|
||
volumes:
|
||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||
- ${NEXTCLOUD_USERS_HOST_PATH:-./photos}:/nextcloud-users:rw
|
||
- thumbs_data:/data/thumbs
|
||
- proxies_data:/data/proxies
|
||
- db_data:/data/db
|
||
- models_data:/data/models
|
||
environment:
|
||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||
- REDIS_URL=redis://redis:6379
|
||
- CELERY_BROKER_URL=redis://redis:6379
|
||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||
- PHOTO_DIRS=/photos
|
||
- NEXTCLOUD_USERS_ROOT=${NEXTCLOUD_USERS_ROOT:-/nextcloud-users}
|
||
- NEXTCLOUD_BASE_URL=${NEXTCLOUD_BASE_URL:-}
|
||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||
- TZ=${TZ:-UTC}
|
||
- MULITA_CELERY_WORKER=1
|
||
# ONNX Runtime execution providers. Set to "auto" to auto-detect
|
||
# GPU (CUDA > ROCm > OpenVINO > CPU), or explicitly:
|
||
# "CUDAExecutionProvider,CPUExecutionProvider"
|
||
# "ROCMExecutionProvider,CPUExecutionProvider"
|
||
# Default: CPU only. To enable GPU, also uncomment the deploy
|
||
# section below and install nvidia-container-toolkit on the host.
|
||
- VISION_EXECUTION_PROVIDERS=${VISION_EXECUTION_PROVIDERS:-CPUExecutionProvider}
|
||
# Pin each ONNX session to one intra-op thread so N prefork children
|
||
# × default-all-cores doesn't oversubscribe the box. With
|
||
# concurrency=5 and OMP=1, vision peaks at 5 busy cores, leaving
|
||
# one for worker-light + system. These env vars cover the three
|
||
# threading runtimes ONNX Runtime might pick up on first use.
|
||
- OMP_NUM_THREADS=1
|
||
- OPENBLAS_NUM_THREADS=1
|
||
- MKL_NUM_THREADS=1
|
||
# Uncomment for NVIDIA GPU passthrough:
|
||
# deploy:
|
||
# resources:
|
||
# reservations:
|
||
# devices:
|
||
# - driver: nvidia
|
||
# count: all
|
||
# capabilities: [gpu]
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "celery -A app.tasks.celery inspect ping -d vision@$$HOSTNAME 2>/dev/null | grep -q OK"]
|
||
interval: 30s
|
||
timeout: 10s
|
||
retries: 3
|
||
start_period: 300s
|
||
depends_on:
|
||
redis:
|
||
condition: service_started
|
||
backend:
|
||
condition: service_started
|
||
db:
|
||
condition: service_healthy
|
||
networks:
|
||
- mulita-network
|
||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||
# docker DNS forwards the lookup to the host's resolver, which
|
||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||
# traefik exposure list, so TLS handshakes against it die with
|
||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||
extra_hosts:
|
||
- "cloud.hubris.network:192.168.8.175"
|
||
restart: unless-stopped
|
||
|
||
db:
|
||
image: pgvector/pgvector:pg16
|
||
container_name: mulita-db
|
||
environment:
|
||
POSTGRES_USER: mulita
|
||
POSTGRES_PASSWORD: mulita
|
||
POSTGRES_DB: mulita
|
||
volumes:
|
||
- pg_data:/var/lib/postgresql/data
|
||
networks:
|
||
- mulita-network
|
||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||
# docker DNS forwards the lookup to the host's resolver, which
|
||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||
# traefik exposure list, so TLS handshakes against it die with
|
||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||
extra_hosts:
|
||
- "cloud.hubris.network:192.168.8.175"
|
||
restart: unless-stopped
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U mulita -d mulita"]
|
||
interval: 5s
|
||
timeout: 5s
|
||
retries: 10
|
||
|
||
redis:
|
||
image: redis:7-alpine
|
||
container_name: mulita-redis
|
||
# Host port exposed only for local debugging; the backend / worker
|
||
# reach Redis via the internal mulita-network on its container name.
|
||
ports:
|
||
- "${REDIS_PORT:-6379}:6379"
|
||
volumes:
|
||
- redis_data:/data
|
||
networks:
|
||
- mulita-network
|
||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||
# docker DNS forwards the lookup to the host's resolver, which
|
||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||
# traefik exposure list, so TLS handshakes against it die with
|
||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||
extra_hosts:
|
||
- "cloud.hubris.network:192.168.8.175"
|
||
restart: unless-stopped
|
||
command: redis-server --appendonly yes
|
||
healthcheck:
|
||
test: ["CMD", "redis-cli", "ping"]
|
||
interval: 10s
|
||
timeout: 5s
|
||
retries: 5
|
||
|
||
networks:
|
||
mulita-network:
|
||
driver: bridge
|
||
|
||
volumes:
|
||
thumbs_data:
|
||
proxies_data:
|
||
db_data:
|
||
redis_data:
|
||
pg_data:
|
||
models_data: |