Compare commits
79 Commits
d5e4f23c0f
...
claude/inf
| Author | SHA1 | Date | |
|---|---|---|---|
| b2b6060872 | |||
| 5be6fd9047 | |||
| 3e164c48d0 | |||
| 259adb6a41 | |||
| ccf2c6b7c7 | |||
| a13e171295 | |||
| 73c36b4817 | |||
| 82f2a40269 | |||
| f6c0f7a507 | |||
| 1df16a6142 | |||
| 5da1022ed1 | |||
|
|
da63ad769a | ||
|
|
86e38e152d | ||
|
|
3757eb0170 | ||
|
|
cfd0c6aa81 | ||
| 243e5d3831 | |||
| 14a1b4e54e | |||
| 7df1c04c0f | |||
| 8f97590d9f | |||
| 4c08eba27a | |||
| 6c96c22b33 | |||
| 70dc1b6bdf | |||
| e3d4f6d92e | |||
| 9fc650fb12 | |||
| 29f7ad7073 | |||
| c134afe023 | |||
| a54d90a2d9 | |||
| 97f51a05c4 | |||
| d1ddc48f81 | |||
| 55c870c155 | |||
| 981328faff | |||
| e1707c314d | |||
| 64c0da794d | |||
| 0f4e2e0b8f | |||
| ea1803ec2f | |||
| fc5f30fad1 | |||
| a7b8a60473 | |||
| 24dfa996b3 | |||
| e364e4128f | |||
| d2a76fa58c | |||
| 0d5f380948 | |||
| 680fa90cbe | |||
| 9bba097d91 | |||
| 155e9bb126 | |||
| c783f129cc | |||
| 24c449f475 | |||
| e36f1939c6 | |||
|
|
2a75896274 | ||
|
|
85847848c4 | ||
|
|
b0c8c06b2b | ||
|
|
986dab7334 | ||
| 829d7bed83 | |||
| d70244f17e | |||
| a38c3c6e9b | |||
|
|
8ac406ac1f | ||
| 70de4b65ec | |||
| 79a9ef49d4 | |||
| bd39d310ab | |||
|
|
f9f276a986 | ||
|
|
ca9f6e6bd2 | ||
| 9d955d6b94 | |||
| af96922e89 | |||
| ccbc1050de | |||
|
|
cfd85a1fe8 | ||
| aa63d4c11d | |||
| 79ec511482 | |||
|
|
505fef5dfc | ||
|
|
8083328f2d | ||
| e669e80a91 | |||
|
|
3d8e050af4 | ||
|
|
cce1d876c3 | ||
| 7e10f0b462 | |||
| 6b8c7abc20 | |||
|
|
9a3ad3e579 | ||
|
|
4abe6d758c | ||
| cb5bc120dc | |||
| a72619e3d1 | |||
| 84e433ff63 | |||
| d35de8a2a9 |
22
.env
22
.env
@@ -1,22 +0,0 @@
|
||||
# Mulita / PhotoVault local environment.
|
||||
# See .env.example for the full list of knobs and their docs.
|
||||
|
||||
# REQUIRED — host path to your photo library.
|
||||
PHOTO_DIRS=/mnt/library/homecloud/admin/files/
|
||||
|
||||
# Ports — change if 3000 / 8001 collide with other services on the host.
|
||||
FRONTEND_PORT=3000
|
||||
BACKEND_PORT=8001
|
||||
REDIS_PORT=6379
|
||||
|
||||
# CORS — wildcard for local dev. Lock down for real deployments.
|
||||
ALLOWED_ORIGINS=*
|
||||
|
||||
# Logging + timezone.
|
||||
LOG_LEVEL=INFO
|
||||
TZ=UTC
|
||||
|
||||
# Celery worker pools — split worker-light (IO) and worker-vision (CPU).
|
||||
# Defaults target a 6-core / 16 GB host.
|
||||
CELERY_LIGHT_CONCURRENCY=2
|
||||
CELERY_VISION_CONCURRENCY=5
|
||||
172
.env.example
172
.env.example
@@ -1,113 +1,83 @@
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Mulita / PhotoVault — example environment file
|
||||
# Example environment file. Copy to `.env` and adjust.
|
||||
#
|
||||
# Copy this file to `.env` and adjust the values for your setup. Every key
|
||||
# below has a sensible default in docker-compose.yml, so you only need to
|
||||
# uncomment the ones you actually want to change.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# podman-compose --env-file .env \
|
||||
# -f docker-compose.yml -f docker-compose.podman.yml up -d
|
||||
|
||||
|
||||
# ── REQUIRED ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# Host path to your photo library. The compose file mounts this at /photos
|
||||
# inside the backend + worker containers. The backend creates a default
|
||||
# source root pointing at /photos on first boot, so once this is set the
|
||||
# library is scanned with zero further configuration.
|
||||
# Host path to your photo library. PhotoPrism reads this in place and
|
||||
# writes EXIF backwrites next to originals (when PP_ORIGINALS_MODE=rw).
|
||||
PHOTO_DIRS=/mnt/library/homecloud/admin/files/
|
||||
|
||||
# Bootstrap admin password. The first PhotoPrism boot creates an `admin`
|
||||
# account with this password. Rotate after first login from the UI.
|
||||
PP_ADMIN_PASSWORD=please-change-me
|
||||
|
||||
# MariaDB passwords. Generate with `openssl rand -hex 24`.
|
||||
PP_DB_PASSWORD=please-change-me
|
||||
PP_DB_ROOT_PASSWORD=please-change-me
|
||||
|
||||
|
||||
# ── OPTIONAL ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# Loopback host port for PhotoPrism's API (and UI, if you tunnel to it).
|
||||
# Vite proxies /api/v1/* here and the host-mode sidecar reaches it on
|
||||
# localhost. Not published on the public interface.
|
||||
PP_PORT=2342
|
||||
|
||||
# Site URL — used for share links, OIDC redirect URI, and reverse-proxy aware
|
||||
# URL generation. Set to the public hostname once the proxy is in front.
|
||||
PP_SITE_URL=http://localhost:2342/
|
||||
|
||||
# Auth mode — "password" for username/password (default), "public" for an
|
||||
# unauthenticated kiosk mode (don't use this on a multi-user library).
|
||||
PP_AUTH_MODE=password
|
||||
|
||||
# Library mount mode. "rw" allows rename / folder mutations / EXIF backwrite;
|
||||
# "ro" is safe-for-archives but disables those sidecar endpoints. Set in
|
||||
# lockstep with PP_READONLY below.
|
||||
PP_ORIGINALS_MODE=rw
|
||||
PP_READONLY=false
|
||||
|
||||
# UID/GID inside the PhotoPrism container. Set these to the host UID/GID that
|
||||
# owns ${PHOTO_DIRS}. `id -u` and `id -g`.
|
||||
PP_UID=1000
|
||||
PP_GID=1000
|
||||
|
||||
|
||||
# ── OIDC SSO (Authentik or equivalent) ───────────────────────────────────────
|
||||
# Leave blank to keep OIDC dormant. Fill in to enable the "Sign in with OIDC"
|
||||
# button on the login page; OIDC_REGISTER=true auto-creates accounts at role
|
||||
# `user` (override to `admin` to grant full access on first SSO login).
|
||||
#
|
||||
# Examples:
|
||||
# macOS / Linux: PHOTO_DIRS=/Users/you/Pictures
|
||||
# Network share: PHOTO_DIRS=/mnt/nas/photos
|
||||
# Windows (WSL): PHOTO_DIRS=/mnt/c/Users/you/Pictures
|
||||
PHOTO_DIRS=./photos
|
||||
# The compose file reads these and maps them to PhotoPrism's actual env-var
|
||||
# names (PHOTOPRISM_OIDC_URI / _CLIENT / _SECRET / _PROVIDER) — see the
|
||||
# comment in docker-compose.yml. The PhotoPrism callback URI is auto-derived
|
||||
# from PP_SITE_URL; do not set it manually.
|
||||
|
||||
# OIDC_PROVIDER_NAME=Authentik
|
||||
# OIDC_ISSUER_URL=https://auth.example.com/application/o/photoprism/
|
||||
# OIDC_CLIENT_ID=...
|
||||
# OIDC_CLIENT_SECRET=...
|
||||
# OIDC_SCOPES=openid profile email
|
||||
# OIDC_REGISTER=true
|
||||
# OIDC_ROLE=user
|
||||
|
||||
|
||||
# ── PORTS ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Host port the SPA is served on. Browse to http://<host>:<FRONTEND_PORT>/.
|
||||
FRONTEND_PORT=3000
|
||||
|
||||
# Host port for the backend API. Almost never needed directly — the frontend
|
||||
# nginx proxies /api/ to the backend over the internal compose network. Kept
|
||||
# exposed for debugging / curl.
|
||||
BACKEND_PORT=8001
|
||||
|
||||
# Redis host port. Internal services reach Redis on its container name; this
|
||||
# is just for local debugging.
|
||||
REDIS_PORT=6379
|
||||
|
||||
|
||||
# ── AUTH ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Secret key used to sign JWT tokens. Generate a strong random value for
|
||||
# production (e.g. `openssl rand -base64 32`). The default is a deterministic
|
||||
# placeholder acceptable only for local/homelab use.
|
||||
# SECRET_KEY=change-me-to-a-random-string
|
||||
|
||||
# How long access and refresh tokens stay valid. Access tokens are short-lived
|
||||
# and silently refreshed by the frontend; refresh tokens let a session survive
|
||||
# across browser restarts.
|
||||
# ACCESS_TOKEN_EXPIRE_MINUTES=60
|
||||
# REFRESH_TOKEN_EXPIRE_DAYS=30
|
||||
|
||||
|
||||
# ── CORS ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Comma-separated list of allowed origins for direct browser access to the
|
||||
# backend. Same-origin requests through the nginx / vite proxy never trip
|
||||
# CORS, so this only matters when something hits the backend port directly
|
||||
# from a different origin (e.g. another machine, dev tools, a reverse proxy
|
||||
# under a different hostname).
|
||||
# ── USER LIBRARY ISOLATION ───────────────────────────────────────────────────
|
||||
# Maps PhotoPrism usernames to originals-relative subdirectories so each
|
||||
# user only sees their own photos. Format: comma-separated user:path pairs.
|
||||
# The sidecar reconciler applies this to auth_users.base_path on boot and
|
||||
# every 60s. Leave empty for single-user deployments.
|
||||
#
|
||||
# Default "*" is permissive, fine for a single-user homelab. Lock it down in
|
||||
# real deployments:
|
||||
# ALLOWED_ORIGINS=https://photos.example.com
|
||||
# ALLOWED_ORIGINS=https://photos.example.com,http://192.168.1.10:3000
|
||||
ALLOWED_ORIGINS=*
|
||||
# USER_BASEPATHS="alice:alice, bob:bob"
|
||||
|
||||
# Sidecar DB password — provisioned by mariadb/init/01-sidecar.sql on first
|
||||
# boot. Rotate before any non-local deployment.
|
||||
# SIDECAR_DB_PASSWORD=replace-at-m4-bringup
|
||||
|
||||
|
||||
# ── LOGGING / TIMEZONE ───────────────────────────────────────────────────────
|
||||
# ── LOGGING ──────────────────────────────────────────────────────────────────
|
||||
|
||||
# Python log level for the backend and Celery worker. Bump to DEBUG when
|
||||
# chasing scan / thumbnail issues.
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# Container timezone. Affects the timestamps in logs and the "added at"
|
||||
# field on newly imported photos. Defaults to UTC.
|
||||
# TZ=Europe/Berlin
|
||||
# TZ=America/New_York
|
||||
TZ=UTC
|
||||
|
||||
|
||||
# ── WORKER CONCURRENCY ───────────────────────────────────────────────────────
|
||||
#
|
||||
# The ingestion pipeline runs on two Celery worker services with separate
|
||||
# concurrency knobs so heavy vision tasks can't starve cheap IO tasks:
|
||||
#
|
||||
# worker-light (default / high / low queues)
|
||||
# Runs: scan, thumbnails, EXIF, pHash, duplicate regrouping.
|
||||
# Mostly IO-bound — 2 prefork children keep a library streaming in.
|
||||
#
|
||||
# worker-vision (vision queue)
|
||||
# Runs: embeddings, object detection, OCR, face extraction, content
|
||||
# classification. Each prefork child loads ~2 GB of ONNX model weights,
|
||||
# so set this to roughly (physical_cores − 1) and watch RAM.
|
||||
#
|
||||
# Defaults target a ~6 core / 16 GB host. Raise these, then
|
||||
# docker compose up -d worker-light worker-vision
|
||||
# to pick them up. Lower for a Pi; go higher on a workstation.
|
||||
#
|
||||
# The old `CELERYD_CONCURRENCY=N` single-worker variable is no longer
|
||||
# read — delete it from your .env if it's set.
|
||||
CELERY_LIGHT_CONCURRENCY=2
|
||||
CELERY_VISION_CONCURRENCY=5
|
||||
|
||||
|
||||
# ── INTERNAL (rarely overridden) ─────────────────────────────────────────────
|
||||
|
||||
# These point at the in-compose Redis and the bind-mounted SQLite db. Override
|
||||
# only if you're running Mulita without docker-compose or against an external
|
||||
# Redis.
|
||||
# REDIS_URL=redis://redis:6379
|
||||
# CELERY_BROKER_URL=redis://redis:6379
|
||||
# CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
# DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
PP_LOG_LEVEL=info
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
# PhotoPrism stack — example environment file.
|
||||
#
|
||||
# Copy to `.env.photoprism` and adjust. The compose file is invoked with
|
||||
# `--env-file .env.photoprism` so this never collides with the legacy
|
||||
# `.env` used by the mule-image stack.
|
||||
#
|
||||
# docker compose --env-file .env.photoprism -f docker-compose.photoprism.yml up -d
|
||||
|
||||
|
||||
# ── REQUIRED ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# Host path to your photo library. PhotoPrism reads this in place and
|
||||
# (post-M2) writes EXIF backwrites next to originals. Same path the legacy
|
||||
# mule-image backend used.
|
||||
PHOTO_DIRS=/mnt/library/homecloud/admin/files/
|
||||
|
||||
# Bootstrap admin password. The first PhotoPrism boot creates an `admin`
|
||||
# account with this password. Rotate after first login from the UI.
|
||||
PP_ADMIN_PASSWORD=please-change-me
|
||||
|
||||
# MariaDB passwords. Generate with `openssl rand -hex 24`.
|
||||
PP_DB_PASSWORD=please-change-me
|
||||
PP_DB_ROOT_PASSWORD=please-change-me
|
||||
|
||||
|
||||
# ── OPTIONAL ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# Host port for direct PhotoPrism UI access during M0–M3. Moves behind a
|
||||
# Caddy reverse proxy at M4; keep this open through then for debugging.
|
||||
PP_PORT=2342
|
||||
|
||||
# Site URL — used for share links, OIDC redirect URI, and reverse-proxy aware
|
||||
# URL generation. Set to the public hostname once the proxy is in front.
|
||||
PP_SITE_URL=http://localhost:2342/
|
||||
|
||||
# Auth mode — "password" for username/password (default), "public" for an
|
||||
# unauthenticated kiosk mode (don't use this on a multi-user library).
|
||||
PP_AUTH_MODE=password
|
||||
|
||||
# Library mount mode. M0 = "ro" (safe initial validation). Flip to "rw" at M2
|
||||
# when the right-sidebar enables EXIF backwrite. Set in lockstep with
|
||||
# PP_READONLY below.
|
||||
PP_ORIGINALS_MODE=ro
|
||||
PP_READONLY=true
|
||||
|
||||
# UID/GID inside the PhotoPrism container. Set these to the host UID/GID that
|
||||
# owns ${PHOTO_DIRS}. `id -u` and `id -g`.
|
||||
PP_UID=1000
|
||||
PP_GID=1000
|
||||
|
||||
|
||||
# ── OIDC SSO (Authentik or equivalent) ───────────────────────────────────────
|
||||
# Leave blank to keep OIDC dormant. Fill in to enable the "Sign in with OIDC"
|
||||
# button on the login page; per plan, OIDC_REGISTER=true auto-creates
|
||||
# accounts at role `user`.
|
||||
|
||||
# OIDC_PROVIDER_NAME=Authentik
|
||||
# OIDC_ISSUER_URL=https://auth.example.com/application/o/photoprism/
|
||||
# OIDC_CLIENT_ID=...
|
||||
# OIDC_CLIENT_SECRET=...
|
||||
# OIDC_REDIRECT_URI=http://localhost:2342/api/v1/oidc/redirect
|
||||
# OIDC_SCOPES=openid profile email
|
||||
# OIDC_REGISTER=true
|
||||
# OIDC_ROLE=user
|
||||
|
||||
|
||||
# ── LOGGING ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PP_LOG_LEVEL=info
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -37,7 +37,6 @@ dist-ssr/
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
.env.photoprism
|
||||
|
||||
# Database
|
||||
*.db
|
||||
@@ -66,9 +65,6 @@ docker-compose.override.yml
|
||||
/pp/storage/
|
||||
/pp/import/
|
||||
|
||||
# Local-dev sample photo library used for M0 PhotoPrism validation.
|
||||
/photos-sample/
|
||||
|
||||
# Sidecar runtime state (per-user marks etc.) — generated, not seed data.
|
||||
/sidecar/data/
|
||||
|
||||
@@ -80,4 +76,3 @@ docker-compose.override.yml
|
||||
|
||||
# Thumbnails
|
||||
/thumbs/
|
||||
/trash/backend/yolov8n.pt
|
||||
|
||||
247
.hermes/plans/2026-06-06_120000-photos_users-label-isolation.md
Normal file
247
.hermes/plans/2026-06-06_120000-photos_users-label-isolation.md
Normal file
@@ -0,0 +1,247 @@
|
||||
# Plan: Populate `photos_users` to fix label isolation in PhotoPrism
|
||||
|
||||
**Date:** 2026-06-06
|
||||
**Author:** Hermes Agent
|
||||
**Status:** Draft
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Fix the label isolation leak where a user with `base_path` set (e.g. `muli`) sees photos from other users' directories (e.g. `dtoro`) in PhotoPrism's labels view.
|
||||
|
||||
## 2. Current Context
|
||||
|
||||
### The problem
|
||||
- PhotoPrism's `base_path` feature correctly scopes the main search (`/api/v1/photos`).
|
||||
- Label views (`/api/v1/labels`) do **not** respect `base_path` — labels show photo counts and thumbnails from the entire library.
|
||||
- User reports: "all photos on the main labels page are a mix of both" muli and dtoro.
|
||||
|
||||
### What we know
|
||||
- **53 files changed** in the sidecar (Go + GORM, gorm.io/gorm v1.31.1).
|
||||
- Sidecar has a working PhotoPrism DB connection via `PpDSN` (user: `photoprism`, schema: `photoprism.*`).
|
||||
- The `photos_users` table exists in PhotoPrism's MariaDB schema but is **empty** (0 rows).
|
||||
- Schema of `photos_users`:
|
||||
|
||||
```
|
||||
photos_users:
|
||||
uid varbinary(42) NOT NULL PRI (composite PK or single?)
|
||||
user_uid varbinary(42) NOT NULL PRI
|
||||
team_uid varbinary(42) YES MUL
|
||||
perm int(10) unsigned YES
|
||||
```
|
||||
|
||||
- Known user UIDs: `dtoro=utfetfdk0so2z9zl`, `muli=utg7jjbd8iwaghn6`
|
||||
- Known base paths: `dtoro→dtoro`, `muli→muli`
|
||||
- The `photo_path` column in `photos` stores paths like `muli/files/Photo Archive...` or `dtoro/Memories/...`
|
||||
- Currently: ~88K photos, ~52K files indexed.
|
||||
|
||||
### The sidecar's current reconciler (`users.go`)
|
||||
- Runs every 60s.
|
||||
- Only calls `UPDATE auth_users SET base_path = ? WHERE user_name = ?`.
|
||||
- Does **not** touch `photos_users`.
|
||||
|
||||
### Unknowns
|
||||
1. **Does PhotoPrism use `photos_users` for general label filtering?** The table appears designed for explicit sharing (e.g. share a specific photo with another user), not for base_path ACL. PhotoPrism may ignore `photos_users` in label queries.
|
||||
2. **Performance impact**: 88K photos × 2 users = up to 176K rows. Could slow label queries.
|
||||
3. **Side effects**: If `photos_users` controls sharing, adding auto-entries might break explicit share workflow.
|
||||
4. **`perm` values**: Unclear what `perm` value grants "view" access. Likely a bitmap (bit 0 = view).
|
||||
|
||||
## 3. Proposed Approach
|
||||
|
||||
### Phase 1: Investigate (prove the approach works before building)
|
||||
|
||||
**Step 1.1: Insert test rows into `photos_users` manually**
|
||||
|
||||
On the production DB, insert a few `photos_users` entries for muli mapping to some of muli's own photos, plus one entry mapping to a dtoro photo. Use a guessed `perm` value (e.g. `1` = view).
|
||||
|
||||
Then check:
|
||||
- Does muli see fewer photos now? (If `photos_users` works as an exclusive ACL, yes.)
|
||||
- Does the dtoro photo with a `photos_users` entry for muli show up for muli?
|
||||
- Does the label view change?
|
||||
|
||||
**Step 1.2: Test with `perm` variations**
|
||||
|
||||
If `perm=1` does nothing, try `perm=2`, `perm=7`, or `perm=15` (common Unix-ish bitmap patterns).
|
||||
|
||||
**Step 1.3: Examine PhotoPrism source**
|
||||
|
||||
Check PhotoPrism's search/label code to confirm whether `photos_users` is joined in label queries. This tells us definitively whether the approach is viable.
|
||||
|
||||
### Phase 2: Build (if Phase 1 confirms the approach works)
|
||||
|
||||
**Step 2.1: Add `photos_users` GORM model**
|
||||
|
||||
New struct in `db.go` or a new file `perms.go`:
|
||||
|
||||
```go
|
||||
type PhotoUser struct {
|
||||
PhotoUID string `gorm:"primaryKey;size:42;column:uid"`
|
||||
UserUID string `gorm:"primaryKey;size:42;column:user_uid"`
|
||||
TeamUID string `gorm:"size:42;column:team_uid"`
|
||||
Perm int `gorm:"column:perm"`
|
||||
}
|
||||
|
||||
func (PhotoUser) TableName() string { return "photos_users" }
|
||||
```
|
||||
|
||||
Note: GORM `AutoMigrate` is called on `mule_sidecar` schema, not `photoprism.*`. The `photos_users` table already exists in the `photoprism` schema — we only query/insert, never migrate.
|
||||
|
||||
**Step 2.2: Add `reconcilePhotoUsers` function**
|
||||
|
||||
New function in a new file `perms.go` alongside `users.go`. Signature:
|
||||
|
||||
```go
|
||||
func reconcilePhotoUsers(ppDSN, originalsRoot string, mapping map[string]string) error
|
||||
```
|
||||
|
||||
Logic:
|
||||
1. For each `username:path` pair in `mapping`, look up the user's `user_uid` in `auth_users`.
|
||||
2. Query `photos` for all `photo_uid` where `photo_path LIKE 'path/%'`.
|
||||
3. Batch-insert entries into `photos_users` with a default `perm` value (to be determined in Phase 1).
|
||||
4. Use `INSERT IGNORE` or `ON DUPLICATE KEY UPDATE` for idempotency.
|
||||
5. Handle deletions: if a photo's path is changed (via rename), the old `photos_users` entry should be cleaned up.
|
||||
|
||||
**Step 2.3: Wire into reconciler loop**
|
||||
|
||||
Extend the existing `startUserBasepathReconciler` to call `reconcilePhotoUsers` after `reconcileUserBasepaths`.
|
||||
|
||||
```go
|
||||
func apply() {
|
||||
reconcileUserBasepaths(...)
|
||||
reconcilePhotoUsers(...)
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2.4: Handle re-index edge cases**
|
||||
|
||||
- When new photos are indexed, they won't have `photos_users` entries until the next 60s tick.
|
||||
- Could add a webhook or a one-shot trigger after PhotoPrism's index completes.
|
||||
- Alternative: accept the 60s lag as a design trade-off (current base_path reconciler already has this lag).
|
||||
|
||||
### Phase 3: Validate (if Phase 1 confirms)
|
||||
|
||||
1. Build the binary: `cd sidecar && CGO_ENABLED=0 go build -o mule-sidecar .`
|
||||
2. Rebuild the Docker image and restart the sidecar.
|
||||
3. Check `photos_users` has expected rows.
|
||||
4. Log in as `muli` via Authentik SSO, browse labels — verify dtoro photos are gone.
|
||||
5. Log in as `dtoro` — verify still sees own photos.
|
||||
6. Verify no regression: search, album, folder views still work for both users.
|
||||
|
||||
## 4. Files Likely to Change
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `sidecar/perms.go` | **New file** — `PhotoUser` model, `reconcilePhotoUsers` function |
|
||||
| `sidecar/db.go` | Add `photos_users`-related constants/helpers (optional) |
|
||||
| `sidecar/users.go` | Extend `reconcileUserBasepaths` or add a phase to the existing reconciler |
|
||||
| `sidecar/main.go` | Wire the new reconciler phase (minor — call from existing ticker) |
|
||||
| `sidecar/Dockerfile` | Unchanged (Go build picks up new `.go` files automatically) |
|
||||
|
||||
## 5. Tests & Validation
|
||||
|
||||
1. **Build check**: `go build ./...` from `sidecar/`
|
||||
2. **Manual DB test** (Phase 1): Insert test `photos_users` rows via `docker exec pp-mariadb mysql ...`
|
||||
3. **Integration test**: After deploy, check `photos_users` row count matches expected photo count per user.
|
||||
4. **Label isolation check**: Browse labels as each user — confirm no cross-user leaks.
|
||||
|
||||
## 6. Source Code Analysis (Completed)
|
||||
|
||||
### How base_path scoping works in PhotoPrism
|
||||
|
||||
Found the critical function `ScopePhotosForSession` in `internal/entity/search/photos_scope.go`:
|
||||
|
||||
```go
|
||||
func ScopePhotosForSession(stmt *gorm.DB, sess *entity.Session) *gorm.DB {
|
||||
// Admin/library role → no scoping needed
|
||||
if sess == nil || acl.Rules.AllowAny(acl.ResourcePhotos, sess.GetUserRole(), acl.Permissions{acl.AccessAll, acl.AccessLibrary}) {
|
||||
return stmt
|
||||
}
|
||||
user := sess.GetUser()
|
||||
if basePath := user.GetBasePath(); basePath == "" {
|
||||
return stmt.Where(sharedAlbums + "photos.created_by = ? OR ...", ...)
|
||||
} else {
|
||||
return stmt.Where(sharedAlbums + "... OR photos.photo_path = ? OR photos.photo_path LIKE ?",
|
||||
..., basePath, basePath + "/%")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Key: base_path filtering is done by adding `WHERE photos.photo_path LIKE 'muli/%'` to the SQL query. It is **NOT** done via `photos_users`.
|
||||
|
||||
### How endpoints use base_path
|
||||
|
||||
| Endpoint | Function | Applies base_path? |
|
||||
|----------|----------|-------------------|
|
||||
| `GET /api/v1/photos` | `SearchPhotos` → `UserPhotos` → `searchPhotos` → **`ScopePhotosForSession`** | ✅ Yes |
|
||||
| `GET /api/v1/labels` | `SearchLabels` → `search.Labels(frm)` — no session passed | ❌ **No** |
|
||||
| Review tab | Uses `GET /api/v1/photos?quality=3` → goes through `ScopePhotosForSession` | ✅ Should scope |
|
||||
| Archive tab | Uses `GET /api/v1/photos?archived=true` → goes through `ScopePhotosForSession` | ✅ Should scope |
|
||||
| Albums | TBD — depends on whether they use `ScopePhotosForSession` | ⚠️ Unknown |
|
||||
|
||||
### The `photos_users` table
|
||||
|
||||
Found in `internal/entity/photo_user.go`:
|
||||
|
||||
```go
|
||||
type PhotoUser struct {
|
||||
UID string // photo_uid
|
||||
UserUID string // user_uid
|
||||
TeamUID string // team_uid
|
||||
Perm uint // permission bitmap
|
||||
}
|
||||
```
|
||||
|
||||
This table is **not referenced** in `ScopePhotosForSession`, `searchPhotos`, or any label/album search function. It is only used for **explicit sharing** (via `FirstOrCreatePhotoUser` called when sharing a specific photo with another user).
|
||||
|
||||
**Conclusion: Populating `photos_users` will NOT fix the label, review, or archive tab isolation.** PhotoPrism does not consult this table for any of these queries.
|
||||
|
||||
### Why review/archive might show cross-user photos
|
||||
|
||||
Since review and archive use `GET /api/v1/photos` which goes through `ScopePhotosForSession`, they **should** be scoped. The issue might be:
|
||||
1. **Pre-computed counts** in the sidebar tabs show total numbers across all users
|
||||
2. **Label thumbnails** and category summaries are computed from the `labels` table which is global
|
||||
3. The actual photo list in review/archive should be correctly scoped — the user may be seeing dtoro photos only in the summary/counts
|
||||
|
||||
### DB experiment results
|
||||
|
||||
Confirmed `photos_users` is empty (0 rows). Inserted 100 muli-photo entries + 1 dtoro-photo entry for muli with `perm=1`. Label API response unchanged — `photo_count` values remained the same (Dog: 733, Cat: 57), confirming labels ignore `photos_users`.
|
||||
|
||||
## 7. Updated Recommendation
|
||||
|
||||
**Abandon the `photos_users` approach.** It won't fix the problem because PhotoPrism never consults this table for labels, review, or archive queries.
|
||||
|
||||
### Real fix options
|
||||
|
||||
1. **Sidecar label filter** (recommended) — The sidecar already validates sessions via `resolveSession()` which returns the user's `BasePath`. Extend the sidecar to expose a **proxied `/api/v1/labels`** endpoint that:
|
||||
- Accepts the caller's `X-Auth-Token` (already validated by `requireSession`)
|
||||
- Forwards the request to PhotoPrism's `/api/v1/labels`
|
||||
- **Filters the response** to remove labels whose `Thumb` belongs to a photo outside the user's `base_path`
|
||||
- Recalculates `PhotoCount` for the user's scope (count photos under `base_path/%` for that label)
|
||||
- Also filter `Count` values in the sidebar summary response
|
||||
|
||||
**Why this works:** The sidecar already has DB access to PhotoPrism's schema (`PpDSN`) and validates sessions. It can query `photos` to count label intersections per base_path.
|
||||
|
||||
2. **Same approach for review/archive sidebar counts** — Intercept the relevant metadata/summary endpoints to scope counts by base_path.
|
||||
|
||||
3. **Accept the limitation** — Labels show cross-user thumbnails/counts but the actual photo list is scoped.
|
||||
|
||||
### Implementation sketch for option 1
|
||||
|
||||
```
|
||||
sidecar/
|
||||
├── proxy.go # New file
|
||||
│ ├── handleLabels(c) → GET /api/sidecar/labels → proxies to PP, filters by base_path
|
||||
│ ├── handleReviewCount(c) → GET /api/sidecar/review → returns scoped count
|
||||
│ └── handleArchiveCount(c) → GET /api/sidecar/archive → returns scoped count
|
||||
```
|
||||
|
||||
The SvelteKit frontend would call `/api/sidecar/labels` instead of `/api/v1/labels`.
|
||||
|
||||
### Clean up: remove test rows from photos_users
|
||||
|
||||
Since the approach won't work, remove the test rows inserted during Phase 1:
|
||||
|
||||
```sql
|
||||
DELETE FROM photoprism.photos_users WHERE user_uid = 'utg7jjbd8iwaghn6';
|
||||
```
|
||||
@@ -0,0 +1,301 @@
|
||||
# Plan: Fix user isolation in PhotoPrism — labels, review, and archive views
|
||||
|
||||
**Date:** 2026-06-06
|
||||
**Author:** Hermes Agent
|
||||
**Status:** Draft
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Fix the three views where a user with `base_path` set (e.g. `muli`) sees photos from other users (e.g. `dtoro`):
|
||||
|
||||
1. **Labels** — labels list + label drill-down show all library photos
|
||||
2. **Review** — photos needing review tab shows cross-user photos
|
||||
3. **Archive** — archived photos tab shows cross-user photos
|
||||
|
||||
## 2. Current Context & Source Analysis
|
||||
|
||||
### 2.1 How base_path scoping works
|
||||
|
||||
PhotoPrism's `ScopePhotosForSession` (in `internal/entity/search/photos_scope.go`) is the only function that enforces user isolation. It adds `WHERE photos.photo_path = '<basePath>' OR photos.photo_path LIKE '<basePath>/%'` to the SQL query.
|
||||
|
||||
This is called by `searchPhotos()` — used by the **`GET /api/v1/photos`** endpoint (search, timeline, folders).
|
||||
|
||||
### 2.2 How endpoints use base_path
|
||||
|
||||
| View | Endpoint | Function chain | Applies base_path? |
|
||||
|------|----------|----------------|-------------------|
|
||||
| Main timeline | `GET /api/v1/photos` | `SearchPhotos` → `searchPhotos` → `ScopePhotosForSession` | ✅ Yes |
|
||||
| Folders | `GET /api/v1/photos` with path filter | Same chain | ✅ Yes |
|
||||
| **Labels** | `GET /api/v1/labels` | `SearchLabels` → `search.Labels(frm)` — **no session** | ❌ **No** — queries `labels` table directly |
|
||||
| Label drill-down | `GET /api/v1/photos?label=X` | Uses `searchPhotos` → `ScopePhotosForSession` | ✅ Should scope (if label= param doesn't bypass) |
|
||||
| **Review tab** | `GET /api/v1/photos?q=review:true` | `searchPhotos` → `ScopePhotosForSession` | ✅ Should scope, BUT... |
|
||||
| **Archive tab** | `GET /api/v1/photos?q=archived:true` | Same | ✅ Should scope, BUT... |
|
||||
|
||||
### 2.3 The review/archive problem: ACL overrides
|
||||
|
||||
In `searchPhotosForm()` (internal/api/photos_search.go):
|
||||
|
||||
```go
|
||||
if acl.Rules.Deny(acl.ResourcePhotos, s.GetUserRole(), acl.ActionManage) {
|
||||
frm.Quality = 3
|
||||
}
|
||||
```
|
||||
|
||||
For role=user, `Deny(ActionManage)` → true → sets `frm.Quality = 3` (minimum quality).
|
||||
|
||||
Then in `searchPhotos()` (internal/entity/search/photos.go):
|
||||
|
||||
```go
|
||||
if acl.Rules.Deny(acl.ResourcePhotos, aclRole, acl.ActionDelete) {
|
||||
frm.Archived = false
|
||||
frm.Review = false
|
||||
}
|
||||
```
|
||||
|
||||
For role=user, `Deny(ActionDelete)` → true → **overrides `review:true` and `archived:true` to false**.
|
||||
|
||||
So the review and archive filters are **completely ignored** for the `user` role. The frontend sends `review:true` but the server discards it. The result: the review/archive tabs show ALL photos scoped by base_path (no quality/review/archive filter), which means basically the same as the main timeline.
|
||||
|
||||
### 2.4 Label problem: no session scoping at all
|
||||
|
||||
`search.Labels()` queries the `labels` table directly with a `WHERE photo_count > 0` clause. There is no session parameter, no `ScopePhotosForSession`, and no base_path or user filtering whatsoever. Labels are **library-wide** in PhotoPrism.
|
||||
|
||||
The label drill-down (click into a label) uses `GET /api/v1/photos?label=X` which DOES go through `ScopePhotosForSession`, so the photo list itself should be scoped — but the label thumbnails, counts, and covers are global.
|
||||
|
||||
### 2.5 ACL GrantDefaults — missing RoleUser entry
|
||||
|
||||
```go
|
||||
var GrantDefaults = Roles{
|
||||
RoleAdmin: GrantFullAccess, // FullAccess = AccessLibrary + everything
|
||||
RoleGuest: GrantReactShared, // Only shared content
|
||||
RoleVisitor: GrantViewShared, // Only shared content
|
||||
RolePortal: GrantFullAccess,
|
||||
RoleClient: GrantFullAccess,
|
||||
// RoleUser and RoleViewer are NOT listed → fallback to RoleDefault (also missing) → denied
|
||||
}
|
||||
```
|
||||
|
||||
Because `RoleUser` is absent from `GrantDefaults`, the `Allow()` function falls back to `RoleDefault` which is also absent → returns `false` for all permissions. This means:
|
||||
- `ScopePhotosForSession` correctly enters the `base_path` branch (good — user is isolated)
|
||||
- BUT `ActionDelete` is denied → review/archive filters are forced off (bad — can't browse review/archive)
|
||||
|
||||
### 2.6 The `photos_users` table — ruled out
|
||||
|
||||
`internal/entity/photo_user.go` defines `PhotoUser` but it is **not referenced** in `ScopePhotosForSession`, `searchPhotos`, or any label/album search function. Populating it won't fix any of these issues.
|
||||
|
||||
## 3. Proposed Approach
|
||||
|
||||
### Phase 1: Sidecar proxy for labels (direct fix)
|
||||
|
||||
Extend the sidecar to expose a **scoped labels endpoint**:
|
||||
|
||||
```
|
||||
GET /api/sidecar/labels → proxies to GET /api/v1/labels → filters by base_path
|
||||
```
|
||||
|
||||
**How it works:**
|
||||
|
||||
1. Sidecar receives the caller's `X-Auth-Token`
|
||||
2. `requireSession` middleware resolves the token → returns user's `BasePath`
|
||||
3. Sidecar makes the same `/api/v1/labels` request to PhotoPrism (using the caller's token)
|
||||
4. **Filter step**: for each label in the response, query the DB to count photos with that label AND `photo_path LIKE '<base_path>/%'`
|
||||
5. Return filtered labels with corrected `PhotoCount` and `Thumb`
|
||||
|
||||
**Implementation:**
|
||||
|
||||
New file `sidecar/handlers_labels.go`:
|
||||
|
||||
```go
|
||||
// handleLabels proxies to PP's /api/v1/labels, then post-filters
|
||||
// counts and thumbnails by the caller's base_path.
|
||||
func handleLabels(pp *ppClient, ppDSN string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
token := ctxToken(c)
|
||||
user := ctxUser(c) // resolved from session, includes BasePath
|
||||
|
||||
// 1. Get raw labels from PhotoPrism
|
||||
resp, _ := pp.call(c, "GET", "/api/v1/labels?"+c.Request.URL.RawQuery, token, nil)
|
||||
|
||||
// 2. Decode labels
|
||||
var labels []PpLabel
|
||||
json.Unmarshal(resp.Body, &labels)
|
||||
|
||||
// 3. For each label, recalculate count for this user's base_path
|
||||
for i, l := range labels {
|
||||
// Count photos with this label AND where photo_path matches base_path
|
||||
var count int
|
||||
db.Raw(`SELECT COUNT(*) FROM photos_labels pl
|
||||
JOIN photos p ON pl.photo_uid = p.photo_uid
|
||||
WHERE pl.label_uid = ? AND p.photo_path LIKE ?`,
|
||||
l.UID, user.BasePath+"/%").Scan(&count)
|
||||
labels[i].PhotoCount = count
|
||||
|
||||
// If count is 0, the thumb from the global label doesn't apply
|
||||
// Could also update thumb to a user-scoped one
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, labels)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Frontend change:** Update the label query in `web/src/routes/tags/[category]/[[value]]/+page.svelte` to call `/api/sidecar/labels` instead of `/api/v1/labels`.
|
||||
|
||||
### Phase 2: Fix review/archive — skip the ACL override
|
||||
|
||||
Two options:
|
||||
|
||||
**Option A (Recommended): Sidecar proxy for photos search**
|
||||
|
||||
Extend the sidecar with:
|
||||
```
|
||||
GET /api/sidecar/photos → proxies to GET /api/v1/photos → adds path filter
|
||||
```
|
||||
|
||||
The sidecar intercepts the photos request and adds the `path:<basePath>` query parameter to PhotoPrism's API call. This forces PhotoPrism to add `WHERE photo_path LIKE '<base_path>/%'`.
|
||||
|
||||
For review/archive, the sidecar also adds `review:true` or `archived:true` BEFORE the ACL override happens (since the sidecar doesn't hit the ACL code).
|
||||
|
||||
**Option B: Custom frontend query**
|
||||
|
||||
The frontend explicitly adds `path:muli` to the query string for review/archive tabs:
|
||||
```
|
||||
GET /api/v1/photos?q=review:true path:muli&count=50
|
||||
```
|
||||
|
||||
The `path` filter is a standard PhotoPrism search operator that adds `WHERE photos.photo_path = '<path>'`. But this only matches the exact path, not `path/%` (subdirectories). The `path:` operator does `photo_path = ?` (exact match) per the code at line 668.
|
||||
|
||||
**Option A is better** because:
|
||||
- Works for all users without frontend changes
|
||||
- Can add the proper `LIKE` prefix match
|
||||
- Centralized logic in the sidecar
|
||||
|
||||
### Phase 3: Sidecar proxy for sidebar counts
|
||||
|
||||
The session response (or `GET /api/v1/config`) includes library-wide counts:
|
||||
```json
|
||||
"count": {
|
||||
"review": 248,
|
||||
"archived": 94,
|
||||
"all": 88203,
|
||||
"photos": 88000
|
||||
}
|
||||
```
|
||||
|
||||
These show the TOTAL across all users. The sidecar can proxy this and recalculate counts per base_path.
|
||||
|
||||
## 4. Step-by-step Plan
|
||||
|
||||
### Step 1: Sidecar — labels proxy
|
||||
|
||||
Files: `sidecar/handlers_labels.go` (new), `sidecar/main.go` (route wiring)
|
||||
|
||||
1. New types: `PpLabel` (mirrors PhotoPrism's label response shape)
|
||||
2. Handler function `handleLabels()` that:
|
||||
- Validates token via `requireSession`
|
||||
- Gets `BasePath` from session
|
||||
- Calls PhotoPrism's `/api/v1/labels`
|
||||
- For each label, queries photos_labels + photos to count user-scoped photos
|
||||
- Returns filtered labels
|
||||
3. Wire route: `auth.GET("/labels", handleLabels(...))` in `main.go`
|
||||
4. Frontend: change label fetch URL from `/api/v1/labels` to `/api/sidecar/labels`
|
||||
|
||||
### Step 2: Sidecar — photos proxy (review/archive fix)
|
||||
|
||||
Files: `sidecar/handlers_photos.go` (new), `sidecar/main.go` (route wiring)
|
||||
|
||||
1. Handler function `handlePhotos()` that:
|
||||
- Validates token
|
||||
- Gets `BasePath` from session
|
||||
- Parses the query string to detect `review:true` or `archived:true`
|
||||
- Forwards to PhotoPrism's `/api/v1/photos` with `path:<basePath>` added to query
|
||||
- For review/archive, also ensures `review/archived` filter is NOT stripped
|
||||
- Returns PhotoPrism's response
|
||||
|
||||
2. Two implementation variants:
|
||||
|
||||
**Variant A** (simpler): add `path:<basePath>` to the forwarded query. This only matches exact path, not subdirs (PhotoPrism's `path:` operator does exact match). Might miss photos in subdirectories.
|
||||
|
||||
**Variant B** (correct): Forward the query without path, then post-filter the response to remove photos whose `photo_path` doesn't match `basePath/%`. This is more robust.
|
||||
|
||||
### Step 3: Validation
|
||||
|
||||
1. Build sidecar: `cd sidecar && CGO_ENABLED=0 go build -o mule-sidecar .`
|
||||
2. Rebuild Docker image: `docker compose build sidecar`
|
||||
3. Restart sidecar: `docker compose up -d sidecar`
|
||||
4. Test labels as muli — verify only muli's labels appear
|
||||
5. Test review tab as muli — verify only muli's photos needing review appear
|
||||
6. Test archive tab as muli — verify only muli's archived photos appear
|
||||
7. Test same views as admin — verify dtoro still sees all
|
||||
|
||||
## 5. Files Likely to Change
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `sidecar/handlers_labels.go` | **New** — label proxy handler |
|
||||
| `sidecar/handlers_photos.go` | **New** — photos proxy handler (or merged into one proxy.go) |
|
||||
| `sidecar/handlers_folder.go` | Reference for existing handler patterns |
|
||||
| `sidecar/main.go` | Wire new routes under `auth` group |
|
||||
| `sidecar/pp.go` | May need new helper methods for label/photo API calls |
|
||||
| `sidecar/users.go` | No change |
|
||||
| `sidecar/db.go` | May add types for PpLabel, PpPhoto |
|
||||
| `web/src/routes/tags/[category]/[[value]]/+page.svelte` | Change label fetch URL |
|
||||
| `web/src/lib/stores/filters.svelte.ts` | Possibly change how review/archive queries are built |
|
||||
|
||||
## 6. Tests & Validation
|
||||
|
||||
**Build**: `cd sidecar && go build ./... && go vet ./...`
|
||||
|
||||
**Manual validation on LXC 120:**
|
||||
|
||||
```bash
|
||||
# Test labels endpoint
|
||||
curl -s "http://localhost:8000/api/sidecar/labels?count=5" \
|
||||
-H "X-Auth-Token: <muli-token>" | python3 -c "import sys,json;d=json.load(sys.stdin);[print(l.get('Name','?'),l.get('PhotoCount')) for l in d[:5]]"
|
||||
|
||||
# Test photos endpoint with review
|
||||
curl -s "http://localhost:8000/api/sidecar/photos?q=review:true&count=5" \
|
||||
-H "X-Auth-Token: <muli-token>" | python3 -c "import sys,json;d=json.load(sys.stdin);print(f'{len(d)} photos')"
|
||||
|
||||
# Verify vs. admin token — counts should differ
|
||||
```
|
||||
|
||||
**Cross-user check:** Log in as `muli` and `dtoro` in separate browser sessions. Verify:
|
||||
- Labels show different counts per user
|
||||
- Review photos are scoped per user
|
||||
- Archive photos are scoped per user
|
||||
|
||||
## 7. Risks, Tradeoffs & Open Questions
|
||||
|
||||
### Risks
|
||||
|
||||
| Risk | Impact | Mitigation |
|
||||
|------|--------|------------|
|
||||
| Sidecar proxying adds latency | Slower page loads | Labels are small payloads; single DB query per label is fast |
|
||||
| Frontend needs URL changes | Breaks if not updated | Do frontend change alongside sidecar deploy |
|
||||
| Photo count queries on every label request | DB load | Cache results for 30s in the sidecar |
|
||||
| PhotoPrism's label `PhotoCount` is stale | Mismatch with actual count | Acceptable — PhotoPrism's count is already cached |
|
||||
| Review/archive fix depends on how PhotoPrism handles `path:` operator | Photos in subdirs missed | Use Variant B (post-filter by path prefix) |
|
||||
|
||||
### Open Questions
|
||||
|
||||
- **Q1**: For review/archive — is the user seeing dtoro's photos in the *grid* or only the *sidebar counts*? Need to verify actual API response vs what the frontend renders.
|
||||
- **Q2**: What's the performance impact of running `SELECT COUNT(*) FROM photos_labels ... JOIN photos ...` for every label in the response? (Labels list is typically short, < 100)
|
||||
- **Q3**: Does the frontend cache the label response aggressively? Need to invalidate cache on user switch.
|
||||
- **Q4**: For the `path:` operator — does it do exact match or LIKE? From source: `WHERE photos.photo_path = ?` — exact match only.
|
||||
|
||||
### Tradeoffs
|
||||
|
||||
- **Sidecar proxy vs. frontend-only**: Proxy centralizes logic but adds network hop. Frontend-only is faster but more complex (every route needs path filtering).
|
||||
- **Label count accuracy**: Recalculated per-user counts will differ from the library-wide counts. This is intentional — labels are scoped now.
|
||||
- **Sidecar vs. patching PhotoPrism**: Sidecar approach is non-invasive (no fork/build of PP). PhotoPrism patch would be cleaner but requires maintaining a fork.
|
||||
|
||||
## 8. Recommendation
|
||||
|
||||
1. **Build the labels proxy** (Phase 1) — it directly solves the label isolation problem and can be done with existing sidecar infrastructure
|
||||
2. **Investigate review/archive leak** first — run the actual API query as muli to confirm whether the photos search is actually scoped. The code analysis says it should be, but the user reports otherwise. If confirmed as a real leak, implement the sidecar photos proxy (Phase 2)
|
||||
3. **Sidebar counts** (Phase 3) — lower priority, can be done after labels and review/archive are fixed
|
||||
|
||||
Before building, confirm with the user whether they see cross-user photos in the actual grid or only in the sidebar counts for review/archive.
|
||||
227
.hermes/plans/2026-06-06_210000-remaining-fixes.md
Normal file
227
.hermes/plans/2026-06-06_210000-remaining-fixes.md
Normal file
@@ -0,0 +1,227 @@
|
||||
# Plan: Fix remaining user isolation issues — 404 errors and folder tree
|
||||
|
||||
**Date:** 2026-06-06
|
||||
**Author:** Hermes Agent
|
||||
**Status:** Draft
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Fix the remaining issues after deploying the sidecar scoping proxy:
|
||||
|
||||
1. **404 on photo grid** — "Request failed with status code 404" in private window
|
||||
2. **Folder tree shows other users** — on first load, the library tree lists other users' folders; a refresh fixes it
|
||||
|
||||
## 2. Current Context
|
||||
|
||||
### What's deployed
|
||||
|
||||
| Component | Status |
|
||||
|-----------|--------|
|
||||
| Sidecar labels proxy (`/api/sidecar/labels`) | ✅ Working |
|
||||
| Sidecar counts proxy (`/api/sidecar/counts`) | ✅ Working |
|
||||
| Sidecar timeline proxy (`/api/sidecar/timeline`) | ✅ Working through Caddy |
|
||||
| Caddy fallback for `/api/v1/api/sidecar/*` | ✅ Working |
|
||||
| Frontend rebuild with `sidecar` axios instance | ✅ Built and deployed |
|
||||
|
||||
### Verified working via Caddy
|
||||
|
||||
```bash
|
||||
# Through public URL with valid admin token
|
||||
curl https://photos.hubris.network/api/sidecar/timeline?count=1 → HTTP 200
|
||||
curl https://photos.hubris.network/api/v1/photos?count=1 → HTTP 200
|
||||
```
|
||||
|
||||
Both endpoints return 200 when tested directly through Caddy with a valid token.
|
||||
|
||||
### Reported issues
|
||||
|
||||
1. **404 on photo grid** — even in private window (no cache interference)
|
||||
2. **Folder tree shows other users' folders** on first load, fixed by refresh
|
||||
|
||||
## 3. Root Cause Analysis
|
||||
|
||||
### Issue 1: 404 on photo grid
|
||||
|
||||
The `sidecar` axios instance (`baseURL: ''`) is missing the **response interceptor** that:
|
||||
- Handles 401 → clears session → redirects to login
|
||||
- Re-throws with meaningful error message
|
||||
|
||||
The `http` instance (for `/api/v1` endpoints) has this interceptor. Without it on `sidecar`:
|
||||
- If the sidecar returns a non-2xx (401, 502 from upstream PP failure, etc.), axios throws a raw error
|
||||
- The TanStack Query error boundary catches it and shows "Request failed with status code <status>"
|
||||
- Very likely the sidecar is returning 401 on some calls (token expired / session not yet established) and the error message might show 404 because Caddy's catch-all returns 404 when a matcher doesn't find a route
|
||||
|
||||
**Hypothesis:** During OIDC login flow, the frontend may make some sidecar calls BEFORE the session is fully established (token loaded into `session.accessToken`). The `sidecar` interceptor checks `session.accessToken` but it might be null. Then the request to `/api/sidecar/timeline` has no auth header → sidecar returns 401 → no response interceptor → raw error.
|
||||
|
||||
**Fix:** Add the same 401 → login redirect interceptor to the `sidecar` instance.
|
||||
|
||||
### Issue 2: Folder tree shows other users
|
||||
|
||||
`listFolders()` calls `http.get('/folders/originals')` which hits PhotoPrism directly. PhotoPrism returns **all folders across the library** regardless of user. The frontend then filters by `userBasePath()` on the result:
|
||||
|
||||
```typescript
|
||||
const bp = userBasePath();
|
||||
if (bp === '') return folders; // On first load, bp might be empty!
|
||||
return folders.filter((f) => f.Path === bp || f.Path.startsWith(bp + '/'))
|
||||
```
|
||||
|
||||
On first load, `userBasePath()` returns `""` because:
|
||||
1. The session data is loaded asynchronously
|
||||
2. `session.user.BasePath` might not yet be populated when `listFolders` fires
|
||||
3. The TanStack Query cache from a previous session might still have old data
|
||||
|
||||
After a refresh, the session is fully loaded, and `userBasePath()` returns the correct value.
|
||||
|
||||
A secondary issue: the `http` interceptor's 401 handler clears the session on 401. If the session expires during the app's lifetime, all subsequent requests fail with 401.
|
||||
|
||||
## 4. Proposed Approach
|
||||
|
||||
### Phase 1: Fix 404 — add response interceptor to sidecar
|
||||
|
||||
**File:** `web/src/lib/services/photoprism.ts`
|
||||
|
||||
Add the same 401 → login redirect interceptor to `sidecar` as already exists on `http`:
|
||||
|
||||
```typescript
|
||||
sidecar.interceptors.response.use(
|
||||
(r) => r,
|
||||
(err: AxiosError) => {
|
||||
if (err.response?.status === 401 && browser) {
|
||||
clearSession();
|
||||
const url = err.config?.url ?? '';
|
||||
if (!url.endsWith('/session')) {
|
||||
void goto('/login', { replaceState: true });
|
||||
}
|
||||
}
|
||||
return Promise.reject(err);
|
||||
}
|
||||
);
|
||||
```
|
||||
|
||||
### Phase 2: Fix folder tree — sidecar folder proxy
|
||||
|
||||
**File:** `sidecar/handlers_folders.go` (new)
|
||||
|
||||
Add a sidecar endpoint that proxies `/folders/originals` and post-filters by BasePath:
|
||||
|
||||
```
|
||||
GET /api/sidecar/folders → proxies to GET /api/v1/folders/originals
|
||||
→ removes folders not under user's base_path
|
||||
→ returns filtered list
|
||||
```
|
||||
|
||||
This avoids the timing issue entirely by filtering on the server side.
|
||||
|
||||
**Alternative (simpler):** Fix the frontend timing issue by ensuring `listFolders` doesn't fire until the session is ready.
|
||||
|
||||
### Phase 3: Change folder tree in frontend
|
||||
|
||||
**File:** `web/src/lib/services/photoprism.ts`
|
||||
|
||||
Change `listFolders()` to use `sidecar` instance and call `/api/sidecar/folders`:
|
||||
|
||||
```typescript
|
||||
export async function listFolders(): Promise<PpFolder[]> {
|
||||
const { data } = await sidecar.get<{ folders?: PpFolder[] }>(
|
||||
'/api/sidecar/folders',
|
||||
{ params: { recursive: true, uncached: true, files: false } }
|
||||
);
|
||||
const bp = userBasePath();
|
||||
const folders = data.folders ?? [];
|
||||
if (bp === '') return folders;
|
||||
return folders
|
||||
.filter((f) => f.Path === bp || f.Path.startsWith(bp + '/'))
|
||||
.map((f) => ({ ...f, Path: toUserPath(f.Path) }));
|
||||
}
|
||||
```
|
||||
|
||||
## 5. Step-by-step Plan
|
||||
|
||||
### Step 1: Add sidecar response interceptor
|
||||
|
||||
1. Edit `web/src/lib/services/photoprism.ts`
|
||||
2. Add the 401-handling response interceptor to the `sidecar` instance
|
||||
3. The interceptor mirrors the existing `http` response interceptor exactly
|
||||
|
||||
### Step 2: Rebuild frontend
|
||||
|
||||
```bash
|
||||
cd /opt/mule-image/web && npm run build
|
||||
```
|
||||
|
||||
### Step 3: (Optional) Add sidecar folder proxy
|
||||
|
||||
1. New file `sidecar/handlers_folders_proxy.go`
|
||||
2. Handler similar to `handlePhotos` — proxies to `/api/v1/folders/originals`, post-filters by `Path` prefix
|
||||
3. Wire route in `main.go`: `auth.GET("/folders", handleFoldersProxy(pp))`
|
||||
4. Build Docker image, restart sidecar
|
||||
|
||||
### Step 4: Update listFolders to use sidecar
|
||||
|
||||
1. Change `listFolders()` to use `sidecar` instance
|
||||
2. Call `/api/sidecar/folders` instead of `/folders/originals`
|
||||
|
||||
### Step 5: Rebuild + validate
|
||||
|
||||
```bash
|
||||
# Rebuild frontend
|
||||
cd /opt/mule-image/web && npm run build
|
||||
|
||||
# Test through Caddy
|
||||
curl -s "https://photos.hubris.network/api/sidecar/timeline?count=1" \
|
||||
-H "X-Auth-Token: <token>" | head -c 200
|
||||
|
||||
# Verify folders
|
||||
curl -s "https://photos.hubris.network/api/sidecar/folders" \
|
||||
-H "X-Auth-Token: <token>" | python3 -c "import sys,json;d=json.load(sys.stdin);print(json.dumps(d[:3],indent=2))"
|
||||
```
|
||||
|
||||
### Step 6: Commit
|
||||
|
||||
```bash
|
||||
git add -A && git commit -m "fix: add sidecar response interceptor + folder proxy" && git push
|
||||
```
|
||||
|
||||
## 6. Files Likely to Change
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `web/src/lib/services/photoprism.ts` | Add response interceptor to sidecar instance; change listFolders URL |
|
||||
| `sidecar/handlers_folders_proxy.go` | **New** — folder proxy handler |
|
||||
| `sidecar/main.go` | Wire folder proxy route |
|
||||
|
||||
## 7. Tests & Validation
|
||||
|
||||
**Manual:**
|
||||
1. Open private window → navigate to photos.hubris.network
|
||||
2. Log in as muli via Authentik OIDC
|
||||
3. Verify photo grid loads without 404
|
||||
4. Verify folder tree shows only muli's folders
|
||||
5. Switch to dtoro account → verify folders/timeline scoped to dtoro
|
||||
|
||||
**API tests:**
|
||||
```bash
|
||||
# Sidecar timeline (no token → 401 redirect)
|
||||
curl -s "https://photos.hubris.network/api/sidecar/timeline?count=1"
|
||||
|
||||
# Sidecar folders
|
||||
curl -s "https://photos.hubris.network/api/sidecar/folders"
|
||||
```
|
||||
|
||||
## 8. Risks & Open Questions
|
||||
|
||||
### Risks
|
||||
|
||||
| Risk | Impact | Mitigation |
|
||||
|------|--------|------------|
|
||||
| Sidecar returns 401 during OIDC login flow before session is ready | 404 showing instead of graceful redirect | Add response interceptor in Phase 1 |
|
||||
| Folder proxy adds latency | Slower folder tree loading | Minimal — single proxy call, same as PP direct |
|
||||
| `userBasePath()` timing issue in listFolders persists even with sidecar | Folder tree still shows wrong folders on first load | Sidecar filter is server-side → no timing dependency |
|
||||
|
||||
### Open Questions
|
||||
|
||||
- **Q1**: Are there other API calls that bypass the `sidecar` instance and might also be unscoped? (e.g., `listSubjects`, `listGeo`, etc.)
|
||||
- **Q2**: Does the sidecar need a folder proxy, or is the timing fix sufficient? The timing fix (delaying `listFolders` until session is ready) is simpler but fragile.
|
||||
- **Q3**: Could the 404 be from Caddy's catch-all returning 404 when the sidecar isn't reachable? The Caddy fallback timeout for the sidecar might need tuning.
|
||||
330
README.md
330
README.md
@@ -1,230 +1,140 @@
|
||||
# Mulita - Self-Hosted Photo Management Application
|
||||
# mule-image
|
||||
|
||||
A self-hosted, Docker-deployed photo management application inspired by Lightroom's workflow. Mulita provides a fast, keyboard-driven interface to browse, organize, tag, and manage your photo library.
|
||||
|
||||
## Features
|
||||
|
||||
- **Photo Organization**: Browse photos in a timeline view with virtual scrolling for performance
|
||||
- **Thumbnail Generation**: Automatic thumbnail generation for all photo formats including RAW
|
||||
- **Metadata Extraction**: Full EXIF/XMP metadata extraction and GPS mapping
|
||||
- **Keyboard Shortcuts**: Lightroom-style keyboard navigation and actions
|
||||
- **File Support**: JPEG, PNG, RAW formats (CR2, CR3, NEF, ARW, etc.), HEIC/HEIF, and videos
|
||||
- **Heaps**: Temporary collections for organizing photos
|
||||
- **Tags & Ratings**: Organize with tags, star ratings, and color labels — each with a card-grid browse view that drills into a full Timeline detail
|
||||
- **Dark Mode**: Photography-optimized dark interface
|
||||
- **Vision Pipeline**: YOLO object detection, OCR text extraction, CLIP embeddings for semantic search, InsightFace face detection and clustering
|
||||
- **People View**: Browse identified people as cards, click to see all photos of a person
|
||||
- **Map View**: Browse GPS-tagged photos on an interactive Leaflet map
|
||||
- **Duplicate Detection**: Perceptual hash-based duplicate grouping with best-pick UI
|
||||
- **Semantic Search**: Natural-language photo search powered by CLIP embeddings
|
||||
|
||||
## Tech Stack
|
||||
|
||||
### Backend
|
||||
- Python 3.12 with FastAPI
|
||||
- PostgreSQL + pgvector with SQLAlchemy (async) and Alembic migrations
|
||||
- Celery + Redis for background tasks
|
||||
- pyvips for fast thumbnail generation
|
||||
- ExifTool for metadata extraction
|
||||
- ONNX Runtime for vision models (YOLO, CLIP, InsightFace)
|
||||
|
||||
### Frontend
|
||||
- React 18 with TypeScript
|
||||
- Vite for fast development
|
||||
- TanStack Query for data fetching
|
||||
- TanStack Virtual for virtualized scrolling
|
||||
- Tailwind CSS for styling
|
||||
- Zustand for state management
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
- Docker and Docker Compose
|
||||
|
||||
### Setup (one variable)
|
||||
|
||||
1. Clone the repo:
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd muleimage
|
||||
```
|
||||
|
||||
2. Copy the example env file and set **one** variable — the **host**
|
||||
directory that contains your photo library. Whatever you point at
|
||||
will become your library inside Mulita.
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# then edit .env and set PHOTO_DIRS:
|
||||
# macOS / Linux: PHOTO_DIRS=/Users/you/Pictures
|
||||
# Network share: PHOTO_DIRS=/mnt/nas/photos
|
||||
# Windows (WSL): PHOTO_DIRS=/mnt/c/Users/you/Pictures
|
||||
```
|
||||
|
||||
3. Start the stack:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
4. Open `http://localhost:3000`. On first boot Mulita will:
|
||||
- Mount your `PHOTO_DIRS` at `/photos` inside the container
|
||||
- Auto-create a source root called **Library** pointing at `/photos`
|
||||
- Queue an initial scan, generate thumbnails, and start serving them
|
||||
|
||||
You don't need to touch `mulita.yml` or the API to get started.
|
||||
|
||||
### Configuration knobs
|
||||
|
||||
Everything is environment-driven. `PHOTO_DIRS` is the only required
|
||||
value; the rest have sensible defaults documented in `.env.example`:
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|----------------------|---------|----------------------------------------------------|
|
||||
| `PHOTO_DIRS` | — | **Required.** Host path mounted at `/photos`. |
|
||||
| `FRONTEND_PORT` | `3000` | SPA host port. Bump if `3000` is taken. |
|
||||
| `BACKEND_PORT` | `8001` | Direct backend port (debug only — frontend uses internal nginx proxy). |
|
||||
| `REDIS_PORT` | `6379` | Redis host port (internal services don't need it). |
|
||||
| `ALLOWED_ORIGINS` | `*` | Comma-separated CORS origins for direct backend access. Lock down for prod, e.g. `https://photos.example.com`. |
|
||||
| `LOG_LEVEL` | `INFO` | Backend + worker log level. `DEBUG` for chasing scan issues. |
|
||||
| `TZ` | `UTC` | Container timezone. Affects log timestamps and "added at". |
|
||||
| `CELERYD_CONCURRENCY`| `4` | Parallel worker processes (scans, thumbs, metadata). Lower on a Pi, higher on a beefy host. |
|
||||
|
||||
### Accessing from another machine
|
||||
|
||||
The frontend talks to the backend through its bundled nginx, which
|
||||
proxies `/api/` to the backend on the internal compose network. That
|
||||
means requests are always **same-origin** as the page, so accessing
|
||||
Mulita from another host works without any CORS dance:
|
||||
|
||||
```
|
||||
http://<your-server-ip>:3000
|
||||
```
|
||||
|
||||
If you want to put it behind a reverse proxy at e.g.
|
||||
`https://photos.your.tld`, set `ALLOWED_ORIGINS` to that host so the
|
||||
backend's direct port (`BACKEND_PORT`) also accepts cross-origin
|
||||
requests if anything bypasses the proxy.
|
||||
|
||||
### How libraries are managed
|
||||
|
||||
Mulita is **config-driven**: the host directory you mount via
|
||||
`PHOTO_DIRS` becomes your library, and the backend automatically
|
||||
registers it as a source root on startup. There is no UI for adding
|
||||
or removing source roots — to change what Mulita scans, edit `.env`
|
||||
(or `docker-compose.yml` for multi-mount setups) and restart the
|
||||
stack.
|
||||
|
||||
This keeps the model simple: **the docker mount IS the library**.
|
||||
No two layers, no confusion about which view to use.
|
||||
|
||||
### Changing or adding libraries
|
||||
|
||||
To point at a different library:
|
||||
1. Edit `PHOTO_DIRS` in `.env`
|
||||
2. `docker compose down`
|
||||
3. (Optional, for a clean slate) `docker volume rm muleimage_db_data muleimage_thumbs_data muleimage_proxies_data`
|
||||
4. `docker compose up -d`
|
||||
|
||||
The new library shows up automatically. Without step 3 the old
|
||||
library's metadata stays in the DB and you'll see a warning at
|
||||
startup that the old source root's path is missing on disk —
|
||||
that's a hint to clean up.
|
||||
|
||||
For multiple libraries, edit `docker-compose.yml` and add additional
|
||||
mount lines:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- ${PHOTO_DIRS}:/photos:rw
|
||||
- /Volumes/Archive:/archive:rw # additional library
|
||||
```
|
||||
|
||||
Each mounted directory will need a corresponding source root row in
|
||||
the DB; today that means `POST /api/v1/folders` via curl, or wait
|
||||
for the multi-mount auto-registration that's on the roadmap.
|
||||
|
||||
### Read-only libraries
|
||||
|
||||
The default mount is `:rw` because file operations (rename, move,
|
||||
empty discard pile) need to mutate the filesystem. If you want a
|
||||
strict read-only library — pointing at a network share, an
|
||||
authoritative archive, etc. — flip `:rw` to `:ro` in
|
||||
`docker-compose.yml`. Mulita will keep working for browsing, rating,
|
||||
color labels, picks, heaps, and the (soft) discard flag, but the
|
||||
following will return an OS error:
|
||||
|
||||
- `PATCH /photos/{id}` with a new `filename` (rename)
|
||||
- `POST /photos/move` (bulk move)
|
||||
- `DELETE /discard/empty` (file unlinks)
|
||||
|
||||
**Heads up**: with `:rw`, Mulita has full write access to whatever
|
||||
host directory you mount. Treat the same way you would Lightroom's
|
||||
catalog folder.
|
||||
Self-hosted photo management built on top of [PhotoPrism][pp]. A SvelteKit
|
||||
frontend ([`web/`](web/)) plus a small Go service ([`sidecar/`](sidecar/))
|
||||
fill in the keyboard-driven UI and the file/folder/mark endpoints
|
||||
PhotoPrism's REST API does not expose. PhotoPrism itself handles
|
||||
indexing, originals, thumbnails, and the database; we never re-implement
|
||||
those.
|
||||
|
||||
## Architecture
|
||||
|
||||
The application consists of 5 Docker services:
|
||||
|
||||
- **frontend**: React SPA served by Nginx
|
||||
- **backend**: FastAPI REST API
|
||||
- **worker**: Celery workers for background tasks (thumbnails, metadata, vision pipeline)
|
||||
- **redis**: Message broker for Celery
|
||||
- **db**: PostgreSQL with pgvector extension (for CLIP/face embeddings)
|
||||
|
||||
## Keyboard Shortcuts
|
||||
|
||||
| Key | Action |
|
||||
|-----|--------|
|
||||
| `←` `→` `↑` `↓` | Navigate photos |
|
||||
| `Space` | Quick preview |
|
||||
| `Enter` | Open loupe view |
|
||||
| `T` | Add to active heap |
|
||||
| `1-5` | Set star rating |
|
||||
| `Tab` | Toggle left sidebar |
|
||||
| `I` | Toggle metadata panel |
|
||||
| `G` | Grid view |
|
||||
| `E` | Loupe view |
|
||||
| `Delete` | Move to trash |
|
||||
|
||||
## Development
|
||||
|
||||
### Backend Development
|
||||
```bash
|
||||
cd backend
|
||||
pip install -r requirements.txt
|
||||
uvicorn app.main:app --reload
|
||||
```text
|
||||
┌──────────────────┐ /api/v1/* ┌──────────────┐
|
||||
│ SvelteKit web/ │ ───────────────▶ │ photoprism │ ──▶ mariadb
|
||||
│ (Vite : 5173) │ /api/sidecar/* │ :2342 │
|
||||
│ │ ─────────┐ └──────────────┘
|
||||
└──────────────────┘ ▼
|
||||
┌──────────────┐
|
||||
│ sidecar │ ──▶ mariadb (mule_sidecar.*)
|
||||
│ :8000 │ ──▶ originals FS (rename / folders / dups)
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
### Frontend Development
|
||||
Three compose services — `mariadb`, `photoprism`, `sidecar` — plus the
|
||||
SvelteKit `web/` app served separately. PhotoPrism's port `2342` is
|
||||
**bound to `127.0.0.1` only**; it isn't a user-facing surface. The
|
||||
SvelteKit app is.
|
||||
|
||||
What the sidecar adds on top of PhotoPrism (full list in
|
||||
[`sidecar/README.md`](sidecar/README.md)):
|
||||
|
||||
- Per-photo marks (rating + color) persisted to `mule_sidecar.marks`
|
||||
- File rename + folder create/rename/delete with PhotoPrism reindex
|
||||
- Heap (album) → folder conversion
|
||||
- Perceptual-hash duplicate scan + archive
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
cp .env.example .env
|
||||
# edit .env: set PHOTO_DIRS to the host path holding your library
|
||||
# rotate PP_ADMIN_PASSWORD, PP_DB_PASSWORD, PP_DB_ROOT_PASSWORD
|
||||
# before any non-local deployment.
|
||||
|
||||
podman-compose --env-file .env \
|
||||
-f docker-compose.yml \
|
||||
-f docker-compose.podman.yml \
|
||||
up -d
|
||||
```
|
||||
|
||||
Then serve the frontend. For local use the simplest path is the Vite
|
||||
dev server:
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npm install
|
||||
npm run dev
|
||||
# open http://localhost:5173
|
||||
```
|
||||
|
||||
For a static deployment, `npm run build` produces a bundle under
|
||||
`web/build/` that any static file host (nginx, Caddy, GitHub Pages-style)
|
||||
can serve. Reverse-proxy `/api/v1/*` to `http://127.0.0.1:2342` and
|
||||
`/api/sidecar/*` to `http://127.0.0.1:8000`.
|
||||
|
||||
PhotoPrism's own UI is still reachable from the host at
|
||||
`http://127.0.0.1:2342` if you need admin features (user management,
|
||||
settings) — set up an SSH tunnel from your laptop if the server is
|
||||
remote.
|
||||
|
||||
## Configuration
|
||||
|
||||
Source roots are managed by the UI / API (the database owns them). Edit
|
||||
`mulita.yml` to configure operational settings only:
|
||||
All knobs live in [`.env.example`](.env.example). The required ones:
|
||||
|
||||
- Thumbnail sizes, quality, and format
|
||||
- Scanner behaviour (watch, batch size, initial scan)
|
||||
- Performance tuning (concurrency, cache TTLs, DB pool)
|
||||
| Variable | Notes |
|
||||
|----------------------|-----------------------------------------------------------------------------------------------|
|
||||
| `PHOTO_DIRS` | Host path mounted at `/photoprism/originals`. The library. |
|
||||
| `PP_ADMIN_PASSWORD` | First-boot admin password. Rotate. |
|
||||
| `PP_DB_PASSWORD` | MariaDB password for the `photoprism` user. Rotate. |
|
||||
| `PP_DB_ROOT_PASSWORD`| MariaDB root password. Rotate. |
|
||||
| `PP_UID` / `PP_GID` | Host UID/GID that owns `PHOTO_DIRS`. PhotoPrism + sidecar drop to this user inside. |
|
||||
| `PP_PORT` | Loopback host port for PhotoPrism (default `2342`). |
|
||||
| `PP_ORIGINALS_MODE` | `rw` (default) or `ro` — see [Read-only libraries](#read-only-libraries). |
|
||||
| `SIDECAR_PORT` | Loopback host port for the sidecar (default `8000`). |
|
||||
|
||||
## Performance
|
||||
Sidecar-specific env (DB DSN, `USER_BASEPATHS`, etc.) is documented in
|
||||
[`sidecar/README.md`](sidecar/README.md).
|
||||
|
||||
- Handles 100,000+ photos efficiently
|
||||
- Virtual scrolling for smooth timeline navigation
|
||||
- Thumbnail generation at 10+ photos/second
|
||||
- PostgreSQL full-text search with tsvector indexing
|
||||
- pgvector for fast nearest-neighbor embedding search
|
||||
## Read-only libraries
|
||||
|
||||
## Future Features
|
||||
The default originals mount is `:rw` because file operations (rename,
|
||||
folder mutations, duplicate archive, heap convert) need to mutate the
|
||||
filesystem. To run against a read-only archive, set
|
||||
`PP_ORIGINALS_MODE=ro` in `.env`. Browsing, marks, ratings, and color
|
||||
labels still work; the following sidecar endpoints return an OS error:
|
||||
|
||||
- Smart albums (auto-populated by saved filters)
|
||||
- Export presets
|
||||
- Multi-user support
|
||||
- `POST /api/sidecar/files/:uid/rename`
|
||||
- `POST /api/sidecar/folders` / `:rel/rename` / `DELETE /:rel`
|
||||
- `POST /api/sidecar/albums/:uid/convert`
|
||||
- `POST /api/sidecar/duplicates/archive`
|
||||
|
||||
## License
|
||||
PhotoPrism's `PHOTOPRISM_READONLY` is controlled separately by
|
||||
`PP_READONLY` and gates its own backwrite / import paths.
|
||||
|
||||
MIT
|
||||
## Dev iteration loop
|
||||
|
||||
For fast iteration on the sidecar without rebuilding its image on every
|
||||
change, run it as a host process — bring up just `mariadb` and
|
||||
`photoprism` from compose, then build and run the Go binary locally.
|
||||
Full instructions in [`sidecar/README.md`](sidecar/README.md#dev-iteration-loop-host-build).
|
||||
|
||||
## Layout
|
||||
|
||||
```text
|
||||
.
|
||||
├── docker-compose.yml base stack: mariadb + photoprism + sidecar
|
||||
├── docker-compose.podman.yml rootless-podman overlay (keep-id mapping)
|
||||
├── docker-compose.gpu.yml opt-in VA-API GPU passthrough overlay
|
||||
├── .env.example required env vars (copy to .env)
|
||||
├── mariadb/init/ first-boot SQL: creates mule_sidecar DB + user
|
||||
├── pp/ PhotoPrism bind-mounted state (storage, import)
|
||||
├── sidecar/ Go service — see sidecar/README.md
|
||||
└── web/ SvelteKit frontend
|
||||
```
|
||||
|
||||
## GPU video acceleration (optional)
|
||||
|
||||
Hosts with a VA-API-capable GPU (Intel iGPU, AMD APU, etc.) can layer
|
||||
[`docker-compose.gpu.yml`](docker-compose.gpu.yml) to hand `/dev/dri/*`
|
||||
to PhotoPrism and switch ffmpeg to hardware encode/decode — a large
|
||||
perf win for video thumbnails and HEVC→H.264 transcodes:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
|
||||
```
|
||||
|
||||
Set `PP_FFMPEG_ENCODER=vaapi` in `.env` (default for the overlay). Verify
|
||||
with `docker exec pp-app photoprism show config | grep -i ffmpeg`.
|
||||
|
||||
[pp]: https://photoprism.app/
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
FROM python:3.12-slim
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
# Build dependencies
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
# Image processing libraries
|
||||
libvips42 \
|
||||
libvips-dev \
|
||||
# ExifTool for metadata extraction
|
||||
libimage-exiftool-perl \
|
||||
# FFmpeg for video processing
|
||||
ffmpeg \
|
||||
# Git for some Python packages
|
||||
git \
|
||||
# PostgreSQL client (for potential future use)
|
||||
postgresql-client \
|
||||
# Clean up
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
# buildkit cache mount keeps pip's download cache on disk across builds
|
||||
# so even when this layer is invalidated, wheels are reused locally.
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p /data/thumbs /data/db /data/proxies /data/models /app/config
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8000
|
||||
|
||||
# Run the application
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
|
||||
@@ -1,48 +0,0 @@
|
||||
# Alembic configuration for PhotoVault.
|
||||
#
|
||||
# The actual database URL is loaded at runtime by alembic/env.py from the
|
||||
# DATABASE_URL environment variable (with the async driver suffix stripped).
|
||||
# The placeholder below is only used for `alembic revision --autogenerate`
|
||||
# when no env var is set.
|
||||
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
version_path_separator = os
|
||||
sqlalchemy.url = postgresql+psycopg2://mulita:mulita@localhost:5432/mulita
|
||||
|
||||
[post_write_hooks]
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -1,95 +0,0 @@
|
||||
"""
|
||||
Alembic environment for PhotoVault.
|
||||
|
||||
Pulls DATABASE_URL from the environment so the same migrations work in
|
||||
docker compose and locally. Strips the async driver suffix because Alembic
|
||||
runs synchronously via psycopg2.
|
||||
|
||||
Future-migration note
|
||||
---------------------
|
||||
Fresh installs run `Base.metadata.create_all` in `app.database.init_db`
|
||||
*before* migrations would normally apply, so any migration that adds a
|
||||
column / index / table to an object the model already declares will see
|
||||
that object already present. Write migrations defensively:
|
||||
|
||||
op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS new_col TEXT")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_foo ON foo(bar)")
|
||||
|
||||
For brand-new tables that the model also declares, the same applies — use
|
||||
`op.execute("CREATE TABLE IF NOT EXISTS ...")` or check first.
|
||||
"""
|
||||
from logging.config import fileConfig
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
from alembic import context
|
||||
|
||||
# Make `app` importable from this script.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.database import Base # noqa: E402
|
||||
# Import all models so they're registered on Base.metadata for autogenerate.
|
||||
from app.models import ( # noqa: E402, F401
|
||||
Photo,
|
||||
Folder,
|
||||
SourceRoot,
|
||||
Tag,
|
||||
Heap,
|
||||
HeapPhoto,
|
||||
)
|
||||
|
||||
config = context.config
|
||||
|
||||
# Resolve DATABASE_URL from env. Strip async driver suffixes — Alembic
|
||||
# uses sync drivers.
|
||||
db_url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url")
|
||||
if db_url:
|
||||
if "+asyncpg" in db_url:
|
||||
db_url = db_url.replace("+asyncpg", "+psycopg2")
|
||||
elif db_url.startswith("postgresql://"):
|
||||
db_url = db_url.replace("postgresql://", "postgresql+psycopg2://", 1)
|
||||
elif "+aiosqlite" in db_url:
|
||||
db_url = db_url.replace("+aiosqlite", "")
|
||||
config.set_main_option("sqlalchemy.url", db_url)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode (emit SQL only)."""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations against a live database."""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -1,26 +0,0 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -1,27 +0,0 @@
|
||||
"""baseline (empty)
|
||||
|
||||
Revision ID: 0001_baseline
|
||||
Revises:
|
||||
Create Date: 2026-04-10
|
||||
|
||||
The current schema is created by SQLAlchemy `Base.metadata.create_all` in
|
||||
`app.database.init_db()` on first boot. Alembic only owns deltas from
|
||||
PR3 onward. This baseline is intentionally empty so `alembic upgrade head`
|
||||
on a fresh DB simply creates the `alembic_version` table and stamps it.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "0001_baseline"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -1,85 +0,0 @@
|
||||
"""extend tags for vision pipeline
|
||||
|
||||
Revision ID: 0002_extend_tags
|
||||
Revises: 0001_baseline
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Add kind, source, representative_photo_id to tags table.
|
||||
Add confidence, bbox, source to photo_tags association.
|
||||
Switch uniqueness from (name) to (name, kind).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision: str = "0002_extend_tags"
|
||||
down_revision: Union[str, None] = "0001_baseline"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ── tags table ────────────────────────────────────────────────────
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS kind VARCHAR NOT NULL DEFAULT 'user'")
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS source VARCHAR")
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS representative_photo_id VARCHAR REFERENCES photos(id) ON DELETE SET NULL")
|
||||
|
||||
# Create index on kind for filtering
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_tags_kind ON tags(kind)")
|
||||
|
||||
# Drop old unique constraint on name (if it exists) and add (name, kind).
|
||||
# SQLAlchemy create_all may have created either — handle both cases.
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Drop the old single-column unique index/constraint if present.
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_indexes
|
||||
WHERE tablename = 'tags' AND indexname = 'ix_tags_name'
|
||||
) THEN
|
||||
DROP INDEX ix_tags_name;
|
||||
END IF;
|
||||
|
||||
-- Some SQLAlchemy versions create a unique constraint directly.
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM information_schema.table_constraints
|
||||
WHERE table_name = 'tags' AND constraint_name = 'tags_name_key'
|
||||
) THEN
|
||||
ALTER TABLE tags DROP CONSTRAINT tags_name_key;
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'uq_tags_name_kind'
|
||||
) THEN
|
||||
ALTER TABLE tags ADD CONSTRAINT uq_tags_name_kind UNIQUE (name, kind);
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
# ── photo_tags table ──────────────────────────────────────────────
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS confidence FLOAT")
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS bbox JSONB")
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS source VARCHAR")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# photo_tags columns
|
||||
op.drop_column("photo_tags", "source")
|
||||
op.drop_column("photo_tags", "bbox")
|
||||
op.drop_column("photo_tags", "confidence")
|
||||
|
||||
# tags: restore old unique constraint
|
||||
op.execute("ALTER TABLE tags DROP CONSTRAINT IF EXISTS uq_tags_name_kind")
|
||||
op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tags_name ON tags(name)")
|
||||
|
||||
# tags columns
|
||||
op.drop_column("tags", "representative_photo_id")
|
||||
op.drop_column("tags", "source")
|
||||
op.drop_column("tags", "kind")
|
||||
@@ -1,52 +0,0 @@
|
||||
"""pgvector embeddings
|
||||
|
||||
Revision ID: 0003_pgvector_embeddings
|
||||
Revises: 0002_extend_tags
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Rewrite the embeddings table to use pgvector Vector(512) instead of
|
||||
LargeBinary. Add composite PK (photo_id, model), created_at, and
|
||||
HNSW index on vector column.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0003_pgvector_embeddings"
|
||||
down_revision: Union[str, None] = "0002_extend_tags"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Drop the old placeholder table and recreate with pgvector types.
|
||||
# No data to preserve — it was never populated.
|
||||
op.execute("DROP TABLE IF EXISTS embeddings")
|
||||
op.execute("""
|
||||
CREATE TABLE embeddings (
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
model VARCHAR(64) NOT NULL,
|
||||
vector vector(512),
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
PRIMARY KEY (photo_id, model)
|
||||
)
|
||||
""")
|
||||
# HNSW index for cosine similarity search.
|
||||
# Defer creation on large backfills — drop and recreate afterward.
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
||||
ON embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS embeddings")
|
||||
op.execute("""
|
||||
CREATE TABLE embeddings (
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
model VARCHAR,
|
||||
vector BYTEA,
|
||||
PRIMARY KEY (photo_id)
|
||||
)
|
||||
""")
|
||||
@@ -1,82 +0,0 @@
|
||||
"""ocr_text table and Postgres FTS
|
||||
|
||||
Revision ID: 0004_ocr_fts
|
||||
Revises: 0003_pgvector_embeddings
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Create ocr_text table for storing OCR results. Add a tsvector column
|
||||
to photos for unified full-text search (filename + user_title +
|
||||
user_notes) with a GIN index. OCR text is rolled up into a materialized
|
||||
view or joined at query time.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0004_ocr_fts"
|
||||
down_revision: Union[str, None] = "0003_pgvector_embeddings"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ── ocr_text table ────────────────────────────────────────────────
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ocr_text (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
language VARCHAR(8) DEFAULT '',
|
||||
confidence FLOAT,
|
||||
bbox JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT now()
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_ocr_text_photo_id ON ocr_text(photo_id)")
|
||||
|
||||
# ── tsvector column on photos ─────────────────────────────────────
|
||||
op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS search_vector tsvector")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_photos_search_vector ON photos USING GIN (search_vector)")
|
||||
|
||||
# Trigger to auto-update search_vector on INSERT/UPDATE
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION photos_search_vector_update() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_vector :=
|
||||
setweight(to_tsvector('english', coalesce(NEW.filename, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.user_title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.user_notes, '')), 'B');
|
||||
RETURN NEW;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_trigger WHERE tgname = 'photos_search_vector_trigger'
|
||||
) THEN
|
||||
CREATE TRIGGER photos_search_vector_trigger
|
||||
BEFORE INSERT OR UPDATE OF filename, user_title, user_notes
|
||||
ON photos
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION photos_search_vector_update();
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
# Backfill existing rows
|
||||
op.execute("""
|
||||
UPDATE photos SET search_vector =
|
||||
setweight(to_tsvector('english', coalesce(filename, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(user_title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(user_notes, '')), 'B')
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TRIGGER IF EXISTS photos_search_vector_trigger ON photos")
|
||||
op.execute("DROP FUNCTION IF EXISTS photos_search_vector_update()")
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_search_vector")
|
||||
op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS search_vector")
|
||||
op.execute("DROP TABLE IF EXISTS ocr_text")
|
||||
@@ -1,41 +0,0 @@
|
||||
"""face_embeddings table
|
||||
|
||||
Revision ID: 0005_face_embeddings
|
||||
Revises: 0004_ocr_fts
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Create face_embeddings table with pgvector Vector(128) for SFace
|
||||
recognition embeddings and HNSW index.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0005_face_embeddings"
|
||||
down_revision: Union[str, None] = "0004_ocr_fts"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS face_embeddings (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
bbox JSONB,
|
||||
vector vector(128),
|
||||
cluster_id VARCHAR REFERENCES tags(id) ON DELETE SET NULL,
|
||||
quality FLOAT,
|
||||
created_at TIMESTAMPTZ DEFAULT now()
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_face_embeddings_photo_id ON face_embeddings(photo_id)")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_face_embeddings_cluster_id ON face_embeddings(cluster_id)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS face_embeddings")
|
||||
@@ -1,39 +0,0 @@
|
||||
"""face_embeddings vector 128 -> 512
|
||||
|
||||
Revision ID: 0006_face_512d
|
||||
Revises: 0005_face_embeddings
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Resize face_embeddings.vector from Vector(128) to Vector(512) for
|
||||
ArcFace embeddings (InsightFace). Drops existing data and HNSW index,
|
||||
recreates both.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0006_face_512d"
|
||||
down_revision: Union[str, None] = "0005_face_embeddings"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Drop index, truncate (old 128-d vectors are incompatible), resize
|
||||
op.execute("DROP INDEX IF EXISTS ix_face_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM face_embeddings")
|
||||
op.execute("ALTER TABLE face_embeddings ALTER COLUMN vector TYPE vector(512)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_face_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM face_embeddings")
|
||||
op.execute("ALTER TABLE face_embeddings ALTER COLUMN vector TYPE vector(128)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
@@ -1,67 +0,0 @@
|
||||
"""folders + photos is_hidden flag
|
||||
|
||||
Revision ID: 0007_folder_hidden
|
||||
Revises: 0006_face_512d
|
||||
Create Date: 2026-04-11
|
||||
|
||||
Adds an "exclude from cross-cutting views" flag:
|
||||
|
||||
folders.is_hidden — user-toggled on a folder or source root. When
|
||||
true, photos in that subtree are hidden from
|
||||
library-wide views (All Photos, Map, Tags,
|
||||
People, Search, Duplicates, sidebar counts) but
|
||||
remain indexed and visible when the user
|
||||
navigates into the folder directly.
|
||||
|
||||
photos.is_hidden — denormalized: true iff any ancestor folder in
|
||||
the photo's folder chain has is_hidden=true.
|
||||
Kept as a real column (rather than a recursive
|
||||
query per read) because the filter runs on
|
||||
essentially every photo query in the app, and
|
||||
the toggle operation that recomputes it is
|
||||
rare. Indexed so `WHERE NOT is_hidden` doesn't
|
||||
fall off the rating/taken_at indexes.
|
||||
|
||||
Both columns default to false so existing rows need no backfill.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0007_folder_hidden"
|
||||
down_revision: Union[str, None] = "0006_face_512d"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"folders",
|
||||
sa.Column(
|
||||
"is_hidden",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"photos",
|
||||
sa.Column(
|
||||
"is_hidden",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_photos_is_hidden",
|
||||
"photos",
|
||||
["is_hidden"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_photos_is_hidden", table_name="photos")
|
||||
op.drop_column("photos", "is_hidden")
|
||||
op.drop_column("folders", "is_hidden")
|
||||
@@ -1,49 +0,0 @@
|
||||
"""photos has_date_warning flag
|
||||
|
||||
Revision ID: 0008_photos_date_warning
|
||||
Revises: 0007_folder_hidden
|
||||
Create Date: 2026-04-11
|
||||
|
||||
Adds `photos.has_date_warning` — a denormalized boolean that's true when
|
||||
the scanner's folder/filename date guesser disagrees with the stored
|
||||
taken_at by more than 24h (or taken_at is missing and the path would
|
||||
provide a date). Surfacing this as a real column means the filter bar
|
||||
can restrict the timeline to suspicious photos without the client
|
||||
recomputing the heuristic for every row.
|
||||
|
||||
Indexed because the filter is meant to run on top of the existing
|
||||
taken_at / folder queries that dominate the timeline, and we want the
|
||||
partial `WHERE has_date_warning` scan to stay cheap as the library
|
||||
grows.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0008_photos_date_warning"
|
||||
down_revision: Union[str, None] = "0007_folder_hidden"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"photos",
|
||||
sa.Column(
|
||||
"has_date_warning",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_photos_has_date_warning",
|
||||
"photos",
|
||||
["has_date_warning"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_photos_has_date_warning", table_name="photos")
|
||||
op.drop_column("photos", "has_date_warning")
|
||||
@@ -1,144 +0,0 @@
|
||||
"""users table and user_id foreign keys
|
||||
|
||||
Revision ID: 0009_users_and_auth
|
||||
Revises: 0008_photos_date_warning
|
||||
Create Date: 2026-04-12
|
||||
|
||||
Introduces multi-user support:
|
||||
1. Creates the `users` table.
|
||||
2. Adds `user_id` FK columns to photos, folders, source_roots, heaps, tags.
|
||||
3. For existing installs: creates a default admin user and assigns all
|
||||
existing rows to that user. The generated password is printed to the
|
||||
backend logs — the admin should change it on first login.
|
||||
4. Replaces the unique constraint on tags (name, kind) with
|
||||
(name, kind, user_id) so each user can have their own tags.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
import uuid
|
||||
import secrets
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0009_users_and_auth"
|
||||
down_revision: Union[str, None] = "0008_photos_date_warning"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# 1. Create users table (IF NOT EXISTS — safe on fresh installs where
|
||||
# init_db's create_all has already laid down the schema).
|
||||
conn.execute(sa.text("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id VARCHAR NOT NULL PRIMARY KEY,
|
||||
username VARCHAR(50) NOT NULL UNIQUE,
|
||||
email VARCHAR UNIQUE,
|
||||
hashed_password VARCHAR NOT NULL,
|
||||
role VARCHAR NOT NULL DEFAULT 'user',
|
||||
is_active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT now(),
|
||||
media_path VARCHAR NOT NULL
|
||||
)
|
||||
"""))
|
||||
conn.execute(sa.text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_users_username ON users (username)"
|
||||
))
|
||||
|
||||
# 2. Add user_id columns (nullable initially for the data migration)
|
||||
for table in ("photos", "folders", "source_roots", "heaps", "tags"):
|
||||
conn.execute(sa.text(
|
||||
f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS user_id VARCHAR"
|
||||
))
|
||||
conn.execute(sa.text(
|
||||
f"CREATE INDEX IF NOT EXISTS ix_{table}_user_id ON {table} (user_id)"
|
||||
))
|
||||
# FK — check if it already exists before adding
|
||||
fk_name = f"fk_{table}_user_id"
|
||||
fk_exists = conn.execute(sa.text(
|
||||
"SELECT 1 FROM information_schema.table_constraints "
|
||||
"WHERE constraint_name = :name AND table_name = :tbl"
|
||||
), {"name": fk_name, "tbl": table}).scalar()
|
||||
if not fk_exists:
|
||||
conn.execute(sa.text(
|
||||
f"ALTER TABLE {table} ADD CONSTRAINT {fk_name} "
|
||||
f"FOREIGN KEY (user_id) REFERENCES users(id)"
|
||||
))
|
||||
|
||||
# 3. Data migration: if rows exist, create a default admin and assign
|
||||
conn = op.get_bind()
|
||||
photo_count = conn.execute(sa.text("SELECT COUNT(*) FROM photos")).scalar()
|
||||
|
||||
if photo_count > 0:
|
||||
admin_id = str(uuid.uuid4())
|
||||
generated_password = secrets.token_urlsafe(16)
|
||||
|
||||
# Hash the password using passlib at migration time
|
||||
from passlib.context import CryptContext
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
hashed = pwd_context.hash(generated_password)
|
||||
|
||||
# Every user gets a subfolder — including the migrated admin.
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"INSERT INTO users (id, username, hashed_password, role, media_path) "
|
||||
"VALUES (:id, :username, :hashed, :role, :media_path)"
|
||||
),
|
||||
{
|
||||
"id": admin_id,
|
||||
"username": "admin",
|
||||
"hashed": hashed,
|
||||
"role": "admin",
|
||||
"media_path": "/photos/admin",
|
||||
},
|
||||
)
|
||||
|
||||
# Assign all existing rows to the default admin
|
||||
for table in ("photos", "folders", "source_roots", "heaps", "tags"):
|
||||
conn.execute(
|
||||
sa.text(f"UPDATE {table} SET user_id = :uid WHERE user_id IS NULL"),
|
||||
{"uid": admin_id},
|
||||
)
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger("alembic.migration")
|
||||
logger.warning(
|
||||
f"=== MIGRATION 0009 === Default admin created. "
|
||||
f"Username: admin | Password: {generated_password} | "
|
||||
f"Change this password on first login!"
|
||||
)
|
||||
|
||||
# 4. Replace tag unique constraint to include user_id
|
||||
# Check whether the old constraint exists before trying to drop it
|
||||
# (on fresh installs create_all creates the new constraint directly).
|
||||
old_uq_exists = conn.execute(sa.text(
|
||||
"SELECT 1 FROM information_schema.table_constraints "
|
||||
"WHERE constraint_name = 'uq_tags_name_kind' AND table_name = 'tags'"
|
||||
)).scalar()
|
||||
if old_uq_exists:
|
||||
op.drop_constraint("uq_tags_name_kind", "tags", type_="unique")
|
||||
|
||||
new_uq_exists = conn.execute(sa.text(
|
||||
"SELECT 1 FROM information_schema.table_constraints "
|
||||
"WHERE constraint_name = 'uq_tags_name_kind_user' AND table_name = 'tags'"
|
||||
)).scalar()
|
||||
if not new_uq_exists:
|
||||
op.create_unique_constraint("uq_tags_name_kind_user", "tags", ["name", "kind", "user_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Reverse the tag constraint
|
||||
op.drop_constraint("uq_tags_name_kind_user", "tags", type_="unique")
|
||||
op.create_unique_constraint("uq_tags_name_kind", "tags", ["name", "kind"])
|
||||
|
||||
# Drop user_id columns and FKs
|
||||
for table in ("photos", "folders", "source_roots", "heaps", "tags"):
|
||||
op.drop_constraint(f"fk_{table}_user_id", table, type_="foreignkey")
|
||||
op.drop_index(f"ix_{table}_user_id", table_name=table)
|
||||
op.drop_column(table, "user_id")
|
||||
|
||||
# Drop users table
|
||||
op.drop_index("ix_users_username", table_name="users")
|
||||
op.drop_table("users")
|
||||
@@ -1,39 +0,0 @@
|
||||
"""embeddings vector 512 -> 768
|
||||
|
||||
Revision ID: 0010_embeddings_768d
|
||||
Revises: 0009_users_and_auth
|
||||
Create Date: 2026-04-12
|
||||
|
||||
Resize embeddings.vector from Vector(512) to Vector(768) for
|
||||
SigLIP2 ViT-B/16 embeddings. Drops existing data and HNSW index,
|
||||
recreates with the new dimension. Existing embeddings will be
|
||||
regenerated by the vision backfill task.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0010_embeddings_768d"
|
||||
down_revision: Union[str, None] = "0009_users_and_auth"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM embeddings")
|
||||
op.execute("ALTER TABLE embeddings ALTER COLUMN vector TYPE vector(768)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
||||
ON embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM embeddings")
|
||||
op.execute("ALTER TABLE embeddings ALTER COLUMN vector TYPE vector(512)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
||||
ON embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
@@ -1,54 +0,0 @@
|
||||
"""Add sharing tables for heaps and folders
|
||||
|
||||
Revision ID: 0011_sharing
|
||||
Revises: 0010_embeddings_768d
|
||||
Create Date: 2026-04-13
|
||||
|
||||
Adds heap_shares and folder_shares tables so users can share
|
||||
heaps and folders with other users (read or read+write).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0011_sharing"
|
||||
down_revision: Union[str, None] = "0010_embeddings_768d"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS heap_shares (
|
||||
id VARCHAR NOT NULL PRIMARY KEY,
|
||||
heap_id VARCHAR NOT NULL REFERENCES heaps(id) ON DELETE CASCADE,
|
||||
owner_id VARCHAR NOT NULL REFERENCES users(id),
|
||||
shared_with_id VARCHAR NOT NULL REFERENCES users(id),
|
||||
permission VARCHAR NOT NULL DEFAULT 'read',
|
||||
created_at TIMESTAMP DEFAULT now(),
|
||||
CONSTRAINT uq_heap_share UNIQUE (heap_id, shared_with_id)
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_heap_shares_shared_with ON heap_shares(shared_with_id)")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_heap_shares_heap_id ON heap_shares(heap_id)")
|
||||
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS folder_shares (
|
||||
id VARCHAR NOT NULL PRIMARY KEY,
|
||||
folder_id VARCHAR NOT NULL,
|
||||
folder_type VARCHAR NOT NULL DEFAULT 'folder',
|
||||
owner_id VARCHAR NOT NULL REFERENCES users(id),
|
||||
shared_with_id VARCHAR NOT NULL REFERENCES users(id),
|
||||
permission VARCHAR NOT NULL DEFAULT 'read',
|
||||
created_at TIMESTAMP DEFAULT now(),
|
||||
CONSTRAINT uq_folder_share UNIQUE (folder_id, shared_with_id)
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_folder_shares_shared_with ON folder_shares(shared_with_id)")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_folder_shares_folder_id ON folder_shares(folder_id)")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS folder_shares")
|
||||
op.execute("DROP TABLE IF EXISTS heap_shares")
|
||||
@@ -1,65 +0,0 @@
|
||||
"""Strip AI pipeline to binary classifier only
|
||||
|
||||
Revision ID: 0012_strip_ai
|
||||
Revises: 0011_sharing
|
||||
Create Date: 2026-04-14
|
||||
|
||||
Removes face recognition, OCR, object detection, and semantic embeddings.
|
||||
The remaining AI is a single binary 'photography' vs 'other' classifier
|
||||
whose output feeds Tag(kind='content_type') and a new Photo.needs_review
|
||||
flag.
|
||||
|
||||
Drops: embeddings, face_embeddings, ocr_text tables.
|
||||
Drops: photo_tags rows produced by 'vision:yolov8n' and 'vision:sface'.
|
||||
Drops: tags with kind IN ('object','scene','face_cluster').
|
||||
Drops: tags.representative_photo_id column.
|
||||
Adds: photos.needs_review (bool, default false) + partial index.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0012_strip_ai"
|
||||
down_revision: Union[str, None] = "0011_sharing"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Drop dropped-AI tables. CASCADE clears any lingering FKs/indices.
|
||||
op.execute("DROP TABLE IF EXISTS embeddings CASCADE")
|
||||
op.execute("DROP TABLE IF EXISTS face_embeddings CASCADE")
|
||||
op.execute("DROP TABLE IF EXISTS ocr_text CASCADE")
|
||||
|
||||
# Clear ML-produced photo_tags rows and their parent tags.
|
||||
op.execute(
|
||||
"DELETE FROM photo_tags WHERE source IN ('vision:yolov8n','vision:sface')"
|
||||
)
|
||||
op.execute(
|
||||
"DELETE FROM tags WHERE kind IN ('object','scene','face_cluster')"
|
||||
)
|
||||
|
||||
# Drop the face-cluster representative column.
|
||||
op.execute("ALTER TABLE tags DROP COLUMN IF EXISTS representative_photo_id")
|
||||
|
||||
# Add the needs_review flag.
|
||||
op.execute(
|
||||
"ALTER TABLE photos ADD COLUMN IF NOT EXISTS needs_review "
|
||||
"BOOLEAN NOT NULL DEFAULT false"
|
||||
)
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS ix_photos_needs_review "
|
||||
"ON photos(needs_review) WHERE needs_review"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Data is not recoverable on downgrade — only the schema stubs are
|
||||
# put back so a future reinstall of the old pipeline can re-populate.
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_needs_review")
|
||||
op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS needs_review")
|
||||
|
||||
op.execute(
|
||||
"ALTER TABLE tags ADD COLUMN IF NOT EXISTS representative_photo_id "
|
||||
"VARCHAR REFERENCES photos(id) ON DELETE SET NULL"
|
||||
)
|
||||
@@ -1,35 +0,0 @@
|
||||
"""Drop legacy content_type tags from the 6-category classifier
|
||||
|
||||
Revision ID: 0013_drop_old_ct
|
||||
Revises: 0012_strip_ai
|
||||
Create Date: 2026-04-14
|
||||
|
||||
The previous classifier wrote Tag(kind='content_type', name IN
|
||||
('photograph','screenshot','document','receipt','meme','artwork')).
|
||||
The new binary classifier writes names ('photography','other'). Both
|
||||
coexisted after the cutover so users saw duplicate groupings like
|
||||
'photography' alongside 'photograph'. Drop the old names — photo_tags
|
||||
rows cascade-delete via the FK.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0013_drop_old_ct"
|
||||
down_revision: Union[str, None] = "0012_strip_ai"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
LEGACY_NAMES = ('photograph', 'screenshot', 'document', 'receipt', 'meme', 'artwork')
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"DELETE FROM tags WHERE kind = 'content_type' "
|
||||
f"AND name IN {LEGACY_NAMES}"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -1,67 +0,0 @@
|
||||
"""Add status + accepted_at to heap_shares and folder_shares
|
||||
|
||||
Revision ID: 0014_share_status
|
||||
Revises: 0013_drop_old_ct
|
||||
Create Date: 2026-04-21
|
||||
|
||||
Shares used to activate instantly on the owner's side. We now want a
|
||||
pending/accepted lifecycle so the recipient gets a notification bell and
|
||||
chooses to accept or decline before the shared item shows up in their
|
||||
sidebar.
|
||||
|
||||
Backfill note: every pre-existing row is treated as `accepted` with
|
||||
accepted_at = created_at. This is a pragmatic fiction — it keeps the
|
||||
sidebar populated after the migration without anyone having to click
|
||||
accept on shares that were already live. Any future "accepted X ago" UI
|
||||
inheriting this backfilled timestamp should be aware it's not a real
|
||||
user-action moment.
|
||||
|
||||
The one-migration trick: we add `status` with `server_default="accepted"`
|
||||
so the backfill happens in-place, then drop the default so new inserts
|
||||
fall through to the Python-side model default ("pending").
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0014_share_status"
|
||||
down_revision: Union[str, None] = "0013_drop_old_ct"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for table in ("heap_shares", "folder_shares"):
|
||||
op.add_column(
|
||||
table,
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.String(),
|
||||
nullable=False,
|
||||
server_default="accepted",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
table,
|
||||
sa.Column("accepted_at", sa.DateTime(), nullable=True),
|
||||
)
|
||||
op.execute(
|
||||
f"UPDATE {table} SET accepted_at = created_at "
|
||||
"WHERE accepted_at IS NULL"
|
||||
)
|
||||
# Drop the DB default so new rows inherit the Python-side
|
||||
# model default ("pending") instead of silently auto-accepting.
|
||||
op.alter_column(table, "status", server_default=None)
|
||||
op.create_index(
|
||||
f"ix_{table}_shared_with_status",
|
||||
table,
|
||||
["shared_with_id", "status"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table in ("heap_shares", "folder_shares"):
|
||||
op.drop_index(f"ix_{table}_shared_with_status", table_name=table)
|
||||
op.drop_column(table, "accepted_at")
|
||||
op.drop_column(table, "status")
|
||||
@@ -1,83 +0,0 @@
|
||||
"""OIDC identity + avatar / display_name on users
|
||||
|
||||
Revision ID: 0015_oidc_and_avatar
|
||||
Revises: 0014_share_status
|
||||
Create Date: 2026-04-22
|
||||
|
||||
Lets users sign in via an OIDC provider (Authentik) and carry a profile
|
||||
image / display name from the provider. Password-only users are
|
||||
unaffected.
|
||||
|
||||
1. Add users.oidc_issuer, users.oidc_sub (identity pair from the IdP).
|
||||
2. Add users.avatar_url, users.display_name (profile bits from claims
|
||||
or manually set).
|
||||
3. Make users.hashed_password nullable — OIDC-only users have no local
|
||||
password. Existing rows all have hashes so the NULLability change
|
||||
is backwards-compatible.
|
||||
4. Partial unique index on (oidc_issuer, oidc_sub) WHERE oidc_sub IS
|
||||
NOT NULL so multiple password-only users (both NULL) don't collide.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0015_oidc_and_avatar"
|
||||
down_revision: Union[str, None] = "0014_share_status"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# 1 + 2. Add new columns (idempotent — create_all on fresh installs
|
||||
# already built them from the model).
|
||||
for col_def in (
|
||||
"oidc_issuer VARCHAR",
|
||||
"oidc_sub VARCHAR",
|
||||
"avatar_url VARCHAR",
|
||||
"display_name VARCHAR",
|
||||
):
|
||||
conn.execute(sa.text(f"ALTER TABLE users ADD COLUMN IF NOT EXISTS {col_def}"))
|
||||
|
||||
# 3. Drop NOT NULL on hashed_password. Postgres only — SQLite can't
|
||||
# alter column nullability in place, but the SQLite escape hatch is
|
||||
# used for fresh local dev where create_all already wrote the new
|
||||
# nullable definition.
|
||||
if conn.dialect.name == "postgresql":
|
||||
conn.execute(sa.text(
|
||||
"ALTER TABLE users ALTER COLUMN hashed_password DROP NOT NULL"
|
||||
))
|
||||
|
||||
# 4. Partial unique index — Postgres supports the WHERE clause so
|
||||
# NULLs don't collide; SQLite treats NULLs as distinct in unique
|
||||
# indexes already, so a plain unique index is safe there too.
|
||||
if conn.dialect.name == "postgresql":
|
||||
conn.execute(sa.text(
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS ix_users_oidc_identity "
|
||||
"ON users (oidc_issuer, oidc_sub) WHERE oidc_sub IS NOT NULL"
|
||||
))
|
||||
else:
|
||||
conn.execute(sa.text(
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS ix_users_oidc_identity "
|
||||
"ON users (oidc_issuer, oidc_sub)"
|
||||
))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
conn.execute(sa.text("DROP INDEX IF EXISTS ix_users_oidc_identity"))
|
||||
|
||||
if conn.dialect.name == "postgresql":
|
||||
# Can't re-apply NOT NULL if any OIDC-only user has NULL — so
|
||||
# only do it when safe.
|
||||
conn.execute(sa.text(
|
||||
"UPDATE users SET hashed_password = '' WHERE hashed_password IS NULL"
|
||||
))
|
||||
conn.execute(sa.text(
|
||||
"ALTER TABLE users ALTER COLUMN hashed_password SET NOT NULL"
|
||||
))
|
||||
|
||||
for col in ("display_name", "avatar_url", "oidc_sub", "oidc_issuer"):
|
||||
conn.execute(sa.text(f"ALTER TABLE users DROP COLUMN IF EXISTS {col}"))
|
||||
@@ -1,52 +0,0 @@
|
||||
"""Nextcloud integration: per-user username override + encrypted app password
|
||||
|
||||
Revision ID: 0016_nextcloud_integration
|
||||
Revises: 0015_oidc_and_avatar
|
||||
Create Date: 2026-04-26
|
||||
|
||||
Lets each mule-image user wire their account to a Nextcloud account so
|
||||
photos can be browsed, indexed, and mutated under their own Nextcloud
|
||||
file tree (`/mnt/library/homecloud/<nc_user>/files/...` mounted into the
|
||||
backend + workers as `/nextcloud-users`). The OIDC `preferred_username`
|
||||
claim is the default mapping; the override field handles cases where the
|
||||
authentik username and the Nextcloud username don't match.
|
||||
|
||||
1. users.nextcloud_username — default sourced from preferred_username
|
||||
on OIDC login (only when null), editable via PATCH /api/v1/auth/me.
|
||||
2. users.nextcloud_app_password_enc — Fernet-encrypted Nextcloud app
|
||||
password used for HTTP Basic auth on WebDAV calls. Set from the
|
||||
Settings UI; the cleartext is never persisted.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0016_nextcloud_integration"
|
||||
down_revision: Union[str, None] = "0015_oidc_and_avatar"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
for col_def in (
|
||||
"nextcloud_username VARCHAR",
|
||||
"nextcloud_app_password_enc VARCHAR",
|
||||
):
|
||||
conn.execute(sa.text(f"ALTER TABLE users ADD COLUMN IF NOT EXISTS {col_def}"))
|
||||
|
||||
# Index the username for the per-user path-scoping check on /browse
|
||||
# and /source-roots — keeps lookups fast even on tiny user tables.
|
||||
conn.execute(sa.text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_users_nextcloud_username "
|
||||
"ON users (nextcloud_username)"
|
||||
))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
conn.execute(sa.text("DROP INDEX IF EXISTS ix_users_nextcloud_username"))
|
||||
for col in ("nextcloud_app_password_enc", "nextcloud_username"):
|
||||
conn.execute(sa.text(f"ALTER TABLE users DROP COLUMN IF EXISTS {col}"))
|
||||
@@ -1,40 +0,0 @@
|
||||
"""Partial index for the photos list query
|
||||
|
||||
Revision ID: 0017_photos_list_index
|
||||
Revises: 0016_nextcloud_integration
|
||||
Create Date: 2026-05-10
|
||||
|
||||
The default photos-list query (GET /api/v1/photos?per_page=N&sort=taken_at&order=desc)
|
||||
filters `NOT is_trashed AND NOT is_hidden` and sorts by
|
||||
`(taken_at DESC NULLS LAST, id DESC)`. EXPLAIN ANALYZE on a 21k-row
|
||||
table showed a seq-scan + top-N heapsort (~20ms standalone, worse under
|
||||
concurrency) — Postgres can't use the single-column ix_photos_taken_at
|
||||
when the leading WHERE clause is two booleans.
|
||||
|
||||
A partial index on the sort key, scoped to the visible subset, lets the
|
||||
planner index-scan in reverse and stop at LIMIT N. Two booleans select
|
||||
~95% of rows, so the partial predicate is tighter than the full table
|
||||
without losing common queries.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0017_photos_list_index"
|
||||
down_revision: Union[str, None] = "0016_nextcloud_integration"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS ix_photos_list_visible
|
||||
ON photos (taken_at DESC NULLS LAST, id DESC)
|
||||
WHERE NOT is_trashed AND NOT is_hidden
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_list_visible")
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Add photos.nextcloud_fileid for NC preview proxying
|
||||
|
||||
Revision ID: 0018_photos_nextcloud_fileid
|
||||
Revises: 0017_photos_list_index
|
||||
Create Date: 2026-05-11
|
||||
|
||||
The thumbnail endpoint will proxy Nextcloud's /index.php/core/preview
|
||||
instead of generating and serving its own WebP cache under /data/thumbs.
|
||||
That requires storing each photo's Nextcloud numeric fileid alongside
|
||||
the row. NULL is allowed because legacy / non-NC photos still exist
|
||||
and the handler keeps the on-disk fallback for them.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0018_photos_nextcloud_fileid"
|
||||
down_revision: Union[str, None] = "0017_photos_list_index"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"photos",
|
||||
sa.Column("nextcloud_fileid", sa.Integer(), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_photos_nextcloud_fileid",
|
||||
"photos",
|
||||
["nextcloud_fileid"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_photos_nextcloud_fileid", table_name="photos")
|
||||
op.drop_column("photos", "nextcloud_fileid")
|
||||
@@ -1,47 +0,0 @@
|
||||
"""Drop AI remnants: photos.needs_review and the pgvector extension
|
||||
|
||||
Revision ID: 0019_drop_ai_remnants
|
||||
Revises: 0018_photos_nextcloud_fileid
|
||||
Create Date: 2026-05-14
|
||||
|
||||
The vision pipeline has been removed entirely (no more classifier, no
|
||||
worker-vision service, no torch/onnxruntime/open-clip-torch deps). The
|
||||
`needs_review` boolean and its partial index were populated only by
|
||||
that classifier and have no remaining writers or readers.
|
||||
|
||||
The pgvector extension was originally added by 0003_pgvector_embeddings
|
||||
for the embeddings table that 0012_strip_ai_pipeline dropped; the
|
||||
extension itself is now unused, and the next deploy switches the
|
||||
Postgres image from pgvector/pgvector:pg16 to plain postgres:16. The
|
||||
extension must be dropped *before* that image swap or the new
|
||||
container will fail to load existing CREATE EXTENSION declarations.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision: str = "0019_drop_ai_remnants"
|
||||
down_revision: Union[str, None] = "0018_photos_nextcloud_fileid"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_needs_review")
|
||||
op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS needs_review")
|
||||
op.execute("DROP EXTENSION IF EXISTS vector")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Re-create the column as a no-op (data is gone). The pgvector
|
||||
# extension is intentionally NOT re-added — matches the precedent
|
||||
# set by 0012_strip_ai_pipeline for its dropped tables.
|
||||
op.execute(
|
||||
"ALTER TABLE photos ADD COLUMN IF NOT EXISTS needs_review "
|
||||
"BOOLEAN NOT NULL DEFAULT FALSE"
|
||||
)
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS ix_photos_needs_review "
|
||||
"ON photos(needs_review) WHERE needs_review"
|
||||
)
|
||||
@@ -1,47 +0,0 @@
|
||||
"""
|
||||
Authentication utilities — password hashing and JWT token management.
|
||||
"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from jose import jwt, JWTError
|
||||
from passlib.context import CryptContext
|
||||
|
||||
from app.config import settings
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
|
||||
ALGORITHM = "HS256"
|
||||
|
||||
|
||||
def hash_password(plain: str) -> str:
|
||||
return pwd_context.hash(plain)
|
||||
|
||||
|
||||
def verify_password(plain: str, hashed: str) -> bool:
|
||||
return pwd_context.verify(plain, hashed)
|
||||
|
||||
|
||||
def create_access_token(user_id: str, role: str) -> str:
|
||||
expire = datetime.now(timezone.utc) + timedelta(minutes=settings.access_token_expire_minutes)
|
||||
payload = {
|
||||
"sub": user_id,
|
||||
"role": role,
|
||||
"exp": expire,
|
||||
"type": "access",
|
||||
}
|
||||
return jwt.encode(payload, settings.secret_key, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def create_refresh_token(user_id: str) -> str:
|
||||
expire = datetime.now(timezone.utc) + timedelta(days=settings.refresh_token_expire_days)
|
||||
payload = {
|
||||
"sub": user_id,
|
||||
"exp": expire,
|
||||
"type": "refresh",
|
||||
}
|
||||
return jwt.encode(payload, settings.secret_key, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def decode_token(token: str) -> dict:
|
||||
"""Decode and validate a JWT. Raises JWTError on any problem."""
|
||||
return jwt.decode(token, settings.secret_key, algorithms=[ALGORITHM])
|
||||
@@ -1,77 +0,0 @@
|
||||
"""OIDC (OpenID Connect) client setup — used for Authentik SSO today,
|
||||
generic enough to register other providers later.
|
||||
|
||||
Authlib handles the Authorization Code + PKCE flow, including discovery
|
||||
via the provider's `.well-known/openid-configuration` document. We keep
|
||||
a single registered client named "authentik" regardless of label, so the
|
||||
router code always knows where to find it.
|
||||
|
||||
If OIDC isn't fully configured the module stays inert — `is_enabled()`
|
||||
returns False and the registry has no client. Callers must guard.
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
from authlib.integrations.starlette_client import OAuth
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
PROVIDER_NAME = "authentik"
|
||||
|
||||
|
||||
def is_configured() -> bool:
|
||||
"""True when every required OIDC setting is present."""
|
||||
return bool(
|
||||
settings.oidc_issuer
|
||||
and settings.oidc_client_id
|
||||
and settings.oidc_client_secret
|
||||
and settings.oidc_redirect_uri
|
||||
)
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
"""True when OIDC is both configured and the admin flipped it on."""
|
||||
return bool(settings.oidc_enabled) and is_configured()
|
||||
|
||||
|
||||
def provider_label() -> str:
|
||||
return settings.oidc_provider_label or "Authentik"
|
||||
|
||||
|
||||
def _build_oauth() -> OAuth:
|
||||
"""Build the Authlib OAuth registry. Always safe to call; only
|
||||
registers the provider when credentials are present so importing
|
||||
this module never fails on a fresh install.
|
||||
"""
|
||||
registry = OAuth()
|
||||
if not is_configured():
|
||||
return registry
|
||||
|
||||
# Authentik exposes discovery at `{issuer}/.well-known/openid-configuration`.
|
||||
# Trailing slash handling varies by Authentik version, so normalise.
|
||||
issuer = settings.oidc_issuer.rstrip("/")
|
||||
discovery_url = f"{issuer}/.well-known/openid-configuration"
|
||||
|
||||
registry.register(
|
||||
name=PROVIDER_NAME,
|
||||
client_id=settings.oidc_client_id,
|
||||
client_secret=settings.oidc_client_secret,
|
||||
server_metadata_url=discovery_url,
|
||||
client_kwargs={
|
||||
"scope": settings.oidc_scopes,
|
||||
# Force PKCE — cheap win for public clients, harmless for
|
||||
# confidential ones.
|
||||
"code_challenge_method": "S256",
|
||||
},
|
||||
)
|
||||
return registry
|
||||
|
||||
|
||||
oauth: OAuth = _build_oauth()
|
||||
|
||||
|
||||
def get_client() -> Optional[object]:
|
||||
"""Return the registered provider client, or None when not configured."""
|
||||
if not is_configured():
|
||||
return None
|
||||
return oauth.create_client(PROVIDER_NAME)
|
||||
@@ -1,189 +0,0 @@
|
||||
"""
|
||||
Application configuration using Pydantic Settings
|
||||
"""
|
||||
from pydantic_settings import BaseSettings
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
|
||||
class ThumbnailSettings(BaseModel):
|
||||
"""Thumbnail generation settings"""
|
||||
small: int = 240
|
||||
medium: int = 640
|
||||
large: int = 1280
|
||||
quality: int = 85
|
||||
format: str = "webp"
|
||||
|
||||
class ScannerSettings(BaseModel):
|
||||
"""File scanner settings"""
|
||||
watch: bool = True
|
||||
initial_scan_on_start: bool = True
|
||||
batch_size: int = 100
|
||||
concurrent_workers: int = 4
|
||||
|
||||
class PerformanceSettings(BaseModel):
|
||||
"""Performance tuning settings"""
|
||||
max_concurrent_thumbnails: int = 10
|
||||
cache_ttl: int = 3600
|
||||
db_pool_size: int = 10
|
||||
db_pool_max_overflow: int = 10
|
||||
db_pool_recycle: int = 3600
|
||||
|
||||
class MulitaConfig(BaseModel):
|
||||
"""Main configuration from YAML file. Source roots and the discard
|
||||
workflow are owned by the database now — only operational settings
|
||||
live here."""
|
||||
thumbnails: ThumbnailSettings = ThumbnailSettings()
|
||||
scanner: ScannerSettings = ScannerSettings()
|
||||
performance: PerformanceSettings = PerformanceSettings()
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings"""
|
||||
# Database — plain Postgres. The SQLite escape hatch remains
|
||||
# supported via the docker-compose.sqlite.yml override and by setting
|
||||
# DATABASE_URL=sqlite+aiosqlite:///... in .env for local dev.
|
||||
database_url: str = Field(
|
||||
default="postgresql+asyncpg://mulita:mulita@db:5432/mulita",
|
||||
env="DATABASE_URL"
|
||||
)
|
||||
|
||||
# Redis
|
||||
redis_url: str = Field(
|
||||
default="redis://localhost:6379",
|
||||
env="REDIS_URL"
|
||||
)
|
||||
|
||||
# Celery
|
||||
celery_broker_url: str = Field(
|
||||
default="redis://localhost:6379",
|
||||
env="CELERY_BROKER_URL"
|
||||
)
|
||||
celery_result_backend: str = Field(
|
||||
default="redis://localhost:6379",
|
||||
env="CELERY_RESULT_BACKEND"
|
||||
)
|
||||
|
||||
# Photo directories
|
||||
photo_dirs: str = Field(
|
||||
default="/photos",
|
||||
env="PHOTO_DIRS"
|
||||
)
|
||||
|
||||
# API settings
|
||||
api_host: str = Field(default="0.0.0.0", env="API_HOST")
|
||||
api_port: int = Field(default=8000, env="API_PORT")
|
||||
|
||||
# CORS — comma-separated list of allowed origins, or "*" for any.
|
||||
# Same-origin requests (the normal case behind nginx / vite proxy)
|
||||
# never trip CORS, so this is only for direct browser access from
|
||||
# other origins (LAN IP, reverse proxy, dev tools).
|
||||
allowed_origins: str = Field(default="*", env="ALLOWED_ORIGINS")
|
||||
|
||||
# Logging — accepts standard python levels (DEBUG, INFO, WARNING,
|
||||
# ERROR, CRITICAL). Bumped from INFO when chasing a problem.
|
||||
log_level: str = Field(default="INFO", env="LOG_LEVEL")
|
||||
|
||||
# Auth — JWT signing key. Set SECRET_KEY in .env for production.
|
||||
# If unset, a deterministic fallback is used (acceptable for
|
||||
# single-machine homelab deploys, but set a real key if the instance
|
||||
# is network-exposed).
|
||||
secret_key: str = Field(
|
||||
default="mulita-dev-secret-change-me",
|
||||
env="SECRET_KEY",
|
||||
)
|
||||
access_token_expire_minutes: int = Field(default=525600, env="ACCESS_TOKEN_EXPIRE_MINUTES") # 1 year
|
||||
refresh_token_expire_days: int = Field(default=3650, env="REFRESH_TOKEN_EXPIRE_DAYS") # 10 years
|
||||
|
||||
# ── OIDC / Authentik single sign-on ────────────────────────────────
|
||||
# Disabled by default; enable by setting OIDC_ENABLED=true and the
|
||||
# issuer + client credentials. When enabled the login page shows a
|
||||
# "Sign in with {label}" button alongside the username/password form.
|
||||
oidc_enabled: bool = Field(default=False, env="OIDC_ENABLED")
|
||||
oidc_issuer: Optional[str] = Field(default=None, env="OIDC_ISSUER")
|
||||
oidc_client_id: Optional[str] = Field(default=None, env="OIDC_CLIENT_ID")
|
||||
oidc_client_secret: Optional[str] = Field(default=None, env="OIDC_CLIENT_SECRET")
|
||||
# Absolute URL the IdP redirects back to. Must match the Redirect URI
|
||||
# configured on the Authentik side exactly.
|
||||
oidc_redirect_uri: Optional[str] = Field(default=None, env="OIDC_REDIRECT_URI")
|
||||
oidc_scopes: str = Field(default="openid profile email", env="OIDC_SCOPES")
|
||||
oidc_provider_label: str = Field(default="Authentik", env="OIDC_PROVIDER_LABEL")
|
||||
# When true, a successful OIDC login for a subject we've never seen
|
||||
# auto-creates a local user + their /photos/{username} folder. When
|
||||
# false, unknown subjects get 403 and must be pre-provisioned.
|
||||
oidc_allow_signup: bool = Field(default=True, env="OIDC_ALLOW_SIGNUP")
|
||||
# Comma-separated Authentik group names. Any group-claim match
|
||||
# promotes the user to role=admin; otherwise role=user. Role is
|
||||
# refreshed on every sign-in so removals demote automatically.
|
||||
oidc_admin_groups: str = Field(default="", env="OIDC_ADMIN_GROUPS")
|
||||
# Last-resort link step: if (issuer, sub) AND email fallback both
|
||||
# miss, try matching the IdP's `preferred_username` claim against
|
||||
# `users.username`. Safe in single-tenant setups where the IdP is
|
||||
# the source of truth for usernames (homelab, family instance).
|
||||
# Leave off in multi-tenant — a name collision would hand someone
|
||||
# else's account to a new SSO user.
|
||||
oidc_link_by_username: bool = Field(default=False, env="OIDC_LINK_BY_USERNAME")
|
||||
# Starlette session cookie secret — only used to hold PKCE/state
|
||||
# during the brief OIDC round-trip. Falls back to secret_key when
|
||||
# unset.
|
||||
session_secret: Optional[str] = Field(default=None, env="SESSION_SECRET")
|
||||
|
||||
@property
|
||||
def oidc_admin_group_list(self) -> list[str]:
|
||||
raw = (self.oidc_admin_groups or "").strip()
|
||||
return [g.strip() for g in raw.split(",") if g.strip()]
|
||||
|
||||
@property
|
||||
def effective_session_secret(self) -> str:
|
||||
return self.session_secret or self.secret_key
|
||||
|
||||
@property
|
||||
def cors_origins(self) -> list[str]:
|
||||
"""Parse the ALLOWED_ORIGINS env var into a list. Accepts:
|
||||
- "*" → wildcard (single-element list ["*"])
|
||||
- "http://a.com,http://b.com" → split + strip
|
||||
Empty entries are dropped.
|
||||
"""
|
||||
raw = (self.allowed_origins or "").strip()
|
||||
if not raw or raw == "*":
|
||||
return ["*"]
|
||||
return [o.strip() for o in raw.split(",") if o.strip()]
|
||||
|
||||
# App configuration from YAML
|
||||
_config: Optional[MulitaConfig] = None
|
||||
|
||||
@property
|
||||
def config(self) -> MulitaConfig:
|
||||
"""Load configuration from YAML file"""
|
||||
if self._config is None:
|
||||
config_path = Path("/app/config/mulita.yml")
|
||||
if not config_path.exists():
|
||||
config_path = Path("mulita.yml")
|
||||
|
||||
if config_path.exists():
|
||||
with open(config_path, "r") as f:
|
||||
config_data = yaml.safe_load(f)
|
||||
self._config = MulitaConfig(**config_data)
|
||||
else:
|
||||
self._config = MulitaConfig()
|
||||
|
||||
return self._config
|
||||
|
||||
@property
|
||||
def thumbnails(self) -> ThumbnailSettings:
|
||||
return self.config.thumbnails
|
||||
|
||||
@property
|
||||
def scanner(self) -> ScannerSettings:
|
||||
return self.config.scanner
|
||||
|
||||
@property
|
||||
def performance(self) -> PerformanceSettings:
|
||||
return self.config.performance
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
case_sensitive = False
|
||||
|
||||
# Global settings instance
|
||||
settings = Settings()
|
||||
@@ -1,204 +0,0 @@
|
||||
"""
|
||||
Database configuration and session management.
|
||||
|
||||
Schema management strategy
|
||||
--------------------------
|
||||
Postgres (default): Alembic owns schema deltas. `alembic upgrade head` is
|
||||
run before the app starts (in the container CMD). `init_db()` calls
|
||||
`create_all` afterward as the source of truth for fresh installs — it is
|
||||
idempotent for existing tables and creates any tables defined on
|
||||
`Base.metadata` that don't yet exist. Future Alembic migrations should be
|
||||
written defensively (`IF NOT EXISTS` etc.) so they remain safe to run on a
|
||||
fresh DB where `create_all` has already laid down the same objects.
|
||||
|
||||
SQLite (escape hatch via docker-compose.sqlite.yml): no Alembic. The
|
||||
historical inline ALTER TABLE block stays in place so existing dev
|
||||
installs keep upgrading.
|
||||
"""
|
||||
import os
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy.pool import NullPool
|
||||
from sqlalchemy import text
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_is_sqlite = settings.database_url.startswith("sqlite")
|
||||
_is_postgres = settings.database_url.startswith("postgresql")
|
||||
|
||||
# When running inside a Celery worker we use NullPool rather than the
|
||||
# default connection pool. The reasons stack up:
|
||||
#
|
||||
# 1. Celery's prefork model forks the master *after* imports, so every
|
||||
# child inherits the same asyncpg Connection objects — they share
|
||||
# a socket, and two children using one concurrently raises
|
||||
# "another operation is in progress".
|
||||
#
|
||||
# 2. Task bodies run under `asyncio.run()`, which spins up a fresh
|
||||
# event loop per invocation. A pooled asyncpg Connection created
|
||||
# on loop A, returned to the pool, and checked out on loop B
|
||||
# raises "Future attached to a different loop".
|
||||
#
|
||||
# NullPool dodges both: every session checkout opens a brand-new
|
||||
# connection on the *current* loop and the connection is closed at
|
||||
# session end. Connection setup is cheap compared to task cost, so this
|
||||
# is the right default for the worker. The FastAPI backend keeps the
|
||||
# normal pool because it serves many short requests on a single long-
|
||||
# lived event loop, where pooling is a clear win.
|
||||
_is_celery_worker = os.environ.get("MULITA_CELERY_WORKER") == "1"
|
||||
|
||||
if _is_sqlite:
|
||||
db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", ""))
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False,
|
||||
connect_args={
|
||||
"check_same_thread": False,
|
||||
"timeout": 30,
|
||||
},
|
||||
)
|
||||
elif _is_celery_worker:
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False,
|
||||
poolclass=NullPool,
|
||||
)
|
||||
else:
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False,
|
||||
pool_size=settings.performance.db_pool_size,
|
||||
max_overflow=settings.performance.db_pool_max_overflow,
|
||||
pool_recycle=settings.performance.db_pool_recycle,
|
||||
pool_pre_ping=True,
|
||||
pool_timeout=10, # fail fast if pool exhausted (default 30)
|
||||
# Kill connections idle in a transaction for >60s. Prevents leaked
|
||||
# sessions from thumbnail requests that disconnect mid-flight.
|
||||
connect_args={"server_settings": {"idle_in_transaction_session_timeout": "60000"}},
|
||||
)
|
||||
|
||||
# Create async session factory
|
||||
AsyncSessionLocal = async_sessionmaker(
|
||||
engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False
|
||||
)
|
||||
|
||||
# Base class for models
|
||||
Base = declarative_base()
|
||||
|
||||
async def get_db() -> AsyncSession:
|
||||
"""Dependency to get database session.
|
||||
|
||||
Rolls back any uncommitted transaction before closing so a client
|
||||
disconnect doesn't leave idle-in-transaction connections in the pool.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
yield session
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
async def init_db():
|
||||
"""Initialize database, create tables if they don't exist"""
|
||||
async with engine.begin() as conn:
|
||||
# Import all models to register them with Base
|
||||
from app.models import User, Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto # noqa: F401
|
||||
|
||||
# Create all tables. Note: create_all only creates *missing* tables —
|
||||
# it does NOT add new columns to existing tables when the model gains
|
||||
# them. On Postgres, Alembic handles deltas; on SQLite, the inline
|
||||
# ALTER block below is the legacy fallback.
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
gps_columns_added = False
|
||||
|
||||
if _is_sqlite:
|
||||
# WAL mode for better concurrency.
|
||||
await conn.execute(text("PRAGMA journal_mode=WAL"))
|
||||
await conn.execute(text("PRAGMA synchronous=NORMAL"))
|
||||
await conn.execute(text("PRAGMA cache_size=10000"))
|
||||
await conn.execute(text("PRAGMA temp_store=MEMORY"))
|
||||
|
||||
# ── Idempotent column adds (SQLite only) ─────────────────────
|
||||
# SQLite supports ADD COLUMN but not "IF NOT EXISTS" for
|
||||
# columns, so introspect via PRAGMA first. Each entry is
|
||||
# (column_name, ALTER statement). Add new columns at the
|
||||
# bottom. On Postgres these live in Alembic migrations.
|
||||
existing_cols = {
|
||||
row[1]
|
||||
for row in (
|
||||
await conn.execute(text("PRAGMA table_info(photos)"))
|
||||
).fetchall()
|
||||
}
|
||||
pending_alters: list[tuple[str, str]] = [
|
||||
("phash", "ALTER TABLE photos ADD COLUMN phash VARCHAR(16)"),
|
||||
(
|
||||
"duplicate_group_id",
|
||||
"ALTER TABLE photos ADD COLUMN duplicate_group_id VARCHAR",
|
||||
),
|
||||
("latitude", "ALTER TABLE photos ADD COLUMN latitude REAL"),
|
||||
("longitude", "ALTER TABLE photos ADD COLUMN longitude REAL"),
|
||||
]
|
||||
for col_name, alter_sql in pending_alters:
|
||||
if col_name not in existing_cols:
|
||||
logger.info(f"Adding photos.{col_name} column")
|
||||
await conn.execute(text(alter_sql))
|
||||
if col_name in ("latitude", "longitude"):
|
||||
gps_columns_added = True
|
||||
await conn.execute(
|
||||
text("CREATE INDEX IF NOT EXISTS ix_photos_phash ON photos(phash)")
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_photos_duplicate_group_id "
|
||||
"ON photos(duplicate_group_id)"
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_photos_lat_lon "
|
||||
"ON photos(latitude, longitude)"
|
||||
)
|
||||
)
|
||||
|
||||
logger.info("Database initialized successfully")
|
||||
|
||||
# If we just introduced the GPS columns on an existing SQLite
|
||||
# install, kick off a one-shot backfill so the Map view is
|
||||
# populated without a manual full re-scan. Postgres installs are
|
||||
# always fresh (no SQLite→PG migration path), so this code path
|
||||
# is SQLite-only.
|
||||
if _is_sqlite and gps_columns_added:
|
||||
try:
|
||||
from app.tasks.scan import backfill_gps
|
||||
backfill_gps.delay()
|
||||
logger.info("Queued one-shot backfill_gps task after column add")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue backfill_gps task: {e}")
|
||||
|
||||
async def create_fts_table():
|
||||
"""Create Full-Text Search table for SQLite. On Postgres this is
|
||||
replaced by a tsvector column on the photos table (added in PR5)."""
|
||||
if _is_sqlite:
|
||||
async with engine.begin() as conn:
|
||||
# Create FTS5 virtual table for full-text search
|
||||
await conn.execute(text("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS photos_fts USING fts5(
|
||||
photo_id UNINDEXED,
|
||||
filename,
|
||||
user_title,
|
||||
user_notes,
|
||||
exif_text,
|
||||
tokenize='unicode61'
|
||||
)
|
||||
"""))
|
||||
logger.info("FTS5 table created successfully")
|
||||
@@ -1,355 +0,0 @@
|
||||
"""
|
||||
FastAPI dependencies for authentication and user-scoped data access.
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, HTTPException, Query, Request, status
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
from jose import JWTError
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.auth import decode_token
|
||||
from app.database import get_db
|
||||
from app.models.user import User
|
||||
from app.models.photos import Photo
|
||||
from app.models.folders import Folder, SourceRoot
|
||||
from app.models.heaps import Heap, heap_photos
|
||||
from app.models.tags import Tag
|
||||
from app.models.sharing import HeapShare, FolderShare
|
||||
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/login")
|
||||
|
||||
|
||||
async def get_current_user(
|
||||
token: str = Depends(oauth2_scheme),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User:
|
||||
"""Decode JWT, look up user, raise 401 if invalid or inactive."""
|
||||
credentials_exception = HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or expired token",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
try:
|
||||
payload = decode_token(token)
|
||||
user_id: str = payload.get("sub")
|
||||
token_type: str = payload.get("type")
|
||||
if user_id is None or token_type != "access":
|
||||
raise credentials_exception
|
||||
except JWTError:
|
||||
raise credentials_exception
|
||||
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None or not user.is_active:
|
||||
raise credentials_exception
|
||||
return user
|
||||
|
||||
|
||||
async def get_current_user_media(
|
||||
request: Request,
|
||||
token: Optional[str] = Query(None, alias="token"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User:
|
||||
"""Authenticate via Authorization header OR ?token= query parameter.
|
||||
|
||||
Used for media endpoints (thumbnails, originals, proxies) where the
|
||||
URL is set as an <img src> or <video src> and the browser can't
|
||||
attach an Authorization header. The frontend appends ?token=JWT to
|
||||
media URLs so they pass auth without custom fetch logic.
|
||||
"""
|
||||
# Try Authorization header first.
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
jwt_token = None
|
||||
if auth_header.startswith("Bearer "):
|
||||
jwt_token = auth_header[7:]
|
||||
elif token:
|
||||
jwt_token = token
|
||||
|
||||
if not jwt_token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Missing token",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
credentials_exception = HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or expired token",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
try:
|
||||
payload = decode_token(jwt_token)
|
||||
user_id: str = payload.get("sub")
|
||||
token_type: str = payload.get("type")
|
||||
if user_id is None or token_type != "access":
|
||||
raise credentials_exception
|
||||
except JWTError:
|
||||
raise credentials_exception
|
||||
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None or not user.is_active:
|
||||
raise credentials_exception
|
||||
return user
|
||||
|
||||
|
||||
async def require_admin(
|
||||
user: User = Depends(get_current_user),
|
||||
) -> User:
|
||||
"""Raise 403 if user is not an admin."""
|
||||
if user.role != "admin":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Admin privileges required",
|
||||
)
|
||||
return user
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User-scoped query helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def user_photos_query(user: User):
|
||||
"""Base select for photos owned by user, with tags eager-loaded."""
|
||||
return (
|
||||
select(Photo)
|
||||
.options(selectinload(Photo.tags))
|
||||
.where(Photo.user_id == user.id)
|
||||
)
|
||||
|
||||
|
||||
async def get_user_photo(
|
||||
photo_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> Photo:
|
||||
"""Fetch a single photo by ID, scoped to the user. Raises 404."""
|
||||
result = await db.execute(
|
||||
select(Photo)
|
||||
.options(selectinload(Photo.tags))
|
||||
.where(Photo.id == photo_id, Photo.user_id == user.id)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
if photo is None:
|
||||
raise HTTPException(status_code=404, detail="Photo not found")
|
||||
return photo
|
||||
|
||||
|
||||
async def get_user_folder(
|
||||
folder_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> Folder:
|
||||
"""Fetch a single folder by ID, scoped to the user. Raises 404."""
|
||||
result = await db.execute(
|
||||
select(Folder).where(Folder.id == folder_id, Folder.user_id == user.id)
|
||||
)
|
||||
folder = result.scalar_one_or_none()
|
||||
if folder is None:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
return folder
|
||||
|
||||
|
||||
async def get_user_heap(
|
||||
heap_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> Heap:
|
||||
"""Fetch a single heap by ID, scoped to the user. Raises 404."""
|
||||
result = await db.execute(
|
||||
select(Heap).where(Heap.id == heap_id, Heap.user_id == user.id)
|
||||
)
|
||||
heap = result.scalar_one_or_none()
|
||||
if heap is None:
|
||||
raise HTTPException(status_code=404, detail="Heap not found")
|
||||
return heap
|
||||
|
||||
|
||||
async def get_user_tag(
|
||||
tag_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> Tag:
|
||||
"""Fetch a single tag by ID, scoped to the user. Raises 404."""
|
||||
result = await db.execute(
|
||||
select(Tag).where(Tag.id == tag_id, Tag.user_id == user.id)
|
||||
)
|
||||
tag = result.scalar_one_or_none()
|
||||
if tag is None:
|
||||
raise HTTPException(status_code=404, detail="Tag not found")
|
||||
return tag
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sharing helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def get_user_or_shared_heap(
|
||||
heap_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> tuple:
|
||||
"""Fetch a heap the user owns OR has a share for.
|
||||
|
||||
Returns ``(heap, permission)`` where *permission* is
|
||||
``'owner'``, ``'read'``, or ``'write'``. Raises 404 if no access.
|
||||
"""
|
||||
# Fast path: owned by current user.
|
||||
result = await db.execute(
|
||||
select(Heap).where(Heap.id == heap_id, Heap.user_id == user.id)
|
||||
)
|
||||
heap = result.scalar_one_or_none()
|
||||
if heap:
|
||||
return heap, "owner"
|
||||
|
||||
# Shared path.
|
||||
result = await db.execute(
|
||||
select(HeapShare).where(
|
||||
HeapShare.heap_id == heap_id,
|
||||
HeapShare.shared_with_id == user.id,
|
||||
)
|
||||
)
|
||||
share = result.scalar_one_or_none()
|
||||
if share:
|
||||
result = await db.execute(select(Heap).where(Heap.id == heap_id))
|
||||
heap = result.scalar_one_or_none()
|
||||
if heap:
|
||||
return heap, share.permission
|
||||
|
||||
raise HTTPException(status_code=404, detail="Heap not found")
|
||||
|
||||
|
||||
async def get_user_or_shared_folder(
|
||||
folder_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> tuple:
|
||||
"""Fetch a folder (or source root) the user owns OR has a share for.
|
||||
|
||||
Returns ``(entity, permission)`` where *entity* is a Folder or
|
||||
SourceRoot and *permission* is ``'owner'``, ``'read'``, or ``'write'``.
|
||||
"""
|
||||
# Try owned folder first.
|
||||
result = await db.execute(
|
||||
select(Folder).where(Folder.id == folder_id, Folder.user_id == user.id)
|
||||
)
|
||||
folder = result.scalar_one_or_none()
|
||||
if folder:
|
||||
return folder, "owner"
|
||||
|
||||
# Try owned source root.
|
||||
result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == user.id)
|
||||
)
|
||||
sr = result.scalar_one_or_none()
|
||||
if sr:
|
||||
return sr, "owner"
|
||||
|
||||
# Shared path.
|
||||
result = await db.execute(
|
||||
select(FolderShare).where(
|
||||
FolderShare.folder_id == folder_id,
|
||||
FolderShare.shared_with_id == user.id,
|
||||
)
|
||||
)
|
||||
share = result.scalar_one_or_none()
|
||||
if share:
|
||||
if share.folder_type == "source_root":
|
||||
result = await db.execute(select(SourceRoot).where(SourceRoot.id == folder_id))
|
||||
else:
|
||||
result = await db.execute(select(Folder).where(Folder.id == folder_id))
|
||||
entity = result.scalar_one_or_none()
|
||||
if entity:
|
||||
return entity, share.permission
|
||||
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
|
||||
|
||||
async def resolve_username(
|
||||
username: str,
|
||||
db: AsyncSession,
|
||||
) -> User:
|
||||
"""Look up an active user by username. Raises 404 if not found."""
|
||||
result = await db.execute(
|
||||
select(User).where(User.username == username, User.is_active.is_(True))
|
||||
)
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
return user
|
||||
|
||||
|
||||
async def can_access_photo_via_share(
|
||||
photo_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> bool:
|
||||
"""Check whether *user* can access *photo_id* through any share.
|
||||
|
||||
Returns True if the photo belongs to a heap or folder that has been
|
||||
shared with the user. Used as a fallback in media-serving endpoints
|
||||
after the direct ownership check fails.
|
||||
"""
|
||||
import os
|
||||
|
||||
# Check heap shares: photo in any heap shared with user?
|
||||
result = await db.execute(
|
||||
select(heap_photos.c.photo_id).where(
|
||||
heap_photos.c.photo_id == photo_id,
|
||||
heap_photos.c.heap_id.in_(
|
||||
select(HeapShare.heap_id).where(HeapShare.shared_with_id == user.id)
|
||||
),
|
||||
).limit(1)
|
||||
)
|
||||
if result.scalar_one_or_none() is not None:
|
||||
return True
|
||||
|
||||
# Check folder shares: photo in any folder (or descendant) shared with user?
|
||||
result = await db.execute(
|
||||
select(Photo.folder_id).where(Photo.id == photo_id)
|
||||
)
|
||||
photo_folder_id = result.scalar_one_or_none()
|
||||
if photo_folder_id is None:
|
||||
return False
|
||||
|
||||
# Get the photo's folder path for prefix matching.
|
||||
result = await db.execute(
|
||||
select(Folder.path, Folder.source_root_id).where(Folder.id == photo_folder_id)
|
||||
)
|
||||
row = result.one_or_none()
|
||||
if row is None:
|
||||
return False
|
||||
photo_path, photo_sr_id = row
|
||||
|
||||
# Check source root shares — photo's source root matches a shared root?
|
||||
result = await db.execute(
|
||||
select(FolderShare.folder_id).where(
|
||||
FolderShare.shared_with_id == user.id,
|
||||
FolderShare.folder_type == "source_root",
|
||||
FolderShare.folder_id == photo_sr_id,
|
||||
).limit(1)
|
||||
)
|
||||
if result.scalar_one_or_none() is not None:
|
||||
return True
|
||||
|
||||
# Check folder shares — photo's folder is at or below a shared folder?
|
||||
# Single query: join folder_shares → folders to get shared paths, then
|
||||
# check if the photo's path starts with any of them.
|
||||
result = await db.execute(
|
||||
select(Folder.path).where(
|
||||
Folder.id.in_(
|
||||
select(FolderShare.folder_id).where(
|
||||
FolderShare.shared_with_id == user.id,
|
||||
FolderShare.folder_type == "folder",
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
for (shared_path,) in result.all():
|
||||
if photo_path == shared_path or photo_path.startswith(shared_path + os.sep):
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -1,135 +0,0 @@
|
||||
"""
|
||||
Mulita - Photo Management Application
|
||||
Main FastAPI application entry point
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
import logging
|
||||
import os
|
||||
|
||||
from app.config import settings
|
||||
from app.database import init_db
|
||||
from app.routers import photos, folders, heaps, tags, discard, library, search, auth, admin, sharing, download, nextcloud, nc_webhook
|
||||
from app.services.scanner import start_initial_scan, bootstrap_default_source_root
|
||||
from app.services.cleanup import cleanup_data_integrity
|
||||
from app.services.nextcloud_dav import init_preview_client, close_preview_client
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Manage application lifecycle"""
|
||||
logger.info("Starting Mulita application...")
|
||||
|
||||
# Initialize database
|
||||
await init_db()
|
||||
|
||||
# Pooled httpx client to Nextcloud — keepalive + HTTP/2 means every
|
||||
# thumbnail / Memories-info call after the first reuses one socket
|
||||
# instead of paying TCP+TLS handshake per request.
|
||||
await init_preview_client()
|
||||
|
||||
# First-boot convenience: if there are no source roots in the DB yet,
|
||||
# create one for the default /photos mount so the user sees their
|
||||
# library immediately without configuring anything in the UI.
|
||||
try:
|
||||
await bootstrap_default_source_root()
|
||||
except Exception as e:
|
||||
logger.error(f"Bootstrap source root failed (continuing): {e}")
|
||||
|
||||
# One-shot cleanup of duplicate source_roots / folders left over from
|
||||
# earlier scanner versions that didn't normalize paths. Idempotent.
|
||||
try:
|
||||
await cleanup_data_integrity()
|
||||
except Exception as e:
|
||||
logger.error(f"Startup cleanup failed (continuing): {e}")
|
||||
|
||||
# Start initial scan if configured
|
||||
if settings.scanner.initial_scan_on_start:
|
||||
logger.info("Starting initial library scan...")
|
||||
await start_initial_scan()
|
||||
|
||||
yield
|
||||
|
||||
logger.info("Shutting down Mulita application...")
|
||||
await close_preview_client()
|
||||
|
||||
# Create FastAPI app
|
||||
app = FastAPI(
|
||||
title="Mulita Photo Management API",
|
||||
description="Self-hosted photo management application inspired by Lightroom",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan
|
||||
)
|
||||
|
||||
# Configure CORS. The frontend normally talks to the backend through the
|
||||
# nginx (prod) or vite (dev) proxy, so requests are same-origin and never
|
||||
# trip CORS. ALLOWED_ORIGINS in .env controls the fallback for direct
|
||||
# browser access from other origins (LAN IP, reverse proxy under a
|
||||
# different host). Defaults to "*" since this is a single-user homelab
|
||||
# tool; lock it down by setting e.g. ALLOWED_ORIGINS=https://photos.your.tld
|
||||
# in production deployments.
|
||||
_origins = settings.cors_origins
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=_origins,
|
||||
# Wildcard origins can't be combined with credentials per the CORS
|
||||
# spec, so credentials get auto-disabled in that case.
|
||||
allow_credentials=_origins != ["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Session middleware — only used by Authlib to hold PKCE state during
|
||||
# the OIDC round-trip. max_age is short because the cookie is only
|
||||
# meaningful between /auth/oidc/login and /auth/oidc/callback; the app
|
||||
# itself still runs on JWTs.
|
||||
app.add_middleware(
|
||||
SessionMiddleware,
|
||||
secret_key=settings.effective_session_secret,
|
||||
session_cookie="mulita_oidc",
|
||||
max_age=600,
|
||||
same_site="lax",
|
||||
https_only=False,
|
||||
)
|
||||
|
||||
# Mount static files for serving thumbnails (with X-Accel-Redirect support)
|
||||
if os.path.exists("/data/thumbs"):
|
||||
app.mount("/thumbs", StaticFiles(directory="/data/thumbs"), name="thumbs")
|
||||
|
||||
# Include routers
|
||||
app.include_router(auth.router, prefix="/api/v1/auth", tags=["auth"])
|
||||
app.include_router(admin.router, prefix="/api/v1/admin", tags=["admin"])
|
||||
app.include_router(sharing.router, prefix="/api/v1", tags=["sharing"])
|
||||
app.include_router(photos.router, prefix="/api/v1/photos", tags=["photos"])
|
||||
app.include_router(folders.router, prefix="/api/v1/folders", tags=["folders"])
|
||||
app.include_router(heaps.router, prefix="/api/v1/heaps", tags=["heaps"])
|
||||
app.include_router(tags.router, prefix="/api/v1/tags", tags=["tags"])
|
||||
app.include_router(discard.router, prefix="/api/v1/discard", tags=["discard"])
|
||||
app.include_router(library.router, prefix="/api/v1/library", tags=["library"])
|
||||
app.include_router(search.router, prefix="/api/v1/photos/search", tags=["search"])
|
||||
app.include_router(download.router, prefix="/api/v1/download", tags=["download"])
|
||||
app.include_router(nextcloud.router, prefix="/api/v1/nextcloud", tags=["nextcloud"])
|
||||
app.include_router(nc_webhook.router, prefix="/api/v1/internal", tags=["nc-webhook"])
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
"""Root endpoint"""
|
||||
return {
|
||||
"name": "Mulita Photo Management API",
|
||||
"version": "1.0.0",
|
||||
"status": "running"
|
||||
}
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
"""Health check endpoint for Docker"""
|
||||
return {"status": "healthy"}
|
||||
@@ -1,22 +0,0 @@
|
||||
"""
|
||||
Database models for Mulita
|
||||
"""
|
||||
from app.models.user import User
|
||||
from app.models.photos import Photo
|
||||
from app.models.folders import Folder, SourceRoot
|
||||
from app.models.tags import Tag, PhotoTag
|
||||
from app.models.heaps import Heap, HeapPhoto
|
||||
from app.models.sharing import HeapShare, FolderShare
|
||||
|
||||
__all__ = [
|
||||
'User',
|
||||
'Photo',
|
||||
'Folder',
|
||||
'SourceRoot',
|
||||
'Tag',
|
||||
'PhotoTag',
|
||||
'Heap',
|
||||
'HeapPhoto',
|
||||
'HeapShare',
|
||||
'FolderShare',
|
||||
]
|
||||
@@ -1,58 +0,0 @@
|
||||
"""
|
||||
Folder and SourceRoot model definitions
|
||||
"""
|
||||
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey, Index
|
||||
from sqlalchemy.sql import func
|
||||
from sqlalchemy.orm import relationship
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
class SourceRoot(Base):
|
||||
__tablename__ = 'source_roots'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False)
|
||||
path = Column(String, unique=True, nullable=False)
|
||||
is_active = Column(Boolean, default=True)
|
||||
added_at = Column(DateTime, server_default=func.now())
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
|
||||
# Relationships
|
||||
folders = relationship("Folder", back_populates="source_root")
|
||||
|
||||
class Folder(Base):
|
||||
__tablename__ = 'folders'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False)
|
||||
path = Column(String, unique=True, nullable=False)
|
||||
parent_id = Column(String, ForeignKey('folders.id'))
|
||||
source_root_id = Column(String, ForeignKey('source_roots.id'))
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
photo_count = Column(Integer, default=0)
|
||||
last_scanned = Column(DateTime)
|
||||
|
||||
# "Hide from views" — when true, photos in this folder (and every
|
||||
# descendant folder) are excluded from cross-cutting views like
|
||||
# All Photos, Map, Tags, People, Search and the sidebar counts.
|
||||
# Photos are still scanned, thumbnailed and indexed — they just
|
||||
# stop showing up unless the user navigates directly to a folder
|
||||
# inside the hidden subtree. The effective flag is materialized
|
||||
# onto Photo.is_hidden so queries don't have to walk parent_id.
|
||||
is_hidden = Column(Boolean, nullable=False, default=False, server_default='false')
|
||||
|
||||
# Relationships
|
||||
source_root = relationship("SourceRoot", back_populates="folders")
|
||||
photos = relationship("Photo", backref="folder")
|
||||
|
||||
# Indexes
|
||||
__table_args__ = (
|
||||
Index('ix_folders_path', 'path'),
|
||||
Index('ix_folders_parent_id', 'parent_id'),
|
||||
Index('ix_folders_source_root_id', 'source_root_id'),
|
||||
)
|
||||
@@ -1,40 +0,0 @@
|
||||
"""
|
||||
Heap model definitions
|
||||
"""
|
||||
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey, Table, Index
|
||||
from sqlalchemy.sql import func
|
||||
from sqlalchemy.orm import relationship
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
# Association table for many-to-many relationship with additional fields
|
||||
heap_photos = Table(
|
||||
'heap_photos',
|
||||
Base.metadata,
|
||||
Column('heap_id', String, ForeignKey('heaps.id', ondelete='CASCADE'), primary_key=True),
|
||||
Column('photo_id', String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True),
|
||||
Column('added_at', DateTime, server_default=func.now()),
|
||||
Column('sort_order', Integer, default=0),
|
||||
Index('ix_heap_photos_heap_id', 'heap_id'),
|
||||
Index('ix_heap_photos_photo_id', 'photo_id'),
|
||||
)
|
||||
|
||||
class Heap(Base):
|
||||
__tablename__ = 'heaps'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False)
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
updated_at = Column(DateTime, onupdate=func.now())
|
||||
is_active = Column(Boolean, default=False) # For active heap feature
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
|
||||
# Relationships
|
||||
photos = relationship("Photo", secondary=heap_photos, backref="heaps")
|
||||
|
||||
class HeapPhoto:
|
||||
"""Helper class for heap-photo associations (not a table model)"""
|
||||
pass
|
||||
@@ -1,136 +0,0 @@
|
||||
"""
|
||||
Photo model definition
|
||||
"""
|
||||
from sqlalchemy import Column, String, Integer, Float, Boolean, DateTime, ForeignKey, Text, Index
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
class Photo(Base):
|
||||
__tablename__ = 'photos'
|
||||
|
||||
# Primary key
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
# Eager-loadable owner relationship. Used by the thumbnail handler so
|
||||
# one photo lookup also yields the NC creds we need to call the
|
||||
# preview endpoint, instead of issuing a second SELECT users WHERE
|
||||
# id=…. No FK change — user_id above already exists.
|
||||
user = relationship("User", lazy="select")
|
||||
|
||||
# File information
|
||||
filepath = Column(String, unique=True, nullable=False)
|
||||
filename = Column(String, nullable=False)
|
||||
folder_id = Column(String, ForeignKey('folders.id'))
|
||||
file_hash = Column(String, index=True) # SHA-256 hash for duplicate detection
|
||||
|
||||
# Nextcloud fileid for the same file. Set by the scanner when the file
|
||||
# lives under a Nextcloud-rooted SourceRoot. Used by the thumbnail
|
||||
# endpoint to proxy /index.php/core/preview instead of generating
|
||||
# and serving thumbs locally — Nextcloud already maintains previews
|
||||
# for the same source file, and duplicating that work was the bulk
|
||||
# of `/data/thumbs/*`. NULL on legacy / non-NC paths; the handler
|
||||
# falls back to on-disk thumbs when this is unset.
|
||||
nextcloud_fileid = Column(Integer, nullable=True, index=True)
|
||||
|
||||
# Media information
|
||||
media_type = Column(String, nullable=False) # 'photo' | 'video' | 'raw' | 'heic'
|
||||
original_format = Column(String) # 'CR3', 'NEF', 'HEIC', 'MP4', 'JPEG', etc.
|
||||
width = Column(Integer)
|
||||
height = Column(Integer)
|
||||
file_size = Column(Integer)
|
||||
|
||||
# Timestamps
|
||||
taken_at = Column(DateTime) # from EXIF DateTimeOriginal, fallback to file mtime
|
||||
taken_at_source = Column(String) # 'exif' | 'filesystem' | 'manual'
|
||||
added_at = Column(DateTime, server_default=func.now())
|
||||
updated_at = Column(DateTime, onupdate=func.now())
|
||||
|
||||
# Discard status. The DB column names stay is_trashed/trashed_at to avoid
|
||||
# a migration; only the Python attribute name reflects the rename.
|
||||
is_discarded = Column('is_trashed', Boolean, default=False)
|
||||
discarded_at = Column('trashed_at', DateTime)
|
||||
|
||||
# "Hidden from views" — materialized from Folder.is_hidden walking
|
||||
# the ancestry chain. True iff any ancestor folder (including the
|
||||
# photo's direct folder) is hidden. Cross-cutting queries filter
|
||||
# `AND NOT is_hidden`; per-folder browses ignore the flag so the
|
||||
# user can still open a hidden folder and see its contents. The
|
||||
# column is maintained by two places: the scanner sets it on new
|
||||
# rows, and POST /folders/{id}/hide recomputes it on toggle.
|
||||
is_hidden = Column(Boolean, nullable=False, default=False, server_default='false', index=True)
|
||||
|
||||
# "Capture date is probably wrong" — denormalized from the folder/filename
|
||||
# date-guesser. Set at scan time and recomputed on every taken_at edit so
|
||||
# the filter bar can query it directly. See services/date_guess.py for
|
||||
# the heuristic; kept as a stored column because recomputing on every
|
||||
# list query would mean running the regex stack across thousands of rows.
|
||||
has_date_warning = Column(Boolean, nullable=False, default=False, server_default='false', index=True)
|
||||
|
||||
# Thumbnail paths
|
||||
thumb_small = Column(String) # path to 240px thumb
|
||||
thumb_medium = Column(String) # path to 640px thumb
|
||||
thumb_large = Column(String) # path to 1280px thumb
|
||||
|
||||
# Processing status
|
||||
processing_status = Column(String, default='pending') # 'pending' | 'processing' | 'completed' | 'failed'
|
||||
processing_error = Column(Text)
|
||||
|
||||
# Metadata
|
||||
exif_json = Column(Text) # full EXIF/XMP blob as JSON
|
||||
|
||||
# GPS coordinates extracted from EXIF, in signed decimal degrees
|
||||
# (S latitude / W longitude are negative). Stored as first-class columns
|
||||
# so the Map view and any future location filters can query/index them
|
||||
# without parsing exif_json on every request.
|
||||
latitude = Column(Float)
|
||||
longitude = Column(Float)
|
||||
|
||||
# User-editable fields
|
||||
user_title = Column(String)
|
||||
user_notes = Column(Text)
|
||||
rating = Column(Integer, default=0) # 0-5 stars
|
||||
color_label = Column(String) # 'red'|'orange'|'yellow'|'green'|'blue'|'purple'|NULL
|
||||
# Note: is_rejected was merged into is_discarded (a single soft "discarded"
|
||||
# concept). is_picked was unified with active-heap membership — picking a
|
||||
# photo just means adding it to the active heap. Both DB columns may still
|
||||
# exist on legacy installs but are no longer read or written.
|
||||
|
||||
# Duplicate detection.
|
||||
#
|
||||
# - file_hash (above): SHA-256 of the raw bytes. Catches byte-identical
|
||||
# copies but not visually-identical re-encodes / resizes / screenshots.
|
||||
# - phash: 16-char hex of a 64-bit perceptual hash, computed by the
|
||||
# thumbs worker from the decoded original frame. Robust to resize and
|
||||
# re-compression — this is what actually identifies "the same photo
|
||||
# saved twice with different JPEG quality".
|
||||
# - duplicate_group_id: shared by every photo in the same duplicate
|
||||
# cluster. Maintained by app.services.duplicates.regroup_duplicates,
|
||||
# not on individual writes — recomputed in batches after scans / on
|
||||
# demand from the Settings panel.
|
||||
# - is_duplicate: derived boolean (group_id IS NOT NULL). Kept as a real
|
||||
# column so the existing PhotoThumbnail badge and /library/stats
|
||||
# duplicates count don't have to change.
|
||||
is_duplicate = Column(Boolean, default=False)
|
||||
phash = Column(String(16), index=True)
|
||||
duplicate_group_id = Column(String, index=True)
|
||||
|
||||
# Live photo support
|
||||
live_photo_video_id = Column(String, ForeignKey('photos.id'))
|
||||
|
||||
# Indexes for performance
|
||||
__table_args__ = (
|
||||
Index('ix_photos_taken_at', 'taken_at'),
|
||||
Index('ix_photos_folder_id', 'folder_id'),
|
||||
Index('ix_photos_is_trashed', 'is_trashed'),
|
||||
Index('ix_photos_rating', 'rating'),
|
||||
Index('ix_photos_color_label', 'color_label'),
|
||||
Index('ix_photos_media_type', 'media_type'),
|
||||
Index('ix_photos_processing_status', 'processing_status'),
|
||||
Index('ix_photos_lat_lon', 'latitude', 'longitude'),
|
||||
)
|
||||
@@ -1,62 +0,0 @@
|
||||
"""
|
||||
Sharing models — cross-user access to heaps and folders.
|
||||
|
||||
HeapShare grants another user read or read+write access to a heap.
|
||||
FolderShare does the same for a folder (or source root).
|
||||
"""
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import (
|
||||
Column, DateTime, ForeignKey, Index, String, UniqueConstraint, func,
|
||||
)
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class HeapShare(Base):
|
||||
__tablename__ = "heap_shares"
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
heap_id = Column(
|
||||
String, ForeignKey("heaps.id", ondelete="CASCADE"), nullable=False,
|
||||
)
|
||||
# Denormalized from heap.user_id for fast "shares I own" lookups.
|
||||
owner_id = Column(String, ForeignKey("users.id"), nullable=False)
|
||||
shared_with_id = Column(String, ForeignKey("users.id"), nullable=False)
|
||||
permission = Column(String, nullable=False, default="read") # 'read' | 'write'
|
||||
# Lifecycle: 'pending' while the recipient hasn't acted, 'accepted'
|
||||
# once they've Accept'd in the notification bell. Decline deletes the
|
||||
# row outright — see migration 0014 for the backfill of pre-existing
|
||||
# rows to 'accepted' so nothing vanishes from existing sidebars.
|
||||
status = Column(String, nullable=False, default="pending")
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
accepted_at = Column(DateTime, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("heap_id", "shared_with_id", name="uq_heap_share"),
|
||||
Index("ix_heap_shares_shared_with", "shared_with_id"),
|
||||
Index("ix_heap_shares_heap_id", "heap_id"),
|
||||
Index("ix_heap_shares_shared_with_status", "shared_with_id", "status"),
|
||||
)
|
||||
|
||||
|
||||
class FolderShare(Base):
|
||||
__tablename__ = "folder_shares"
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
# Can reference either a Folder.id or a SourceRoot.id.
|
||||
folder_id = Column(String, nullable=False)
|
||||
folder_type = Column(String, nullable=False, default="folder") # 'folder' | 'source_root'
|
||||
owner_id = Column(String, ForeignKey("users.id"), nullable=False)
|
||||
shared_with_id = Column(String, ForeignKey("users.id"), nullable=False)
|
||||
permission = Column(String, nullable=False, default="read") # 'read' | 'write'
|
||||
status = Column(String, nullable=False, default="pending")
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
accepted_at = Column(DateTime, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("folder_id", "shared_with_id", name="uq_folder_share"),
|
||||
Index("ix_folder_shares_shared_with", "shared_with_id"),
|
||||
Index("ix_folder_shares_folder_id", "folder_id"),
|
||||
Index("ix_folder_shares_shared_with_status", "shared_with_id", "status"),
|
||||
)
|
||||
@@ -1,56 +0,0 @@
|
||||
"""
|
||||
Tag model definitions.
|
||||
|
||||
Tags are unified across user-created tags, ML-detected objects, scene
|
||||
labels, and face clusters via the `kind` column. The `photo_tags`
|
||||
association carries per-photo ML metadata (confidence, bounding box,
|
||||
source model).
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, Table, Index, UniqueConstraint # noqa: F401
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
# Association table for many-to-many relationship
|
||||
photo_tags = Table(
|
||||
'photo_tags',
|
||||
Base.metadata,
|
||||
Column('photo_id', String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True),
|
||||
Column('tag_id', String, ForeignKey('tags.id', ondelete='CASCADE'), primary_key=True),
|
||||
# ML metadata — null for user-applied tags
|
||||
Column('confidence', Float, nullable=True),
|
||||
Column('bbox', JSONB, nullable=True), # [x1, y1, x2, y2] normalized 0-1
|
||||
Column('source', String, nullable=True), # null for user-applied tags
|
||||
Index('ix_photo_tags_photo_id', 'photo_id'),
|
||||
Index('ix_photo_tags_tag_id', 'tag_id'),
|
||||
)
|
||||
|
||||
class Tag(Base):
|
||||
__tablename__ = 'tags'
|
||||
__table_args__ = (
|
||||
UniqueConstraint('name', 'kind', 'user_id', name='uq_tags_name_kind_user'),
|
||||
)
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False, index=True)
|
||||
color = Column(String) # Hex color code for UI display
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
|
||||
# Tag classification
|
||||
kind = Column(String, nullable=False, default='user', index=True)
|
||||
# kind values: 'user' | 'content_type'
|
||||
|
||||
# Which producer wrote this tag (null for user-created)
|
||||
source = Column(String, nullable=True)
|
||||
|
||||
# Relationships
|
||||
photos = relationship("Photo", secondary=photo_tags, backref="tags")
|
||||
|
||||
|
||||
class PhotoTag:
|
||||
"""Helper class for photo-tag associations (not a table model)"""
|
||||
pass
|
||||
@@ -1,49 +0,0 @@
|
||||
"""
|
||||
User model definition
|
||||
"""
|
||||
from sqlalchemy import Column, String, Boolean, DateTime
|
||||
from sqlalchemy.sql import func
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = 'users'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
username = Column(String(50), unique=True, nullable=False, index=True)
|
||||
email = Column(String, unique=True, nullable=True)
|
||||
# Nullable: OIDC-only users have no local password. Local accounts
|
||||
# still always have one.
|
||||
hashed_password = Column(String, nullable=True)
|
||||
role = Column(String, nullable=False, default='user') # 'admin' | 'user'
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
|
||||
# Absolute path to this user's photo directory (e.g., "/photos/daniel")
|
||||
media_path = Column(String, nullable=False)
|
||||
|
||||
# OIDC identity — populated when a user signs in via Authentik (or any
|
||||
# other OIDC provider later). `oidc_sub` is stable per provider, so
|
||||
# lookups key on (oidc_issuer, oidc_sub). NULL for password-only users.
|
||||
oidc_issuer = Column(String, nullable=True)
|
||||
oidc_sub = Column(String, nullable=True)
|
||||
|
||||
# Profile bits that can come from OIDC claims or be filled in later.
|
||||
# avatar_url wins over Gravatar when set; the /auth/me response
|
||||
# computes the final avatar URL for the frontend.
|
||||
avatar_url = Column(String, nullable=True)
|
||||
display_name = Column(String, nullable=True)
|
||||
|
||||
# Nextcloud integration. `nextcloud_username` defaults to the
|
||||
# `preferred_username` OIDC claim on first login but can be overridden
|
||||
# in Settings (the local mule-image username doesn't always match the
|
||||
# Nextcloud user — e.g. authentik `dtoro` ↔ Nextcloud `admin`).
|
||||
# `nextcloud_app_password_enc` is the user's Nextcloud app password
|
||||
# (created from Nextcloud → Settings → Security), Fernet-encrypted at
|
||||
# rest with a key derived from settings.secret_key. Used as HTTP Basic
|
||||
# auth on outgoing WebDAV calls when the user mutates a file under
|
||||
# their Nextcloud-rooted SourceRoot.
|
||||
nextcloud_username = Column(String, nullable=True, index=True)
|
||||
nextcloud_app_password_enc = Column(String, nullable=True)
|
||||
@@ -1,260 +0,0 @@
|
||||
"""
|
||||
Admin router — user management and app configuration.
|
||||
All endpoints require admin role.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from typing import Optional, List
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func as sa_func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth import hash_password
|
||||
from app.database import get_db
|
||||
from app.dependencies import require_admin
|
||||
from app.models.user import User
|
||||
from app.models.photos import Photo
|
||||
from app.models.folders import SourceRoot
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class CreateUserRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
role: str = "user" # 'admin' | 'user'
|
||||
|
||||
class UpdateUserRequest(BaseModel):
|
||||
role: Optional[str] = None
|
||||
is_active: Optional[bool] = None
|
||||
new_password: Optional[str] = None
|
||||
|
||||
class UserDetailResponse(BaseModel):
|
||||
id: str
|
||||
username: str
|
||||
email: Optional[str]
|
||||
role: str
|
||||
is_active: bool
|
||||
media_path: str
|
||||
created_at: Optional[str]
|
||||
photo_count: int = 0
|
||||
|
||||
class UserListResponse(BaseModel):
|
||||
users: List[UserDetailResponse]
|
||||
total: int
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/users", response_model=UserListResponse)
|
||||
async def list_users(
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all users with their photo counts."""
|
||||
result = await db.execute(select(User).order_by(User.created_at))
|
||||
users = result.scalars().all()
|
||||
|
||||
user_list = []
|
||||
for u in users:
|
||||
count_result = await db.execute(
|
||||
select(sa_func.count(Photo.id)).where(Photo.user_id == u.id)
|
||||
)
|
||||
photo_count = count_result.scalar() or 0
|
||||
|
||||
user_list.append(UserDetailResponse(
|
||||
id=u.id,
|
||||
username=u.username,
|
||||
email=u.email,
|
||||
role=u.role,
|
||||
is_active=u.is_active,
|
||||
media_path=u.media_path,
|
||||
created_at=u.created_at.isoformat() if u.created_at else None,
|
||||
photo_count=photo_count,
|
||||
))
|
||||
|
||||
return UserListResponse(users=user_list, total=len(user_list))
|
||||
|
||||
|
||||
@router.post("/users", status_code=201, response_model=UserDetailResponse)
|
||||
async def create_user(
|
||||
body: CreateUserRequest,
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Create a new user. Creates their media directory and source root."""
|
||||
if body.role not in ("admin", "user"):
|
||||
raise HTTPException(status_code=400, detail="Role must be 'admin' or 'user'")
|
||||
if len(body.username.strip()) < 2:
|
||||
raise HTTPException(status_code=400, detail="Username must be at least 2 characters")
|
||||
if len(body.password) < 6:
|
||||
raise HTTPException(status_code=400, detail="Password must be at least 6 characters")
|
||||
|
||||
# Check for duplicate username
|
||||
existing = await db.execute(
|
||||
select(User).where(User.username == body.username.strip())
|
||||
)
|
||||
if existing.scalar_one_or_none() is not None:
|
||||
raise HTTPException(status_code=409, detail="Username already taken")
|
||||
|
||||
media_path = os.path.join(settings.photo_dirs, body.username.strip())
|
||||
os.makedirs(media_path, exist_ok=True)
|
||||
|
||||
user = User(
|
||||
username=body.username.strip(),
|
||||
hashed_password=hash_password(body.password),
|
||||
role=body.role,
|
||||
media_path=media_path,
|
||||
)
|
||||
db.add(user)
|
||||
await db.flush() # get user.id before creating source root
|
||||
|
||||
source_root = SourceRoot(
|
||||
name=f"{user.username}'s Library",
|
||||
path=media_path,
|
||||
user_id=user.id,
|
||||
)
|
||||
db.add(source_root)
|
||||
await db.commit()
|
||||
|
||||
logger.info(f"Admin '{admin.username}' created user '{user.username}' (role={user.role})")
|
||||
|
||||
return UserDetailResponse(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
email=user.email,
|
||||
role=user.role,
|
||||
is_active=user.is_active,
|
||||
media_path=user.media_path,
|
||||
created_at=user.created_at.isoformat() if user.created_at else None,
|
||||
photo_count=0,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/users/{user_id}", response_model=UserDetailResponse)
|
||||
async def get_user(
|
||||
user_id: str,
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get a single user's details."""
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
|
||||
count_result = await db.execute(
|
||||
select(sa_func.count(Photo.id)).where(Photo.user_id == user.id)
|
||||
)
|
||||
photo_count = count_result.scalar() or 0
|
||||
|
||||
return UserDetailResponse(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
email=user.email,
|
||||
role=user.role,
|
||||
is_active=user.is_active,
|
||||
media_path=user.media_path,
|
||||
created_at=user.created_at.isoformat() if user.created_at else None,
|
||||
photo_count=photo_count,
|
||||
)
|
||||
|
||||
|
||||
@router.patch("/users/{user_id}", response_model=UserDetailResponse)
|
||||
async def update_user(
|
||||
user_id: str,
|
||||
body: UpdateUserRequest,
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update a user's role, active status, or password."""
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
|
||||
if body.role is not None:
|
||||
if body.role not in ("admin", "user"):
|
||||
raise HTTPException(status_code=400, detail="Role must be 'admin' or 'user'")
|
||||
# Prevent demoting the last admin
|
||||
if user.role == "admin" and body.role == "user":
|
||||
admin_count = (await db.execute(
|
||||
select(sa_func.count(User.id)).where(User.role == "admin", User.is_active == True)
|
||||
)).scalar()
|
||||
if admin_count <= 1:
|
||||
raise HTTPException(status_code=400, detail="Cannot demote the last admin")
|
||||
user.role = body.role
|
||||
|
||||
if body.is_active is not None:
|
||||
# Prevent deactivating the last admin
|
||||
if user.role == "admin" and not body.is_active:
|
||||
admin_count = (await db.execute(
|
||||
select(sa_func.count(User.id)).where(User.role == "admin", User.is_active == True)
|
||||
)).scalar()
|
||||
if admin_count <= 1:
|
||||
raise HTTPException(status_code=400, detail="Cannot deactivate the last admin")
|
||||
user.is_active = body.is_active
|
||||
|
||||
if body.new_password is not None:
|
||||
if len(body.new_password) < 6:
|
||||
raise HTTPException(status_code=400, detail="Password must be at least 6 characters")
|
||||
user.hashed_password = hash_password(body.new_password)
|
||||
|
||||
await db.commit()
|
||||
|
||||
count_result = await db.execute(
|
||||
select(sa_func.count(Photo.id)).where(Photo.user_id == user.id)
|
||||
)
|
||||
photo_count = count_result.scalar() or 0
|
||||
|
||||
return UserDetailResponse(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
email=user.email,
|
||||
role=user.role,
|
||||
is_active=user.is_active,
|
||||
media_path=user.media_path,
|
||||
created_at=user.created_at.isoformat() if user.created_at else None,
|
||||
photo_count=photo_count,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/users/{user_id}")
|
||||
async def delete_user(
|
||||
user_id: str,
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Soft-delete a user by deactivating them. Media is preserved."""
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
|
||||
if user.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="Cannot delete yourself")
|
||||
|
||||
# Prevent deleting the last admin
|
||||
if user.role == "admin":
|
||||
admin_count = (await db.execute(
|
||||
select(sa_func.count(User.id)).where(User.role == "admin", User.is_active == True)
|
||||
)).scalar()
|
||||
if admin_count <= 1:
|
||||
raise HTTPException(status_code=400, detail="Cannot delete the last admin")
|
||||
|
||||
user.is_active = False
|
||||
await db.commit()
|
||||
|
||||
logger.info(f"Admin '{admin.username}' deactivated user '{user.username}'")
|
||||
return {"status": "ok", "detail": f"User '{user.username}' deactivated"}
|
||||
@@ -1,552 +0,0 @@
|
||||
"""
|
||||
Authentication router — login, token refresh, profile, first-run setup,
|
||||
and optional OIDC (Authentik) sign-in.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from fastapi.responses import RedirectResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func as sa_func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth import hash_password, verify_password, create_access_token, create_refresh_token, decode_token
|
||||
from app.auth_oidc import get_client as get_oidc_client, is_enabled as oidc_is_enabled, provider_label, PROVIDER_NAME
|
||||
from app.database import get_db
|
||||
from app.dependencies import get_current_user
|
||||
from app.models.user import User
|
||||
from app.models.folders import SourceRoot
|
||||
from app.services.gravatar import gravatar_url
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / response schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
access_token: str
|
||||
refresh_token: str
|
||||
token_type: str = "bearer"
|
||||
|
||||
class RefreshRequest(BaseModel):
|
||||
refresh_token: str
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
id: str
|
||||
username: str
|
||||
email: Optional[str]
|
||||
role: str
|
||||
is_active: bool
|
||||
created_at: Optional[str]
|
||||
avatar_url: Optional[str] = None
|
||||
display_name: Optional[str] = None
|
||||
# Nextcloud integration — username override (defaults to OIDC
|
||||
# preferred_username) and a flag for whether the user has stored
|
||||
# an app password. Cleartext passwords are never serialized.
|
||||
nextcloud_username: Optional[str] = None
|
||||
has_nextcloud_app_password: bool = False
|
||||
|
||||
|
||||
class UpdateMeRequest(BaseModel):
|
||||
"""PATCH /me payload. Every field is optional — only what's set
|
||||
gets touched. Setting `nextcloud_app_password` to "" clears it."""
|
||||
nextcloud_username: Optional[str] = None
|
||||
nextcloud_app_password: Optional[str] = None
|
||||
|
||||
class SetupRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
current_password: str
|
||||
new_password: str
|
||||
|
||||
|
||||
class OidcConfig(BaseModel):
|
||||
enabled: bool
|
||||
label: str
|
||||
login_url: str
|
||||
|
||||
|
||||
class AuthConfigResponse(BaseModel):
|
||||
# None when OIDC is disabled / not configured — the frontend uses
|
||||
# that to hide the SSO button.
|
||||
oidc: Optional[OidcConfig] = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def serialize_user(user: User) -> UserResponse:
|
||||
"""Build the user-facing payload, computing the avatar URL with the
|
||||
`provider picture > Gravatar > None` fallback chain."""
|
||||
avatar = user.avatar_url or gravatar_url(user.email)
|
||||
return UserResponse(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
email=user.email,
|
||||
role=user.role,
|
||||
is_active=user.is_active,
|
||||
created_at=user.created_at.isoformat() if user.created_at else None,
|
||||
avatar_url=avatar,
|
||||
display_name=user.display_name,
|
||||
nextcloud_username=user.nextcloud_username,
|
||||
has_nextcloud_app_password=bool(user.nextcloud_app_password_enc),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/config", response_model=AuthConfigResponse)
|
||||
async def auth_config():
|
||||
"""Public — tells the frontend which login options to render.
|
||||
|
||||
Returns `oidc: null` when OIDC is disabled or not fully configured,
|
||||
so the login page can hide the SSO button without a round-trip to
|
||||
the IdP. The `login_url` is browser-navigable (full redirect); it
|
||||
starts the Authlib flow that sets the PKCE cookie.
|
||||
"""
|
||||
if not oidc_is_enabled():
|
||||
return AuthConfigResponse(oidc=None)
|
||||
return AuthConfigResponse(
|
||||
oidc=OidcConfig(
|
||||
enabled=True,
|
||||
label=provider_label(),
|
||||
login_url="/api/v1/auth/oidc/login",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.post("/login", response_model=TokenResponse)
|
||||
async def login(body: LoginRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""Authenticate with username + password, receive JWT tokens."""
|
||||
result = await db.execute(
|
||||
select(User).where(User.username == body.username)
|
||||
)
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
# OIDC-only users (hashed_password IS NULL) can't sign in via this
|
||||
# endpoint; they must go through the SSO flow. Treat as auth failure
|
||||
# so we don't leak account existence.
|
||||
if (
|
||||
user is None
|
||||
or not user.hashed_password
|
||||
or not verify_password(body.password, user.hashed_password)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid username or password",
|
||||
)
|
||||
if not user.is_active:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Account is deactivated",
|
||||
)
|
||||
|
||||
return TokenResponse(
|
||||
access_token=create_access_token(user.id, user.role),
|
||||
refresh_token=create_refresh_token(user.id),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/refresh", response_model=TokenResponse)
|
||||
async def refresh_token(body: RefreshRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""Exchange a valid refresh token for a new access + refresh pair."""
|
||||
try:
|
||||
payload = decode_token(body.refresh_token)
|
||||
if payload.get("type") != "refresh":
|
||||
raise ValueError("not a refresh token")
|
||||
user_id = payload["sub"]
|
||||
except Exception:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or expired refresh token",
|
||||
)
|
||||
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None or not user.is_active:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="User not found or deactivated",
|
||||
)
|
||||
|
||||
return TokenResponse(
|
||||
access_token=create_access_token(user.id, user.role),
|
||||
refresh_token=create_refresh_token(user.id),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserResponse)
|
||||
async def get_me(current_user: User = Depends(get_current_user)):
|
||||
"""Return the authenticated user's profile."""
|
||||
return serialize_user(current_user)
|
||||
|
||||
|
||||
_NC_USERNAME_RE = re.compile(r"^[a-zA-Z0-9._@-]{1,64}$")
|
||||
|
||||
|
||||
@router.patch("/me", response_model=UserResponse)
|
||||
async def update_me(
|
||||
body: UpdateMeRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update the authenticated user's Nextcloud integration settings.
|
||||
|
||||
`nextcloud_username` overrides the OIDC `preferred_username` default
|
||||
so e.g. the local mule-image user `dtoro` can map to Nextcloud user
|
||||
`admin`. `nextcloud_app_password` is encrypted at rest via the
|
||||
Fernet helper in `services/secrets.py`; passing an empty string
|
||||
clears it.
|
||||
"""
|
||||
from app.services.secrets import encrypt
|
||||
|
||||
changed = False
|
||||
if body.nextcloud_username is not None:
|
||||
candidate = body.nextcloud_username.strip()
|
||||
if candidate and not _NC_USERNAME_RE.match(candidate):
|
||||
raise HTTPException(status_code=400, detail="Invalid Nextcloud username")
|
||||
current_user.nextcloud_username = candidate or None
|
||||
changed = True
|
||||
|
||||
if body.nextcloud_app_password is not None:
|
||||
if body.nextcloud_app_password == "":
|
||||
current_user.nextcloud_app_password_enc = None
|
||||
else:
|
||||
current_user.nextcloud_app_password_enc = encrypt(body.nextcloud_app_password)
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
await db.commit()
|
||||
await db.refresh(current_user)
|
||||
return serialize_user(current_user)
|
||||
|
||||
|
||||
@router.post("/change-password")
|
||||
async def change_password(
|
||||
body: ChangePasswordRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Change the authenticated user's password."""
|
||||
if not current_user.hashed_password or not verify_password(
|
||||
body.current_password, current_user.hashed_password
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Current password is incorrect",
|
||||
)
|
||||
current_user.hashed_password = hash_password(body.new_password)
|
||||
await db.commit()
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.post("/setup", response_model=TokenResponse, status_code=201)
|
||||
async def setup(body: SetupRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""First-run only: create the initial admin account.
|
||||
|
||||
Returns 409 if any user already exists. This endpoint is
|
||||
unauthenticated by design — it can only run once.
|
||||
"""
|
||||
count = (await db.execute(select(sa_func.count(User.id)))).scalar()
|
||||
if count > 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="Setup already completed — users exist",
|
||||
)
|
||||
|
||||
if len(body.username.strip()) < 2:
|
||||
raise HTTPException(status_code=400, detail="Username must be at least 2 characters")
|
||||
if len(body.password) < 6:
|
||||
raise HTTPException(status_code=400, detail="Password must be at least 6 characters")
|
||||
|
||||
# Every user — including the initial admin — gets their own subfolder
|
||||
# under the photo mount root. Nobody owns the root directory itself.
|
||||
media_path = os.path.join(settings.photo_dirs, body.username.strip())
|
||||
os.makedirs(media_path, exist_ok=True)
|
||||
|
||||
user = User(
|
||||
username=body.username.strip(),
|
||||
hashed_password=hash_password(body.password),
|
||||
role="admin",
|
||||
media_path=media_path,
|
||||
)
|
||||
db.add(user)
|
||||
await db.flush() # get user.id before creating source root
|
||||
|
||||
source_root = SourceRoot(
|
||||
name=f"{user.username}'s Library",
|
||||
path=media_path,
|
||||
user_id=user.id,
|
||||
)
|
||||
db.add(source_root)
|
||||
await db.commit()
|
||||
|
||||
logger.info(f"Initial admin account created: {user.username}")
|
||||
|
||||
return TokenResponse(
|
||||
access_token=create_access_token(user.id, user.role),
|
||||
refresh_token=create_refresh_token(user.id),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def auth_status(db: AsyncSession = Depends(get_db)):
|
||||
"""Public endpoint: returns whether setup has been completed.
|
||||
|
||||
The frontend calls this to decide whether to show the setup page
|
||||
or the login page.
|
||||
"""
|
||||
count = (await db.execute(select(sa_func.count(User.id)))).scalar()
|
||||
return {"setup_completed": count > 0}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OIDC (Authentik) sign-in
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_USERNAME_SANITIZER = re.compile(r"[^a-zA-Z0-9_.-]+")
|
||||
|
||||
|
||||
def _derive_username(claims: dict, existing_usernames: set[str]) -> str:
|
||||
"""Pick a local username from OIDC claims.
|
||||
|
||||
Order of preference:
|
||||
1. `preferred_username` claim (Authentik's usual choice)
|
||||
2. local-part of `email`
|
||||
3. `sub` claim (always present)
|
||||
|
||||
Strips characters the rest of the app doesn't like in paths/URLs,
|
||||
trims to 50 chars (User.username column limit), and appends a short
|
||||
suffix on collision so two Authentik users can't land on the same
|
||||
local row.
|
||||
"""
|
||||
raw = (
|
||||
claims.get("preferred_username")
|
||||
or (claims.get("email") or "").split("@", 1)[0]
|
||||
or claims.get("sub")
|
||||
or "user"
|
||||
)
|
||||
base = _USERNAME_SANITIZER.sub("", str(raw)).strip("._-") or "user"
|
||||
base = base[:40]
|
||||
candidate = base
|
||||
suffix = 0
|
||||
while candidate in existing_usernames:
|
||||
suffix += 1
|
||||
candidate = f"{base}-{suffix}"[:50]
|
||||
return candidate
|
||||
|
||||
|
||||
def _frontend_origin(request: Request) -> str:
|
||||
"""Best guess at where the SPA lives so the callback redirect lands
|
||||
back on the app origin. Uses the configured redirect URI's scheme +
|
||||
host (strips /api/... path) when available, falling back to the
|
||||
request's own origin."""
|
||||
if settings.oidc_redirect_uri:
|
||||
parsed = urlparse(settings.oidc_redirect_uri)
|
||||
return f"{parsed.scheme}://{parsed.netloc}"
|
||||
return f"{request.url.scheme}://{request.url.netloc}"
|
||||
|
||||
|
||||
@router.get("/oidc/login")
|
||||
async def oidc_login(request: Request):
|
||||
"""Start the OIDC flow — redirect to Authentik's authorization URL."""
|
||||
if not oidc_is_enabled():
|
||||
raise HTTPException(status_code=404, detail="OIDC login is not enabled")
|
||||
client = get_oidc_client()
|
||||
if client is None:
|
||||
raise HTTPException(status_code=500, detail="OIDC client not configured")
|
||||
|
||||
redirect_uri = settings.oidc_redirect_uri
|
||||
return await client.authorize_redirect(request, redirect_uri)
|
||||
|
||||
|
||||
@router.get("/oidc/callback")
|
||||
async def oidc_callback(request: Request, db: AsyncSession = Depends(get_db)):
|
||||
"""Handle the OIDC redirect — exchange code, provision/link user,
|
||||
issue our own JWTs, bounce back to the SPA."""
|
||||
if not oidc_is_enabled():
|
||||
raise HTTPException(status_code=404, detail="OIDC login is not enabled")
|
||||
client = get_oidc_client()
|
||||
if client is None:
|
||||
raise HTTPException(status_code=500, detail="OIDC client not configured")
|
||||
|
||||
try:
|
||||
token = await client.authorize_access_token(request)
|
||||
except Exception as exc:
|
||||
logger.warning("OIDC callback: authorize_access_token failed: %s", exc)
|
||||
return _oidc_error_redirect(request, "oidc_exchange_failed")
|
||||
|
||||
# `parse_id_token` verifies signature + nonce; `userinfo` fills in
|
||||
# claims some IdPs don't put in the ID token (e.g. picture). We
|
||||
# merge both, preferring userinfo when both are present.
|
||||
claims = dict(token.get("userinfo") or {})
|
||||
if not claims:
|
||||
try:
|
||||
claims = dict(await client.userinfo(token=token))
|
||||
except Exception:
|
||||
claims = {}
|
||||
id_token_claims = token.get("id_token_claims") or {}
|
||||
for k, v in id_token_claims.items():
|
||||
claims.setdefault(k, v)
|
||||
|
||||
sub = claims.get("sub")
|
||||
if not sub:
|
||||
logger.warning("OIDC callback: claims missing `sub` — %r", claims)
|
||||
return _oidc_error_redirect(request, "oidc_missing_sub")
|
||||
|
||||
issuer = claims.get("iss") or (settings.oidc_issuer or "").rstrip("/")
|
||||
email = claims.get("email")
|
||||
display_name = claims.get("name") or claims.get("preferred_username")
|
||||
picture = claims.get("picture")
|
||||
groups = claims.get("groups") or []
|
||||
if isinstance(groups, str):
|
||||
groups = [groups]
|
||||
|
||||
admin_groups = set(settings.oidc_admin_group_list)
|
||||
role = "admin" if admin_groups and admin_groups.intersection(groups) else "user"
|
||||
|
||||
# 1. Match by (issuer, sub) first — stable identity key.
|
||||
user = (await db.execute(
|
||||
select(User).where(
|
||||
User.oidc_issuer == issuer,
|
||||
User.oidc_sub == sub,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
# 2. Fall back to email so a pre-existing local account can be
|
||||
# linked on first SSO login (homelab admin keeps their row).
|
||||
if user is None and email:
|
||||
user = (await db.execute(
|
||||
select(User).where(User.email == email)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
# 3. Last-resort link by preferred_username. Off by default; only
|
||||
# used in trusted single-tenant setups where local accounts
|
||||
# predate OIDC and never collected email (the app has no UI for
|
||||
# it). Guarded by OIDC_LINK_BY_USERNAME to avoid hijacking
|
||||
# accounts in shared instances.
|
||||
if user is None and settings.oidc_link_by_username:
|
||||
preferred = claims.get("preferred_username")
|
||||
if preferred:
|
||||
user = (await db.execute(
|
||||
select(User).where(User.username == preferred)
|
||||
)).scalar_one_or_none()
|
||||
if user is not None:
|
||||
logger.info(
|
||||
"OIDC linked existing user %s by preferred_username",
|
||||
preferred,
|
||||
)
|
||||
|
||||
if user is None:
|
||||
if not settings.oidc_allow_signup:
|
||||
logger.info("OIDC signup disabled — rejecting unknown sub=%s email=%s", sub, email)
|
||||
return _oidc_error_redirect(request, "oidc_signup_disabled")
|
||||
|
||||
# JIT provision.
|
||||
existing = {
|
||||
u for (u,) in (await db.execute(select(User.username))).all()
|
||||
}
|
||||
username = _derive_username(claims, existing)
|
||||
media_path = os.path.join(settings.photo_dirs, username)
|
||||
os.makedirs(media_path, exist_ok=True)
|
||||
|
||||
user = User(
|
||||
username=username,
|
||||
email=email,
|
||||
hashed_password=None,
|
||||
role=role,
|
||||
is_active=True,
|
||||
media_path=media_path,
|
||||
oidc_issuer=issuer,
|
||||
oidc_sub=sub,
|
||||
avatar_url=picture,
|
||||
display_name=display_name,
|
||||
# Default the Nextcloud username from preferred_username so
|
||||
# the common case "same name on both sides" needs zero
|
||||
# configuration. Override is exposed in Settings for the
|
||||
# mismatch case (e.g. authentik dtoro ↔ Nextcloud admin).
|
||||
nextcloud_username=(claims.get("preferred_username") or None),
|
||||
)
|
||||
db.add(user)
|
||||
await db.flush()
|
||||
|
||||
db.add(SourceRoot(
|
||||
name=f"{user.username}'s Library",
|
||||
path=media_path,
|
||||
user_id=user.id,
|
||||
))
|
||||
await db.commit()
|
||||
logger.info("OIDC JIT-created user %s (role=%s)", user.username, role)
|
||||
else:
|
||||
# Refresh profile bits + link identity if needed. We do *not*
|
||||
# demote admins created locally; only touch role when admin
|
||||
# group mapping is configured.
|
||||
changed = False
|
||||
if user.oidc_sub != sub or user.oidc_issuer != issuer:
|
||||
user.oidc_issuer = issuer
|
||||
user.oidc_sub = sub
|
||||
changed = True
|
||||
if email and user.email != email:
|
||||
user.email = email
|
||||
changed = True
|
||||
if display_name and user.display_name != display_name:
|
||||
user.display_name = display_name
|
||||
changed = True
|
||||
if picture and user.avatar_url != picture:
|
||||
user.avatar_url = picture
|
||||
changed = True
|
||||
# Backfill nextcloud_username on first OIDC login for users that
|
||||
# predate the column. NEVER overwrites a value the user already
|
||||
# set in Settings — once the override is non-null, it wins.
|
||||
if not user.nextcloud_username:
|
||||
preferred = claims.get("preferred_username")
|
||||
if preferred:
|
||||
user.nextcloud_username = preferred
|
||||
changed = True
|
||||
if admin_groups:
|
||||
new_role = "admin" if admin_groups.intersection(groups) else "user"
|
||||
if user.role != new_role:
|
||||
user.role = new_role
|
||||
changed = True
|
||||
if not user.is_active:
|
||||
# Don't resurrect a deactivated account — surface an error.
|
||||
logger.info("OIDC login rejected — user %s is deactivated", user.username)
|
||||
return _oidc_error_redirect(request, "oidc_deactivated")
|
||||
if changed:
|
||||
await db.commit()
|
||||
|
||||
# Mint our own JWTs and bounce back to the SPA. Tokens ride in the
|
||||
# URL fragment-free for simplicity; the frontend callback page
|
||||
# strips them from the location bar immediately.
|
||||
access = create_access_token(user.id, user.role)
|
||||
refresh = create_refresh_token(user.id)
|
||||
|
||||
params = urlencode({"access_token": access, "refresh_token": refresh})
|
||||
target = f"{_frontend_origin(request)}/auth/callback?{params}"
|
||||
return RedirectResponse(url=target, status_code=302)
|
||||
|
||||
|
||||
def _oidc_error_redirect(request: Request, code: str) -> RedirectResponse:
|
||||
"""Bounce back to the SPA with an `error=` query so the login page
|
||||
can render something meaningful instead of a stack trace."""
|
||||
target = f"{_frontend_origin(request)}/auth/callback?error={code}"
|
||||
return RedirectResponse(url=target, status_code=302)
|
||||
@@ -1,117 +0,0 @@
|
||||
"""
|
||||
Discard API router
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from fastapi import APIRouter, Depends, HTTPException, Body
|
||||
from sqlalchemy import select, and_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user
|
||||
from app.services.nextcloud_dav import delete_for_user, is_nextcloud_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("")
|
||||
async def list_discarded(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""List discarded photos"""
|
||||
result = await db.execute(
|
||||
select(Photo).where(Photo.is_discarded == True, Photo.user_id == current_user.id)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
return photos
|
||||
|
||||
@router.post("/restore")
|
||||
async def restore_photos(photo_ids: list[str], db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Restore photos from the discard pile"""
|
||||
result = await db.execute(
|
||||
select(Photo).where(and_(Photo.id.in_(photo_ids), Photo.is_discarded == True, Photo.user_id == current_user.id))
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
|
||||
for photo in photos:
|
||||
photo.is_discarded = False
|
||||
photo.discarded_at = None
|
||||
|
||||
await db.commit()
|
||||
return {"status": "success", "restored": len(photos)}
|
||||
|
||||
@router.delete("/empty")
|
||||
async def empty_discard(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Permanently delete all discarded photos and unlink their files from
|
||||
disk. Failures on individual files are logged but don't abort the batch.
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(Photo).where(Photo.is_discarded == True, Photo.user_id == current_user.id)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
return await _permanently_delete(db, photos, current_user)
|
||||
|
||||
|
||||
@router.delete("")
|
||||
async def delete_discarded(
|
||||
photo_ids: list[str] = Body(..., embed=True),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Permanently delete a specific subset of discarded photos. The photos
|
||||
must already be in the discard pile — non-discarded ids are skipped so
|
||||
this can never bypass the soft-delete safety net.
|
||||
"""
|
||||
if not photo_ids:
|
||||
return {"status": "success", "deleted": 0, "file_errors": 0}
|
||||
result = await db.execute(
|
||||
select(Photo).where(
|
||||
and_(Photo.id.in_(photo_ids), Photo.is_discarded == True, Photo.user_id == current_user.id)
|
||||
)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
return await _permanently_delete(db, photos, current_user)
|
||||
|
||||
|
||||
async def _permanently_delete(db: AsyncSession, photos: list[Photo], user: User) -> dict:
|
||||
"""Shared helper: unlink files for the given photos and delete their
|
||||
rows. Per-file errors are counted but don't abort the batch.
|
||||
|
||||
For files inside a Nextcloud-rooted SourceRoot the unlink is dispatched
|
||||
through Nextcloud's WebDAV `DELETE` so Nextcloud moves the file into
|
||||
the user's trashbin and updates `oc_filecache`. For everything else we
|
||||
fall back to plain `os.unlink`.
|
||||
"""
|
||||
deleted = 0
|
||||
file_errors = 0
|
||||
for photo in photos:
|
||||
try:
|
||||
if photo.filepath:
|
||||
if is_nextcloud_path(photo.filepath):
|
||||
# WebDAV DELETE — Nextcloud moves to trashbin and
|
||||
# updates oc_filecache. The bind mount will then
|
||||
# reflect the file's absence (Nextcloud writes
|
||||
# synchronously). 404 from NC is treated as already
|
||||
# gone (idempotent).
|
||||
delete_for_user(user, photo.filepath)
|
||||
elif os.path.exists(photo.filepath):
|
||||
os.unlink(photo.filepath)
|
||||
except HTTPException as e:
|
||||
# WebDAV-side error — surface to caller via the file_errors
|
||||
# counter rather than aborting the whole batch.
|
||||
file_errors += 1
|
||||
logger.error(f"Failed to delete {photo.filepath} via Nextcloud: {e.detail}")
|
||||
continue
|
||||
except OSError as e:
|
||||
file_errors += 1
|
||||
logger.error(f"Failed to unlink {photo.filepath}: {e}")
|
||||
await db.delete(photo)
|
||||
deleted += 1
|
||||
|
||||
await db.commit()
|
||||
return {
|
||||
"status": "success",
|
||||
"deleted": deleted,
|
||||
"file_errors": file_errors,
|
||||
}
|
||||
@@ -1,247 +0,0 @@
|
||||
"""
|
||||
Download router — streams a .zip of every photo in a folder (recursively)
|
||||
or a heap back to the browser.
|
||||
|
||||
Auth: both endpoints accept the regular Authorization header *or* a
|
||||
``?token=JWT`` query string, mirroring the media endpoints. That lets the
|
||||
frontend trigger a download with a plain ``<a href>`` (which can't set a
|
||||
header), keeping the client side a one-liner.
|
||||
|
||||
Implementation: we build the zip into a ``NamedTemporaryFile`` and then
|
||||
stream its bytes back, deleting the temp file on the way out. Stored
|
||||
(uncompressed) mode because photos and videos are already compressed —
|
||||
deflating them again just burns CPU for a fraction of a percent. For
|
||||
very large libraries the temp-file route is mildly wasteful vs. a true
|
||||
streaming zip (zipstream-ng etc), but it avoids a new dependency and
|
||||
handles arbitrary folder sizes without blowing out RAM.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import zipfile
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.dependencies import get_current_user_media
|
||||
from app.models import Folder, Heap, Photo, SourceRoot
|
||||
from app.models.heaps import heap_photos
|
||||
from app.models.user import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _safe_filename(name: str) -> str:
|
||||
"""Strip characters that Content-Disposition or Windows filesystems
|
||||
would choke on. Keeps the download's filename readable without
|
||||
needing any escaping on the client side."""
|
||||
cleaned = re.sub(r'[\\/:*?"<>|\r\n\t]', '_', name).strip().strip('.')
|
||||
return cleaned or 'download'
|
||||
|
||||
|
||||
async def _collect_folder_photos(
|
||||
folder_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> tuple[str, str, List[Photo]]:
|
||||
"""Resolve a folder or source-root id → (base_path, display_name,
|
||||
photos). ``base_path`` is the prefix we strip off each photo's
|
||||
filepath when naming zip entries, so the archive mirrors the user's
|
||||
on-disk structure under that folder.
|
||||
"""
|
||||
folder = (await db.execute(
|
||||
select(Folder).where(Folder.id == folder_id, Folder.user_id == user.id)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
base_path: str
|
||||
display_name: str
|
||||
if folder is not None:
|
||||
base_path = os.path.normpath(folder.path)
|
||||
display_name = folder.name or os.path.basename(base_path)
|
||||
else:
|
||||
sr = (await db.execute(
|
||||
select(SourceRoot).where(
|
||||
SourceRoot.id == folder_id,
|
||||
SourceRoot.user_id == user.id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if sr is None:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
base_path = os.path.normpath(sr.path)
|
||||
display_name = sr.name or os.path.basename(base_path)
|
||||
|
||||
# Every photo whose filepath is at or below the base path — matches
|
||||
# the same prefix convention folders.py uses for recursive deletes.
|
||||
descendant_prefix = base_path.rstrip(os.sep) + os.sep
|
||||
result = await db.execute(
|
||||
select(Photo).where(
|
||||
Photo.user_id == user.id,
|
||||
Photo.is_discarded == False, # noqa: E712
|
||||
(Photo.filepath == base_path) | (Photo.filepath.like(descendant_prefix + '%')),
|
||||
)
|
||||
)
|
||||
photos = list(result.scalars().all())
|
||||
return base_path, display_name, photos
|
||||
|
||||
|
||||
def _build_zip(
|
||||
photos: List[Photo],
|
||||
arcname_fn,
|
||||
) -> tempfile.NamedTemporaryFile:
|
||||
"""Write ``photos`` into a fresh ZIP_STORED temp file.
|
||||
|
||||
``arcname_fn(photo, used_names)`` returns the entry name to use for
|
||||
the given photo; the caller supplies it because folder downloads
|
||||
want path-preserving names while heap downloads flatten to bare
|
||||
filenames (with a collision suffix).
|
||||
"""
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.zip')
|
||||
try:
|
||||
used: set[str] = set()
|
||||
with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_STORED, allowZip64=True) as zf:
|
||||
for p in photos:
|
||||
if not p.filepath or not os.path.exists(p.filepath):
|
||||
# Silent skip: the scanner may have indexed files
|
||||
# that have since been moved / unlinked by a shell.
|
||||
continue
|
||||
name = arcname_fn(p, used)
|
||||
used.add(name)
|
||||
try:
|
||||
zf.write(p.filepath, name)
|
||||
except OSError as e:
|
||||
logger.warning(f"Skipping {p.filepath} in zip: {e}")
|
||||
tmp.close()
|
||||
return tmp
|
||||
except Exception:
|
||||
tmp.close()
|
||||
try:
|
||||
os.unlink(tmp.name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _stream_and_cleanup(path: str):
|
||||
"""Yield the temp zip in 1 MiB chunks and unlink it when the
|
||||
iterator is exhausted (or GC'd, if the client disconnects early)."""
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
while True:
|
||||
chunk = f.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError as e:
|
||||
logger.debug(f"Temp zip cleanup failed for {path}: {e}")
|
||||
|
||||
|
||||
def _dedupe(name: str, used: set[str]) -> str:
|
||||
"""Return ``name`` (or ``name (2)``, ``name (3)`` ...) such that the
|
||||
result doesn't collide with anything in ``used``. Needed for heap
|
||||
downloads where two members can have identical filenames from
|
||||
different folders."""
|
||||
if name not in used:
|
||||
return name
|
||||
stem, ext = os.path.splitext(name)
|
||||
n = 2
|
||||
while True:
|
||||
cand = f"{stem} ({n}){ext}"
|
||||
if cand not in used:
|
||||
return cand
|
||||
n += 1
|
||||
|
||||
|
||||
@router.get("/folders/{folder_id}")
|
||||
async def download_folder(
|
||||
folder_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user_media),
|
||||
):
|
||||
"""Zip every (non-discarded) photo under a folder/source-root and
|
||||
stream it back. Entries preserve the folder structure relative to
|
||||
the downloaded root so the resulting archive is a faithful snapshot.
|
||||
"""
|
||||
base_path, display_name, photos = await _collect_folder_photos(
|
||||
folder_id, current_user, db
|
||||
)
|
||||
if not photos:
|
||||
raise HTTPException(status_code=404, detail="No photos to download")
|
||||
|
||||
def arcname(p: Photo, _used: set[str]) -> str:
|
||||
# Relative path from the download root, falling back to the
|
||||
# bare filename if the photo somehow lives outside base_path.
|
||||
abs_path = os.path.normpath(p.filepath)
|
||||
if abs_path.startswith(base_path + os.sep):
|
||||
rel = abs_path[len(base_path) + 1:]
|
||||
elif abs_path == base_path:
|
||||
rel = os.path.basename(abs_path)
|
||||
else:
|
||||
rel = p.filename or os.path.basename(abs_path)
|
||||
# Nest everything under display_name so users see one top-level
|
||||
# folder inside the zip rather than loose files.
|
||||
return os.path.join(_safe_filename(display_name), rel)
|
||||
|
||||
tmp = _build_zip(photos, arcname)
|
||||
filename = _safe_filename(display_name) + '.zip'
|
||||
return StreamingResponse(
|
||||
_stream_and_cleanup(tmp.name),
|
||||
media_type='application/zip',
|
||||
headers={
|
||||
'Content-Disposition': f'attachment; filename="{filename}"',
|
||||
'Content-Length': str(os.path.getsize(tmp.name)),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/heaps/{heap_id}")
|
||||
async def download_heap(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user_media),
|
||||
):
|
||||
"""Zip every photo in a heap. Heaps are flat collections, so entries
|
||||
use the original filename (with a ``(2)`` collision suffix when
|
||||
two members share a name)."""
|
||||
heap = (await db.execute(
|
||||
select(Heap).where(Heap.id == heap_id, Heap.user_id == current_user.id)
|
||||
)).scalar_one_or_none()
|
||||
if heap is None:
|
||||
raise HTTPException(status_code=404, detail="Heap not found")
|
||||
|
||||
result = await db.execute(
|
||||
select(Photo)
|
||||
.join(heap_photos, heap_photos.c.photo_id == Photo.id)
|
||||
.where(
|
||||
heap_photos.c.heap_id == heap_id,
|
||||
Photo.is_discarded == False, # noqa: E712
|
||||
)
|
||||
)
|
||||
photos = list(result.scalars().all())
|
||||
if not photos:
|
||||
raise HTTPException(status_code=404, detail="Heap is empty")
|
||||
|
||||
def arcname(p: Photo, used: set[str]) -> str:
|
||||
bare = p.filename or os.path.basename(p.filepath or 'photo')
|
||||
entry = os.path.join(_safe_filename(heap.name), _dedupe(bare, used))
|
||||
return entry
|
||||
|
||||
tmp = _build_zip(photos, arcname)
|
||||
filename = _safe_filename(heap.name) + '.zip'
|
||||
return StreamingResponse(
|
||||
_stream_and_cleanup(tmp.name),
|
||||
media_type='application/zip',
|
||||
headers={
|
||||
'Content-Disposition': f'attachment; filename="{filename}"',
|
||||
'Content-Length': str(os.path.getsize(tmp.name)),
|
||||
},
|
||||
)
|
||||
@@ -1,584 +0,0 @@
|
||||
"""
|
||||
Folders API router. Source roots themselves are config-driven (PHOTO_DIRS
|
||||
in .env → backend bootstrap on startup) — adding or removing one is a
|
||||
docker-compose change. Sub-folders inside a source root can be created,
|
||||
renamed, and deleted from the UI; those changes are mirrored to disk.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from typing import Literal, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func, update as sql_update, delete as sql_delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Folder, SourceRoot, Photo
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user, get_user_folder
|
||||
from app.services.nextcloud_dav import (
|
||||
delete_for_user as nc_delete,
|
||||
is_nextcloud_path,
|
||||
mkcol_for_user,
|
||||
move_for_user as nc_move,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class FolderRename(BaseModel):
|
||||
name: str
|
||||
|
||||
|
||||
class FolderCreate(BaseModel):
|
||||
name: str
|
||||
parent_id: str # Folder.id (NOT a SourceRoot id)
|
||||
|
||||
|
||||
class FolderHide(BaseModel):
|
||||
hidden: bool
|
||||
|
||||
|
||||
def _validate_folder_name(name: str) -> str:
|
||||
"""Trim + sanity-check a folder name. Rejects names that contain a
|
||||
path separator or that resolve to a parent traversal — those would
|
||||
let the user escape the parent directory through this endpoint.
|
||||
"""
|
||||
name = (name or '').strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Name cannot be empty")
|
||||
if '/' in name or '\\' in name or name in ('.', '..'):
|
||||
raise HTTPException(status_code=400, detail="Invalid folder name")
|
||||
return name
|
||||
|
||||
@router.get("")
|
||||
async def get_folders(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Get all source folders"""
|
||||
# Get source roots instead of regular folders
|
||||
result = await db.execute(select(SourceRoot).where(SourceRoot.is_active == True, SourceRoot.user_id == current_user.id))
|
||||
source_roots = result.scalars().all()
|
||||
|
||||
folders_list = []
|
||||
for root in source_roots:
|
||||
# Get photo count for this source root
|
||||
folder_result = await db.execute(
|
||||
select(Folder).where(Folder.source_root_id == root.id)
|
||||
)
|
||||
folders = folder_result.scalars().all()
|
||||
photo_count = sum(f.photo_count for f in folders)
|
||||
|
||||
folders_list.append({
|
||||
"id": root.id,
|
||||
"name": root.name or os.path.basename(root.path),
|
||||
"path": root.path,
|
||||
"photo_count": photo_count
|
||||
})
|
||||
|
||||
return {"folders": folders_list}
|
||||
|
||||
@router.get("/tree")
|
||||
async def get_folder_tree(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Recursive folder tree, one root per active SourceRoot. The tree
|
||||
starts at the Folder row matching the SourceRoot.path (the scanner
|
||||
creates one for every walked directory), with the SourceRoot's
|
||||
display name overlaid so the top-level entry reads as "Library"
|
||||
instead of "/photos".
|
||||
|
||||
Returns a list of root nodes; each node has:
|
||||
{ id, name, path, photo_count, children: [...] }
|
||||
|
||||
photo_count is **recursive** — every node reports the total non-
|
||||
discarded photos in its own subtree, so the badge matches what the
|
||||
user sees when they click the row (which also filters recursively).
|
||||
|
||||
The stored Folder.photo_count column is intentionally NOT trusted;
|
||||
the scanner's bookkeeping for that field has historically been
|
||||
wrong (it leaks the global total into whichever folder os.walk
|
||||
visited last). We compute counts here from the photos table.
|
||||
|
||||
Sub-folders that physically belong to the same source root but
|
||||
weren't created on disk (e.g. the / row the scanner sometimes
|
||||
creates as a parent walk) are skipped via path-prefix filtering.
|
||||
"""
|
||||
sr_result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active == True, SourceRoot.user_id == current_user.id) # noqa: E712
|
||||
)
|
||||
source_roots = sr_result.scalars().all()
|
||||
|
||||
out = []
|
||||
for sr in source_roots:
|
||||
# Folders physically inside this source root, by path prefix.
|
||||
prefix = os.path.normpath(sr.path).rstrip(os.sep)
|
||||
f_result = await db.execute(
|
||||
select(Folder).where(
|
||||
Folder.source_root_id == sr.id,
|
||||
# Either the folder IS the source root, or it sits beneath it.
|
||||
(Folder.path == prefix) | (Folder.path.like(prefix + os.sep + '%'))
|
||||
)
|
||||
)
|
||||
folders = f_result.scalars().all()
|
||||
if not folders:
|
||||
continue
|
||||
|
||||
# Direct (non-recursive) photo counts per folder, computed from
|
||||
# the photos table. Excludes discarded AND hidden photos so the
|
||||
# sidebar badge matches the "All Photos"-style cross-cutting
|
||||
# views. Users can still click into a hidden folder and see its
|
||||
# contents; the badge count simply won't reflect those photos.
|
||||
folder_ids = [f.id for f in folders]
|
||||
direct_counts: dict[str, int] = {}
|
||||
if folder_ids:
|
||||
count_result = await db.execute(
|
||||
select(Photo.folder_id, func.count(Photo.id))
|
||||
.where(
|
||||
Photo.is_discarded == False, # noqa: E712
|
||||
Photo.is_hidden == False, # noqa: E712
|
||||
Photo.folder_id.in_(folder_ids),
|
||||
)
|
||||
.group_by(Photo.folder_id)
|
||||
)
|
||||
direct_counts = {row[0]: int(row[1]) for row in count_result.all()}
|
||||
|
||||
# Build a path → node map so we can attach children regardless of
|
||||
# parent_id consistency. We populate photo_count with the direct
|
||||
# count first, then accumulate descendants in a post-order pass.
|
||||
# `is_hidden` on each node carries the user-set folder flag (NOT
|
||||
# the effective ancestry flag) so the frontend can render the
|
||||
# hidden icon on the exact folder the user toggled.
|
||||
nodes = {
|
||||
f.path: {
|
||||
"id": f.id,
|
||||
"name": f.name or os.path.basename(f.path),
|
||||
"path": f.path,
|
||||
"photo_count": direct_counts.get(f.id, 0),
|
||||
"is_hidden": bool(f.is_hidden),
|
||||
"children": [],
|
||||
}
|
||||
for f in folders
|
||||
}
|
||||
|
||||
root_node = None
|
||||
for f in folders:
|
||||
node = nodes[f.path]
|
||||
if f.path == prefix:
|
||||
root_node = node
|
||||
# Override the display name with the source root's label.
|
||||
node["name"] = sr.name or node["name"]
|
||||
continue
|
||||
parent_path = os.path.normpath(os.path.dirname(f.path))
|
||||
parent = nodes.get(parent_path)
|
||||
if parent is not None:
|
||||
parent["children"].append(node)
|
||||
# If parent isn't in the set (orphan from a partial scan), drop
|
||||
# the node — it can't be rendered consistently.
|
||||
|
||||
if root_node is not None:
|
||||
# Sort children alphabetically at every level.
|
||||
def sort_recursive(n):
|
||||
n["children"].sort(key=lambda c: c["name"].lower())
|
||||
for c in n["children"]:
|
||||
sort_recursive(c)
|
||||
sort_recursive(root_node)
|
||||
|
||||
# Post-order: each node's recursive count is its own direct
|
||||
# count plus the sum of every descendant's recursive count.
|
||||
def accumulate(n) -> int:
|
||||
total = n["photo_count"]
|
||||
for c in n["children"]:
|
||||
total += accumulate(c)
|
||||
n["photo_count"] = total
|
||||
return total
|
||||
accumulate(root_node)
|
||||
|
||||
out.append(root_node)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
@router.patch("/{folder_id}")
|
||||
async def rename_folder(
|
||||
folder_id: str,
|
||||
body: FolderRename,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Rename a folder. Two cases:
|
||||
|
||||
- SourceRoot id → just change the display label. The on-disk path
|
||||
is owned by the docker mount and never moves.
|
||||
- Folder id → rename the directory on disk AND update every
|
||||
descendant Folder.path + Photo.filepath that
|
||||
lived under the old prefix. Refuses to rename
|
||||
the source-root folder itself (= the row that
|
||||
matches the SourceRoot.path) because that would
|
||||
require renaming the docker mount.
|
||||
"""
|
||||
name = _validate_folder_name(body.name)
|
||||
|
||||
# Try SourceRoot first (display-only rename).
|
||||
sr_result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
source_root = sr_result.scalar_one_or_none()
|
||||
if source_root:
|
||||
source_root.name = name
|
||||
await db.commit()
|
||||
return {
|
||||
"id": source_root.id,
|
||||
"name": source_root.name,
|
||||
"path": source_root.path,
|
||||
}
|
||||
|
||||
# Otherwise it's a Folder row.
|
||||
folder = await get_user_folder(folder_id, current_user, db)
|
||||
|
||||
# Refuse to rename the bare source root mount through here.
|
||||
sr_check = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder.source_root_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
sr = sr_check.scalar_one_or_none()
|
||||
if sr and os.path.normpath(folder.path) == os.path.normpath(sr.path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot rename the source root mount; rename the docker mount instead.",
|
||||
)
|
||||
|
||||
old_path = os.path.normpath(folder.path).rstrip(os.sep)
|
||||
parent_dir = os.path.dirname(old_path)
|
||||
new_path = os.path.join(parent_dir, name)
|
||||
|
||||
if os.path.exists(new_path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"A folder named '{name}' already exists here",
|
||||
)
|
||||
|
||||
if is_nextcloud_path(old_path):
|
||||
# WebDAV MOVE keeps Nextcloud's oc_filecache + sharing metadata
|
||||
# consistent. NC's MOVE is recursive — descendants come along,
|
||||
# exactly like shutil.move.
|
||||
nc_move(current_user, old_path, new_path)
|
||||
else:
|
||||
try:
|
||||
shutil.move(old_path, new_path)
|
||||
except OSError as e:
|
||||
raise HTTPException(status_code=500, detail=f"Rename failed: {e}")
|
||||
|
||||
# Update folder paths: this row + every descendant. SQLite REPLACE
|
||||
# rewrites the prefix; we use the trailing separator on the LIKE
|
||||
# pattern so a folder named "foo" doesn't accidentally match "foobar".
|
||||
await db.execute(
|
||||
sql_update(Folder)
|
||||
.where(Folder.id == folder.id)
|
||||
.values(path=new_path, name=name)
|
||||
)
|
||||
descendant_prefix = old_path + os.sep
|
||||
descendants = await db.execute(
|
||||
select(Folder).where(Folder.path.like(descendant_prefix + '%'))
|
||||
)
|
||||
for d in descendants.scalars().all():
|
||||
d.path = new_path + d.path[len(old_path):]
|
||||
|
||||
# Update every photo whose filepath lives under the old prefix.
|
||||
photos_result = await db.execute(
|
||||
select(Photo).where(Photo.filepath.like(descendant_prefix + '%'))
|
||||
)
|
||||
for p in photos_result.scalars().all():
|
||||
p.filepath = new_path + p.filepath[len(old_path):]
|
||||
# Photos directly inside this folder (not in a subdir) won't match
|
||||
# the descendant_prefix LIKE if their old path was old_path + '/file'
|
||||
# — actually they DO match, since 'oldpath/file' starts with
|
||||
# 'oldpath/'. So the loop above already covers them.
|
||||
|
||||
await db.commit()
|
||||
return {
|
||||
"id": folder.id,
|
||||
"name": folder.name,
|
||||
"path": folder.path,
|
||||
}
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_folder(body: FolderCreate, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Create a new sub-folder under an existing Folder. Mirrors the
|
||||
create to disk so the next scan sees it. Body: { name, parent_id }.
|
||||
parent_id MUST be an existing Folder row id (any descendant of a
|
||||
source root); creating a brand-new top-level mount is a docker
|
||||
operation, not a UI one.
|
||||
"""
|
||||
name = _validate_folder_name(body.name)
|
||||
|
||||
parent = await get_user_folder(body.parent_id, current_user, db)
|
||||
|
||||
new_path = os.path.join(parent.path, name)
|
||||
if os.path.exists(new_path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"A folder named '{name}' already exists here",
|
||||
)
|
||||
|
||||
if is_nextcloud_path(new_path):
|
||||
# MKCOL via WebDAV so Nextcloud knows about the new collection.
|
||||
mkcol_for_user(current_user, new_path)
|
||||
else:
|
||||
try:
|
||||
os.makedirs(new_path, exist_ok=False)
|
||||
except OSError as e:
|
||||
raise HTTPException(status_code=500, detail=f"Create failed: {e}")
|
||||
|
||||
new_folder = Folder(
|
||||
name=name,
|
||||
path=new_path,
|
||||
source_root_id=parent.source_root_id,
|
||||
user_id=current_user.id,
|
||||
photo_count=0,
|
||||
)
|
||||
db.add(new_folder)
|
||||
await db.commit()
|
||||
await db.refresh(new_folder)
|
||||
return {
|
||||
"id": new_folder.id,
|
||||
"name": new_folder.name,
|
||||
"path": new_folder.path,
|
||||
"parent_id": parent.id,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{folder_id}")
|
||||
async def delete_folder(
|
||||
folder_id: str,
|
||||
mode: Literal['discard', 'permanent'] = Query('discard'),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Delete a folder. Behavior depends on mode:
|
||||
|
||||
- mode=discard (default): mark every photo whose filepath lives
|
||||
under this folder as is_discarded=true. The folder row, its
|
||||
descendant rows, and the on-disk directory are LEFT INTACT —
|
||||
the user can still recover photos from the discard pile, and
|
||||
a re-scan won't double-import them.
|
||||
|
||||
- mode=permanent: unlink every photo file under this folder,
|
||||
remove the photo + folder rows from the DB, and rmtree the
|
||||
on-disk directory. Irreversible.
|
||||
|
||||
Refuses to delete the bare source-root mount in either mode (deleting
|
||||
the docker mount through the UI would be a footgun).
|
||||
"""
|
||||
folder = await get_user_folder(folder_id, current_user, db)
|
||||
|
||||
sr_check = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder.source_root_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
sr = sr_check.scalar_one_or_none()
|
||||
if sr and os.path.normpath(folder.path) == os.path.normpath(sr.path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot delete the source root mount through the UI",
|
||||
)
|
||||
|
||||
folder_path = os.path.normpath(folder.path).rstrip(os.sep)
|
||||
descendant_prefix = folder_path + os.sep
|
||||
|
||||
# Collect every photo under this folder OR any descendant. We match
|
||||
# by filepath prefix instead of folder_id because that catches photos
|
||||
# in nested subfolders without a recursive folder walk.
|
||||
photos_result = await db.execute(
|
||||
select(Photo).where(
|
||||
(Photo.filepath == folder_path)
|
||||
| (Photo.filepath.like(descendant_prefix + '%'))
|
||||
)
|
||||
)
|
||||
photos = photos_result.scalars().all()
|
||||
|
||||
if mode == 'discard':
|
||||
from datetime import datetime
|
||||
now = datetime.utcnow()
|
||||
for p in photos:
|
||||
p.is_discarded = True
|
||||
p.discarded_at = now
|
||||
await db.commit()
|
||||
return {
|
||||
"status": "success",
|
||||
"mode": "discard",
|
||||
"discarded": len(photos),
|
||||
}
|
||||
|
||||
# mode == 'permanent'
|
||||
file_errors = 0
|
||||
folder_is_nc = is_nextcloud_path(folder_path)
|
||||
|
||||
if folder_is_nc:
|
||||
# One WebDAV DELETE on the folder itself does the recursive
|
||||
# delete (NC moves the whole tree to trashbin and updates
|
||||
# oc_filecache for everything inside). Skip per-photo unlinks.
|
||||
try:
|
||||
nc_delete(current_user, folder_path)
|
||||
except HTTPException as e:
|
||||
logger.error(f"Nextcloud DELETE failed for {folder_path}: {e.detail}")
|
||||
raise
|
||||
for p in photos:
|
||||
await db.delete(p)
|
||||
else:
|
||||
for p in photos:
|
||||
try:
|
||||
if p.filepath and os.path.exists(p.filepath):
|
||||
os.unlink(p.filepath)
|
||||
except OSError as e:
|
||||
file_errors += 1
|
||||
logger.error(f"Failed to unlink {p.filepath}: {e}")
|
||||
await db.delete(p)
|
||||
|
||||
# Delete this folder + every descendant Folder row.
|
||||
await db.execute(
|
||||
sql_delete(Folder).where(
|
||||
(Folder.id == folder.id)
|
||||
| (Folder.path.like(descendant_prefix + '%'))
|
||||
)
|
||||
)
|
||||
|
||||
if not folder_is_nc:
|
||||
try:
|
||||
if os.path.isdir(folder_path):
|
||||
shutil.rmtree(folder_path)
|
||||
except OSError as e:
|
||||
logger.error(f"Failed to rmtree {folder_path}: {e}")
|
||||
# Don't raise — DB rows are already gone, leaving an orphan
|
||||
# directory is the lesser evil.
|
||||
|
||||
await db.commit()
|
||||
return {
|
||||
"status": "success",
|
||||
"mode": "permanent",
|
||||
"deleted_photos": len(photos),
|
||||
"file_errors": file_errors,
|
||||
}
|
||||
|
||||
|
||||
async def _recompute_photo_hidden_flags(db: AsyncSession) -> None:
|
||||
"""Rematerialize photos.is_hidden from the full folder ancestry.
|
||||
|
||||
`photos.is_hidden` is true iff any ancestor folder in the photo's
|
||||
folder chain (including the folder the photo is directly in) has
|
||||
`folders.is_hidden = true`. Rather than do a recursive walk in
|
||||
Python, we lean on Postgres's WITH RECURSIVE to compute each
|
||||
folder's effective hidden state in a single query, then join on
|
||||
photos to bulk-update the flag.
|
||||
|
||||
Called after any folders.is_hidden toggle AND after moving photos
|
||||
between folders, since the photo's effective-hidden state can
|
||||
change even when no folder flag changes. Cheap — one O(folders)
|
||||
CTE + one O(photos) UPDATE. On a 13k-photo library this runs in
|
||||
under 50ms.
|
||||
"""
|
||||
from sqlalchemy import text as _text
|
||||
|
||||
await db.execute(
|
||||
_text("""
|
||||
WITH RECURSIVE folder_chain AS (
|
||||
-- Base: source-root folders (no parent_id). Their own
|
||||
-- is_hidden is the starting effective value.
|
||||
SELECT id, is_hidden AS effective_hidden
|
||||
FROM folders
|
||||
WHERE parent_id IS NULL
|
||||
UNION ALL
|
||||
-- Step: a child folder inherits from its parent. The
|
||||
-- effective flag is true if the parent's effective flag
|
||||
-- is true OR the child's own flag is true. Short-circuit
|
||||
-- would be nice but a plain OR does the job.
|
||||
SELECT f.id, (f.is_hidden OR fc.effective_hidden) AS effective_hidden
|
||||
FROM folders f
|
||||
JOIN folder_chain fc ON f.parent_id = fc.id
|
||||
)
|
||||
UPDATE photos p
|
||||
SET is_hidden = fc.effective_hidden
|
||||
FROM folder_chain fc
|
||||
WHERE p.folder_id = fc.id
|
||||
AND p.is_hidden IS DISTINCT FROM fc.effective_hidden
|
||||
""")
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{folder_id}/hide")
|
||||
async def set_folder_hidden(
|
||||
folder_id: str,
|
||||
body: FolderHide,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Toggle the "hide from views" flag on a folder or source root.
|
||||
|
||||
A hidden folder's photos are excluded from every cross-cutting view
|
||||
(All Photos, Map, Tags, People, Search, sidebar counts, duplicates)
|
||||
but remain fully indexed and visible when the user navigates
|
||||
directly into the folder. The flag cascades to every descendant
|
||||
folder via the photos.is_hidden recompute — the child folder's own
|
||||
`is_hidden` column stays where the user set it, but a photo under a
|
||||
hidden ancestor will still be marked hidden.
|
||||
|
||||
Accepts both Folder ids and SourceRoot ids. For a SourceRoot, we
|
||||
look up the root Folder row (the one matching source_root.path) and
|
||||
flip that — source roots themselves don't carry the column because
|
||||
the whole subtree lives on a single Folder row anyway.
|
||||
"""
|
||||
# SourceRoot path — resolve to the Folder row at the mount point.
|
||||
sr_result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
source_root = sr_result.scalar_one_or_none()
|
||||
|
||||
folder: Optional[Folder]
|
||||
if source_root:
|
||||
root_folder_result = await db.execute(
|
||||
select(Folder).where(
|
||||
Folder.source_root_id == source_root.id,
|
||||
Folder.user_id == current_user.id,
|
||||
Folder.path == os.path.normpath(source_root.path),
|
||||
)
|
||||
)
|
||||
folder = root_folder_result.scalar_one_or_none()
|
||||
if folder is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Source root has no indexed Folder row yet; scan first.",
|
||||
)
|
||||
else:
|
||||
folder = await get_user_folder(folder_id, current_user, db)
|
||||
|
||||
folder.is_hidden = bool(body.hidden)
|
||||
await db.flush()
|
||||
|
||||
# Rematerialize photos.is_hidden across the whole tree. Cheap
|
||||
# enough (tens of ms on a typical library) that we don't need to
|
||||
# scope the update to just this folder's subtree — doing it
|
||||
# globally also fixes any drift introduced by earlier moves.
|
||||
await _recompute_photo_hidden_flags(db)
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"id": folder.id,
|
||||
"name": folder.name,
|
||||
"path": folder.path,
|
||||
"is_hidden": folder.is_hidden,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{folder_id}/scan")
|
||||
async def scan_folder(folder_id: str, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Trigger manual re-scan of source root folder"""
|
||||
from app.tasks.celery import celery_app
|
||||
|
||||
result = await db.execute(select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == current_user.id))
|
||||
source_root = result.scalar_one_or_none()
|
||||
|
||||
if not source_root:
|
||||
raise HTTPException(status_code=404, detail="Source folder not found")
|
||||
|
||||
# Queue scan task using the task name defined in the decorator
|
||||
task = celery_app.send_task('scan_folder', args=[source_root.path, source_root.id])
|
||||
return {"status": "success", "message": f"Scan queued for {source_root.path}", "task_id": task.id}
|
||||
@@ -1,448 +0,0 @@
|
||||
"""
|
||||
Heaps API router
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import logging
|
||||
from typing import Optional, Literal
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func, update, insert, delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Heap, Photo, Folder
|
||||
from app.models.folders import SourceRoot
|
||||
from app.models.heaps import heap_photos
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user, get_user_heap, get_user_or_shared_heap
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Schemas ───────────────────────────────────────────────────────────────
|
||||
|
||||
class HeapCreate(BaseModel):
|
||||
name: str
|
||||
|
||||
|
||||
class HeapUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
is_active: Optional[bool] = None
|
||||
|
||||
|
||||
class HeapPhotosBody(BaseModel):
|
||||
photo_ids: list[str]
|
||||
|
||||
|
||||
class HeapConvertBody(BaseModel):
|
||||
target_id: str # folder id OR source root id
|
||||
mode: Literal['move', 'copy'] = 'move'
|
||||
delete_heap: bool = False
|
||||
# Optional subfolder name to create inside the target. If provided, the
|
||||
# actual destination is target_dir/subfolder_name (created if missing).
|
||||
# Path separators and dot-segments are rejected.
|
||||
subfolder_name: Optional[str] = None
|
||||
|
||||
|
||||
# ── Endpoints ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("")
|
||||
async def list_heaps(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List all heaps with photo counts."""
|
||||
# LEFT JOIN heap_photos and group so we can return counts in one query.
|
||||
count_subq = (
|
||||
select(
|
||||
heap_photos.c.heap_id,
|
||||
func.count(heap_photos.c.photo_id).label("photo_count"),
|
||||
)
|
||||
.group_by(heap_photos.c.heap_id)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
stmt = (
|
||||
select(Heap, count_subq.c.photo_count)
|
||||
.outerjoin(count_subq, Heap.id == count_subq.c.heap_id)
|
||||
.where(Heap.user_id == current_user.id)
|
||||
.order_by(Heap.created_at.asc())
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
rows = result.all()
|
||||
|
||||
return [
|
||||
{
|
||||
"id": h.id,
|
||||
"name": h.name,
|
||||
"is_active": bool(h.is_active),
|
||||
"created_at": h.created_at,
|
||||
"updated_at": h.updated_at,
|
||||
"photo_count": int(count or 0),
|
||||
}
|
||||
for h, count in rows
|
||||
]
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_heap(
|
||||
body: HeapCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Create a new heap."""
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Heap name is required")
|
||||
heap = Heap(name=name, user_id=current_user.id)
|
||||
db.add(heap)
|
||||
await db.commit()
|
||||
await db.refresh(heap)
|
||||
return {
|
||||
"id": heap.id,
|
||||
"name": heap.name,
|
||||
"is_active": bool(heap.is_active),
|
||||
"created_at": heap.created_at,
|
||||
"updated_at": heap.updated_at,
|
||||
"photo_count": 0,
|
||||
}
|
||||
|
||||
|
||||
@router.patch("/{heap_id}")
|
||||
async def update_heap(
|
||||
heap_id: str,
|
||||
body: HeapUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Rename a heap and/or toggle active state. Setting is_active=true on
|
||||
one heap deactivates all others (single-active invariant)."""
|
||||
heap = await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
if body.name is not None:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Heap name is required")
|
||||
heap.name = name
|
||||
|
||||
if body.is_active is not None:
|
||||
if body.is_active:
|
||||
# Clear active flag on all other heaps for this user
|
||||
await db.execute(
|
||||
update(Heap)
|
||||
.where(Heap.user_id == current_user.id)
|
||||
.values(is_active=False)
|
||||
)
|
||||
heap.is_active = True
|
||||
else:
|
||||
heap.is_active = False
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(heap)
|
||||
return {
|
||||
"id": heap.id,
|
||||
"name": heap.name,
|
||||
"is_active": bool(heap.is_active),
|
||||
"created_at": heap.created_at,
|
||||
"updated_at": heap.updated_at,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{heap_id}/duplicate", status_code=201)
|
||||
async def duplicate_heap(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Create a new heap with the same membership as an existing one. The
|
||||
new heap is named "{original} (copy)" and is never the active target —
|
||||
duplicating shouldn't quietly steal the user's T-key destination.
|
||||
"""
|
||||
source = await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
new_heap = Heap(name=f"{source.name} (copy)", is_active=False, user_id=current_user.id)
|
||||
db.add(new_heap)
|
||||
await db.flush() # populate new_heap.id without committing yet
|
||||
|
||||
# Bulk-copy the membership rows. SELECT photo_id FROM heap_photos WHERE
|
||||
# heap_id = :src — INSERT each into the new heap. Done as a single
|
||||
# INSERT...SELECT to avoid round-tripping ids through Python.
|
||||
member_rows = await db.execute(
|
||||
select(heap_photos.c.photo_id).where(heap_photos.c.heap_id == heap_id)
|
||||
)
|
||||
photo_ids = [row[0] for row in member_rows.all()]
|
||||
if photo_ids:
|
||||
await db.execute(
|
||||
insert(heap_photos),
|
||||
[{"heap_id": new_heap.id, "photo_id": pid} for pid in photo_ids],
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(new_heap)
|
||||
return {
|
||||
"id": new_heap.id,
|
||||
"name": new_heap.name,
|
||||
"is_active": False,
|
||||
"photo_count": len(photo_ids),
|
||||
"created_at": new_heap.created_at,
|
||||
"updated_at": new_heap.updated_at,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{heap_id}", status_code=204)
|
||||
async def delete_heap(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Delete a heap. Photos themselves are unaffected — only the membership
|
||||
rows in heap_photos cascade-delete."""
|
||||
heap = await get_user_heap(heap_id, current_user, db)
|
||||
await db.delete(heap)
|
||||
await db.commit()
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/{heap_id}/photo_ids")
|
||||
async def get_heap_photo_ids(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Return just the photo ids belonging to a heap. Used by the frontend
|
||||
to maintain a fast client-side membership lookup for the active heap
|
||||
(for the basket affordance on thumbnails) without fetching full photo
|
||||
records."""
|
||||
await get_user_or_shared_heap(heap_id, current_user, db)
|
||||
result = await db.execute(
|
||||
select(heap_photos.c.photo_id).where(heap_photos.c.heap_id == heap_id)
|
||||
)
|
||||
return [row[0] for row in result.all()]
|
||||
|
||||
|
||||
@router.post("/{heap_id}/photos")
|
||||
async def add_photos_to_heap(
|
||||
heap_id: str,
|
||||
body: HeapPhotosBody,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Add photos to a heap. Idempotent: re-adding existing members is a
|
||||
no-op (handled by an INSERT OR IGNORE-style filter on duplicates).
|
||||
Shared users with write permission can add their own photos."""
|
||||
_heap, permission = await get_user_or_shared_heap(heap_id, current_user, db)
|
||||
if permission == "read":
|
||||
raise HTTPException(status_code=403, detail="Read-only access to this heap")
|
||||
|
||||
if not body.photo_ids:
|
||||
return {"status": "success", "added": 0}
|
||||
|
||||
# Find which ids are already members so we don't violate the PK.
|
||||
existing = await db.execute(
|
||||
select(heap_photos.c.photo_id).where(
|
||||
heap_photos.c.heap_id == heap_id,
|
||||
heap_photos.c.photo_id.in_(body.photo_ids),
|
||||
)
|
||||
)
|
||||
existing_ids = {row[0] for row in existing.all()}
|
||||
new_ids = [pid for pid in body.photo_ids if pid not in existing_ids]
|
||||
|
||||
if new_ids:
|
||||
await db.execute(
|
||||
insert(heap_photos),
|
||||
[{"heap_id": heap_id, "photo_id": pid} for pid in new_ids],
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
return {"status": "success", "added": len(new_ids), "already_present": len(existing_ids)}
|
||||
|
||||
|
||||
@router.post("/{heap_id}/convert")
|
||||
async def convert_heap_to_folder(
|
||||
heap_id: str,
|
||||
body: HeapConvertBody,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Convert a heap into a folder by moving (or copying) every member
|
||||
photo into the target directory. Optionally deletes the heap row at
|
||||
the end.
|
||||
|
||||
target_id may be a Folder id or a SourceRoot id (matches the
|
||||
/photos/move convention so the same dropdown can populate it).
|
||||
"""
|
||||
heap = await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
# Resolve target_id → (target_dir, target_folder)
|
||||
sr_check = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == body.target_id)
|
||||
)
|
||||
source_root = sr_check.scalar_one_or_none()
|
||||
|
||||
if source_root is not None:
|
||||
parent_dir = source_root.path
|
||||
parent_source_root_id = source_root.id
|
||||
else:
|
||||
folder_check = await db.execute(
|
||||
select(Folder).where(Folder.id == body.target_id)
|
||||
)
|
||||
parent_folder = folder_check.scalar_one_or_none()
|
||||
if parent_folder is None:
|
||||
raise HTTPException(status_code=404, detail="Target folder not found")
|
||||
parent_dir = parent_folder.path
|
||||
parent_source_root_id = parent_folder.source_root_id
|
||||
|
||||
if not os.path.isdir(parent_dir):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Target parent does not exist: {parent_dir}",
|
||||
)
|
||||
|
||||
# Resolve target_dir, creating an optional subfolder if requested.
|
||||
if body.subfolder_name is not None:
|
||||
sub = body.subfolder_name.strip()
|
||||
if not sub:
|
||||
raise HTTPException(status_code=400, detail="Subfolder name cannot be empty")
|
||||
if '/' in sub or '\\' in sub or sub in ('.', '..'):
|
||||
raise HTTPException(status_code=400, detail="Invalid subfolder name")
|
||||
target_dir = os.path.join(parent_dir, sub)
|
||||
if not os.path.exists(target_dir):
|
||||
try:
|
||||
os.makedirs(target_dir)
|
||||
except OSError as e:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to create subfolder: {e}",
|
||||
)
|
||||
elif not os.path.isdir(target_dir):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"{target_dir} exists but is not a directory",
|
||||
)
|
||||
else:
|
||||
target_dir = parent_dir
|
||||
|
||||
# Ensure a Folder row for the target, reusing the scanner helper so
|
||||
# path normalization + dedupe stay consistent.
|
||||
from app.tasks.scan import get_or_create_folder
|
||||
target_folder = await get_or_create_folder(db, target_dir, parent_source_root_id)
|
||||
|
||||
# Fetch the heap's photos via the join table.
|
||||
photo_result = await db.execute(
|
||||
select(Photo)
|
||||
.join(heap_photos, Photo.id == heap_photos.c.photo_id)
|
||||
.where(heap_photos.c.heap_id == heap_id)
|
||||
)
|
||||
photos = photo_result.scalars().all()
|
||||
|
||||
moved = 0
|
||||
copied = 0
|
||||
errors: list[dict] = []
|
||||
|
||||
def _unique_target_name(directory: str, filename: str) -> Optional[str]:
|
||||
if not os.path.exists(os.path.join(directory, filename)):
|
||||
return filename
|
||||
stem, ext = os.path.splitext(filename)
|
||||
for i in range(1, 100):
|
||||
suffix = '' if i == 1 else f' {i}'
|
||||
candidate = f"{stem} (copy{suffix}){ext}"
|
||||
if not os.path.exists(os.path.join(directory, candidate)):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
for photo in photos:
|
||||
if not os.path.exists(photo.filepath):
|
||||
errors.append({"id": photo.id, "error": "source file missing"})
|
||||
continue
|
||||
|
||||
if body.mode == 'move':
|
||||
if photo.folder_id == target_folder.id:
|
||||
continue # already there
|
||||
new_path = os.path.join(target_dir, photo.filename)
|
||||
if os.path.exists(new_path):
|
||||
errors.append({"id": photo.id, "error": f"name collision: {photo.filename}"})
|
||||
continue
|
||||
try:
|
||||
shutil.move(photo.filepath, new_path)
|
||||
except OSError as e:
|
||||
errors.append({"id": photo.id, "error": str(e)})
|
||||
continue
|
||||
photo.filepath = new_path
|
||||
photo.folder_id = target_folder.id
|
||||
moved += 1
|
||||
else: # copy
|
||||
new_name = _unique_target_name(target_dir, photo.filename)
|
||||
if new_name is None:
|
||||
errors.append({"id": photo.id, "error": "too many name collisions"})
|
||||
continue
|
||||
new_path = os.path.join(target_dir, new_name)
|
||||
try:
|
||||
shutil.copy2(photo.filepath, new_path)
|
||||
except OSError as e:
|
||||
errors.append({"id": photo.id, "error": str(e)})
|
||||
continue
|
||||
new_photo = Photo(
|
||||
filepath=new_path,
|
||||
filename=new_name,
|
||||
folder_id=target_folder.id,
|
||||
file_hash=photo.file_hash,
|
||||
media_type=photo.media_type,
|
||||
original_format=photo.original_format,
|
||||
width=photo.width,
|
||||
height=photo.height,
|
||||
file_size=photo.file_size,
|
||||
taken_at=photo.taken_at,
|
||||
taken_at_source=photo.taken_at_source,
|
||||
user_title=photo.user_title,
|
||||
user_notes=photo.user_notes,
|
||||
rating=photo.rating,
|
||||
color_label=photo.color_label,
|
||||
exif_json=photo.exif_json,
|
||||
is_duplicate=True,
|
||||
processing_status='pending',
|
||||
)
|
||||
db.add(new_photo)
|
||||
copied += 1
|
||||
|
||||
if body.delete_heap:
|
||||
await db.delete(heap)
|
||||
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"mode": body.mode,
|
||||
"moved": moved,
|
||||
"copied": copied,
|
||||
"errors": errors,
|
||||
"heap_deleted": body.delete_heap,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{heap_id}/photos")
|
||||
async def remove_photos_from_heap(
|
||||
heap_id: str,
|
||||
body: HeapPhotosBody,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Remove photos from a heap. Removing a non-member is a no-op.
|
||||
Shared users with write permission can remove photos."""
|
||||
_heap, permission = await get_user_or_shared_heap(heap_id, current_user, db)
|
||||
if permission == "read":
|
||||
raise HTTPException(status_code=403, detail="Read-only access to this heap")
|
||||
|
||||
if not body.photo_ids:
|
||||
return {"status": "success", "removed": 0}
|
||||
|
||||
res = await db.execute(
|
||||
delete(heap_photos).where(
|
||||
heap_photos.c.heap_id == heap_id,
|
||||
heap_photos.c.photo_id.in_(body.photo_ids),
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
return {"status": "success", "removed": res.rowcount or 0}
|
||||
@@ -1,879 +0,0 @@
|
||||
"""
|
||||
Library API router for stats, scanning, and maintenance.
|
||||
|
||||
The /maintenance/* endpoints are surfaced through the frontend Settings
|
||||
panel. They're intentionally idempotent and operate by re-queueing the
|
||||
existing Celery tasks rather than doing any heavy lifting in the
|
||||
request thread.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select, func, update, or_, true as sa_true
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo
|
||||
from app.models.folders import SourceRoot, Folder
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _owner_filter(user: User, scope: str | None):
|
||||
"""Return a column expression scoping photos to the current user,
|
||||
or a pass-through true() when an admin requests global scope."""
|
||||
if scope == "global" and user.role == "admin":
|
||||
return sa_true()
|
||||
return Photo.user_id == user.id
|
||||
|
||||
|
||||
async def _active_source_root_paths(
|
||||
db: AsyncSession, user: User, scope: str | None
|
||||
) -> list[str]:
|
||||
"""Active SourceRoot.path values visible to this user, honouring the
|
||||
admin ?scope=global escape hatch."""
|
||||
q = select(SourceRoot.path).where(SourceRoot.is_active.is_(True))
|
||||
if not (scope == "global" and user.role == "admin"):
|
||||
q = q.where(SourceRoot.user_id == user.id)
|
||||
return (await db.execute(q)).scalars().all()
|
||||
|
||||
# Media types we accept in the regenerate-thumbnails request body. Mirrors
|
||||
# the values produced by `app.tasks.scan.get_media_type`.
|
||||
_VALID_MEDIA_TYPES = {'photo', 'raw', 'heic', 'video'}
|
||||
|
||||
@router.get("/stats")
|
||||
async def get_library_stats(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Get library statistics. Pass ?scope=global (admin only) for
|
||||
cross-user totals (used by the Settings page)."""
|
||||
owner = _owner_filter(current_user, scope)
|
||||
visible = owner & (Photo.is_discarded.is_(False)) & (Photo.is_hidden.is_(False))
|
||||
|
||||
all_photos_count = (
|
||||
await db.execute(select(func.count(Photo.id)).where(visible))
|
||||
).scalar() or 0
|
||||
|
||||
rated_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(visible, Photo.rating >= 1)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
colored_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
visible, Photo.color_label.is_not(None)
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
with_gps_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
visible, Photo.latitude.is_not(None)
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
duplicates_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
visible, Photo.is_duplicate.is_(True)
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
discarded_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
owner, Photo.is_hidden.is_(False), Photo.is_discarded.is_(True)
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
# Legacy split (kept for the existing /stats consumers).
|
||||
photo_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
owner,
|
||||
Photo.media_type.in_(['photo', 'heic', 'raw'])
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
video_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(owner, Photo.media_type == 'video')
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
size = (await db.execute(select(func.sum(Photo.file_size)).where(owner))).scalar() or 0
|
||||
|
||||
roots = sorted(await _active_source_root_paths(db, current_user, scope))
|
||||
|
||||
return {
|
||||
"all_photos": all_photos_count,
|
||||
"rated": rated_count,
|
||||
"colored": colored_count,
|
||||
"with_gps": with_gps_count,
|
||||
"duplicates": duplicates_count,
|
||||
"discarded": discarded_count,
|
||||
"total_photos": photo_count,
|
||||
"total_videos": video_count,
|
||||
"total_size": size,
|
||||
"total_size_gb": round(size / (1024**3), 2) if size else 0,
|
||||
"source_dirs": roots,
|
||||
}
|
||||
|
||||
@router.post("/scan")
|
||||
async def trigger_scan(current_user: User = Depends(get_current_user)):
|
||||
"""Trigger full library re-scan"""
|
||||
from app.tasks.scan import scan_all_source_roots
|
||||
|
||||
scan_all_source_roots.delay()
|
||||
|
||||
return {"status": "success", "message": "Library scan started"}
|
||||
|
||||
|
||||
@router.post("/maintenance/recover-stuck")
|
||||
async def recover_stuck_photos(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Reset photos stuck in 'processing' for more than 30 minutes back to
|
||||
'pending' so the pipeline can retry them. Returns the count of recovered
|
||||
photos."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=30)
|
||||
result = await db.execute(
|
||||
update(Photo)
|
||||
.where(
|
||||
Photo.processing_status == 'processing',
|
||||
Photo.updated_at < cutoff,
|
||||
)
|
||||
.values(
|
||||
processing_status='pending',
|
||||
processing_error='Auto-recovered from stuck processing state',
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
count = result.rowcount
|
||||
if count:
|
||||
logger.info("Recovered %d stuck photos back to pending", count)
|
||||
return {"status": "success", "recovered": count}
|
||||
|
||||
|
||||
@router.post("/backfill-gps")
|
||||
async def trigger_backfill_gps(current_user: User = Depends(get_current_user)):
|
||||
"""Re-run EXIF metadata extraction on every photo that's still missing
|
||||
GPS coordinates. Useful after fixing the EXIF parser, or any time the
|
||||
Map view looks emptier than expected. Returns immediately — work runs
|
||||
on the Celery worker."""
|
||||
from app.tasks.scan import backfill_gps
|
||||
|
||||
backfill_gps.delay()
|
||||
return {"status": "success", "message": "GPS backfill queued"}
|
||||
|
||||
|
||||
@router.post("/maintenance/backfill-taken-at")
|
||||
async def trigger_backfill_taken_at(current_user: User = Depends(get_current_user)):
|
||||
"""Re-run extract_metadata on every non-manual photo to recompute
|
||||
taken_at with the current EXIF-priority list and path-based fallback.
|
||||
Useful after the date-extraction logic changes (e.g. dropping the
|
||||
ModifyDate fallback). Manual edits are preserved."""
|
||||
from app.services.metadata import backfill_taken_at
|
||||
|
||||
backfill_taken_at.delay()
|
||||
return {"status": "success", "message": "taken_at backfill queued"}
|
||||
|
||||
|
||||
@router.get("/scan/status")
|
||||
async def get_scan_status(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Get current scan status"""
|
||||
import redis
|
||||
from app.config import settings
|
||||
|
||||
# Connect to Redis to get scan status
|
||||
r = redis.Redis.from_url(settings.redis_url)
|
||||
|
||||
# Get scan status from Redis (set by worker tasks)
|
||||
is_scanning = r.get('scan:active') == b'true'
|
||||
current_folder = r.get('scan:current_folder')
|
||||
processed_files = int(r.get('scan:processed_files') or 0)
|
||||
total_files = int(r.get('scan:total_files') or 0)
|
||||
errors = r.lrange('scan:errors', 0, -1)
|
||||
|
||||
return {
|
||||
"is_scanning": is_scanning,
|
||||
"current_folder": current_folder.decode() if current_folder else None,
|
||||
"processed_files": processed_files,
|
||||
"total_files": total_files,
|
||||
"errors": [e.decode() for e in errors] if errors else []
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Maintenance endpoints — surfaced via the Settings panel.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class RegenerateThumbnailsRequest(BaseModel):
|
||||
"""Optional filters narrowing which photos get re-queued. With both
|
||||
fields omitted the request resets every photo in the library."""
|
||||
media_types: Optional[List[str]] = Field(
|
||||
default=None,
|
||||
description="Restrict to these media_type values (photo/raw/heic/video).",
|
||||
)
|
||||
only_failed: bool = Field(
|
||||
default=False,
|
||||
description="If true, only re-queue photos whose processing_status is 'failed'.",
|
||||
)
|
||||
only_pending: bool = Field(
|
||||
default=False,
|
||||
description="If true, only (re-)queue photos whose processing_status is 'pending'. "
|
||||
"Useful for kicking rows that were created by a scan but never had "
|
||||
"their thumbnail task picked up.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/maintenance/thumbnail-stats")
|
||||
async def get_thumbnail_stats(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Counts of photos by processing_status, plus a media-type breakdown
|
||||
so the Settings panel can show the user what's outstanding."""
|
||||
owner = _owner_filter(current_user, scope)
|
||||
status_rows = (
|
||||
await db.execute(
|
||||
select(Photo.processing_status, func.count(Photo.id))
|
||||
.where(owner)
|
||||
.group_by(Photo.processing_status)
|
||||
)
|
||||
).all()
|
||||
|
||||
media_rows = (
|
||||
await db.execute(
|
||||
select(Photo.media_type, func.count(Photo.id))
|
||||
.where(owner)
|
||||
.group_by(Photo.media_type)
|
||||
)
|
||||
).all()
|
||||
|
||||
by_status = {status or 'unknown': count for status, count in status_rows}
|
||||
by_media_type = {media or 'unknown': count for media, count in media_rows}
|
||||
total = sum(by_status.values())
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"pending": by_status.get('pending', 0),
|
||||
"processing": by_status.get('processing', 0),
|
||||
"completed": by_status.get('completed', 0),
|
||||
"failed": by_status.get('failed', 0),
|
||||
"by_media_type": by_media_type,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/maintenance/regenerate-thumbnails")
|
||||
async def regenerate_thumbnails(
|
||||
body: RegenerateThumbnailsRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Reset matching photos' on-disk thumbnail directories and re-queue
|
||||
Celery thumbnail generation. Used by the Settings panel for the
|
||||
'regenerate video thumbnails' / 'regenerate failed' buttons.
|
||||
|
||||
Files on disk are removed under /data/thumbs/<photo_id>/ so the next
|
||||
request to /photos/{id}/thumb/{size} actually re-generates instead of
|
||||
serving the stale placeholder.
|
||||
"""
|
||||
from app.tasks.thumbs import generate_thumbnails
|
||||
|
||||
owner = _owner_filter(current_user, scope)
|
||||
|
||||
# Validate media_types early so a typo can't silently match nothing.
|
||||
media_types = body.media_types
|
||||
if media_types is not None:
|
||||
invalid = [m for m in media_types if m not in _VALID_MEDIA_TYPES]
|
||||
if invalid:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Invalid media_types: {invalid}. "
|
||||
f"Allowed: {sorted(_VALID_MEDIA_TYPES)}",
|
||||
}
|
||||
|
||||
query = select(Photo).where(owner)
|
||||
if media_types:
|
||||
query = query.where(Photo.media_type.in_(media_types))
|
||||
if body.only_failed:
|
||||
query = query.where(Photo.processing_status == 'failed')
|
||||
if body.only_pending:
|
||||
query = query.where(Photo.processing_status == 'pending')
|
||||
|
||||
photos = (await db.execute(query)).scalars().all()
|
||||
|
||||
cleared_dirs = 0
|
||||
file_errors = 0
|
||||
for photo in photos:
|
||||
thumb_dir = f"/data/thumbs/{photo.id}"
|
||||
if os.path.isdir(thumb_dir):
|
||||
try:
|
||||
shutil.rmtree(thumb_dir)
|
||||
cleared_dirs += 1
|
||||
except OSError as e:
|
||||
file_errors += 1
|
||||
logger.warning(f"Could not clear thumb dir {thumb_dir}: {e}")
|
||||
photo.processing_status = 'pending'
|
||||
photo.processing_error = None
|
||||
photo.thumb_small = None
|
||||
photo.thumb_medium = None
|
||||
photo.thumb_large = None
|
||||
|
||||
await db.commit()
|
||||
|
||||
# Queue celery tasks AFTER the commit so the worker sees the reset
|
||||
# state when it picks the job up.
|
||||
queued = 0
|
||||
for photo in photos:
|
||||
try:
|
||||
generate_thumbnails.delay(photo.id)
|
||||
queued += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue thumbnail job for {photo.id}: {e}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"matched": len(photos),
|
||||
"queued": queued,
|
||||
"cleared_dirs": cleared_dirs,
|
||||
"file_errors": file_errors,
|
||||
"filters": {
|
||||
"media_types": media_types,
|
||||
"only_failed": body.only_failed,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/worker-status")
|
||||
async def get_worker_status(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Diagnostics for the Celery worker fleet + recent task failures.
|
||||
|
||||
Surfaced in the Settings panel so the user can spot a stuck queue or
|
||||
a worker that's gone away without tailing container logs. Returns:
|
||||
|
||||
- workers: list of {name, status, active, concurrency, queues}
|
||||
derived from celery_app.control.inspect(). `status` is 'online'
|
||||
when ping succeeds, 'unreachable' otherwise. Empty list means no
|
||||
workers are responding at all (broker down, container crashed,
|
||||
wrong queue routing, etc.).
|
||||
- queues: per-queue depth read from Redis (LLEN of each queue key
|
||||
used by celery.kombu). Mirrors what tasks are waiting to be
|
||||
picked up.
|
||||
- failures: aggregate count of photos with processing_status='failed'
|
||||
plus the most recent N error messages so the user can see *why*
|
||||
things failed without opening the DB.
|
||||
- broker_ok: bool — could we even reach Redis?
|
||||
"""
|
||||
owner = _owner_filter(current_user, scope)
|
||||
from app.tasks.celery import celery_app
|
||||
from app.config import settings
|
||||
import redis as _redis
|
||||
|
||||
# ----- Celery inspect (workers + active tasks) -------------------------
|
||||
# Each inspect.* call is a separate broadcast-and-wait with its own
|
||||
# timeout, so running them serially multiplies the wait. Fan them out
|
||||
# to threads and gather, collapsing 6 × timeout into ~1 × timeout.
|
||||
# Timeout dropped to 0.5s — a responsive worker answers within a few
|
||||
# ms; anything past that is effectively "not responding" for the
|
||||
# purposes of a settings dashboard.
|
||||
import asyncio
|
||||
workers: list[dict] = []
|
||||
inspect_error: Optional[str] = None
|
||||
try:
|
||||
inspect = celery_app.control.inspect(timeout=0.5)
|
||||
ping, active, reserved, scheduled, stats, active_queues = await asyncio.gather(
|
||||
asyncio.to_thread(inspect.ping),
|
||||
asyncio.to_thread(inspect.active),
|
||||
asyncio.to_thread(inspect.reserved),
|
||||
asyncio.to_thread(inspect.scheduled),
|
||||
asyncio.to_thread(inspect.stats),
|
||||
asyncio.to_thread(inspect.active_queues),
|
||||
)
|
||||
ping = ping or {}
|
||||
active = active or {}
|
||||
reserved = reserved or {}
|
||||
scheduled = scheduled or {}
|
||||
stats = stats or {}
|
||||
active_queues = active_queues or {}
|
||||
|
||||
worker_names = set(ping) | set(active) | set(stats)
|
||||
for name in sorted(worker_names):
|
||||
wstats = stats.get(name) or {}
|
||||
pool = wstats.get('pool') or {}
|
||||
workers.append({
|
||||
"name": name,
|
||||
"status": "online" if name in ping else "unreachable",
|
||||
"active": len(active.get(name, []) or []),
|
||||
"reserved": len(reserved.get(name, []) or []),
|
||||
"scheduled": len(scheduled.get(name, []) or []),
|
||||
"concurrency": pool.get('max-concurrency'),
|
||||
"processed": (wstats.get('total') or {}),
|
||||
"queues": [q.get('name') for q in (active_queues.get(name) or [])],
|
||||
"active_tasks": [
|
||||
{
|
||||
"id": t.get('id'),
|
||||
"name": t.get('name'),
|
||||
"args": t.get('args'),
|
||||
"time_start": t.get('time_start'),
|
||||
}
|
||||
for t in (active.get(name) or [])[:10]
|
||||
],
|
||||
})
|
||||
except Exception as e:
|
||||
inspect_error = str(e)
|
||||
logger.warning(f"Celery inspect failed: {e}")
|
||||
|
||||
# ----- Broker / queue depth --------------------------------------------
|
||||
broker_ok = False
|
||||
queue_depths: dict[str, int] = {}
|
||||
broker_error: Optional[str] = None
|
||||
try:
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
r.ping()
|
||||
broker_ok = True
|
||||
for q in ('default', 'high', 'low'):
|
||||
try:
|
||||
queue_depths[q] = int(r.llen(q) or 0)
|
||||
except Exception:
|
||||
queue_depths[q] = 0
|
||||
except Exception as e:
|
||||
broker_error = str(e)
|
||||
logger.warning(f"Redis broker unreachable: {e}")
|
||||
|
||||
# ----- Recent task failures from the photos table ----------------------
|
||||
failed_total = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(owner, Photo.processing_status == 'failed')
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
recent_failed_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Photo.id,
|
||||
Photo.filename,
|
||||
Photo.media_type,
|
||||
Photo.processing_error,
|
||||
Photo.updated_at,
|
||||
)
|
||||
.where(owner, Photo.processing_status == 'failed')
|
||||
.order_by(Photo.updated_at.desc().nullslast())
|
||||
.limit(20)
|
||||
)
|
||||
).all()
|
||||
|
||||
recent_failures = [
|
||||
{
|
||||
"photo_id": row[0],
|
||||
"filename": row[1],
|
||||
"media_type": row[2],
|
||||
"error": (row[3] or '')[:500],
|
||||
"updated_at": row[4].isoformat() if row[4] else None,
|
||||
}
|
||||
for row in recent_failed_rows
|
||||
]
|
||||
|
||||
# ----- Most recent scan errors (Redis list) ----------------------------
|
||||
scan_errors: list[str] = []
|
||||
try:
|
||||
if broker_ok:
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
raw = r.lrange('scan:errors', 0, 19) or []
|
||||
scan_errors = [e.decode(errors='replace') for e in raw]
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not read scan:errors: {e}")
|
||||
|
||||
return {
|
||||
"broker_ok": broker_ok,
|
||||
"broker_error": broker_error,
|
||||
"inspect_error": inspect_error,
|
||||
"workers": workers,
|
||||
"worker_count": len(workers),
|
||||
"queues": queue_depths,
|
||||
"failures": {
|
||||
"total": failed_total,
|
||||
"recent": recent_failures,
|
||||
},
|
||||
"scan_errors": scan_errors,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/pipeline-stats")
|
||||
async def get_pipeline_stats(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Per-stage progress across the ingestion pipeline.
|
||||
|
||||
Returns a `{stage_key: {done, total, label}}` map so the Settings
|
||||
panel can render one progress bar per stage. `total` is the number
|
||||
of non-discarded photos the stage is *expected* to run on — which is
|
||||
every non-discarded photo for most stages, or a narrower subset when
|
||||
a stage is image-only (e.g. embeddings don't run on videos).
|
||||
|
||||
Keep the shape flat + serialisable; the frontend turns it straight
|
||||
into a list of rows without needing to know about the models.
|
||||
"""
|
||||
from app.models.tags import photo_tags # association Table, not a model
|
||||
|
||||
owner = _owner_filter(current_user, scope)
|
||||
not_discarded = owner & Photo.is_discarded.is_(False)
|
||||
|
||||
async def scalar_count(query):
|
||||
return (await db.execute(query)).scalar() or 0
|
||||
|
||||
# Total non-discarded photos — the denominator for most stages.
|
||||
total_photos = await scalar_count(
|
||||
select(func.count(Photo.id)).where(not_discarded)
|
||||
)
|
||||
|
||||
# Image-only denominator (embeddings, tags, faces, OCR, phash). We
|
||||
# exclude videos because those stages either don't apply or run off
|
||||
# the extracted video frame which is treated separately.
|
||||
total_images = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.media_type != 'video'
|
||||
)
|
||||
)
|
||||
|
||||
completed = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.processing_status == 'completed'
|
||||
)
|
||||
)
|
||||
with_exif = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.exif_json.is_not(None)
|
||||
)
|
||||
)
|
||||
with_gps = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded,
|
||||
Photo.latitude.is_not(None),
|
||||
Photo.longitude.is_not(None),
|
||||
)
|
||||
)
|
||||
with_phash = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.phash.is_not(None)
|
||||
)
|
||||
)
|
||||
|
||||
duplicate_groups = await scalar_count(
|
||||
select(func.count(func.distinct(Photo.duplicate_group_id))).where(
|
||||
not_discarded, Photo.duplicate_group_id.is_not(None)
|
||||
)
|
||||
)
|
||||
duplicate_members = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.duplicate_group_id.is_not(None)
|
||||
)
|
||||
)
|
||||
|
||||
# Ordered list so the frontend renders stages in pipeline order
|
||||
# without needing to know the sequence itself.
|
||||
stages = [
|
||||
{
|
||||
"key": "thumbnails",
|
||||
"label": "Thumbnails & pHash",
|
||||
"done": completed,
|
||||
"total": total_photos,
|
||||
"hint": "Generated on scan. Unlocks every downstream stage.",
|
||||
},
|
||||
{
|
||||
"key": "exif",
|
||||
"label": "EXIF metadata",
|
||||
"done": with_exif,
|
||||
"total": total_photos,
|
||||
"hint": "Camera, lens, capture time. Required for GPS + taken_at.",
|
||||
},
|
||||
{
|
||||
"key": "gps",
|
||||
"label": "GPS coordinates",
|
||||
"done": with_gps,
|
||||
"total": total_photos,
|
||||
"hint": "Subset of EXIF. Drives the map view; many photos legitimately have none.",
|
||||
"partial": True, # not every photo is expected to have GPS
|
||||
},
|
||||
{
|
||||
"key": "phash",
|
||||
"label": "Perceptual hashes",
|
||||
"done": with_phash,
|
||||
"total": total_images,
|
||||
"hint": "Feeds duplicate detection.",
|
||||
},
|
||||
{
|
||||
"key": "duplicates",
|
||||
"label": "Duplicate groups",
|
||||
"done": duplicate_groups,
|
||||
"total": duplicate_groups, # same — current count, not a progress ratio
|
||||
"hint": f"{duplicate_members} photos in {duplicate_groups} groups. Run regroup_duplicates after new imports.",
|
||||
"standalone": True,
|
||||
},
|
||||
]
|
||||
|
||||
return {
|
||||
"total_photos": total_photos,
|
||||
"total_images": total_images,
|
||||
"stages": stages,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/missing-stats")
|
||||
async def get_missing_stats(current_user: User = Depends(get_current_user)):
|
||||
"""Count photos whose files no longer exist on disk under a mounted
|
||||
source root. Surfaced in Settings so the user can see a number before
|
||||
pulling the trigger on prune-missing. Cheap enough to call freely."""
|
||||
from app.services.cleanup import prune_missing_photos
|
||||
return await prune_missing_photos(dry_run=True)
|
||||
|
||||
|
||||
@router.post("/maintenance/prune-missing")
|
||||
async def run_prune_missing(current_user: User = Depends(get_current_user)):
|
||||
"""Actually delete the orphaned photo rows reported by /missing-stats.
|
||||
Common cause: PHOTO_DIRS in .env was repointed at a different library
|
||||
leaving every old row dangling. Skips any photo whose source root
|
||||
isn't currently mounted (almost always means an unmounted drive)."""
|
||||
from app.services.cleanup import prune_missing_photos
|
||||
try:
|
||||
return {"status": "success", **(await prune_missing_photos(dry_run=False))}
|
||||
except Exception as e:
|
||||
logger.error(f"Prune missing failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@router.post("/maintenance/cleanup")
|
||||
async def run_data_integrity_cleanup(current_user: User = Depends(get_current_user)):
|
||||
"""Re-run the source-roots / folders / photos data-integrity cleanup
|
||||
that normally only runs on backend startup. Idempotent."""
|
||||
from app.services.cleanup import cleanup_data_integrity
|
||||
|
||||
try:
|
||||
await cleanup_data_integrity()
|
||||
return {"status": "success"}
|
||||
except Exception as e:
|
||||
logger.error(f"Manual cleanup failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Duplicate detection
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/duplicates/groups")
|
||||
async def get_duplicate_groups(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Return every duplicate group with its members.
|
||||
|
||||
Drives the frontend grouped grid view in the Duplicates section. One
|
||||
SQL query, bucketed in Python — no N+1, no per-member fetch. Groups
|
||||
are sorted by member_count DESC then earliest taken_at DESC so the
|
||||
biggest / most recent clusters bubble to the top.
|
||||
|
||||
Each group also carries a `reason` field:
|
||||
* "exact" — every member shares the same SHA-256 (true byte
|
||||
duplicates that the perceptual hash trivially caught)
|
||||
* "similar" — members differ at the byte level but match perceptually
|
||||
"""
|
||||
owner = _owner_filter(current_user, scope)
|
||||
|
||||
# Restrict to photos under an active SourceRoot in the user's
|
||||
# settings. Folder.source_root_id alone isn't trustworthy: Nextcloud's
|
||||
# "move to trash" flow leaves Folder rows like `…/files/.delete/
|
||||
# purge-1` still wired to the original source_root_id, leaking their
|
||||
# photos into this view as ghost paths the user never configured.
|
||||
active_root_paths = await _active_source_root_paths(db, current_user, scope)
|
||||
if not active_root_paths:
|
||||
return {"groups": [], "total_groups": 0, "total_members": 0}
|
||||
|
||||
folder_in_scope = or_(
|
||||
*[
|
||||
(Folder.path == p) | (Folder.path.like(p + '/%'))
|
||||
for p in active_root_paths
|
||||
]
|
||||
)
|
||||
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Photo.id,
|
||||
Photo.filename,
|
||||
Photo.filepath,
|
||||
Photo.taken_at,
|
||||
Photo.file_size,
|
||||
Photo.width,
|
||||
Photo.height,
|
||||
Photo.thumb_small,
|
||||
Photo.file_hash,
|
||||
Photo.folder_id,
|
||||
Photo.media_type,
|
||||
Photo.duplicate_group_id,
|
||||
)
|
||||
.join(Folder, Folder.id == Photo.folder_id)
|
||||
.where(owner)
|
||||
.where(Photo.duplicate_group_id.is_not(None))
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.where(Photo.is_hidden.is_(False))
|
||||
.where(folder_in_scope)
|
||||
.order_by(Photo.duplicate_group_id)
|
||||
)
|
||||
).all()
|
||||
|
||||
# Bucket members by group_id. filepath is included so the
|
||||
# Duplicates view can show "which folder does this copy live in"
|
||||
# — the discriminator the user needs to pick a winner.
|
||||
groups: dict[str, list[dict]] = {}
|
||||
for row in rows:
|
||||
member = {
|
||||
"id": row[0],
|
||||
"filename": row[1],
|
||||
"filepath": row[2],
|
||||
"taken_at": row[3].isoformat() if row[3] else None,
|
||||
"file_size": row[4],
|
||||
"width": row[5],
|
||||
"height": row[6],
|
||||
"thumb_small": row[7],
|
||||
"file_hash": row[8],
|
||||
"folder_id": row[9],
|
||||
"media_type": row[10],
|
||||
}
|
||||
groups.setdefault(row[11], []).append(member)
|
||||
|
||||
def earliest(g: list[dict]) -> str:
|
||||
# Used as a secondary sort key. Photos with no taken_at sort last
|
||||
# by returning a far-future sentinel.
|
||||
taken = [m["taken_at"] for m in g if m["taken_at"]]
|
||||
return min(taken) if taken else "9999"
|
||||
|
||||
out = []
|
||||
for group_id, members in groups.items():
|
||||
if len(members) < 2:
|
||||
# Defensive: a regroup race could leave a singleton briefly.
|
||||
# Skip it so the UI never shows a "group of 1".
|
||||
continue
|
||||
# exact iff every member shares the same non-null file_hash
|
||||
# (true byte-identical copies that pHash also caught). Anything
|
||||
# else — different hashes, missing hashes — counts as "similar".
|
||||
all_hashes = [m["file_hash"] for m in members]
|
||||
reason = (
|
||||
"exact"
|
||||
if len(set(all_hashes)) == 1 and all_hashes[0] is not None
|
||||
else "similar"
|
||||
)
|
||||
out.append({
|
||||
"group_id": group_id,
|
||||
"member_count": len(members),
|
||||
"reason": reason,
|
||||
"members": members,
|
||||
})
|
||||
|
||||
out.sort(key=lambda g: (-g["member_count"], earliest(g["members"])))
|
||||
return {
|
||||
"groups": out,
|
||||
"total_groups": len(out),
|
||||
"total_members": sum(g["member_count"] for g in out),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/maintenance/regroup-duplicates")
|
||||
async def trigger_regroup_duplicates(current_user: User = Depends(get_current_user)):
|
||||
"""Recompute duplicate groups from current perceptual hashes.
|
||||
|
||||
Fires the celery `regroup_duplicates` task which walks every photo's
|
||||
phash, clusters by Hamming distance, and rewrites duplicate_group_id /
|
||||
is_duplicate columns. Idempotent."""
|
||||
from app.tasks.thumbs import regroup_duplicates_task
|
||||
try:
|
||||
regroup_duplicates_task.delay()
|
||||
return {"status": "queued"}
|
||||
except Exception as e:
|
||||
logger.error(f"Regroup queue failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@router.post("/maintenance/backfill-phashes")
|
||||
async def trigger_backfill_phashes(current_user: User = Depends(get_current_user)):
|
||||
"""Compute perceptual hashes for every photo currently missing one.
|
||||
|
||||
One-shot recovery path for libraries that existed before the phash
|
||||
column was added — the thumbs worker computes phash for everything
|
||||
new, but old rows need a backfill pass."""
|
||||
from app.tasks.thumbs import backfill_phashes
|
||||
try:
|
||||
backfill_phashes.delay()
|
||||
return {"status": "queued"}
|
||||
except Exception as e:
|
||||
logger.error(f"Backfill queue failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@router.post("/maintenance/backfill-video-cache")
|
||||
async def trigger_backfill_video_cache(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Pre-transcode every active video in the library so /playback hits
|
||||
the cache on first user click instead of paying the encode cost
|
||||
inline. Idempotent — pretranscode_video skips photos whose cache is
|
||||
already populated and current. CPU-bound; runs on the low-priority
|
||||
queue so it doesn't fight thumbnails or other user-facing tasks."""
|
||||
from app.tasks.video import pretranscode_video
|
||||
result = await db.execute(
|
||||
select(Photo.id, Photo.filepath).where(
|
||||
Photo.media_type == 'video',
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
)
|
||||
rows = result.all()
|
||||
queued = 0
|
||||
for photo_id, filepath in rows:
|
||||
try:
|
||||
pretranscode_video.delay(photo_id, filepath)
|
||||
queued += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"failed to queue pretranscode for {photo_id}: {e}")
|
||||
return {"status": "queued", "count": queued}
|
||||
|
||||
|
||||
@router.post("/maintenance/start-watcher")
|
||||
async def start_file_watcher(current_user: User = Depends(get_current_user)):
|
||||
"""Start the filesystem watcher. Uses a Redis lock so only one
|
||||
instance runs at a time — safe to call repeatedly."""
|
||||
from app.tasks.scan import watch_folders
|
||||
try:
|
||||
watch_folders.apply_async(countdown=2)
|
||||
return {"status": "queued"}
|
||||
except Exception as e:
|
||||
logger.error(f"Watcher queue failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
@@ -1,224 +0,0 @@
|
||||
"""Internal webhook receiver for Nextcloud file events.
|
||||
|
||||
Replaces the watchfiles-based `watch_folders` Celery task: instead of
|
||||
mule polling the bind-mount with inotify, Nextcloud's `webhook_listeners`
|
||||
app POSTs here on every NodeCreated / NodeWritten / NodeDeleted /
|
||||
NodeRenamed event, and we dispatch the same scan_folder /
|
||||
handle_file_deletion machinery that the watcher used.
|
||||
|
||||
Auth: `Authorization: Bearer <NEXTCLOUD_WEBHOOK_SECRET>` header.
|
||||
constant_time compare. 401 on mismatch, 401 also when the secret isn't
|
||||
configured (fail closed).
|
||||
|
||||
The route is intentionally outside `/api/v1/photos/...` so it doesn't
|
||||
get caught by the per-user auth middleware — webhook requests come
|
||||
from Nextcloud as a service principal, not as a logged-in user. They
|
||||
get NO mule app session.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Header, HTTPException, Request, status
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.folders import SourceRoot
|
||||
from app.services.nextcloud_dav import NEXTCLOUD_USERS_ROOT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# Same list the on-disk watcher used (app.tasks.scan.SUPPORTED_EXTENSIONS).
|
||||
# Imported lazily inside the handler so this module can load without
|
||||
# pulling in the tasks package at startup.
|
||||
def _supported_extensions() -> set[str]:
|
||||
from app.tasks.scan import SUPPORTED_EXTENSIONS
|
||||
return SUPPORTED_EXTENSIONS
|
||||
|
||||
|
||||
def _expected_secret() -> str | None:
|
||||
return os.environ.get("NEXTCLOUD_WEBHOOK_SECRET") or None
|
||||
|
||||
|
||||
def _nc_path_to_abs(nc_path: str) -> str | None:
|
||||
"""Map a Nextcloud-internal path (`/admin/files/Photos/foo.jpg`) to
|
||||
the absolute bind-mount path mule's workers operate on
|
||||
(`/nextcloud-users/admin/files/Photos/foo.jpg`).
|
||||
|
||||
Returns None for paths that don't sit under `<user>/files/...`
|
||||
(NC also emits events for trashbin, versions, etc — we ignore
|
||||
those).
|
||||
"""
|
||||
if not nc_path or not nc_path.startswith("/"):
|
||||
return None
|
||||
parts = nc_path.lstrip("/").split("/", 2)
|
||||
if len(parts) < 3 or parts[1] != "files":
|
||||
return None
|
||||
return os.path.join(NEXTCLOUD_USERS_ROOT, parts[0], "files", parts[2])
|
||||
|
||||
|
||||
def _classify(event_class: str) -> str | None:
|
||||
"""Bucket the full event class string into the four buckets we act
|
||||
on. Returns None for events we don't care about (Before*, copy,
|
||||
touched, etc)."""
|
||||
short = event_class.rsplit("\\", 1)[-1]
|
||||
return {
|
||||
"NodeCreatedEvent": "created",
|
||||
"NodeWrittenEvent": "written",
|
||||
"NodeDeletedEvent": "deleted",
|
||||
"NodeRenamedEvent": "renamed",
|
||||
}.get(short)
|
||||
|
||||
|
||||
async def _source_root_for(parent_dir: str) -> str | None:
|
||||
"""Find the SourceRoot id whose path contains `parent_dir`."""
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active == True) # noqa: E712
|
||||
)
|
||||
roots = result.scalars().all()
|
||||
normalized = os.path.normpath(parent_dir)
|
||||
for sr in roots:
|
||||
root_path = os.path.normpath(sr.path)
|
||||
if normalized == root_path or normalized.startswith(root_path + os.sep):
|
||||
return sr.id
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/nc-webhook")
|
||||
async def nc_webhook(
|
||||
request: Request,
|
||||
authorization: str | None = Header(default=None),
|
||||
):
|
||||
"""Receive a Nextcloud file event and dispatch the matching
|
||||
scan_folder / handle_file_deletion task. Returns 204 on success
|
||||
(Nextcloud doesn't care about the body)."""
|
||||
expected = _expected_secret()
|
||||
if not expected:
|
||||
# Fail closed: a misconfigured server should reject webhooks
|
||||
# rather than accept arbitrary POSTs.
|
||||
logger.error("nc-webhook hit but NEXTCLOUD_WEBHOOK_SECRET is not set")
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
|
||||
|
||||
if not authorization or not authorization.startswith("Bearer "):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
|
||||
presented = authorization[len("Bearer "):]
|
||||
if not hmac.compare_digest(presented, expected):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
|
||||
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="malformed json")
|
||||
|
||||
event = payload.get("event") or {}
|
||||
event_class = event.get("class") or ""
|
||||
bucket = _classify(event_class)
|
||||
if bucket is None:
|
||||
return {"status": "ignored", "reason": "unwanted event"}
|
||||
|
||||
# Import lazily so this router can load before the celery app is
|
||||
# ready — important when the backend boots before broker is up.
|
||||
from app.tasks.scan import (
|
||||
scan_folder,
|
||||
handle_file_deletion,
|
||||
handle_directory_deletion,
|
||||
handle_directory_rename,
|
||||
)
|
||||
|
||||
supported = _supported_extensions()
|
||||
|
||||
if bucket in ("created", "written"):
|
||||
node = event.get("node") or {}
|
||||
nc_path = node.get("path")
|
||||
abs_path = _nc_path_to_abs(nc_path) if nc_path else None
|
||||
if not abs_path:
|
||||
return {"status": "ignored", "reason": "non-user-files path"}
|
||||
if Path(abs_path).suffix.lower() not in supported:
|
||||
return {"status": "ignored", "reason": "unsupported extension"}
|
||||
parent_dir = str(Path(abs_path).parent)
|
||||
source_root_id = await _source_root_for(parent_dir)
|
||||
if source_root_id is None:
|
||||
# Outside any registered source root — we don't index this
|
||||
# part of Nextcloud at all.
|
||||
return {"status": "ignored", "reason": "outside source root"}
|
||||
scan_folder.delay(parent_dir, source_root_id)
|
||||
logger.info("nc-webhook %s: queued scan_folder for %s", bucket, parent_dir)
|
||||
return {"status": "queued", "action": "scan_folder", "path": parent_dir}
|
||||
|
||||
if bucket == "deleted":
|
||||
node = event.get("node") or {}
|
||||
nc_path = node.get("path")
|
||||
abs_path = _nc_path_to_abs(nc_path) if nc_path else None
|
||||
if not abs_path:
|
||||
return {"status": "ignored", "reason": "non-user-files path"}
|
||||
# Folder deletes: NC fires exactly one NodeDeletedEvent for the
|
||||
# folder, not one per child file. Detect the directory case by
|
||||
# the absence of a supported image extension and recursively
|
||||
# discard every Photo under that prefix.
|
||||
if Path(abs_path).suffix.lower() not in supported:
|
||||
n = await handle_directory_deletion(abs_path)
|
||||
logger.info(
|
||||
"nc-webhook deleted (dir): %s -> %s photos discarded",
|
||||
abs_path, n,
|
||||
)
|
||||
return {
|
||||
"status": "applied",
|
||||
"action": "discard_subtree",
|
||||
"path": abs_path,
|
||||
"discarded": n,
|
||||
}
|
||||
await handle_file_deletion(abs_path)
|
||||
logger.info("nc-webhook deleted: marked %s as discarded", abs_path)
|
||||
return {"status": "applied", "action": "discard", "path": abs_path}
|
||||
|
||||
if bucket == "renamed":
|
||||
source = event.get("source") or {}
|
||||
target = event.get("target") or {}
|
||||
old_abs = _nc_path_to_abs(source.get("path") or "")
|
||||
new_abs = _nc_path_to_abs(target.get("path") or "")
|
||||
if not old_abs or not new_abs:
|
||||
return {"status": "ignored", "reason": "non-user-files path"}
|
||||
|
||||
old_is_dir = Path(old_abs).suffix.lower() not in supported
|
||||
new_is_dir = Path(new_abs).suffix.lower() not in supported
|
||||
|
||||
# Directory rename: NC fires one event for the directory; the
|
||||
# children's paths change implicitly. Prefix-rewrite in mule.
|
||||
# Same handler covers the feedback case where the PATCH
|
||||
# /folders/{id}/rename endpoint already updated the DB — the
|
||||
# SQL UPDATE matches zero rows the second time around.
|
||||
if old_is_dir and new_is_dir:
|
||||
result = await handle_directory_rename(old_abs, new_abs)
|
||||
logger.info(
|
||||
"nc-webhook renamed (dir): %s -> %s : %s",
|
||||
old_abs, new_abs, result,
|
||||
)
|
||||
return {
|
||||
"status": "applied",
|
||||
"action": "rename_subtree",
|
||||
"from": old_abs,
|
||||
"to": new_abs,
|
||||
**result,
|
||||
}
|
||||
|
||||
# File rename (existing logic).
|
||||
if Path(old_abs).suffix.lower() in supported:
|
||||
await handle_file_deletion(old_abs)
|
||||
if Path(new_abs).suffix.lower() in supported:
|
||||
parent_dir = str(Path(new_abs).parent)
|
||||
source_root_id = await _source_root_for(parent_dir)
|
||||
if source_root_id is not None:
|
||||
scan_folder.delay(parent_dir, source_root_id)
|
||||
logger.info("nc-webhook renamed: %s -> %s", old_abs, new_abs)
|
||||
return {"status": "applied", "action": "rename", "from": old_abs, "to": new_abs}
|
||||
|
||||
# Shouldn't reach here — classify() already filtered.
|
||||
return {"status": "ignored"}
|
||||
@@ -1,387 +0,0 @@
|
||||
"""Nextcloud integration router — folder picker + per-user SourceRoots.
|
||||
|
||||
Exposes three things:
|
||||
|
||||
- GET /api/v1/nextcloud/whoami?candidate=<name>
|
||||
Validate that a Nextcloud username actually has a files/ tree
|
||||
on the mounted homecloud volume. Used by the Settings UI to
|
||||
sanity-check the override field before saving.
|
||||
|
||||
- GET /api/v1/nextcloud/browse?path=<rel>
|
||||
List immediate subdirectories of the current user's Nextcloud
|
||||
files tree, scoped server-side to their nextcloud_username.
|
||||
Powers the folder picker.
|
||||
|
||||
- POST /api/v1/nextcloud/source-roots {name, nextcloud_path}
|
||||
DELETE /api/v1/nextcloud/source-roots/{id}
|
||||
Add or remove a per-user SourceRoot pointing at a Nextcloud
|
||||
subfolder. Adding kicks off an immediate scan_folder task so
|
||||
photos start appearing without a full library re-scan.
|
||||
|
||||
All paths are normalized with realpath and rejected if they escape the
|
||||
user's allowed root — defense-in-depth against `..` and symlink
|
||||
shenanigans.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.dependencies import get_current_user
|
||||
from app.models.folders import SourceRoot, Folder
|
||||
from app.models.photos import Photo
|
||||
from app.models.sharing import FolderShare
|
||||
from app.models.user import User
|
||||
from app.services.nextcloud_dav import (
|
||||
NEXTCLOUD_USERS_ROOT,
|
||||
is_nextcloud_path,
|
||||
whoami_dir_exists,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _user_root(nc_username: str) -> str:
|
||||
"""Absolute path of `<NEXTCLOUD_USERS_ROOT>/<user>/files`."""
|
||||
return os.path.join(NEXTCLOUD_USERS_ROOT, nc_username, "files")
|
||||
|
||||
|
||||
def _resolve_under_user_root(nc_username: str, rel: str) -> str:
|
||||
"""Resolve `rel` (a relative path the client supplied) under the
|
||||
user's Nextcloud `files/` directory and ensure the result is still
|
||||
inside that root. Returns the absolute, realpath-normalized path.
|
||||
|
||||
Raises 400 on traversal attempts (`..`, absolute paths, symlinks
|
||||
that point outside the root)."""
|
||||
rel = (rel or "").lstrip("/")
|
||||
if any(seg in ("..",) for seg in rel.split("/") if seg):
|
||||
raise HTTPException(status_code=400, detail="Invalid path")
|
||||
base = _user_root(nc_username)
|
||||
candidate = os.path.realpath(os.path.join(base, rel))
|
||||
base_real = os.path.realpath(base)
|
||||
if candidate != base_real and not candidate.startswith(base_real + os.sep):
|
||||
raise HTTPException(status_code=400, detail="Path escapes Nextcloud root")
|
||||
return candidate
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class BrowseEntry(BaseModel):
|
||||
name: str
|
||||
path: str # path relative to the user's files/ root
|
||||
has_children: bool # True if it contains at least one sub-directory
|
||||
|
||||
|
||||
class BrowseResponse(BaseModel):
|
||||
nc_username: str
|
||||
rel_path: str
|
||||
parent_rel: Optional[str] # None at the root
|
||||
entries: list[BrowseEntry]
|
||||
|
||||
|
||||
@router.get("/whoami")
|
||||
async def whoami(
|
||||
candidate: Optional[str] = Query(None, description="Nextcloud username to validate"),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Public to authenticated users — returns whether a Nextcloud
|
||||
username has a real files/ directory on the bind mount. Used by
|
||||
the Settings UI to validate the override field before save."""
|
||||
name = (candidate or current_user.nextcloud_username or "").strip()
|
||||
if not name:
|
||||
return {
|
||||
"configured": bool(current_user.nextcloud_username),
|
||||
"candidate": None,
|
||||
"valid": False,
|
||||
"reason": "no_username",
|
||||
}
|
||||
valid = whoami_dir_exists(name)
|
||||
return {
|
||||
"configured": bool(current_user.nextcloud_username),
|
||||
"candidate": name,
|
||||
"valid": valid,
|
||||
"reason": None if valid else "no_files_dir",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/browse", response_model=BrowseResponse)
|
||||
async def browse(
|
||||
path: str = Query("", description="Path relative to the user's NC files/ root"),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List immediate subdirectories of the current user's NC tree."""
|
||||
nc_user = (current_user.nextcloud_username or "").strip()
|
||||
if not nc_user:
|
||||
raise HTTPException(
|
||||
status_code=412,
|
||||
detail="Set your Nextcloud username in Settings → Library first.",
|
||||
)
|
||||
if not whoami_dir_exists(nc_user):
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=(
|
||||
f"Nextcloud user '{nc_user}' has no files/ directory on the "
|
||||
"mounted homecloud volume. Check your nextcloud_username override."
|
||||
),
|
||||
)
|
||||
|
||||
abs_path = _resolve_under_user_root(nc_user, path)
|
||||
if not os.path.isdir(abs_path):
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
|
||||
base = _user_root(nc_user)
|
||||
entries: list[BrowseEntry] = []
|
||||
try:
|
||||
with os.scandir(abs_path) as it:
|
||||
for de in it:
|
||||
# Skip hidden and Nextcloud's appdata noise.
|
||||
if de.name.startswith("."):
|
||||
continue
|
||||
if not de.is_dir(follow_symlinks=False):
|
||||
continue
|
||||
child_abs = os.path.join(abs_path, de.name)
|
||||
rel = os.path.relpath(child_abs, base)
|
||||
# Quick has_children probe: any subdir that's a real
|
||||
# directory. Cap at first hit so deep trees don't slow
|
||||
# the picker.
|
||||
has_children = False
|
||||
try:
|
||||
with os.scandir(child_abs) as sub:
|
||||
for s in sub:
|
||||
if s.name.startswith("."):
|
||||
continue
|
||||
if s.is_dir(follow_symlinks=False):
|
||||
has_children = True
|
||||
break
|
||||
except OSError:
|
||||
has_children = False
|
||||
entries.append(BrowseEntry(name=de.name, path=rel, has_children=has_children))
|
||||
except PermissionError:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=(
|
||||
"Cannot read Nextcloud folder — backend container lacks "
|
||||
"filesystem permissions on the mount. Apply ACL fix on the host."
|
||||
),
|
||||
)
|
||||
|
||||
entries.sort(key=lambda e: e.name.lower())
|
||||
|
||||
rel_path = os.path.relpath(abs_path, _user_root(nc_user)) if abs_path != _user_root(nc_user) else ""
|
||||
if rel_path == ".":
|
||||
rel_path = ""
|
||||
parent_rel: Optional[str] = None
|
||||
if rel_path:
|
||||
parent = os.path.dirname(rel_path)
|
||||
parent_rel = parent
|
||||
|
||||
return BrowseResponse(
|
||||
nc_username=nc_user,
|
||||
rel_path=rel_path,
|
||||
parent_rel=parent_rel,
|
||||
entries=entries,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SourceRootCreate(BaseModel):
|
||||
name: str
|
||||
nextcloud_path: str # relative to the user's files/ root
|
||||
|
||||
|
||||
@router.post("/source-roots", status_code=201)
|
||||
async def create_nextcloud_source_root(
|
||||
body: SourceRootCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Register a Nextcloud subfolder as a SourceRoot for the current user
|
||||
and kick off the initial scan."""
|
||||
nc_user = (current_user.nextcloud_username or "").strip()
|
||||
if not nc_user:
|
||||
raise HTTPException(
|
||||
status_code=412,
|
||||
detail="Set your Nextcloud username in Settings → Library first.",
|
||||
)
|
||||
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Name is required")
|
||||
|
||||
abs_path = _resolve_under_user_root(nc_user, body.nextcloud_path)
|
||||
if not os.path.isdir(abs_path):
|
||||
raise HTTPException(status_code=404, detail="Folder not found in Nextcloud tree")
|
||||
|
||||
# Don't allow registering the user's `files/` root itself as a
|
||||
# SourceRoot — it'd index everything they own (Documents, Notes,
|
||||
# appdata noise). Force them to pick a subfolder.
|
||||
if abs_path == os.path.realpath(_user_root(nc_user)):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Pick a subfolder; the whole files/ root is too broad.",
|
||||
)
|
||||
|
||||
# Refuse duplicates — the path is uniquely indexed but a clean error
|
||||
# beats a 500 from the unique constraint.
|
||||
existing = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.path == abs_path)
|
||||
)
|
||||
if existing.scalar_one_or_none() is not None:
|
||||
raise HTTPException(status_code=409, detail="A SourceRoot for that path already exists")
|
||||
|
||||
sr = SourceRoot(
|
||||
name=name,
|
||||
path=abs_path,
|
||||
user_id=current_user.id,
|
||||
is_active=True,
|
||||
)
|
||||
db.add(sr)
|
||||
await db.flush()
|
||||
await db.commit()
|
||||
await db.refresh(sr)
|
||||
|
||||
# Kick off the initial scan. Failures here shouldn't block the
|
||||
# SourceRoot creation — the user can hit "Re-scan source folders"
|
||||
# from Settings if Celery is wedged.
|
||||
try:
|
||||
from app.tasks.celery import celery_app
|
||||
celery_app.send_task("scan_folder", args=[sr.path, sr.id])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Failed to queue initial scan for new SourceRoot %s: %s", sr.id, exc)
|
||||
|
||||
return {
|
||||
"id": sr.id,
|
||||
"name": sr.name,
|
||||
"path": sr.path,
|
||||
"user_id": sr.user_id,
|
||||
"is_active": sr.is_active,
|
||||
"is_nextcloud": True,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/source-roots/{source_root_id}")
|
||||
async def delete_nextcloud_source_root(
|
||||
source_root_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Hard-delete: drop the SourceRoot row and cascade-delete every
|
||||
Folder and Photo underneath it. The actual files in Nextcloud are
|
||||
NOT touched — this is just unwiring the user's library in mule-image.
|
||||
|
||||
Implementation notes:
|
||||
- photo_tags and heap_photos cascade via DB-level ON DELETE CASCADE,
|
||||
so deleting Photo rows is enough to clean those up.
|
||||
- FolderShare uses a stringly-typed folder_id (no FK), so we have
|
||||
to clean those rows by hand for both the SourceRoot itself and
|
||||
every Folder we're about to delete.
|
||||
- Chunked at 500 to mirror prune_missing_photos so postgres doesn't
|
||||
choke on a 21k-photo source root.
|
||||
"""
|
||||
sr = (await db.execute(
|
||||
select(SourceRoot).where(
|
||||
SourceRoot.id == source_root_id,
|
||||
SourceRoot.user_id == current_user.id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if sr is None:
|
||||
raise HTTPException(status_code=404, detail="SourceRoot not found")
|
||||
if not is_nextcloud_path(sr.path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="This endpoint only manages Nextcloud-rooted SourceRoots",
|
||||
)
|
||||
|
||||
folder_ids = (await db.execute(
|
||||
select(Folder.id).where(Folder.source_root_id == sr.id)
|
||||
)).scalars().all()
|
||||
|
||||
photo_ids: list[str] = []
|
||||
if folder_ids:
|
||||
photo_ids = (await db.execute(
|
||||
select(Photo.id).where(Photo.folder_id.in_(folder_ids))
|
||||
)).scalars().all()
|
||||
|
||||
CHUNK = 500
|
||||
for i in range(0, len(photo_ids), CHUNK):
|
||||
chunk = photo_ids[i:i + CHUNK]
|
||||
await db.execute(delete(Photo).where(Photo.id.in_(chunk)))
|
||||
|
||||
# FolderShare rows: not a real FK, clean both 'source_root' and
|
||||
# 'folder' typed shares pointing at anything we're tearing down.
|
||||
await db.execute(
|
||||
delete(FolderShare).where(
|
||||
FolderShare.folder_id == sr.id,
|
||||
FolderShare.folder_type == 'source_root',
|
||||
)
|
||||
)
|
||||
if folder_ids:
|
||||
await db.execute(
|
||||
delete(FolderShare).where(
|
||||
FolderShare.folder_id.in_(folder_ids),
|
||||
FolderShare.folder_type == 'folder',
|
||||
)
|
||||
)
|
||||
# Folders have a self-referential parent_id FK with no
|
||||
# ON DELETE rule. NULL parent_id on every folder that points
|
||||
# *into* our delete set — that includes children within this
|
||||
# source root AND any folder under a different SourceRoot whose
|
||||
# path happens to nest inside this one (e.g. a 'Leóns 1st Year'
|
||||
# SourceRoot at `.../Taco and Muli - 2024 onward/Leóns 1st Year`
|
||||
# has folder rows whose parent_id points at folder rows under
|
||||
# 'Taco and Muli - 2024 onward'). Without this, deleting the
|
||||
# outer SourceRoot trips folders_parent_id_fkey from the inner
|
||||
# SourceRoot's still-live rows.
|
||||
await db.execute(
|
||||
update(Folder)
|
||||
.where(Folder.parent_id.in_(folder_ids))
|
||||
.values(parent_id=None)
|
||||
)
|
||||
await db.execute(delete(Folder).where(Folder.id.in_(folder_ids)))
|
||||
|
||||
await db.delete(sr)
|
||||
await db.commit()
|
||||
|
||||
logger.info(
|
||||
f"Deleted SourceRoot {sr.id} ({sr.name}): "
|
||||
f"{len(photo_ids)} photos, {len(folder_ids)} folders"
|
||||
)
|
||||
return {
|
||||
"deleted_photos": len(photo_ids),
|
||||
"deleted_folders": len(folder_ids),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/source-roots")
|
||||
async def list_nextcloud_source_roots(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""All Nextcloud-rooted SourceRoots owned by the current user.
|
||||
Used by the Settings panel to render the manage list."""
|
||||
rows = (await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.user_id == current_user.id)
|
||||
)).scalars().all()
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"name": r.name,
|
||||
"path": r.path,
|
||||
"is_active": r.is_active,
|
||||
"is_nextcloud": True,
|
||||
}
|
||||
for r in rows
|
||||
if is_nextcloud_path(r.path)
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,70 +0,0 @@
|
||||
"""
|
||||
Search API router — unified hybrid search endpoint.
|
||||
"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo
|
||||
from app.services.search import hybrid_search
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
q: Optional[str] = None
|
||||
filters: Optional[dict] = None
|
||||
limit: int = 50
|
||||
offset: int = 0
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def search_photos(body: SearchRequest, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""FTS search over photo metadata with optional tag and date filters."""
|
||||
filters = body.filters or {}
|
||||
|
||||
results = await hybrid_search(
|
||||
db=db,
|
||||
q=body.q,
|
||||
tag_ids=filters.get("tag_ids"),
|
||||
date_from=filters.get("date_from"),
|
||||
date_to=filters.get("date_to"),
|
||||
limit=body.limit,
|
||||
offset=body.offset,
|
||||
)
|
||||
|
||||
if not results:
|
||||
return {"results": [], "total": 0}
|
||||
|
||||
# Hydrate with photo data
|
||||
photo_ids = [r["photo_id"] for r in results]
|
||||
stmt = select(Photo).where(Photo.id.in_(photo_ids), Photo.user_id == current_user.id)
|
||||
rows = (await db.execute(stmt)).scalars().all()
|
||||
photo_map = {p.id: p for p in rows}
|
||||
|
||||
hydrated = []
|
||||
for r in results:
|
||||
photo = photo_map.get(r["photo_id"])
|
||||
if not photo:
|
||||
continue
|
||||
hydrated.append({
|
||||
"id": photo.id,
|
||||
"filename": photo.filename,
|
||||
"filepath": photo.filepath,
|
||||
"media_type": photo.media_type,
|
||||
"width": photo.width,
|
||||
"height": photo.height,
|
||||
"taken_at": photo.taken_at.isoformat() if photo.taken_at else None,
|
||||
"rating": photo.rating,
|
||||
"color_label": photo.color_label,
|
||||
"thumb_small": photo.thumb_small,
|
||||
"thumb_medium": photo.thumb_medium,
|
||||
"score": r["score"],
|
||||
})
|
||||
|
||||
return {"results": hydrated, "total": len(hydrated)}
|
||||
@@ -1,613 +0,0 @@
|
||||
"""
|
||||
Sharing API router — manage cross-user access to heaps and folders.
|
||||
"""
|
||||
import logging
|
||||
from typing import Literal, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.heaps import Heap, heap_photos
|
||||
from app.models.folders import Folder, SourceRoot
|
||||
from app.models.photos import Photo
|
||||
from app.models.sharing import HeapShare, FolderShare
|
||||
from app.models.user import User
|
||||
from app.dependencies import (
|
||||
get_current_user,
|
||||
get_user_heap,
|
||||
get_user_folder,
|
||||
resolve_username,
|
||||
)
|
||||
from app.services.gravatar import gravatar_url
|
||||
|
||||
|
||||
def _user_avatar(user: User) -> Optional[str]:
|
||||
"""OIDC `picture` claim wins, Gravatar fills the gap. Returns None
|
||||
when neither source can produce a URL so the frontend can fall back
|
||||
to the initials bubble."""
|
||||
return user.avatar_url or gravatar_url(user.email)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/sharing", tags=["sharing"])
|
||||
|
||||
|
||||
# ── Schemas ──────────────────────────────────────────────────────────────
|
||||
|
||||
class ShareCreate(BaseModel):
|
||||
username: str
|
||||
permission: Literal["read", "write"] = "read"
|
||||
|
||||
|
||||
class ShareResponse(BaseModel):
|
||||
id: str
|
||||
shared_with_id: str
|
||||
shared_with_username: str
|
||||
shared_with_avatar_url: Optional[str] = None
|
||||
shared_with_display_name: Optional[str] = None
|
||||
permission: str
|
||||
status: str # 'pending' | 'accepted'
|
||||
created_at: str
|
||||
|
||||
|
||||
class SharedHeapResponse(BaseModel):
|
||||
# `id` is the heap id (used for navigation). `share_id` is the
|
||||
# heap_shares row id, needed so the recipient can "Leave" via the
|
||||
# existing DELETE endpoint without a separate lookup.
|
||||
id: str
|
||||
share_id: str
|
||||
name: str
|
||||
owner_username: str
|
||||
owner_avatar_url: Optional[str] = None
|
||||
owner_display_name: Optional[str] = None
|
||||
permission: str
|
||||
photo_count: int
|
||||
|
||||
|
||||
class SharedFolderResponse(BaseModel):
|
||||
id: str
|
||||
share_id: str
|
||||
name: str
|
||||
folder_type: str
|
||||
owner_username: str
|
||||
owner_avatar_url: Optional[str] = None
|
||||
owner_display_name: Optional[str] = None
|
||||
permission: str
|
||||
photo_count: int
|
||||
|
||||
|
||||
class PendingInvite(BaseModel):
|
||||
"""A share that exists in the DB but hasn't been accepted yet. Powers
|
||||
the notification bell in the left-sidebar user section."""
|
||||
share_id: str
|
||||
target_id: str # heap id or folder id
|
||||
target_name: str
|
||||
owner_username: str
|
||||
owner_avatar_url: Optional[str] = None
|
||||
owner_display_name: Optional[str] = None
|
||||
permission: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class PendingInvitesResponse(BaseModel):
|
||||
heaps: list[PendingInvite]
|
||||
folders: list[PendingInvite]
|
||||
|
||||
|
||||
class ShareableUser(BaseModel):
|
||||
id: str
|
||||
username: str
|
||||
avatar_url: Optional[str] = None
|
||||
display_name: Optional[str] = None
|
||||
|
||||
|
||||
# ── Shareable users ──────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/users", response_model=list[ShareableUser])
|
||||
async def list_shareable_users(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List every active user other than the caller, for the share-picker
|
||||
dropdown. Sharing only requires knowing a username today, so surfacing
|
||||
the list is no wider an attack surface than the free-text input it
|
||||
replaces. Inactive accounts are filtered out."""
|
||||
result = await db.execute(
|
||||
select(User)
|
||||
.where(User.id != current_user.id)
|
||||
.where(User.is_active.is_(True))
|
||||
.order_by(User.username)
|
||||
)
|
||||
return [
|
||||
ShareableUser(
|
||||
id=str(u.id),
|
||||
username=u.username,
|
||||
avatar_url=_user_avatar(u),
|
||||
display_name=u.display_name,
|
||||
)
|
||||
for u in result.scalars().all()
|
||||
]
|
||||
|
||||
|
||||
# ── Pending invites (recipient-facing, cross-type) ───────────────────────
|
||||
|
||||
@router.get("/pending", response_model=PendingInvitesResponse)
|
||||
async def list_pending_invites(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Every share targeting the current user that's still waiting on
|
||||
them to accept. Feeds the notification bell in the sidebar."""
|
||||
heap_rows = (await db.execute(
|
||||
select(HeapShare, Heap, User)
|
||||
.join(Heap, HeapShare.heap_id == Heap.id)
|
||||
.join(User, HeapShare.owner_id == User.id)
|
||||
.where(HeapShare.shared_with_id == current_user.id)
|
||||
.where(HeapShare.status == "pending")
|
||||
)).all()
|
||||
|
||||
heaps = [
|
||||
PendingInvite(
|
||||
share_id=share.id,
|
||||
target_id=heap.id,
|
||||
target_name=heap.name,
|
||||
owner_username=owner.username,
|
||||
owner_avatar_url=_user_avatar(owner),
|
||||
owner_display_name=owner.display_name,
|
||||
permission=share.permission,
|
||||
created_at=share.created_at.isoformat() if share.created_at else "",
|
||||
)
|
||||
for share, heap, owner in heap_rows
|
||||
]
|
||||
|
||||
folder_rows = (await db.execute(
|
||||
select(FolderShare, User)
|
||||
.join(User, FolderShare.owner_id == User.id)
|
||||
.where(FolderShare.shared_with_id == current_user.id)
|
||||
.where(FolderShare.status == "pending")
|
||||
)).all()
|
||||
|
||||
folders: list[PendingInvite] = []
|
||||
for share, owner in folder_rows:
|
||||
if share.folder_type == "source_root":
|
||||
entity = (await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == share.folder_id)
|
||||
)).scalar_one_or_none()
|
||||
else:
|
||||
entity = (await db.execute(
|
||||
select(Folder).where(Folder.id == share.folder_id)
|
||||
)).scalar_one_or_none()
|
||||
# If the underlying folder was deleted while an invite was
|
||||
# still pending, just skip — the share is effectively orphaned
|
||||
# and the owner's revoke path will clean it up.
|
||||
if entity is None:
|
||||
continue
|
||||
folders.append(PendingInvite(
|
||||
share_id=share.id,
|
||||
target_id=share.folder_id,
|
||||
target_name=entity.name,
|
||||
owner_username=owner.username,
|
||||
owner_avatar_url=_user_avatar(owner),
|
||||
owner_display_name=owner.display_name,
|
||||
permission=share.permission,
|
||||
created_at=share.created_at.isoformat() if share.created_at else "",
|
||||
))
|
||||
|
||||
return PendingInvitesResponse(heaps=heaps, folders=folders)
|
||||
|
||||
|
||||
# ── Heap sharing ─────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/heaps/shared-with-me")
|
||||
async def list_shared_heaps(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List all accepted heap shares for the current user. Pending
|
||||
invites are hidden here and surfaced via /sharing/pending instead."""
|
||||
result = await db.execute(
|
||||
select(HeapShare, Heap, User)
|
||||
.join(Heap, HeapShare.heap_id == Heap.id)
|
||||
.join(User, HeapShare.owner_id == User.id)
|
||||
.where(HeapShare.shared_with_id == current_user.id)
|
||||
.where(HeapShare.status == "accepted")
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
items = []
|
||||
for share, heap, owner in rows:
|
||||
# Count photos in this heap.
|
||||
count_result = await db.execute(
|
||||
select(func.count()).select_from(heap_photos).where(
|
||||
heap_photos.c.heap_id == heap.id
|
||||
)
|
||||
)
|
||||
count = count_result.scalar() or 0
|
||||
|
||||
items.append(SharedHeapResponse(
|
||||
id=heap.id,
|
||||
share_id=share.id,
|
||||
name=heap.name,
|
||||
owner_username=owner.username,
|
||||
owner_avatar_url=_user_avatar(owner),
|
||||
owner_display_name=owner.display_name,
|
||||
permission=share.permission,
|
||||
photo_count=count,
|
||||
))
|
||||
return items
|
||||
|
||||
|
||||
@router.get("/heaps/{heap_id}")
|
||||
async def list_heap_shares(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List all shares for a heap (owner only)."""
|
||||
heap = await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
result = await db.execute(
|
||||
select(HeapShare, User)
|
||||
.join(User, HeapShare.shared_with_id == User.id)
|
||||
.where(HeapShare.heap_id == heap.id)
|
||||
)
|
||||
return [
|
||||
ShareResponse(
|
||||
id=share.id,
|
||||
shared_with_id=user.id,
|
||||
shared_with_username=user.username,
|
||||
shared_with_avatar_url=_user_avatar(user),
|
||||
shared_with_display_name=user.display_name,
|
||||
permission=share.permission,
|
||||
status=share.status,
|
||||
created_at=share.created_at.isoformat() if share.created_at else "",
|
||||
)
|
||||
for share, user in result.all()
|
||||
]
|
||||
|
||||
|
||||
@router.post("/heaps/{heap_id}", status_code=201)
|
||||
async def share_heap(
|
||||
heap_id: str,
|
||||
body: ShareCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Share a heap with another user (owner only)."""
|
||||
heap = await get_user_heap(heap_id, current_user, db)
|
||||
target_user = await resolve_username(body.username, db)
|
||||
|
||||
if target_user.id == current_user.id:
|
||||
raise HTTPException(status_code=400, detail="Cannot share with yourself")
|
||||
|
||||
# Check for existing share.
|
||||
existing = await db.execute(
|
||||
select(HeapShare).where(
|
||||
HeapShare.heap_id == heap.id,
|
||||
HeapShare.shared_with_id == target_user.id,
|
||||
)
|
||||
)
|
||||
if existing.scalar_one_or_none():
|
||||
raise HTTPException(status_code=409, detail="Already shared with this user")
|
||||
|
||||
share = HeapShare(
|
||||
heap_id=heap.id,
|
||||
owner_id=current_user.id,
|
||||
shared_with_id=target_user.id,
|
||||
permission=body.permission,
|
||||
)
|
||||
db.add(share)
|
||||
await db.commit()
|
||||
|
||||
logger.info("Heap %s shared with %s (%s)", heap.name, target_user.username, body.permission)
|
||||
return {"status": "shared", "share_id": share.id}
|
||||
|
||||
|
||||
@router.post("/heaps/{heap_id}/accept", status_code=200)
|
||||
async def accept_heap_share(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Recipient accepts a pending heap invite. Idempotent — if the
|
||||
share is already accepted, returns 200 anyway so double-clicks in
|
||||
the notification popover are harmless."""
|
||||
result = await db.execute(
|
||||
select(HeapShare).where(
|
||||
HeapShare.heap_id == heap_id,
|
||||
HeapShare.shared_with_id == current_user.id,
|
||||
)
|
||||
)
|
||||
share = result.scalar_one_or_none()
|
||||
if share is None:
|
||||
raise HTTPException(status_code=404, detail="Invite not found")
|
||||
if share.status != "accepted":
|
||||
share.status = "accepted"
|
||||
share.accepted_at = func.now()
|
||||
await db.commit()
|
||||
return {"status": "accepted"}
|
||||
|
||||
|
||||
@router.post("/heaps/{heap_id}/decline", status_code=200)
|
||||
async def decline_heap_share(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Recipient declines a pending heap invite. The share row is
|
||||
deleted — there's no separate 'declined' status. A re-invite just
|
||||
creates a fresh pending row."""
|
||||
result = await db.execute(
|
||||
select(HeapShare).where(
|
||||
HeapShare.heap_id == heap_id,
|
||||
HeapShare.shared_with_id == current_user.id,
|
||||
)
|
||||
)
|
||||
share = result.scalar_one_or_none()
|
||||
if share is None:
|
||||
raise HTTPException(status_code=404, detail="Invite not found")
|
||||
await db.delete(share)
|
||||
await db.commit()
|
||||
return {"status": "declined"}
|
||||
|
||||
|
||||
@router.delete("/heaps/{heap_id}/{share_id}", status_code=204)
|
||||
async def revoke_heap_share(
|
||||
heap_id: str,
|
||||
share_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Revoke a heap share. The owner can revoke any share; a recipient
|
||||
can revoke their own share (i.e. leave)."""
|
||||
result = await db.execute(
|
||||
select(HeapShare).where(HeapShare.id == share_id, HeapShare.heap_id == heap_id)
|
||||
)
|
||||
share = result.scalar_one_or_none()
|
||||
if share is None:
|
||||
raise HTTPException(status_code=404, detail="Share not found")
|
||||
|
||||
# Must be the owner or the recipient themselves.
|
||||
if share.owner_id != current_user.id and share.shared_with_id != current_user.id:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
|
||||
await db.delete(share)
|
||||
await db.commit()
|
||||
|
||||
|
||||
# ── Folder sharing ───────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/folders/shared-with-me")
|
||||
async def list_shared_folders(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List all accepted folder/source-root shares for the current
|
||||
user. Pending invites are hidden here and surfaced via
|
||||
/sharing/pending instead."""
|
||||
result = await db.execute(
|
||||
select(FolderShare, User)
|
||||
.join(User, FolderShare.owner_id == User.id)
|
||||
.where(FolderShare.shared_with_id == current_user.id)
|
||||
.where(FolderShare.status == "accepted")
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
items = []
|
||||
for share, owner in rows:
|
||||
# Resolve the folder/source root name and photo count.
|
||||
if share.folder_type == "source_root":
|
||||
sr_result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == share.folder_id)
|
||||
)
|
||||
entity = sr_result.scalar_one_or_none()
|
||||
if not entity:
|
||||
continue
|
||||
name = entity.name
|
||||
# Count all photos under this source root's folders.
|
||||
count_result = await db.execute(
|
||||
select(func.count()).select_from(Photo).where(
|
||||
Photo.folder_id.in_(
|
||||
select(Folder.id).where(Folder.source_root_id == entity.id)
|
||||
),
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
)
|
||||
else:
|
||||
folder_result = await db.execute(
|
||||
select(Folder).where(Folder.id == share.folder_id)
|
||||
)
|
||||
entity = folder_result.scalar_one_or_none()
|
||||
if not entity:
|
||||
continue
|
||||
name = entity.name
|
||||
import os
|
||||
target_path = os.path.normpath(entity.path).rstrip(os.sep)
|
||||
count_result = await db.execute(
|
||||
select(func.count()).select_from(Photo).where(
|
||||
Photo.folder_id.in_(
|
||||
select(Folder.id).where(
|
||||
(Folder.path == target_path)
|
||||
| (Folder.path.like(target_path + os.sep + "%"))
|
||||
)
|
||||
),
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
)
|
||||
|
||||
count = count_result.scalar() or 0
|
||||
items.append(SharedFolderResponse(
|
||||
id=share.folder_id,
|
||||
share_id=share.id,
|
||||
name=name,
|
||||
folder_type=share.folder_type,
|
||||
owner_username=owner.username,
|
||||
owner_avatar_url=_user_avatar(owner),
|
||||
owner_display_name=owner.display_name,
|
||||
permission=share.permission,
|
||||
photo_count=count,
|
||||
))
|
||||
return items
|
||||
|
||||
|
||||
@router.get("/folders/{folder_id}")
|
||||
async def list_folder_shares(
|
||||
folder_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List all shares for a folder (owner only)."""
|
||||
# Verify ownership — try folder then source root.
|
||||
owned = False
|
||||
result = await db.execute(
|
||||
select(Folder).where(Folder.id == folder_id, Folder.user_id == current_user.id)
|
||||
)
|
||||
if result.scalar_one_or_none():
|
||||
owned = True
|
||||
else:
|
||||
result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
if result.scalar_one_or_none():
|
||||
owned = True
|
||||
|
||||
if not owned:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
|
||||
result = await db.execute(
|
||||
select(FolderShare, User)
|
||||
.join(User, FolderShare.shared_with_id == User.id)
|
||||
.where(FolderShare.folder_id == folder_id)
|
||||
)
|
||||
return [
|
||||
ShareResponse(
|
||||
id=share.id,
|
||||
shared_with_id=user.id,
|
||||
shared_with_username=user.username,
|
||||
shared_with_avatar_url=_user_avatar(user),
|
||||
shared_with_display_name=user.display_name,
|
||||
permission=share.permission,
|
||||
status=share.status,
|
||||
created_at=share.created_at.isoformat() if share.created_at else "",
|
||||
)
|
||||
for share, user in result.all()
|
||||
]
|
||||
|
||||
|
||||
@router.post("/folders/{folder_id}", status_code=201)
|
||||
async def share_folder(
|
||||
folder_id: str,
|
||||
body: ShareCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Share a folder or source root with another user (owner only)."""
|
||||
# Determine folder_type and verify ownership.
|
||||
folder_type = "folder"
|
||||
result = await db.execute(
|
||||
select(Folder).where(Folder.id == folder_id, Folder.user_id == current_user.id)
|
||||
)
|
||||
entity = result.scalar_one_or_none()
|
||||
if entity is None:
|
||||
result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
entity = result.scalar_one_or_none()
|
||||
if entity is None:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
folder_type = "source_root"
|
||||
|
||||
target_user = await resolve_username(body.username, db)
|
||||
if target_user.id == current_user.id:
|
||||
raise HTTPException(status_code=400, detail="Cannot share with yourself")
|
||||
|
||||
existing = await db.execute(
|
||||
select(FolderShare).where(
|
||||
FolderShare.folder_id == folder_id,
|
||||
FolderShare.shared_with_id == target_user.id,
|
||||
)
|
||||
)
|
||||
if existing.scalar_one_or_none():
|
||||
raise HTTPException(status_code=409, detail="Already shared with this user")
|
||||
|
||||
share = FolderShare(
|
||||
folder_id=folder_id,
|
||||
folder_type=folder_type,
|
||||
owner_id=current_user.id,
|
||||
shared_with_id=target_user.id,
|
||||
permission=body.permission,
|
||||
)
|
||||
db.add(share)
|
||||
await db.commit()
|
||||
|
||||
logger.info("Folder %s shared with %s (%s)", entity.name, target_user.username, body.permission)
|
||||
return {"status": "shared", "share_id": share.id}
|
||||
|
||||
|
||||
@router.post("/folders/{folder_id}/accept", status_code=200)
|
||||
async def accept_folder_share(
|
||||
folder_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Recipient accepts a pending folder invite. Idempotent."""
|
||||
result = await db.execute(
|
||||
select(FolderShare).where(
|
||||
FolderShare.folder_id == folder_id,
|
||||
FolderShare.shared_with_id == current_user.id,
|
||||
)
|
||||
)
|
||||
share = result.scalar_one_or_none()
|
||||
if share is None:
|
||||
raise HTTPException(status_code=404, detail="Invite not found")
|
||||
if share.status != "accepted":
|
||||
share.status = "accepted"
|
||||
share.accepted_at = func.now()
|
||||
await db.commit()
|
||||
return {"status": "accepted"}
|
||||
|
||||
|
||||
@router.post("/folders/{folder_id}/decline", status_code=200)
|
||||
async def decline_folder_share(
|
||||
folder_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Recipient declines a pending folder invite. Row is deleted."""
|
||||
result = await db.execute(
|
||||
select(FolderShare).where(
|
||||
FolderShare.folder_id == folder_id,
|
||||
FolderShare.shared_with_id == current_user.id,
|
||||
)
|
||||
)
|
||||
share = result.scalar_one_or_none()
|
||||
if share is None:
|
||||
raise HTTPException(status_code=404, detail="Invite not found")
|
||||
await db.delete(share)
|
||||
await db.commit()
|
||||
return {"status": "declined"}
|
||||
|
||||
|
||||
@router.delete("/folders/{folder_id}/{share_id}", status_code=204)
|
||||
async def revoke_folder_share(
|
||||
folder_id: str,
|
||||
share_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Revoke a folder share (owner or self-remove)."""
|
||||
result = await db.execute(
|
||||
select(FolderShare).where(FolderShare.id == share_id, FolderShare.folder_id == folder_id)
|
||||
)
|
||||
share = result.scalar_one_or_none()
|
||||
if share is None:
|
||||
raise HTTPException(status_code=404, detail="Share not found")
|
||||
|
||||
if share.owner_id != current_user.id and share.shared_with_id != current_user.id:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
|
||||
await db.delete(share)
|
||||
await db.commit()
|
||||
@@ -1,155 +0,0 @@
|
||||
"""
|
||||
Tags API router.
|
||||
|
||||
Unified across user tags and the binary content-type classifier
|
||||
('photography' | 'other') via the `kind` column.
|
||||
"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo, Tag
|
||||
from app.models.tags import photo_tags
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Schemas ───────────────────────────────────────────────────────────────
|
||||
|
||||
class TagCreate(BaseModel):
|
||||
name: str
|
||||
color: Optional[str] = None
|
||||
kind: str = "user"
|
||||
|
||||
|
||||
class TagUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
color: Optional[str] = None
|
||||
|
||||
|
||||
# ── Endpoints ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("")
|
||||
async def list_tags(
|
||||
kind: Optional[str] = Query(None, description="Filter by kind: user, content_type"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List all tags with their photo counts, optionally filtered by kind.
|
||||
|
||||
Photo counts here drive the Tags / People sidebar badges, so they
|
||||
exclude discarded + hidden-folder photos to match the rest of the
|
||||
cross-cutting views. A tag that only appears on hidden-folder
|
||||
photos will still show up with count=0 — we don't drop empty tags
|
||||
because the user may want to see them in the management UI.
|
||||
"""
|
||||
count_subq = (
|
||||
select(
|
||||
photo_tags.c.tag_id,
|
||||
func.count(photo_tags.c.photo_id).label("photo_count"),
|
||||
func.min(photo_tags.c.photo_id).label("first_photo_id"),
|
||||
)
|
||||
.select_from(
|
||||
photo_tags.join(Photo, Photo.id == photo_tags.c.photo_id)
|
||||
)
|
||||
.where(
|
||||
Photo.user_id == current_user.id,
|
||||
Photo.is_discarded.is_(False),
|
||||
Photo.is_hidden.is_(False),
|
||||
)
|
||||
.group_by(photo_tags.c.tag_id)
|
||||
.subquery()
|
||||
)
|
||||
stmt = (
|
||||
select(Tag, count_subq.c.photo_count, count_subq.c.first_photo_id)
|
||||
.outerjoin(count_subq, Tag.id == count_subq.c.tag_id)
|
||||
.where(Tag.user_id == current_user.id)
|
||||
)
|
||||
if kind:
|
||||
stmt = stmt.where(Tag.kind == kind)
|
||||
stmt = stmt.order_by(Tag.name.asc())
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = result.all()
|
||||
|
||||
return [
|
||||
{
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"color": tag.color,
|
||||
"kind": tag.kind,
|
||||
"source": tag.source,
|
||||
"representative_photo_id": first_photo_id,
|
||||
"photo_count": int(count or 0),
|
||||
}
|
||||
for tag, count, first_photo_id in rows
|
||||
]
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_tag(body: TagCreate, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Create a new tag. The (name, kind) pair is unique — re-creating an
|
||||
existing pair returns the existing row (idempotent for autocomplete)."""
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Tag name is required")
|
||||
|
||||
existing = await db.execute(
|
||||
select(Tag).where(Tag.name == name, Tag.kind == body.kind, Tag.user_id == current_user.id)
|
||||
)
|
||||
found = existing.scalar_one_or_none()
|
||||
if found:
|
||||
return {
|
||||
"id": found.id, "name": found.name, "color": found.color,
|
||||
"kind": found.kind, "photo_count": 0,
|
||||
}
|
||||
|
||||
tag = Tag(name=name, color=body.color, kind=body.kind, user_id=current_user.id)
|
||||
db.add(tag)
|
||||
await db.commit()
|
||||
await db.refresh(tag)
|
||||
return {
|
||||
"id": tag.id, "name": tag.name, "color": tag.color,
|
||||
"kind": tag.kind, "photo_count": 0,
|
||||
}
|
||||
|
||||
|
||||
@router.patch("/{tag_id}")
|
||||
async def update_tag(
|
||||
tag_id: str, body: TagUpdate, db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Rename or recolor a tag."""
|
||||
result = await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id))
|
||||
tag = result.scalar_one_or_none()
|
||||
if not tag:
|
||||
raise HTTPException(status_code=404, detail="Tag not found")
|
||||
|
||||
if body.name is not None:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Tag name is required")
|
||||
tag.name = name
|
||||
if body.color is not None:
|
||||
tag.color = body.color or None
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(tag)
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "kind": tag.kind}
|
||||
|
||||
|
||||
@router.delete("/{tag_id}", status_code=204)
|
||||
async def delete_tag(tag_id: str, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Delete a tag. Photo associations cascade-delete via the FK."""
|
||||
result = await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id))
|
||||
tag = result.scalar_one_or_none()
|
||||
if not tag:
|
||||
raise HTTPException(status_code=404, detail="Tag not found")
|
||||
await db.delete(tag)
|
||||
await db.commit()
|
||||
return None
|
||||
@@ -1,73 +0,0 @@
|
||||
"""
|
||||
Pydantic schemas for photos
|
||||
"""
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
class PhotoBase(BaseModel):
|
||||
"""Base photo schema"""
|
||||
filename: str
|
||||
media_type: str
|
||||
original_format: Optional[str] = None
|
||||
width: Optional[int] = None
|
||||
height: Optional[int] = None
|
||||
file_size: Optional[int] = None
|
||||
taken_at: Optional[datetime] = None
|
||||
taken_at_source: Optional[str] = None
|
||||
user_title: Optional[str] = None
|
||||
user_notes: Optional[str] = None
|
||||
rating: int = 0
|
||||
color_label: Optional[str] = None
|
||||
|
||||
class PhotoResponse(PhotoBase):
|
||||
"""Photo response schema"""
|
||||
id: str
|
||||
filepath: str
|
||||
folder_id: Optional[str] = None
|
||||
file_hash: Optional[str] = None
|
||||
added_at: datetime
|
||||
updated_at: Optional[datetime] = None
|
||||
is_discarded: bool = False
|
||||
discarded_at: Optional[datetime] = None
|
||||
thumb_small: Optional[str] = None
|
||||
thumb_medium: Optional[str] = None
|
||||
thumb_large: Optional[str] = None
|
||||
processing_status: str = 'pending'
|
||||
processing_error: Optional[str] = None
|
||||
exif_json: Optional[str] = None
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
is_duplicate: bool = False
|
||||
has_date_warning: bool = False
|
||||
live_photo_video_id: Optional[str] = None
|
||||
owner_username: Optional[str] = None
|
||||
# tags: List[Dict[str, Any]] = [] # TODO: Enable when using eager loading
|
||||
|
||||
class Config:
|
||||
orm_mode = True
|
||||
from_attributes = True
|
||||
|
||||
class PhotoUpdate(BaseModel):
|
||||
"""Photo update schema"""
|
||||
filename: Optional[str] = None
|
||||
user_title: Optional[str] = None
|
||||
user_notes: Optional[str] = None
|
||||
rating: Optional[int] = Field(None, ge=0, le=5)
|
||||
color_label: Optional[str] = None
|
||||
is_discarded: Optional[bool] = None
|
||||
taken_at: Optional[datetime] = None
|
||||
|
||||
class PhotoListResponse(BaseModel):
|
||||
"""Photo list response with pagination"""
|
||||
photos: List[PhotoResponse]
|
||||
total: int
|
||||
page: int
|
||||
per_page: int
|
||||
pages: int
|
||||
|
||||
class BulkAction(BaseModel):
|
||||
"""Bulk action on photos"""
|
||||
ids: List[str]
|
||||
action: str # 'discard', 'restore', 'delete_permanent', 'move', 'copy', 'add_tag', 'remove_tag', 'set_rating', 'set_color'
|
||||
value: Optional[Any] = None # For actions that need a value (rating, color, tag_id, folder_id)
|
||||
@@ -1,447 +0,0 @@
|
||||
"""
|
||||
One-shot data integrity cleanup for source_roots / folders / photos.
|
||||
|
||||
Earlier versions of the scanner stored paths verbatim, so trailing slashes
|
||||
and redundant separators produced duplicate SourceRoot and Folder rows for
|
||||
the same physical directory. The watcher also auto-created source roots
|
||||
when fired with a parent dir. This module merges the duplicates and
|
||||
re-points photos to the canonical folder so the data lines up with the
|
||||
post-fix scanner.
|
||||
|
||||
Idempotent: safe to run on every backend startup.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from sqlalchemy import select, update, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo, Folder, SourceRoot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
return os.path.normpath(path)
|
||||
|
||||
|
||||
async def _dedupe_source_roots(session: AsyncSession) -> int:
|
||||
"""Group source roots by normalized path and merge duplicates. Returns
|
||||
the number of rows deleted."""
|
||||
result = await session.execute(select(SourceRoot))
|
||||
rows = result.scalars().all()
|
||||
|
||||
groups: dict[str, list[SourceRoot]] = {}
|
||||
for sr in rows:
|
||||
norm = _normalize_path(sr.path)
|
||||
groups.setdefault(norm, []).append(sr)
|
||||
|
||||
deleted = 0
|
||||
for norm, srs in groups.items():
|
||||
if len(srs) == 1:
|
||||
# Make sure the canonical row's path is normalized too.
|
||||
if srs[0].path != norm:
|
||||
srs[0].path = norm
|
||||
continue
|
||||
# Pick the canonical row: prefer one with a non-empty name and the
|
||||
# earliest added_at (most likely the original).
|
||||
canonical = sorted(
|
||||
srs,
|
||||
key=lambda s: (not bool(s.name), s.added_at or datetime.max),
|
||||
)[0]
|
||||
canonical.path = norm
|
||||
for sr in srs:
|
||||
if sr.id == canonical.id:
|
||||
continue
|
||||
# Re-point folders that referenced the duplicate root.
|
||||
await session.execute(
|
||||
update(Folder)
|
||||
.where(Folder.source_root_id == sr.id)
|
||||
.values(source_root_id=canonical.id)
|
||||
)
|
||||
await session.delete(sr)
|
||||
deleted += 1
|
||||
|
||||
return deleted
|
||||
|
||||
|
||||
async def _dedupe_folders(session: AsyncSession) -> int:
|
||||
"""Group folders by normalized path and merge duplicates. Returns the
|
||||
number of rows deleted."""
|
||||
result = await session.execute(select(Folder))
|
||||
rows = result.scalars().all()
|
||||
|
||||
groups: dict[str, list[Folder]] = {}
|
||||
for f in rows:
|
||||
norm = _normalize_path(f.path)
|
||||
groups.setdefault(norm, []).append(f)
|
||||
|
||||
deleted = 0
|
||||
for norm, folders in groups.items():
|
||||
if len(folders) == 1:
|
||||
if folders[0].path != norm:
|
||||
folders[0].path = norm
|
||||
continue
|
||||
# Canonical = the one with the most photos already attached, then
|
||||
# the lowest-id (deterministic tiebreaker).
|
||||
canonical = sorted(
|
||||
folders,
|
||||
key=lambda f: (-(f.photo_count or 0), f.id),
|
||||
)[0]
|
||||
canonical.path = norm
|
||||
for f in folders:
|
||||
if f.id == canonical.id:
|
||||
continue
|
||||
# Re-point photos to the canonical folder.
|
||||
await session.execute(
|
||||
update(Photo)
|
||||
.where(Photo.folder_id == f.id)
|
||||
.values(folder_id=canonical.id)
|
||||
)
|
||||
await session.delete(f)
|
||||
deleted += 1
|
||||
|
||||
return deleted
|
||||
|
||||
|
||||
async def _recompute_folder_counts(session: AsyncSession) -> None:
|
||||
"""Set folder.photo_count to the actual non-discarded photo count."""
|
||||
result = await session.execute(select(Folder))
|
||||
folders = result.scalars().all()
|
||||
for f in folders:
|
||||
count_result = await session.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
Photo.folder_id == f.id,
|
||||
Photo.is_discarded == False, # noqa: E712
|
||||
)
|
||||
)
|
||||
f.photo_count = int(count_result.scalar() or 0)
|
||||
|
||||
|
||||
def _parent_is_accessible(path: str) -> bool:
|
||||
"""True if the parent directory of `path` is readable. Used to
|
||||
distinguish 'user renamed/deleted the source root folder' (parent
|
||||
mount fine, leaf gone) from 'drive unmounted' (whole subtree
|
||||
inaccessible). The former is safe to prune from; the latter is
|
||||
not."""
|
||||
parent = os.path.dirname(path.rstrip(os.sep))
|
||||
if not parent:
|
||||
return False
|
||||
try:
|
||||
os.listdir(parent)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _sr_state(sr_path: str) -> str:
|
||||
"""Classify a source root path as one of:
|
||||
'present' — directory exists, business as usual
|
||||
'renamed' — leaf missing but parent mount is accessible (user
|
||||
renamed/deleted the folder in their file manager)
|
||||
'unmounted'— parent itself inaccessible (drive not mounted)
|
||||
"""
|
||||
if os.path.isdir(sr_path):
|
||||
return 'present'
|
||||
if _parent_is_accessible(sr_path):
|
||||
return 'renamed'
|
||||
return 'unmounted'
|
||||
|
||||
|
||||
async def _warn_stale_source_roots(session: AsyncSession) -> int:
|
||||
"""Log a warning for any active source root whose path no longer exists
|
||||
on disk. Doesn't delete — a missing path could be a temporarily
|
||||
unmounted drive, and silently dropping user data is worse than
|
||||
surfacing a noisy log line. Logs different hints for renamed-vs-
|
||||
unmounted so the user knows which knob to turn.
|
||||
"""
|
||||
result = await session.execute(select(SourceRoot))
|
||||
rows = result.scalars().all()
|
||||
stale = 0
|
||||
for sr in rows:
|
||||
state = _sr_state(sr.path)
|
||||
if state == 'present':
|
||||
continue
|
||||
stale += 1
|
||||
if state == 'renamed':
|
||||
logger.warning(
|
||||
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
||||
f"— parent mount is fine, looks like the folder was renamed "
|
||||
f"or deleted. Photos under it can be cleared via "
|
||||
f"POST /api/v1/library/maintenance/prune-missing."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
||||
f"— parent directory is also inaccessible; is the docker "
|
||||
f"mount still in place? (Edit docker-compose.yml or "
|
||||
f"PHOTO_DIRS in .env to fix.)"
|
||||
)
|
||||
return stale
|
||||
|
||||
|
||||
async def find_missing(
|
||||
session: AsyncSession,
|
||||
) -> tuple[list[str], list[str], list[str]]:
|
||||
"""Walk every non-discarded photo + every folder and check whether
|
||||
they still resolve on disk. Returns
|
||||
(deletable_photo_ids, deletable_folder_ids, skipped_photo_ids).
|
||||
|
||||
Skipped rows are photos/folders whose owning source_root is truly
|
||||
inaccessible (parent mount missing) — that's almost always an
|
||||
unmounted drive, and silently deleting those rows would be data
|
||||
loss. Photos under a source root whose leaf is missing but whose
|
||||
parent mount IS accessible (user renamed/deleted the folder) are
|
||||
treated as deletable, since their files are genuinely gone from
|
||||
the user's library.
|
||||
"""
|
||||
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
||||
# "Available" = source root path exists OR parent mount is accessible.
|
||||
# Only truly-unmounted source roots skip pruning.
|
||||
sr_mounted: dict[str, bool] = {
|
||||
sr.id: _sr_state(sr.path) != 'unmounted' for sr in sr_rows
|
||||
}
|
||||
|
||||
photos = (await session.execute(
|
||||
select(Photo.id, Photo.filepath, Photo.folder_id)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
)).all()
|
||||
|
||||
folders = (await session.execute(
|
||||
select(Folder.id, Folder.path, Folder.source_root_id)
|
||||
)).all()
|
||||
folder_to_sr = {fid: srid for fid, _path, srid in folders}
|
||||
|
||||
deletable_photos: list[str] = []
|
||||
skipped: list[str] = []
|
||||
for pid, fp, folder_id in photos:
|
||||
sr_id = folder_to_sr.get(folder_id)
|
||||
if sr_id is None or not sr_mounted.get(sr_id, False):
|
||||
skipped.append(pid)
|
||||
continue
|
||||
if not os.path.exists(fp):
|
||||
deletable_photos.append(pid)
|
||||
|
||||
deletable_folders: list[str] = []
|
||||
for fid, fpath, sr_id in folders:
|
||||
if sr_id is None or not sr_mounted.get(sr_id, False):
|
||||
continue
|
||||
if not os.path.isdir(fpath):
|
||||
deletable_folders.append(fid)
|
||||
|
||||
return deletable_photos, deletable_folders, skipped
|
||||
|
||||
|
||||
async def prune_missing_photos(dry_run: bool = True) -> dict:
|
||||
"""Delete photo + folder rows whose paths are no longer on disk *and*
|
||||
whose source root is currently mounted. Common cause: PHOTO_DIRS in
|
||||
.env was repointed at a different library, leaving every old row
|
||||
orphaned.
|
||||
|
||||
Set dry_run=False to actually delete. The default is intentionally
|
||||
safe so the matching count can be surfaced in the UI before the
|
||||
user commits to it.
|
||||
|
||||
Function name kept for backwards compatibility — it now also prunes
|
||||
folders, not just photos.
|
||||
"""
|
||||
from sqlalchemy import delete
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
deletable_photos, deletable_folders, skipped = await find_missing(session)
|
||||
if not dry_run:
|
||||
CHUNK = 500
|
||||
# Photos first (folders may FK from them via folder_id).
|
||||
for i in range(0, len(deletable_photos), CHUNK):
|
||||
await session.execute(
|
||||
delete(Photo).where(
|
||||
Photo.id.in_(deletable_photos[i:i + CHUNK])
|
||||
)
|
||||
)
|
||||
# Then drop folders that ALSO no longer have any photos
|
||||
# pointing at them. We re-check after the photo delete so
|
||||
# we don't strand a folder that legitimately exists on
|
||||
# disk but happened to match the orphan list.
|
||||
if deletable_folders:
|
||||
for i in range(0, len(deletable_folders), CHUNK):
|
||||
chunk = deletable_folders[i:i + CHUNK]
|
||||
# Only delete folders that now have zero photos
|
||||
# left attached (defensive — should always be 0
|
||||
# if the path is gone, but a concurrent scan
|
||||
# could re-create rows).
|
||||
still_used = (await session.execute(
|
||||
select(Photo.folder_id)
|
||||
.where(Photo.folder_id.in_(chunk))
|
||||
.distinct()
|
||||
)).scalars().all()
|
||||
safe = [f for f in chunk if f not in set(still_used)]
|
||||
if safe:
|
||||
await session.execute(
|
||||
delete(Folder).where(Folder.id.in_(safe))
|
||||
)
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"Pruned {len(deletable_photos)} photo rows + "
|
||||
f"{len(deletable_folders)} folder rows"
|
||||
)
|
||||
key_p = "would_delete" if dry_run else "deleted"
|
||||
key_f = "would_delete_folders" if dry_run else "deleted_folders"
|
||||
return {
|
||||
key_p: len(deletable_photos),
|
||||
key_f: len(deletable_folders),
|
||||
"skipped_unmounted": len(skipped),
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"prune_missing_photos failed: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def prune_orphan_thumbnails(
|
||||
thumbs_root: str = "/data/thumbs",
|
||||
dry_run: bool = True,
|
||||
) -> dict:
|
||||
"""Remove `/data/thumbs/{user_id}/{photo_id}/` directories whose
|
||||
photo_id no longer exists in the photos table.
|
||||
|
||||
Layout was per-Phase-4 set up by app.tasks.thumbs and is keyed by
|
||||
`{user_id}/{photo_id}/`. The thumbs worker never deletes its own
|
||||
output on photo removal, so over the lifetime of a library these
|
||||
directories accumulate.
|
||||
|
||||
Set dry_run=False to actually `rm -rf` each matched directory.
|
||||
Returns counts of matched / removed dirs and any per-dir errors.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
if not os.path.isdir(thumbs_root):
|
||||
return {
|
||||
"would_remove": 0,
|
||||
"removed": 0,
|
||||
"skipped_no_root": True,
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
live_ids = {
|
||||
row[0]
|
||||
for row in (await session.execute(select(Photo.id))).all()
|
||||
}
|
||||
|
||||
matched: list[str] = []
|
||||
errors: list[str] = []
|
||||
for user_dir in os.listdir(thumbs_root):
|
||||
user_path = os.path.join(thumbs_root, user_dir)
|
||||
if not os.path.isdir(user_path):
|
||||
continue
|
||||
for photo_dir in os.listdir(user_path):
|
||||
if photo_dir in live_ids:
|
||||
continue
|
||||
matched.append(os.path.join(user_path, photo_dir))
|
||||
|
||||
removed = 0
|
||||
if not dry_run:
|
||||
for path in matched:
|
||||
try:
|
||||
shutil.rmtree(path)
|
||||
removed += 1
|
||||
except OSError as e:
|
||||
errors.append(f"{path}: {e}")
|
||||
|
||||
key = "would_remove" if dry_run else "removed"
|
||||
return {
|
||||
key: len(matched) if dry_run else removed,
|
||||
"errors": errors,
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
|
||||
|
||||
async def discard_missing_photos() -> dict:
|
||||
"""Soft variant of prune_missing_photos for the periodic beat
|
||||
catch-up. Walks every active source root that is currently
|
||||
`present` (not 'renamed' — the user-driven manual flow handles
|
||||
those — and not 'unmounted'), and for each Photo whose file is
|
||||
gone from disk, sets is_discarded=True so it shows up in the
|
||||
in-app trash. Idempotent: skips photos that are already
|
||||
discarded.
|
||||
|
||||
Hard-deletion stays manual via prune_missing_photos so users
|
||||
can review the list before committing.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
||||
present_sr_ids = {
|
||||
sr.id for sr in sr_rows if _sr_state(sr.path) == 'present'
|
||||
}
|
||||
if not present_sr_ids:
|
||||
return {"discarded": 0, "checked": 0}
|
||||
|
||||
folder_to_sr = {
|
||||
fid: srid
|
||||
for fid, _path, srid in (await session.execute(
|
||||
select(Folder.id, Folder.path, Folder.source_root_id)
|
||||
)).all()
|
||||
}
|
||||
|
||||
photo_rows = (await session.execute(
|
||||
select(Photo.id, Photo.filepath, Photo.folder_id)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
)).all()
|
||||
|
||||
missing_ids: list[str] = []
|
||||
checked = 0
|
||||
for pid, fp, folder_id in photo_rows:
|
||||
sr_id = folder_to_sr.get(folder_id)
|
||||
if sr_id not in present_sr_ids:
|
||||
continue
|
||||
checked += 1
|
||||
if not os.path.exists(fp):
|
||||
missing_ids.append(pid)
|
||||
|
||||
if missing_ids:
|
||||
CHUNK = 500
|
||||
now = datetime.utcnow()
|
||||
for i in range(0, len(missing_ids), CHUNK):
|
||||
await session.execute(
|
||||
update(Photo)
|
||||
.where(Photo.id.in_(missing_ids[i:i + CHUNK]))
|
||||
.values(is_discarded=True, discarded_at=now)
|
||||
)
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"discard_missing_photos: discarded {len(missing_ids)} "
|
||||
f"of {checked} photos under {len(present_sr_ids)} present source roots"
|
||||
)
|
||||
|
||||
return {"discarded": len(missing_ids), "checked": checked}
|
||||
except Exception as e:
|
||||
logger.error(f"discard_missing_photos failed: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def cleanup_data_integrity() -> dict:
|
||||
"""Top-level entry point. Runs the dedupe + count refresh in a single
|
||||
transaction. Returns a small summary dict for logging."""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
sr_deleted = await _dedupe_source_roots(session)
|
||||
f_deleted = await _dedupe_folders(session)
|
||||
await _recompute_folder_counts(session)
|
||||
stale = await _warn_stale_source_roots(session)
|
||||
await session.commit()
|
||||
summary = {
|
||||
"source_roots_merged": sr_deleted,
|
||||
"folders_merged": f_deleted,
|
||||
"source_roots_stale": stale,
|
||||
}
|
||||
if sr_deleted or f_deleted:
|
||||
logger.info(f"Cleanup merged duplicates: {summary}")
|
||||
return summary
|
||||
except Exception as e:
|
||||
logger.error(f"Cleanup failed: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
@@ -1,239 +0,0 @@
|
||||
"""
|
||||
Folder/filename-based date guessing and "taken_at looks wrong" detection.
|
||||
|
||||
Direct Python port of `frontend/src/lib/guessDateFromPath.ts` — the logic
|
||||
must stay in sync because the frontend renders the suggestion hint in the
|
||||
info panel while the backend owns the `has_date_warning` flag that the
|
||||
filter bar queries. Any heuristic change has to be applied to both files.
|
||||
|
||||
The guesser walks a filepath, tries the filename first as the source of
|
||||
truth, then falls back to folder segments (deepest first) and multi-
|
||||
segment layouts. Returns ``None`` when no recognisable date can be
|
||||
extracted. `has_date_warning()` compares the guess to a stored `taken_at`
|
||||
and reports whether the difference is large enough to flag.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Literal, Optional
|
||||
|
||||
|
||||
Confidence = Literal["high", "medium", "low"]
|
||||
Source = Literal["folder", "filename"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DateGuess:
|
||||
date: datetime
|
||||
confidence: Confidence
|
||||
matched: str
|
||||
source: Source
|
||||
|
||||
|
||||
_MIN_YEAR = 1970
|
||||
# Bump the ceiling annually via `datetime.now()` rather than a literal so
|
||||
# we don't ship a time bomb. `+1` allows near-future timestamps (cameras
|
||||
# with a slightly advanced clock at year end) without opening the door to
|
||||
# 4-digit serial numbers that happen to start with "30xx".
|
||||
def _max_year() -> int:
|
||||
return datetime.now().year + 1
|
||||
|
||||
|
||||
def _valid_year(y: int) -> bool:
|
||||
return _MIN_YEAR <= y <= _max_year()
|
||||
|
||||
|
||||
def _make_date(y: int, m: int, d: int) -> Optional[datetime]:
|
||||
if not _valid_year(y):
|
||||
return None
|
||||
if not (1 <= m <= 12):
|
||||
return None
|
||||
if not (1 <= d <= 31):
|
||||
return None
|
||||
try:
|
||||
# Noon local so downstream day-bucketing is stable across timezone
|
||||
# rounding. The frontend mirrors this.
|
||||
return datetime(y, m, d, 12, 0, 0)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _segments(filepath: str) -> list[str]:
|
||||
return [s for s in re.split(r"[\\/]+", filepath) if s]
|
||||
|
||||
|
||||
# Synology Photos export: `YY-MM-DD HH-MM-SS NNNN.ext`. The explicit
|
||||
# HH-MM-SS half is what makes the 2-digit year safe to trust — a random
|
||||
# digit triple won't satisfy the hour/minute/second range checks below.
|
||||
# YY is mapped to 2000+YY (this format is a recent export convention).
|
||||
_SYNOLOGY_RE = re.compile(
|
||||
r"(?<!\d)(\d{2})-(\d{2})-(\d{2})[\s_](\d{2})-(\d{2})-(\d{2})(?!\d)"
|
||||
)
|
||||
_COMPACT_RE = re.compile(r"(?<!\d)(\d{4})(\d{2})(\d{2})(?!\d)")
|
||||
_DASHED_RE = re.compile(r"(?<!\d)(\d{4})[-_.](\d{1,2})[-_.](\d{1,2})(?!\d)")
|
||||
_MONTH_RE = re.compile(r"(?<!\d)(\d{4})[-_.](\d{1,2})(?!\d)")
|
||||
_YEAR_RE = re.compile(r"(?<!\d)(\d{4})(?!\d)")
|
||||
_FOUR_DIGITS = re.compile(r"^\d{4}$")
|
||||
_ONE_OR_TWO = re.compile(r"^\d{1,2}$")
|
||||
|
||||
|
||||
def _guess_from_string(
|
||||
input: str,
|
||||
source: Source,
|
||||
allow_year_only: bool,
|
||||
) -> Optional[DateGuess]:
|
||||
if not input:
|
||||
return None
|
||||
|
||||
m = _SYNOLOGY_RE.search(input)
|
||||
if m:
|
||||
yy, mm, dd = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
hh, mi, ss = int(m.group(4)), int(m.group(5)), int(m.group(6))
|
||||
if hh < 24 and mi < 60 and ss < 60:
|
||||
d = _make_date(2000 + yy, mm, dd)
|
||||
if d:
|
||||
d = d.replace(hour=hh, minute=mi, second=ss)
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="high",
|
||||
matched=(
|
||||
f"{m.group(1)}-{m.group(2)}-{m.group(3)} "
|
||||
f"{m.group(4)}:{m.group(5)}:{m.group(6)}"
|
||||
),
|
||||
source=source,
|
||||
)
|
||||
|
||||
m = _COMPACT_RE.search(input)
|
||||
if m:
|
||||
d = _make_date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="high",
|
||||
matched=f"{m.group(1)}-{m.group(2)}-{m.group(3)}",
|
||||
source=source,
|
||||
)
|
||||
|
||||
m = _DASHED_RE.search(input)
|
||||
if m:
|
||||
d = _make_date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="high",
|
||||
matched=f"{m.group(1)}-{m.group(2)}-{m.group(3)}",
|
||||
source=source,
|
||||
)
|
||||
|
||||
m = _MONTH_RE.search(input)
|
||||
if m:
|
||||
d = _make_date(int(m.group(1)), int(m.group(2)), 15)
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="medium",
|
||||
matched=f"{m.group(1)}-{m.group(2)}",
|
||||
source=source,
|
||||
)
|
||||
|
||||
if allow_year_only:
|
||||
m = _YEAR_RE.search(input)
|
||||
if m:
|
||||
d = _make_date(int(m.group(1)), 7, 1)
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="low",
|
||||
matched=m.group(1),
|
||||
source=source,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _guess_from_folder_layout(folders: list[str]) -> Optional[DateGuess]:
|
||||
# YYYY / MM / DD
|
||||
for i in range(len(folders) - 2):
|
||||
a, b, c = folders[i], folders[i + 1], folders[i + 2]
|
||||
if _FOUR_DIGITS.match(a) and _ONE_OR_TWO.match(b) and _ONE_OR_TWO.match(c):
|
||||
d = _make_date(int(a), int(b), int(c))
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="high",
|
||||
matched=f"{a}/{b}/{c}",
|
||||
source="folder",
|
||||
)
|
||||
# YYYY / MM
|
||||
for i in range(len(folders) - 1):
|
||||
a, b = folders[i], folders[i + 1]
|
||||
if _FOUR_DIGITS.match(a) and _ONE_OR_TWO.match(b):
|
||||
d = _make_date(int(a), int(b), 15)
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="medium",
|
||||
matched=f"{a}/{b}",
|
||||
source="folder",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
_CONFIDENCE_RANK: dict[Confidence, int] = {"high": 3, "medium": 2, "low": 1}
|
||||
|
||||
|
||||
def guess_date_from_path(filepath: str) -> Optional[DateGuess]:
|
||||
"""Filename wins when it has any viable match; otherwise walk folder
|
||||
segments deepest-first and pick the strongest hit."""
|
||||
if not filepath:
|
||||
return None
|
||||
|
||||
segs = _segments(filepath)
|
||||
if not segs:
|
||||
return None
|
||||
filename = segs[-1]
|
||||
folders = segs[:-1]
|
||||
|
||||
from_filename = _guess_from_string(filename, "filename", allow_year_only=False)
|
||||
if from_filename:
|
||||
return from_filename
|
||||
|
||||
best: Optional[DateGuess] = None
|
||||
for seg in reversed(folders):
|
||||
hit = _guess_from_string(seg, "folder", allow_year_only=True)
|
||||
if not hit:
|
||||
continue
|
||||
if not best or _CONFIDENCE_RANK[hit.confidence] > _CONFIDENCE_RANK[best.confidence]:
|
||||
best = hit
|
||||
if hit.confidence == "high":
|
||||
break
|
||||
|
||||
from_layout = _guess_from_folder_layout(folders)
|
||||
if from_layout and (
|
||||
not best or _CONFIDENCE_RANK[from_layout.confidence] > _CONFIDENCE_RANK[best.confidence]
|
||||
):
|
||||
best = from_layout
|
||||
|
||||
return best
|
||||
|
||||
|
||||
_ONE_DAY = 24 * 60 * 60
|
||||
|
||||
|
||||
def has_date_warning(filepath: str, taken_at: Optional[datetime]) -> bool:
|
||||
"""True when the path-based guess disagrees with ``taken_at`` by more
|
||||
than 24h, or when ``taken_at`` is missing and the path would supply
|
||||
one. This is the authoritative flag stored on `photos.has_date_warning`
|
||||
and queried by the timeline filter."""
|
||||
guess = guess_date_from_path(filepath)
|
||||
if not guess:
|
||||
return False
|
||||
if taken_at is None:
|
||||
return True
|
||||
try:
|
||||
diff = abs((taken_at - guess.date).total_seconds())
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return diff > _ONE_DAY
|
||||
@@ -1,298 +0,0 @@
|
||||
"""
|
||||
Duplicate detection: group photos by perceptual-hash + CLIP similarity.
|
||||
|
||||
Strategy
|
||||
--------
|
||||
Two complementary signals are fused into a single grouping:
|
||||
|
||||
1. **Perceptual hash (pHash)** — 16-char hex hash from the thumbnail
|
||||
worker. Catches byte-identical copies and mild re-encodes via
|
||||
Hamming distance (threshold ≤ 6 bits out of 64).
|
||||
|
||||
2. **CLIP embedding similarity** — cosine distance over 512-d vectors
|
||||
stored in the `embeddings` table with an HNSW index. Catches
|
||||
visually similar photos even when pHash diverges (e.g. crops,
|
||||
different formats, screenshots of the same content).
|
||||
|
||||
Both signals feed a union-find structure that merges overlapping matches
|
||||
into connected components.
|
||||
|
||||
Incremental mode (default post-scan)
|
||||
-------------------------------------
|
||||
`incremental_regroup` only compares *newly added* photos (those whose
|
||||
`added_at` > watermark) against the entire library. Each new photo does:
|
||||
|
||||
- An HNSW vector similarity query: O(log N) via the index.
|
||||
- A pHash comparison against a small candidate set (same group members
|
||||
or nearby CLIP results) rather than the full N² sweep.
|
||||
|
||||
This makes the post-scan cost O(new × log N) instead of O(N²).
|
||||
|
||||
Full regroup
|
||||
------------
|
||||
`regroup_duplicates` still performs the full pairwise pHash pass +
|
||||
CLIP sweep, used for initial setup and manual re-detection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.photos import Photo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# pHash Hamming distance threshold (6 out of 64 bits).
|
||||
DEFAULT_PHASH_THRESHOLD = 6
|
||||
|
||||
|
||||
def _hex_to_int(h: str) -> int:
|
||||
"""Parse a 16-char hex pHash to a Python int. Returns -1 on bad input
|
||||
so the pairwise loop can skip the row without raising."""
|
||||
try:
|
||||
return int(h, 16)
|
||||
except (TypeError, ValueError):
|
||||
return -1
|
||||
|
||||
|
||||
def _hamming(a: int, b: int) -> int:
|
||||
"""Population count of XOR — the canonical hash distance metric."""
|
||||
x = a ^ b
|
||||
try:
|
||||
return x.bit_count() # type: ignore[attr-defined]
|
||||
except AttributeError:
|
||||
return bin(x).count('1')
|
||||
|
||||
|
||||
class _UnionFind:
|
||||
"""Tiny union-find / disjoint-set used to merge similar photos into
|
||||
connected components."""
|
||||
|
||||
def __init__(self, keys: list[str]) -> None:
|
||||
self._index = {k: i for i, k in enumerate(keys)}
|
||||
n = len(keys)
|
||||
self.parent = list(range(n))
|
||||
self.rank = [0] * n
|
||||
|
||||
def find(self, x: int) -> int:
|
||||
while self.parent[x] != x:
|
||||
self.parent[x] = self.parent[self.parent[x]]
|
||||
x = self.parent[x]
|
||||
return x
|
||||
|
||||
def union_by_key(self, key_a: str, key_b: str) -> None:
|
||||
ia, ib = self._index.get(key_a), self._index.get(key_b)
|
||||
if ia is None or ib is None:
|
||||
return
|
||||
ra, rb = self.find(ia), self.find(ib)
|
||||
if ra == rb:
|
||||
return
|
||||
if self.rank[ra] < self.rank[rb]:
|
||||
ra, rb = rb, ra
|
||||
self.parent[rb] = ra
|
||||
if self.rank[ra] == self.rank[rb]:
|
||||
self.rank[ra] += 1
|
||||
|
||||
def components(self, keys: list[str]) -> dict[int, list[str]]:
|
||||
"""Return {root_idx: [photo_ids...]} for groups of size >= 2."""
|
||||
groups: dict[int, list[str]] = {}
|
||||
for key in keys:
|
||||
idx = self._index[key]
|
||||
root = self.find(idx)
|
||||
groups.setdefault(root, []).append(key)
|
||||
return {r: members for r, members in groups.items() if len(members) >= 2}
|
||||
|
||||
|
||||
async def regroup_duplicates(
|
||||
phash_threshold: int = DEFAULT_PHASH_THRESHOLD,
|
||||
**_ignored,
|
||||
) -> dict:
|
||||
"""Full recompute of duplicate groups using pHash similarity.
|
||||
|
||||
Idempotent — safe to call as often as you like. Returns a summary dict.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Pull all visible photos with a phash or embedding.
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Photo.id, Photo.phash)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.where(Photo.is_hidden.is_(False))
|
||||
)
|
||||
).all()
|
||||
|
||||
if not rows:
|
||||
await _clear_all_groups(session)
|
||||
await session.commit()
|
||||
return {'photos_considered': 0, 'groups': 0, 'members': 0}
|
||||
|
||||
ids = [row[0] for row in rows]
|
||||
phash_map = {row[0]: _hex_to_int(row[1]) for row in rows if row[1]}
|
||||
|
||||
uf = _UnionFind(ids)
|
||||
|
||||
# ── Phase 1: pHash pairwise (O(N²) on photos with phash) ──
|
||||
phash_ids = [pid for pid in ids if pid in phash_map]
|
||||
phash_vals = [phash_map[pid] for pid in phash_ids]
|
||||
n = len(phash_ids)
|
||||
for i in range(n):
|
||||
hi = phash_vals[i]
|
||||
if hi < 0:
|
||||
continue
|
||||
for j in range(i + 1, n):
|
||||
hj = phash_vals[j]
|
||||
if hj < 0:
|
||||
continue
|
||||
if _hamming(hi, hj) <= phash_threshold:
|
||||
uf.union_by_key(phash_ids[i], phash_ids[j])
|
||||
|
||||
# ── Write results ──
|
||||
await _clear_all_groups(session)
|
||||
|
||||
groups = uf.components(ids)
|
||||
groups_created = 0
|
||||
members_total = 0
|
||||
for member_ids in groups.values():
|
||||
group_id = str(uuid.uuid4())
|
||||
await session.execute(
|
||||
update(Photo)
|
||||
.where(Photo.id.in_(member_ids))
|
||||
.values(duplicate_group_id=group_id, is_duplicate=True)
|
||||
)
|
||||
groups_created += 1
|
||||
members_total += len(member_ids)
|
||||
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"regroup_duplicates: {len(ids)} photos, "
|
||||
f"{groups_created} group(s), {members_total} member(s)"
|
||||
)
|
||||
return {
|
||||
'photos_considered': len(ids),
|
||||
'groups': groups_created,
|
||||
'members': members_total,
|
||||
}
|
||||
|
||||
|
||||
async def incremental_regroup(
|
||||
since: Optional[datetime] = None,
|
||||
phash_threshold: int = DEFAULT_PHASH_THRESHOLD,
|
||||
**_ignored,
|
||||
) -> dict:
|
||||
"""Incremental duplicate detection for newly added photos using pHash."""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# If no watermark, fall back to full regroup.
|
||||
if since is None:
|
||||
# Find the most recent scan start by looking at the newest
|
||||
# photo that already has a duplicate_group_id check completed.
|
||||
# As a simple heuristic, use photos added in the last hour.
|
||||
from datetime import timedelta
|
||||
since = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
|
||||
# Photo.added_at is stored as TIMESTAMP WITHOUT TIME ZONE, so
|
||||
# asyncpg rejects aware datetimes with "can't subtract offset-naive
|
||||
# and offset-aware". Normalise: if `since` has a tzinfo, convert
|
||||
# it to UTC and drop the tzinfo so the bind parameter is naive.
|
||||
if since.tzinfo is not None:
|
||||
since = since.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
# Get newly added photos (the "new" set).
|
||||
new_rows = (
|
||||
await session.execute(
|
||||
select(Photo.id, Photo.phash)
|
||||
.where(Photo.added_at >= since)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.where(Photo.is_hidden.is_(False))
|
||||
)
|
||||
).all()
|
||||
|
||||
if not new_rows:
|
||||
return {'photos_considered': 0, 'new_photos': 0, 'groups_updated': 0, 'members_added': 0}
|
||||
|
||||
new_ids = [r[0] for r in new_rows]
|
||||
new_phash = {r[0]: _hex_to_int(r[1]) for r in new_rows if r[1]}
|
||||
|
||||
# Get ALL existing photos for union-find (we need to merge into
|
||||
# existing groups).
|
||||
all_rows = (
|
||||
await session.execute(
|
||||
select(Photo.id, Photo.phash, Photo.duplicate_group_id)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.where(Photo.is_hidden.is_(False))
|
||||
)
|
||||
).all()
|
||||
|
||||
all_ids = [r[0] for r in all_rows]
|
||||
all_phash = {r[0]: _hex_to_int(r[1]) for r in all_rows if r[1]}
|
||||
existing_groups: dict[str, str] = {
|
||||
r[0]: r[2] for r in all_rows if r[2]
|
||||
}
|
||||
|
||||
uf = _UnionFind(all_ids)
|
||||
|
||||
# Pre-seed existing groups into the union-find so we merge into
|
||||
# them rather than creating parallel groups.
|
||||
group_to_members: dict[str, list[str]] = {}
|
||||
for pid, gid in existing_groups.items():
|
||||
group_to_members.setdefault(gid, []).append(pid)
|
||||
for members in group_to_members.values():
|
||||
for i in range(1, len(members)):
|
||||
uf.union_by_key(members[0], members[i])
|
||||
|
||||
# ── Phase 1: pHash — compare each new photo against ALL photos ──
|
||||
for new_id in new_ids:
|
||||
nh = new_phash.get(new_id, -1)
|
||||
if nh < 0:
|
||||
continue
|
||||
for existing_id, eh in all_phash.items():
|
||||
if existing_id == new_id or eh < 0:
|
||||
continue
|
||||
if _hamming(nh, eh) <= phash_threshold:
|
||||
uf.union_by_key(new_id, existing_id)
|
||||
|
||||
# ── Write results ──
|
||||
# Only update groups that contain at least one new photo.
|
||||
# Clear all groups first, then rewrite.
|
||||
await _clear_all_groups(session)
|
||||
|
||||
groups = uf.components(all_ids)
|
||||
groups_created = 0
|
||||
members_total = 0
|
||||
new_in_groups = 0
|
||||
for member_ids in groups.values():
|
||||
group_id = str(uuid.uuid4())
|
||||
await session.execute(
|
||||
update(Photo)
|
||||
.where(Photo.id.in_(member_ids))
|
||||
.values(duplicate_group_id=group_id, is_duplicate=True)
|
||||
)
|
||||
groups_created += 1
|
||||
members_total += len(member_ids)
|
||||
if any(m in new_ids for m in member_ids):
|
||||
new_in_groups += len([m for m in member_ids if m in new_ids])
|
||||
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"incremental_regroup: {len(new_ids)} new photos, "
|
||||
f"{groups_created} group(s), {new_in_groups} new member(s) grouped"
|
||||
)
|
||||
return {
|
||||
'photos_considered': len(all_ids),
|
||||
'new_photos': len(new_ids),
|
||||
'groups_updated': groups_created,
|
||||
'members_added': new_in_groups,
|
||||
}
|
||||
|
||||
|
||||
async def _clear_all_groups(session) -> None:
|
||||
"""Reset duplicate_group_id / is_duplicate on every photo."""
|
||||
await session.execute(
|
||||
update(Photo).values(duplicate_group_id=None, is_duplicate=False)
|
||||
)
|
||||
@@ -1,72 +0,0 @@
|
||||
"""
|
||||
EXIF write-back helpers.
|
||||
|
||||
The rest of the app reads EXIF at scan time and stashes the result in Postgres
|
||||
(see `services/metadata.py`). This module handles the reverse direction: when
|
||||
the user corrects a date in the UI we also rewrite the relevant EXIF tags on
|
||||
disk so a later rescan won't clobber the fix and external tools see the same
|
||||
truth the DB does.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EXIFTOOL_TIMEOUT_SECONDS = 30
|
||||
|
||||
|
||||
class ExifWriteError(RuntimeError):
|
||||
"""Raised when exiftool fails to write tags to a file."""
|
||||
|
||||
|
||||
def _format_exif_dt(dt: datetime) -> str:
|
||||
return dt.strftime("%Y:%m:%d %H:%M:%S")
|
||||
|
||||
|
||||
async def write_taken_at(filepath: str, dt: datetime) -> None:
|
||||
"""Rewrite DateTimeOriginal / CreateDate / ModifyDate on the file.
|
||||
|
||||
- ``-overwrite_original`` so we don't litter the library with
|
||||
``<name>_original`` sidecars.
|
||||
- ``-P`` preserves the file's mtime so the scanner's mtime-based
|
||||
change detection stays quiet.
|
||||
- We set all three common date tags together because different viewers
|
||||
read different ones; keeping them in lockstep avoids confusing
|
||||
downstream tools and our own re-extraction pass.
|
||||
"""
|
||||
if not Path(filepath).exists():
|
||||
raise ExifWriteError(f"File not found: {filepath}")
|
||||
|
||||
stamp = _format_exif_dt(dt)
|
||||
cmd = [
|
||||
"exiftool",
|
||||
"-overwrite_original",
|
||||
"-P",
|
||||
f"-DateTimeOriginal={stamp}",
|
||||
f"-CreateDate={stamp}",
|
||||
f"-ModifyDate={stamp}",
|
||||
filepath,
|
||||
]
|
||||
|
||||
def _run() -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=EXIFTOOL_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(_run)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise ExifWriteError(f"exiftool timed out writing {filepath}") from exc
|
||||
except FileNotFoundError as exc:
|
||||
raise ExifWriteError("exiftool binary not available") from exc
|
||||
|
||||
if result.returncode != 0:
|
||||
msg = (result.stderr or result.stdout or "unknown error").strip()
|
||||
logger.warning("exiftool write failed for %s: %s", filepath, msg)
|
||||
raise ExifWriteError(msg)
|
||||
@@ -1,28 +0,0 @@
|
||||
"""Gravatar URL helper.
|
||||
|
||||
Pure function — no HTTP calls. The browser does the actual image
|
||||
fetch. We just build the deterministic URL from the user's email and
|
||||
let Gravatar serve an identicon when no account exists for that hash,
|
||||
so the avatar is never a broken image.
|
||||
|
||||
Current Gravatar guidance is SHA-256 of the trimmed, lower-cased email.
|
||||
MD5 still works but is deprecated, so we prefer SHA-256.
|
||||
"""
|
||||
import hashlib
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def gravatar_url(email: Optional[str], size: int = 240) -> Optional[str]:
|
||||
"""Return a Gravatar image URL for `email`, or None when email is empty.
|
||||
|
||||
The `d=identicon` fallback guarantees a deterministic placeholder when
|
||||
the address has no Gravatar account, so callers can treat the result
|
||||
as a valid image URL.
|
||||
"""
|
||||
if not email:
|
||||
return None
|
||||
normalized = email.strip().lower()
|
||||
if not normalized:
|
||||
return None
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
return f"https://gravatar.com/avatar/{digest}?d=identicon&s={size}"
|
||||
@@ -1,536 +0,0 @@
|
||||
"""
|
||||
Metadata extraction service using ExifTool
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from celery import shared_task
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo
|
||||
from app.services.date_guess import has_date_warning
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def parse_exif_datetime(date_str: str) -> Optional[datetime]:
|
||||
"""Parse EXIF datetime string to Python datetime.
|
||||
|
||||
Returns a tz-naive datetime — the photos.taken_at column is
|
||||
`timestamp without time zone`. Tz-aware inputs (e.g. SubSec
|
||||
fields with `+02:00` or QuickTime UTC `Z`) are converted to UTC
|
||||
and stripped. Cameras that wrote the all-zero placeholder
|
||||
return None.
|
||||
"""
|
||||
if not date_str:
|
||||
return None
|
||||
s = str(date_str).strip()
|
||||
# All-zero placeholder some cameras emit when the clock isn't set.
|
||||
if s.startswith("0000:00:00") or s.startswith("0000-00-00"):
|
||||
return None
|
||||
|
||||
formats = [
|
||||
"%Y:%m:%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y:%m:%d %H:%M:%S.%f",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%dT%H:%M:%S.%f",
|
||||
# Tz-aware variants: SubSecDateTimeOriginal often looks like
|
||||
# "2023:11:30 14:30:45.123+02:00", QuickTime CreateDate as
|
||||
# "2023:11:30 14:30:45Z" or with offsets.
|
||||
"%Y:%m:%d %H:%M:%S%z",
|
||||
"%Y:%m:%d %H:%M:%S.%f%z",
|
||||
"%Y-%m-%dT%H:%M:%S%z",
|
||||
"%Y-%m-%dT%H:%M:%S.%f%z",
|
||||
]
|
||||
|
||||
for fmt in formats:
|
||||
try:
|
||||
dt = datetime.strptime(s, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
if dt.tzinfo is not None:
|
||||
from datetime import timezone
|
||||
dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
return dt
|
||||
|
||||
return None
|
||||
|
||||
_DMS_RE = re.compile(
|
||||
r"""\s*
|
||||
(?P<deg>-?\d+(?:\.\d+)?)\s*(?:deg|°|d)?\s*
|
||||
(?:(?P<min>\d+(?:\.\d+)?)\s*[\'’m]?\s*)?
|
||||
(?:(?P<sec>\d+(?:\.\d+)?)\s*[\"”s]?\s*)?
|
||||
(?P<ref>[NSEW])?\s*$""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_coord(value, ref: str | None) -> float | None:
|
||||
"""Coerce a single GPS coordinate from any form ExifTool may emit.
|
||||
|
||||
ExifTool's ``-j`` JSON output applies print conversion by default, so
|
||||
coordinates can come back as:
|
||||
|
||||
* a number (``48.1278``) — happens for some sources / when ``-n`` is set
|
||||
* a plain DMS string (``"48 deg 7' 39.96\\""``) — bare ``EXIF:GPSLatitude``
|
||||
* a DMS-with-ref string (``"48 deg 7' 39.96\\" N"``) — ``Composite:GPSLatitude``
|
||||
|
||||
The optional ``ref`` argument lets the caller pass an explicit
|
||||
``GPSLatitudeRef`` / ``GPSLongitudeRef`` ('N'/'S'/'E'/'W') when the
|
||||
string itself doesn't carry one. Returns signed decimal degrees, or
|
||||
``None`` if the value is unparseable.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
# Numeric path — already decimal degrees, possibly already signed.
|
||||
if isinstance(value, (int, float)):
|
||||
out = float(value)
|
||||
else:
|
||||
m = _DMS_RE.match(str(value))
|
||||
if not m:
|
||||
return None
|
||||
deg = float(m.group('deg'))
|
||||
minutes = float(m.group('min') or 0)
|
||||
seconds = float(m.group('sec') or 0)
|
||||
out = abs(deg) + minutes / 60.0 + seconds / 3600.0
|
||||
if deg < 0:
|
||||
out = -out
|
||||
embedded_ref = m.group('ref')
|
||||
if embedded_ref:
|
||||
ref = embedded_ref
|
||||
if ref:
|
||||
r = ref[0].upper()
|
||||
if r in ('S', 'W'):
|
||||
out = -abs(out)
|
||||
elif r in ('N', 'E'):
|
||||
out = abs(out)
|
||||
return out
|
||||
|
||||
|
||||
def extract_gps(exif_data: Dict) -> tuple:
|
||||
"""Return (lat, lon) in signed decimal degrees, or (None, None).
|
||||
|
||||
With ``exiftool -G -j`` GPS values are keyed under their group.
|
||||
``Composite:GPSLatitude`` / ``Composite:GPSLongitude`` carry the
|
||||
hemisphere reference inline (``"48 deg 7' 39.96\\" N"``) while the bare
|
||||
``EXIF:GPSLatitude`` / ``EXIF:GPSLongitude`` need the separate
|
||||
``EXIF:GPSLatitudeRef`` / ``EXIF:GPSLongitudeRef`` to know the sign.
|
||||
|
||||
Pre-fix this function read the *unprefixed* keys ``GPSLatitude`` /
|
||||
``GPSLongitude`` (which never exist in ``-G`` output) AND assumed
|
||||
they were already floats — so it silently dropped every photo's GPS.
|
||||
"""
|
||||
lat = _parse_coord(exif_data.get('Composite:GPSLatitude'), None)
|
||||
lon = _parse_coord(exif_data.get('Composite:GPSLongitude'), None)
|
||||
if lat is None or lon is None:
|
||||
lat = _parse_coord(
|
||||
exif_data.get('EXIF:GPSLatitude'),
|
||||
exif_data.get('EXIF:GPSLatitudeRef'),
|
||||
)
|
||||
lon = _parse_coord(
|
||||
exif_data.get('EXIF:GPSLongitude'),
|
||||
exif_data.get('EXIF:GPSLongitudeRef'),
|
||||
)
|
||||
if lat is None or lon is None:
|
||||
return None, None
|
||||
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
|
||||
return None, None
|
||||
# Some cameras emit (0, 0) when they have no GPS lock — treat as missing
|
||||
if lat == 0 and lon == 0:
|
||||
return None, None
|
||||
return lat, lon
|
||||
|
||||
|
||||
def extract_key_metadata(exif_data: Dict) -> Dict:
|
||||
"""Extract key metadata fields for FTS indexing"""
|
||||
key_fields = []
|
||||
|
||||
# Camera information
|
||||
if 'EXIF:Make' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:Make'])
|
||||
if 'EXIF:Model' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:Model'])
|
||||
if 'EXIF:LensModel' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:LensModel'])
|
||||
|
||||
# Location information
|
||||
lat, lon = extract_gps(exif_data)
|
||||
if lat is not None and lon is not None:
|
||||
key_fields.append(f"GPS: {lat}, {lon}")
|
||||
|
||||
# IPTC/XMP keywords
|
||||
keywords = exif_data.get('IPTC:Keywords') or exif_data.get('XMP:Subject')
|
||||
if keywords:
|
||||
if isinstance(keywords, list):
|
||||
key_fields.extend(keywords)
|
||||
else:
|
||||
key_fields.append(keywords)
|
||||
|
||||
# Copyright and creator
|
||||
if 'EXIF:Copyright' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:Copyright'])
|
||||
if 'XMP:Creator' in exif_data:
|
||||
key_fields.append(exif_data['XMP:Creator'])
|
||||
if 'EXIF:Artist' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:Artist'])
|
||||
|
||||
return {
|
||||
'exif_text': ' '.join(str(f) for f in key_fields),
|
||||
'camera_make': exif_data.get('EXIF:Make'),
|
||||
'camera_model': exif_data.get('EXIF:Model'),
|
||||
'lens_model': exif_data.get('EXIF:LensModel'),
|
||||
'gps_latitude': lat,
|
||||
'gps_longitude': lon,
|
||||
}
|
||||
|
||||
@shared_task(name='extract_metadata')
|
||||
def extract_metadata(photo_id: str):
|
||||
"""Extract metadata from a photo using ExifTool"""
|
||||
return asyncio.run(_extract_metadata_async(photo_id))
|
||||
|
||||
def _apply_memories_metadata(photo: Photo, data: dict) -> None:
|
||||
"""Apply a Memories API `/image/info/{id}` response to a Photo row.
|
||||
|
||||
Replicates the side-effects of the ExifTool path (width, height,
|
||||
latitude, longitude, taken_at, taken_at_source, has_date_warning,
|
||||
exif_json) without spawning a subprocess. Mule's date-fallback chain
|
||||
(SubSec → DateTimeOriginal → CreateDate → MediaCreateDate → path)
|
||||
is preserved — Memories itself only stores the resolved datetaken
|
||||
and we still need to honour `taken_at_source='manual'` and recover
|
||||
filename-encoded dates for archive photos that lack EXIF.
|
||||
|
||||
The frontend PhotoInfoPanel reads `Make`/`Model`/`ISO`/`FNumber`
|
||||
out of `exif_json`. Memories' `exif` dict uses those exact plain
|
||||
key names (no `EXIF:` prefix), so storing it directly keeps the
|
||||
info panel working without a format adapter.
|
||||
"""
|
||||
from app.services.date_guess import guess_date_from_path
|
||||
|
||||
exif: Dict = data.get('exif') or {}
|
||||
|
||||
# Dimensions
|
||||
w = data.get('w')
|
||||
h = data.get('h')
|
||||
if w:
|
||||
photo.width = int(w)
|
||||
if h:
|
||||
photo.height = int(h)
|
||||
|
||||
# GPS — Memories stores plain decimal-degree values in the exif
|
||||
# dict (no DMS/composite parsing needed).
|
||||
gps_lat = exif.get('GPSLatitude')
|
||||
gps_lon = exif.get('GPSLongitude')
|
||||
if isinstance(gps_lat, (int, float)) and isinstance(gps_lon, (int, float)):
|
||||
photo.latitude = float(gps_lat)
|
||||
photo.longitude = float(gps_lon)
|
||||
else:
|
||||
# Memories omits GPS when not present; clear cleanly.
|
||||
photo.latitude = None
|
||||
photo.longitude = None
|
||||
|
||||
# Store the EXIF dict for the info panel + full-text search.
|
||||
photo.exif_json = json.dumps(exif)
|
||||
|
||||
# Date extraction — only when the user hasn't pinned it manually.
|
||||
if photo.taken_at_source != 'manual':
|
||||
date_fields = [
|
||||
'SubSecDateTimeOriginal',
|
||||
'DateTimeOriginal',
|
||||
'CreateDate',
|
||||
'MediaCreateDate',
|
||||
'TrackCreateDate',
|
||||
]
|
||||
new_taken_at = None
|
||||
for field in date_fields:
|
||||
val = exif.get(field)
|
||||
if not val:
|
||||
continue
|
||||
parsed = parse_exif_datetime(val)
|
||||
if parsed:
|
||||
new_taken_at = parsed
|
||||
photo.taken_at = parsed
|
||||
photo.taken_at_source = 'exif'
|
||||
break
|
||||
if new_taken_at is None:
|
||||
# Filename / folder fallback — same heuristic as the
|
||||
# ExifTool path uses for stripped JPEGs and archive scans.
|
||||
guess = guess_date_from_path(photo.filepath)
|
||||
if guess is not None:
|
||||
photo.taken_at = guess.date
|
||||
photo.taken_at_source = 'path'
|
||||
|
||||
photo.has_date_warning = has_date_warning(photo.filepath, photo.taken_at)
|
||||
|
||||
|
||||
async def _extract_metadata_async(photo_id: str):
|
||||
"""Async implementation of metadata extraction.
|
||||
|
||||
Primary path: Memories' HTTP API (~1-2 ms per photo, no
|
||||
subprocess). Falls back to ExifTool when Memories returns 404
|
||||
(file not yet indexed by NC's scan) or any non-success response.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# Get photo from database
|
||||
result = await session.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
|
||||
if not photo:
|
||||
logger.error(f"Photo not found: {photo_id}")
|
||||
return {'status': 'error', 'message': 'Photo not found'}
|
||||
|
||||
# Resolve the owner once — we need it for both the fileid
|
||||
# lookup and the Memories API call.
|
||||
owner = None
|
||||
if photo.user_id:
|
||||
from app.models.user import User
|
||||
owner = (
|
||||
await session.execute(
|
||||
select(User).where(User.id == photo.user_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
# Backfill nextcloud_fileid if missing (same behaviour as
|
||||
# before — the thumb handler depends on this column).
|
||||
if (
|
||||
photo.nextcloud_fileid is None
|
||||
and owner is not None
|
||||
and owner.nextcloud_app_password_enc
|
||||
):
|
||||
from app.services.nextcloud_dav import (
|
||||
fetch_fileid, is_nextcloud_path,
|
||||
)
|
||||
if photo.filepath and is_nextcloud_path(photo.filepath):
|
||||
try:
|
||||
fid = fetch_fileid(owner, photo.filepath)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"fileid lookup failed for %s: %s", photo_id, e
|
||||
)
|
||||
fid = None
|
||||
if fid is not None:
|
||||
photo.nextcloud_fileid = fid
|
||||
|
||||
# Primary path: ask Memories for the metadata it has
|
||||
# already extracted. Replaces a ~80–100 ms ExifTool
|
||||
# subprocess with a single ~1–2 ms HTTP call.
|
||||
if (
|
||||
photo.nextcloud_fileid is not None
|
||||
and owner is not None
|
||||
and owner.nextcloud_app_password_enc
|
||||
):
|
||||
from app.services.nextcloud_dav import (
|
||||
fetch_memories_info_async,
|
||||
)
|
||||
memories_data = None
|
||||
try:
|
||||
memories_data = await fetch_memories_info_async(
|
||||
owner, photo.nextcloud_fileid
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Memories info call failed for %s: %s",
|
||||
photo_id, e,
|
||||
)
|
||||
if memories_data:
|
||||
_apply_memories_metadata(photo, memories_data)
|
||||
photo.processing_status = 'completed'
|
||||
photo.processing_error = None
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"Metadata extracted via Memories for photo {photo_id}"
|
||||
)
|
||||
return {
|
||||
'status': 'success',
|
||||
'source': 'memories',
|
||||
'photo_id': photo_id,
|
||||
'taken_at': (
|
||||
photo.taken_at.isoformat() if photo.taken_at else None
|
||||
),
|
||||
}
|
||||
logger.info(
|
||||
"Memories had no info for fileid %s; falling back to ExifTool",
|
||||
photo.nextcloud_fileid,
|
||||
)
|
||||
|
||||
# Fallback path: ExifTool subprocess. Used when Memories
|
||||
# hasn't indexed the file yet (brand-new uploads racing the
|
||||
# NC scan), or for non-NC photos that bypass the Memories
|
||||
# pipeline entirely.
|
||||
|
||||
# Check if file exists
|
||||
if not Path(photo.filepath).exists():
|
||||
logger.error(f"File not found: {photo.filepath}")
|
||||
return {'status': 'error', 'message': 'File not found'}
|
||||
|
||||
# Run ExifTool to extract metadata
|
||||
cmd = [
|
||||
'exiftool',
|
||||
'-j', # JSON output
|
||||
'-G', # Group names
|
||||
'-s', # Short output format
|
||||
'-All', # All metadata
|
||||
photo.filepath
|
||||
]
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error(f"ExifTool error: {result.stderr}")
|
||||
photo.processing_error = f"ExifTool: {result.stderr[:500]}"
|
||||
await session.commit()
|
||||
return {'status': 'error', 'message': result.stderr}
|
||||
|
||||
# Parse JSON output
|
||||
metadata = json.loads(result.stdout)
|
||||
if metadata and len(metadata) > 0:
|
||||
exif_data = metadata[0]
|
||||
|
||||
# Store full metadata as JSON
|
||||
photo.exif_json = json.dumps(exif_data)
|
||||
|
||||
# Extract taken_at date — but only if the user hasn't
|
||||
# explicitly set it via the UI. Manual edits are the
|
||||
# source of truth and must survive any rescan.
|
||||
if photo.taken_at_source != 'manual':
|
||||
# Trusted EXIF fields, in order of preference.
|
||||
# SubSecDateTimeOriginal includes sub-second
|
||||
# precision and often a tz offset, so it's the
|
||||
# most accurate when present. ModifyDate is NOT
|
||||
# in this list — it's set every time the file
|
||||
# is re-saved (Lightroom export, EXIF strip,
|
||||
# batch resize) and routinely overwrote correct
|
||||
# capture dates with edit-time dates.
|
||||
date_fields = [
|
||||
'EXIF:SubSecDateTimeOriginal',
|
||||
'EXIF:DateTimeOriginal',
|
||||
'EXIF:CreateDate',
|
||||
'QuickTime:MediaCreateDate',
|
||||
'QuickTime:CreateDate',
|
||||
]
|
||||
|
||||
new_taken_at = None
|
||||
for field in date_fields:
|
||||
if field in exif_data:
|
||||
parsed = parse_exif_datetime(exif_data[field])
|
||||
if parsed:
|
||||
new_taken_at = parsed
|
||||
photo.taken_at = parsed
|
||||
photo.taken_at_source = 'exif'
|
||||
break
|
||||
|
||||
# Fallback: if the file has no trusted EXIF date,
|
||||
# try to extract one from the filename / folder
|
||||
# path. The same date_guess module powers the
|
||||
# has_date_warning flag — reusing it here means
|
||||
# photos without EXIF (scanned prints, stripped
|
||||
# JPEGs, re-saved exports) get a sensible date
|
||||
# instead of falling back to filesystem mtime
|
||||
# (which on Nextcloud-mounted files is just the
|
||||
# upload time).
|
||||
if new_taken_at is None:
|
||||
from app.services.date_guess import guess_date_from_path
|
||||
guess = guess_date_from_path(photo.filepath)
|
||||
if guess is not None:
|
||||
photo.taken_at = guess.date
|
||||
photo.taken_at_source = 'path'
|
||||
|
||||
# Re-run the path-vs-date heuristic now that we know
|
||||
# whether EXIF provided a real capture date. A true EXIF
|
||||
# date that matches the folder clears the warning the
|
||||
# scanner set during the filesystem-mtime pass.
|
||||
photo.has_date_warning = has_date_warning(
|
||||
photo.filepath, photo.taken_at
|
||||
)
|
||||
|
||||
# Extract dimensions if not already set
|
||||
if not photo.width:
|
||||
photo.width = exif_data.get('EXIF:ImageWidth') or exif_data.get('File:ImageWidth')
|
||||
if not photo.height:
|
||||
photo.height = exif_data.get('EXIF:ImageHeight') or exif_data.get('File:ImageHeight')
|
||||
|
||||
# Extract GPS coordinates into first-class columns so the
|
||||
# Map view can query them without parsing exif_json.
|
||||
lat, lon = extract_gps(exif_data)
|
||||
photo.latitude = lat
|
||||
photo.longitude = lon
|
||||
|
||||
# Extract and store key metadata for search
|
||||
key_metadata = extract_key_metadata(exif_data)
|
||||
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"Metadata extracted for photo {photo_id}")
|
||||
return {
|
||||
'status': 'success',
|
||||
'photo_id': photo_id,
|
||||
'taken_at': photo.taken_at.isoformat() if photo.taken_at else None
|
||||
}
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error(f"ExifTool timeout for {photo.filepath}")
|
||||
photo.processing_error = 'ExifTool timeout'
|
||||
await session.commit()
|
||||
return {'status': 'error', 'message': 'ExifTool timeout'}
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse ExifTool output: {e}")
|
||||
photo.processing_error = f"Invalid ExifTool output: {e}"
|
||||
await session.commit()
|
||||
return {'status': 'error', 'message': 'Invalid ExifTool output'}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting metadata for {photo_id}: {e}")
|
||||
return {'status': 'error', 'message': str(e)}
|
||||
|
||||
|
||||
@shared_task(name='backfill_taken_at')
|
||||
def backfill_taken_at():
|
||||
"""Re-enqueue extract_metadata for every non-manual photo.
|
||||
|
||||
Used after fixing the date-extraction logic (removing ModifyDate
|
||||
fallback, adding path-based fallback) to re-derive taken_at across
|
||||
the whole library without touching photos the user has manually
|
||||
corrected. Each enqueued task is fast (~90ms) and runs on the
|
||||
default queue; ~21k photos finish in ~15 min on the existing
|
||||
worker-light concurrency.
|
||||
"""
|
||||
return asyncio.run(_backfill_taken_at_async())
|
||||
|
||||
|
||||
async def _backfill_taken_at_async():
|
||||
from sqlalchemy import or_
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(Photo.id).where(
|
||||
# NULL taken_at_source predates the column default and
|
||||
# should still be re-extracted; only 'manual' is sacred.
|
||||
or_(
|
||||
Photo.taken_at_source != 'manual',
|
||||
Photo.taken_at_source.is_(None),
|
||||
),
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
)
|
||||
photo_ids = [row[0] for row in result.all()]
|
||||
|
||||
for pid in photo_ids:
|
||||
extract_metadata.delay(pid)
|
||||
|
||||
logger.info(f"backfill_taken_at: queued extract_metadata for {len(photo_ids)} photos")
|
||||
return {'queued': len(photo_ids)}
|
||||
@@ -1,512 +0,0 @@
|
||||
"""Nextcloud WebDAV client — only the verbs we actually need.
|
||||
|
||||
Outgoing mutations (upload, delete, rename/move) on files that live
|
||||
under a user's Nextcloud-rooted SourceRoot route through this client
|
||||
instead of touching the filesystem directly. That way Nextcloud's
|
||||
oc_filecache, trashbin, sharing/comments metadata, and desktop sync
|
||||
clients all stay coherent — the price of bypassing it is a stale
|
||||
Nextcloud and resurrected files when sync clients re-upload.
|
||||
|
||||
Reads (scanning, hashing, EXIF, ML pipelines) keep using the bind
|
||||
mount at NEXTCLOUD_USERS_ROOT. WebDAV is far too slow for every byte
|
||||
of every photo, and the read side has no consistency cost — Nextcloud
|
||||
is the writer, the bind mount is the reader, that's it.
|
||||
|
||||
Auth: HTTP Basic with the user's Nextcloud app password (set via the
|
||||
Settings UI, stored Fernet-encrypted at rest). OIDC bearer reuse is a
|
||||
later optimization; app passwords work today and are well-supported.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import BinaryIO, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
from app.config import settings
|
||||
from app.models.user import User
|
||||
from app.services.secrets import decrypt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Top-level mount inside the backend container. The Nextcloud user tree
|
||||
# `/mnt/library/homecloud/<nc_user>/files/...` shows up here as
|
||||
# `/nextcloud-users/<nc_user>/files/...`.
|
||||
NEXTCLOUD_USERS_ROOT = os.environ.get("NEXTCLOUD_USERS_ROOT", "/nextcloud-users")
|
||||
|
||||
|
||||
def is_nextcloud_path(path: str) -> bool:
|
||||
"""True iff `path` resolves under the configured NC users mount."""
|
||||
if not path:
|
||||
return False
|
||||
norm = os.path.normpath(path)
|
||||
root = os.path.normpath(NEXTCLOUD_USERS_ROOT)
|
||||
return norm == root or norm.startswith(root + os.sep)
|
||||
|
||||
|
||||
def split_nextcloud_path(path: str) -> Tuple[str, str]:
|
||||
"""Return (nc_username, rel_path) for a file/dir under the NC mount.
|
||||
|
||||
rel_path is the path relative to `<NEXTCLOUD_USERS_ROOT>/<user>/files/`,
|
||||
suitable for appending to the WebDAV base URL. Raises if `path`
|
||||
isn't a Nextcloud-rooted path or doesn't sit under a `files/`
|
||||
directory.
|
||||
"""
|
||||
norm = os.path.normpath(path)
|
||||
root = os.path.normpath(NEXTCLOUD_USERS_ROOT)
|
||||
if not (norm == root or norm.startswith(root + os.sep)):
|
||||
raise ValueError(f"Not a Nextcloud-rooted path: {path!r}")
|
||||
rest = norm[len(root):].lstrip(os.sep) # "<user>/files/foo/bar.jpg"
|
||||
parts = rest.split(os.sep, 2)
|
||||
if len(parts) < 3 or parts[1] != "files":
|
||||
# Either we got just /<user>, /<user>/files (no rel), or a
|
||||
# different second segment — only the user's `files/` tree is
|
||||
# safe to mutate via WebDAV.
|
||||
if len(parts) == 2 and parts[1] == "files":
|
||||
return parts[0], ""
|
||||
raise ValueError(
|
||||
f"Path doesn't live under <user>/files/: {path!r}"
|
||||
)
|
||||
return parts[0], parts[2]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Client
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class NextcloudCredentialsMissing(HTTPException):
|
||||
"""The user hasn't set their Nextcloud app password yet, but the
|
||||
request needs it to mutate a Nextcloud-managed file. 412 because
|
||||
the precondition (credentials) is missing rather than the request
|
||||
itself being malformed."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
status_code=status.HTTP_412_PRECONDITION_FAILED,
|
||||
detail=(
|
||||
"Set your Nextcloud app password in Settings → Library "
|
||||
"before mutating files in your Nextcloud library."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _credentials_for(user: User) -> tuple[str, str]:
|
||||
"""Resolve the (nc_username, app_password) pair for a user.
|
||||
Raises NextcloudCredentialsMissing when either is missing."""
|
||||
nc_user = (user.nextcloud_username or "").strip()
|
||||
app_pw = decrypt(user.nextcloud_app_password_enc)
|
||||
if not nc_user or not app_pw:
|
||||
raise NextcloudCredentialsMissing()
|
||||
return nc_user, app_pw
|
||||
|
||||
|
||||
def _base_url() -> str:
|
||||
"""The Nextcloud WebDAV base URL (without trailing slash, without
|
||||
user-suffixed path). Resolved per-call so a config reload picks up
|
||||
a new value without restarting workers."""
|
||||
base = (
|
||||
os.environ.get("NEXTCLOUD_BASE_URL")
|
||||
or getattr(settings, "nextcloud_base_url", None)
|
||||
or ""
|
||||
).rstrip("/")
|
||||
if not base:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="NEXTCLOUD_BASE_URL is not configured on the backend",
|
||||
)
|
||||
return base
|
||||
|
||||
|
||||
def _dav_url(nc_username: str, rel_path: str) -> str:
|
||||
"""Compose the absolute WebDAV URL for a relative path under the
|
||||
user's `files/` collection."""
|
||||
base = _base_url()
|
||||
rel = (rel_path or "").lstrip("/")
|
||||
# Each segment must be URL-encoded. httpx encodes path segments at
|
||||
# request time, so we hand it the raw join — but we explicitly drop
|
||||
# `..` traversals here as defense in depth.
|
||||
if any(seg in ("", "..") for seg in rel.split("/") if seg):
|
||||
raise HTTPException(status_code=400, detail="Invalid relative path")
|
||||
parts = [base, "remote.php/dav/files", nc_username]
|
||||
if rel:
|
||||
parts.append(rel)
|
||||
return "/".join(parts)
|
||||
|
||||
|
||||
# httpx Client TTL: short, since a single request is the unit of work.
|
||||
_TIMEOUT = httpx.Timeout(30.0, connect=10.0)
|
||||
|
||||
|
||||
def _client(auth: tuple[str, str]) -> httpx.Client:
|
||||
return httpx.Client(timeout=_TIMEOUT, auth=httpx.BasicAuth(*auth), follow_redirects=False)
|
||||
|
||||
|
||||
def _async_client(auth: tuple[str, str]) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(timeout=_TIMEOUT, auth=httpx.BasicAuth(*auth), follow_redirects=False)
|
||||
|
||||
|
||||
# Pooled async client for the read-heavy NC endpoints (preview proxy,
|
||||
# Memories info). Auth is per-user, so it's passed at call time via
|
||||
# `auth=BasicAuth(...)`; the pool itself is auth-less. Keepalive +
|
||||
# HTTP/2 cuts the TCP+TLS handshake from every thumbnail request and
|
||||
# multiplexes the dozens of concurrent grid fetches over one socket.
|
||||
_PREVIEW_LIMITS = httpx.Limits(
|
||||
max_connections=64, max_keepalive_connections=32, keepalive_expiry=120.0,
|
||||
)
|
||||
_preview_client: httpx.AsyncClient | None = None
|
||||
|
||||
|
||||
async def init_preview_client() -> None:
|
||||
"""Called from the FastAPI lifespan startup hook."""
|
||||
global _preview_client
|
||||
if _preview_client is None:
|
||||
_preview_client = httpx.AsyncClient(
|
||||
timeout=_TIMEOUT,
|
||||
limits=_PREVIEW_LIMITS,
|
||||
http2=True,
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
|
||||
async def close_preview_client() -> None:
|
||||
"""Called from the FastAPI lifespan shutdown hook."""
|
||||
global _preview_client
|
||||
if _preview_client is not None:
|
||||
await _preview_client.aclose()
|
||||
_preview_client = None
|
||||
|
||||
|
||||
def _shared_preview_client() -> httpx.AsyncClient:
|
||||
"""Return the pooled client. Falls back to a one-shot AsyncClient if
|
||||
init wasn't called (tests, scripts) — caller must aclose it."""
|
||||
if _preview_client is not None:
|
||||
return _preview_client
|
||||
return httpx.AsyncClient(
|
||||
timeout=_TIMEOUT, http2=True, follow_redirects=False,
|
||||
)
|
||||
|
||||
|
||||
def _raise_for_dav(resp: httpx.Response, action: str) -> None:
|
||||
"""Translate Nextcloud WebDAV errors into FastAPI HTTPExceptions
|
||||
the frontend can show. We surface Nextcloud's body verbatim when
|
||||
it's small enough, since it tends to carry the actually-useful
|
||||
detail (quota, permission denied, etc.)."""
|
||||
if resp.is_success:
|
||||
return
|
||||
body = resp.text or ""
|
||||
if len(body) > 400:
|
||||
body = body[:400] + "…"
|
||||
logger.warning("Nextcloud %s failed: %s %s — %s", action, resp.status_code, resp.reason_phrase, body[:200])
|
||||
if resp.status_code in (401, 403):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail=f"Nextcloud rejected the {action}: {resp.reason_phrase}. "
|
||||
f"Check your app password under Settings → Library.",
|
||||
)
|
||||
if resp.status_code == 404:
|
||||
raise HTTPException(status_code=404, detail=f"Not found in Nextcloud during {action}")
|
||||
if resp.status_code == 507:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_507_INSUFFICIENT_STORAGE,
|
||||
detail="Nextcloud quota exceeded",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail=f"Nextcloud error during {action}: {resp.status_code} {resp.reason_phrase}",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verbs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def delete_for_user(user: User, abs_path: str) -> None:
|
||||
"""WebDAV DELETE — moves the file/dir into the user's NC trashbin.
|
||||
`abs_path` is the absolute filesystem path under the bind mount."""
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
expected_user, rel = split_nextcloud_path(abs_path)
|
||||
if expected_user != nc_user:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Path belongs to a different Nextcloud user",
|
||||
)
|
||||
url = _dav_url(nc_user, rel)
|
||||
with _client((nc_user, app_pw)) as c:
|
||||
resp = c.request("DELETE", url)
|
||||
# 204 = deleted. 404 = already gone (treat as success, idempotent).
|
||||
if resp.status_code == 404:
|
||||
logger.info("Nextcloud DELETE %s already gone, treating as success", rel)
|
||||
return
|
||||
_raise_for_dav(resp, "delete")
|
||||
|
||||
|
||||
def move_for_user(user: User, src_abs: str, dst_abs: str) -> None:
|
||||
"""WebDAV MOVE — rename or move within the same Nextcloud user."""
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
src_user, src_rel = split_nextcloud_path(src_abs)
|
||||
dst_user, dst_rel = split_nextcloud_path(dst_abs)
|
||||
if src_user != nc_user or dst_user != nc_user:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="MOVE across Nextcloud users is not supported",
|
||||
)
|
||||
src_url = _dav_url(nc_user, src_rel)
|
||||
dst_url = _dav_url(nc_user, dst_rel)
|
||||
with _client((nc_user, app_pw)) as c:
|
||||
resp = c.request(
|
||||
"MOVE",
|
||||
src_url,
|
||||
headers={"Destination": dst_url, "Overwrite": "F"},
|
||||
)
|
||||
_raise_for_dav(resp, "move")
|
||||
|
||||
|
||||
def mkcol_for_user(user: User, abs_path: str) -> None:
|
||||
"""WebDAV MKCOL — create a directory. Idempotent: a 405 (Method Not
|
||||
Allowed) means the collection already exists, treat as success."""
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
expected_user, rel = split_nextcloud_path(abs_path)
|
||||
if expected_user != nc_user:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Path belongs to a different Nextcloud user",
|
||||
)
|
||||
url = _dav_url(nc_user, rel)
|
||||
with _client((nc_user, app_pw)) as c:
|
||||
resp = c.request("MKCOL", url)
|
||||
if resp.status_code == 405:
|
||||
return
|
||||
_raise_for_dav(resp, "mkcol")
|
||||
|
||||
|
||||
def put_for_user(
|
||||
user: User,
|
||||
abs_path: str,
|
||||
fileobj: BinaryIO,
|
||||
content_type: Optional[str] = None,
|
||||
) -> None:
|
||||
"""WebDAV PUT — upload `fileobj` to `abs_path`. Caller is
|
||||
responsible for ensuring intermediate collections exist via
|
||||
`mkcol_for_user`. Streams the body, no in-memory copy."""
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
expected_user, rel = split_nextcloud_path(abs_path)
|
||||
if expected_user != nc_user:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Path belongs to a different Nextcloud user",
|
||||
)
|
||||
url = _dav_url(nc_user, rel)
|
||||
headers = {}
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
with _client((nc_user, app_pw)) as c:
|
||||
resp = c.request("PUT", url, content=fileobj, headers=headers)
|
||||
_raise_for_dav(resp, "upload")
|
||||
|
||||
|
||||
def ensure_parents_for_user(user: User, abs_path: str) -> None:
|
||||
"""Walk the parent chain of `abs_path` under the user's NC root and
|
||||
`mkcol` any missing collection. Stops at the user's `files/`
|
||||
directory — never tries to create that, which is owned by Nextcloud
|
||||
itself."""
|
||||
nc_user, _ = _credentials_for(user)
|
||||
expected_user, rel = split_nextcloud_path(abs_path)
|
||||
if expected_user != nc_user:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Path belongs to a different Nextcloud user",
|
||||
)
|
||||
if not rel:
|
||||
return
|
||||
parts = rel.split("/")
|
||||
if len(parts) <= 1:
|
||||
return # no intermediate dirs to make
|
||||
accum: list[str] = []
|
||||
for seg in parts[:-1]:
|
||||
accum.append(seg)
|
||||
sub_rel = "/".join(accum)
|
||||
sub_abs = os.path.join(NEXTCLOUD_USERS_ROOT, nc_user, "files", sub_rel)
|
||||
mkcol_for_user(user, sub_abs)
|
||||
|
||||
|
||||
_FILEID_PROPFIND = (
|
||||
b'<?xml version="1.0"?>'
|
||||
b'<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns">'
|
||||
b'<d:prop><oc:fileid/></d:prop>'
|
||||
b'</d:propfind>'
|
||||
)
|
||||
|
||||
|
||||
def fetch_fileid(user: User, abs_path: str) -> Optional[int]:
|
||||
"""Look up Nextcloud's numeric fileid for the file at `abs_path`.
|
||||
|
||||
`abs_path` is the absolute filesystem path under the bind mount,
|
||||
e.g. `/nextcloud-users/admin/files/Photos/2024/01/foo.jpg`. Returns
|
||||
None when the file isn't under a Nextcloud-rooted tree, the user
|
||||
has no app password set, or Nextcloud returns 404 — callers should
|
||||
treat None as "skip this row" rather than an error.
|
||||
|
||||
Used by `scripts/backfill_nextcloud_fileid.py`. The hot path (the
|
||||
thumbnail handler) reads `Photo.nextcloud_fileid` directly so it
|
||||
doesn't round-trip to Nextcloud per request.
|
||||
"""
|
||||
if not is_nextcloud_path(abs_path):
|
||||
return None
|
||||
try:
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
except NextcloudCredentialsMissing:
|
||||
return None
|
||||
try:
|
||||
expected_user, rel = split_nextcloud_path(abs_path)
|
||||
except ValueError:
|
||||
return None
|
||||
if expected_user != nc_user:
|
||||
return None
|
||||
url = _dav_url(nc_user, rel)
|
||||
with _client((nc_user, app_pw)) as c:
|
||||
resp = c.request(
|
||||
"PROPFIND",
|
||||
url,
|
||||
headers={"Depth": "0", "Content-Type": "application/xml"},
|
||||
content=_FILEID_PROPFIND,
|
||||
)
|
||||
if resp.status_code == 404:
|
||||
return None
|
||||
if not resp.is_success:
|
||||
logger.warning(
|
||||
"Nextcloud PROPFIND %s returned %s", rel, resp.status_code
|
||||
)
|
||||
return None
|
||||
import re as _re
|
||||
m = _re.search(rb"<oc:fileid>(\d+)</oc:fileid>", resp.content)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def get_preview_bytes(
|
||||
user: User, fileid: int, x: int, y: int
|
||||
) -> Optional[bytes]:
|
||||
"""Sync sibling of `get_preview_async` for callers in non-async
|
||||
contexts.
|
||||
|
||||
Returns the preview body on success, None on 404 / non-success /
|
||||
missing credentials. Caller is expected to feed the bytes into
|
||||
PIL or similar.
|
||||
"""
|
||||
try:
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
except NextcloudCredentialsMissing:
|
||||
return None
|
||||
url = f"{_base_url()}/index.php/core/preview"
|
||||
params = {
|
||||
"fileId": str(fileid),
|
||||
"x": str(x),
|
||||
"y": str(y),
|
||||
"a": "true",
|
||||
"forceIcon": "false",
|
||||
}
|
||||
with _client((nc_user, app_pw)) as c:
|
||||
resp = c.get(url, params=params)
|
||||
if resp.status_code == 404 or not resp.is_success:
|
||||
return None
|
||||
return resp.content
|
||||
|
||||
|
||||
async def fetch_memories_info_async(
|
||||
user: User, fileid: int
|
||||
) -> Optional[dict]:
|
||||
"""Fetch the Memories app's per-file metadata blob.
|
||||
|
||||
`GET /index.php/apps/memories/api/image/info/{fileid}` returns
|
||||
Memories' pre-extracted view of the file: `w`, `h`, `datetaken`
|
||||
(unix epoch), `mtime`, `mimetype`, `size`, plus an `exif` dict
|
||||
of plain-named EXIF fields (Make, Model, ISO, FNumber,
|
||||
DateTimeOriginal, GPSLatitude, GPSLongitude, etc.). The endpoint
|
||||
is `#[NoAdminRequired] #[PublicPage]` but CSRF-checked, so we
|
||||
send `OCS-APIRequest: true` to bypass the check the same way
|
||||
OCS API clients do.
|
||||
|
||||
Returns None on 404 (file not yet indexed by Memories, or
|
||||
fileid stale) or any non-success response — callers should fall
|
||||
back to ExifTool extraction in that case.
|
||||
"""
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
url = (
|
||||
f"{_base_url()}/index.php/apps/memories/api/image/info/{int(fileid)}"
|
||||
)
|
||||
client = _shared_preview_client()
|
||||
try:
|
||||
resp = await client.get(
|
||||
url,
|
||||
headers={
|
||||
"OCS-APIRequest": "true",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
auth=httpx.BasicAuth(nc_user, app_pw),
|
||||
)
|
||||
finally:
|
||||
# Only close if we got a one-shot fallback client; the pooled
|
||||
# one is owned by the lifespan hook.
|
||||
if client is not _preview_client:
|
||||
await client.aclose()
|
||||
if resp.status_code == 404:
|
||||
return None
|
||||
if not resp.is_success:
|
||||
logger.warning(
|
||||
"Memories info for fileid %s returned %s", fileid, resp.status_code
|
||||
)
|
||||
return None
|
||||
try:
|
||||
return resp.json()
|
||||
except Exception as e:
|
||||
logger.warning("Memories info parse failed for fileid %s: %s", fileid, e)
|
||||
return None
|
||||
|
||||
|
||||
async def get_preview_async(
|
||||
user: User, fileid: int, x: int, y: int
|
||||
) -> httpx.Response:
|
||||
"""Fetch a Nextcloud preview for `fileid` sized up to (x, y).
|
||||
|
||||
Nextcloud's `/index.php/core/preview` endpoint returns a JPEG (or
|
||||
icon fallback) sized so the longest edge fits within the requested
|
||||
box. `a=true` preserves the source aspect ratio; `forceIcon=false`
|
||||
makes it 404 rather than returning a placeholder if no real preview
|
||||
can be produced.
|
||||
|
||||
Auth uses the user's encrypted app password — same path as every
|
||||
other mutation in this module. The caller streams the body back
|
||||
to the frontend; we don't buffer the bytes here.
|
||||
"""
|
||||
nc_user, app_pw = _credentials_for(user)
|
||||
url = f"{_base_url()}/index.php/core/preview"
|
||||
params = {
|
||||
"fileId": str(fileid),
|
||||
"x": str(x),
|
||||
"y": str(y),
|
||||
"a": "true",
|
||||
"forceIcon": "false",
|
||||
}
|
||||
client = _shared_preview_client()
|
||||
try:
|
||||
return await client.get(
|
||||
url, params=params, auth=httpx.BasicAuth(nc_user, app_pw),
|
||||
)
|
||||
finally:
|
||||
# Only close if we got a one-shot fallback client; the pooled
|
||||
# one is owned by the lifespan hook.
|
||||
if client is not _preview_client:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
def whoami_dir_exists(nc_username: str) -> bool:
|
||||
"""True iff the bind-mounted `<NEXTCLOUD_USERS_ROOT>/<user>/files`
|
||||
directory exists. Used by the UI to validate the override field
|
||||
without round-tripping to Nextcloud — the bind mount is enough to
|
||||
confirm Nextcloud actually has that user."""
|
||||
if not nc_username or "/" in nc_username or nc_username in (".", ".."):
|
||||
return False
|
||||
target = os.path.join(NEXTCLOUD_USERS_ROOT, nc_username, "files")
|
||||
return os.path.isdir(target)
|
||||
@@ -1,93 +0,0 @@
|
||||
"""
|
||||
Scanner service for initial library scan and per-user source root bootstrap.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import SourceRoot
|
||||
from app.models.user import User
|
||||
from app.tasks.scan import scan_all_source_roots
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def bootstrap_user_source_root(user: User, session=None) -> None:
|
||||
"""Create the media directory and a source root for a user.
|
||||
|
||||
Called when a new user is created (by the admin or the setup endpoint).
|
||||
If the user already has a source root, this is a no-op.
|
||||
"""
|
||||
own_session = session is None
|
||||
if own_session:
|
||||
session = AsyncSessionLocal()
|
||||
|
||||
try:
|
||||
# Check if user already has a source root
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.user_id == user.id)
|
||||
)
|
||||
if result.scalar_one_or_none() is not None:
|
||||
return
|
||||
|
||||
os.makedirs(user.media_path, exist_ok=True)
|
||||
|
||||
source_root = SourceRoot(
|
||||
name=f"{user.username}'s Library",
|
||||
path=user.media_path,
|
||||
user_id=user.id,
|
||||
)
|
||||
session.add(source_root)
|
||||
if own_session:
|
||||
await session.commit()
|
||||
else:
|
||||
await session.flush()
|
||||
|
||||
logger.info(
|
||||
f"Bootstrapped source root for user '{user.username}': "
|
||||
f"{user.media_path}"
|
||||
)
|
||||
finally:
|
||||
if own_session:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def bootstrap_default_source_root() -> None:
|
||||
"""Legacy bootstrap — for existing installs that have source roots
|
||||
without user_id (pre-auth migration). On fresh installs, source roots
|
||||
are created per-user via bootstrap_user_source_root. If there are
|
||||
already source roots in the DB, this is a no-op.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(select(SourceRoot))
|
||||
if result.scalars().first() is not None:
|
||||
return # Already have source roots.
|
||||
|
||||
# No source roots and no users means fresh install — the setup
|
||||
# endpoint will create the first user + source root.
|
||||
user_count = (await session.execute(
|
||||
select(User)
|
||||
)).scalars().first()
|
||||
if user_count is None:
|
||||
logger.info(
|
||||
"No users or source roots — waiting for first-run setup."
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
async def start_initial_scan():
|
||||
"""Start the initial library scan.
|
||||
|
||||
The watchfiles-based watcher has been retired in favour of Nextcloud
|
||||
`webhook_listeners` (see `app.routers.nc_webhook`). NC POSTs every
|
||||
file event directly to mule, so we no longer keep a long-running
|
||||
inotify task. The periodic `discard_missing_photos_beat` Celery job
|
||||
is still there as a safety net for deletions a webhook might miss.
|
||||
"""
|
||||
try:
|
||||
scan_all_source_roots.delay()
|
||||
logger.info("Initial scan queued successfully")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start initial scan: {e}")
|
||||
@@ -1,77 +0,0 @@
|
||||
"""
|
||||
FTS search over photos.search_vector with optional tag/date filters.
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Photo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def hybrid_search(
|
||||
db: AsyncSession,
|
||||
q: Optional[str] = None,
|
||||
tag_ids: Optional[list[str]] = None,
|
||||
date_from: Optional[str] = None,
|
||||
date_to: Optional[str] = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
"""Full-text search using photos.search_vector. No embeddings, no OCR."""
|
||||
if q:
|
||||
try:
|
||||
fts_stmt = text("""
|
||||
SELECT id, ts_rank(search_vector, plainto_tsquery('english', :q)) AS rank
|
||||
FROM photos
|
||||
WHERE search_vector @@ plainto_tsquery('english', :q)
|
||||
AND is_trashed = false
|
||||
AND is_hidden = false
|
||||
ORDER BY rank DESC
|
||||
LIMIT 500
|
||||
""")
|
||||
rows = (await db.execute(fts_stmt, {"q": q})).fetchall()
|
||||
except Exception as e:
|
||||
logger.warning("FTS search failed: %s", e)
|
||||
rows = []
|
||||
|
||||
scored = [(pid, float(rank)) for pid, rank in rows]
|
||||
|
||||
if tag_ids:
|
||||
from app.models.tags import photo_tags
|
||||
photo_ids = [pid for pid, _ in scored]
|
||||
if not photo_ids:
|
||||
return []
|
||||
stmt = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.photo_id.in_(photo_ids),
|
||||
photo_tags.c.tag_id.in_(tag_ids),
|
||||
).distinct()
|
||||
valid = {row[0] for row in (await db.execute(stmt)).fetchall()}
|
||||
scored = [(pid, s) for pid, s in scored if pid in valid]
|
||||
|
||||
page = scored[offset : offset + limit]
|
||||
return [{"photo_id": pid, "score": s} for pid, s in page]
|
||||
|
||||
# No text query — recent photos with tag/date filters.
|
||||
if tag_ids:
|
||||
from app.models.tags import photo_tags
|
||||
sub = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.tag_id.in_(tag_ids)
|
||||
).distinct().subquery()
|
||||
stmt = select(Photo.id).join(sub, Photo.id == sub.c.photo_id)
|
||||
else:
|
||||
stmt = select(Photo.id)
|
||||
stmt = stmt.where(
|
||||
Photo.is_discarded.is_(False),
|
||||
Photo.is_hidden.is_(False),
|
||||
)
|
||||
if date_from:
|
||||
stmt = stmt.where(Photo.taken_at >= date_from)
|
||||
if date_to:
|
||||
stmt = stmt.where(Photo.taken_at <= date_to)
|
||||
stmt = stmt.order_by(Photo.added_at.desc()).offset(offset).limit(limit)
|
||||
rows = (await db.execute(stmt)).fetchall()
|
||||
return [{"photo_id": row[0], "score": 0.0} for row in rows]
|
||||
@@ -1,45 +0,0 @@
|
||||
"""Symmetric encryption for credentials we have to store.
|
||||
|
||||
Used today for the per-user Nextcloud app password — we need the
|
||||
plaintext to put it in an outgoing HTTP Basic header, so a one-way
|
||||
hash won't do. Key is derived from `settings.secret_key` via SHA-256
|
||||
so existing deployments don't need a separate KMS dance, and a stable
|
||||
SECRET_KEY rotates these credentials automatically.
|
||||
|
||||
Fernet is symmetric AES-128-CBC + HMAC-SHA256 with a versioned
|
||||
ciphertext envelope; good enough for column-level secrecy in a
|
||||
single-host homelab. Rotate by setting a new SECRET_KEY and asking
|
||||
users to re-enter their app password.
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
from typing import Optional
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
def _fernet() -> Fernet:
|
||||
# Fernet requires a 32-byte url-safe base64 key. SHA-256 of the
|
||||
# configured secret gives us exactly 32 bytes; b64-urlsafe-encode
|
||||
# to fit the API contract.
|
||||
digest = hashlib.sha256(settings.secret_key.encode("utf-8")).digest()
|
||||
return Fernet(base64.urlsafe_b64encode(digest))
|
||||
|
||||
|
||||
def encrypt(plaintext: str) -> str:
|
||||
"""Return a base64 token that can be stored in a VARCHAR column."""
|
||||
return _fernet().encrypt(plaintext.encode("utf-8")).decode("ascii")
|
||||
|
||||
|
||||
def decrypt(token: Optional[str]) -> Optional[str]:
|
||||
"""Inverse of encrypt. Returns None for None / empty input. Raises
|
||||
on tampered or wrong-key tokens — callers should treat that as
|
||||
"credential unset" rather than crashing the request."""
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
return _fernet().decrypt(token.encode("ascii")).decode("utf-8")
|
||||
except InvalidToken:
|
||||
return None
|
||||
@@ -1,132 +0,0 @@
|
||||
"""Shared video helpers used by the /playback endpoint and the
|
||||
pretranscode celery task.
|
||||
|
||||
The actual ffmpeg/ffprobe work is sync (subprocess.run); FastAPI
|
||||
handlers wrap calls in asyncio.to_thread, celery just calls them
|
||||
directly. Keeping a single sync implementation avoids drift between
|
||||
the request-time fallback and the background pre-transcode."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# On-disk cache of browser-playable transcodes. Backed by /data which is
|
||||
# the persistent volume in mulita-backend / mulita-worker-light.
|
||||
VIDEO_CACHE_DIR = Path('/data/video-cache')
|
||||
VIDEO_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Codecs the browser can play in <video> across Chrome / Firefox / Safari
|
||||
# without re-encoding. h264 covers everything practical we have on disk;
|
||||
# add av1 / vp9 here if the source library ever picks those up.
|
||||
PLAYBACK_OK_VCODECS = {'h264', 'avc1'}
|
||||
|
||||
# Container extensions that we trust to passthrough when the codec is
|
||||
# OK. .mov is intentionally excluded — Chrome and Firefox refuse to
|
||||
# play even h264-in-mov reliably, so .mov always goes through the cache.
|
||||
PLAYBACK_OK_EXTS = {'.mp4', '.m4v', '.webm'}
|
||||
|
||||
|
||||
def cache_path_for(photo_id: str) -> Path:
|
||||
"""Where the transcoded MP4 lives for a given photo id."""
|
||||
return VIDEO_CACHE_DIR / f'{photo_id}.mp4'
|
||||
|
||||
|
||||
def ffprobe_video_codec(path: str) -> Optional[str]:
|
||||
"""Return the video stream's codec_name (lowercased) or None on
|
||||
probe failure. ~50ms for typical files."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
'ffprobe', '-v', 'error',
|
||||
'-select_streams', 'v:0',
|
||||
'-show_entries', 'stream=codec_name',
|
||||
'-of', 'default=noprint_wrappers=1:nokey=1',
|
||||
path,
|
||||
],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError) as e:
|
||||
logger.warning("ffprobe failed for %s: %s", path, e)
|
||||
return None
|
||||
if result.returncode != 0:
|
||||
logger.warning(
|
||||
"ffprobe rc=%s for %s: %s",
|
||||
result.returncode, path, (result.stderr or '').strip()[:200],
|
||||
)
|
||||
return None
|
||||
return ((result.stdout or '').strip().lower()) or None
|
||||
|
||||
|
||||
def needs_transcode(src_path: str) -> bool:
|
||||
"""True when /playback would have to encode rather than passthrough.
|
||||
Uses extension first (cheap), only ffprobes when the container is
|
||||
plausibly web-safe."""
|
||||
ext = Path(src_path).suffix.lower()
|
||||
if ext not in PLAYBACK_OK_EXTS:
|
||||
return True
|
||||
return ffprobe_video_codec(src_path) not in PLAYBACK_OK_VCODECS
|
||||
|
||||
|
||||
def transcode_to_h264_mp4(src: str, dst: str, *, timeout: int = 3600) -> bool:
|
||||
"""Transcode `src` to H.264 8-bit MP4 at `dst`. Returns True on
|
||||
success.
|
||||
|
||||
-pix_fmt yuv420p forces 8-bit output so 10-bit HEVC sources still
|
||||
play on browsers without 10-bit decode. Audio is always re-encoded
|
||||
to AAC because iPhone 16 ships APAC audio that browsers can't
|
||||
decode, and the audio pass is cheap next to the video pass.
|
||||
+faststart relocates the moov atom so progressive playback works.
|
||||
|
||||
Atomic publish via tmp + os.replace so a failed run never leaves a
|
||||
half-written .mp4 in the cache. -f mp4 forces the muxer because the
|
||||
.tmp suffix isn't a format hint ffmpeg recognises."""
|
||||
tmp = dst + '.tmp'
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
'ffmpeg', '-y', '-loglevel', 'error',
|
||||
'-i', src,
|
||||
'-map', '0:v:0',
|
||||
'-map', '0:a:0?',
|
||||
'-c:v', 'libx264',
|
||||
'-preset', 'veryfast',
|
||||
'-crf', '23',
|
||||
'-pix_fmt', 'yuv420p',
|
||||
'-c:a', 'aac',
|
||||
'-b:a', '160k',
|
||||
'-movflags', '+faststart',
|
||||
'-f', 'mp4',
|
||||
tmp,
|
||||
],
|
||||
capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError) as e:
|
||||
logger.error("ffmpeg failed for %s: %s", src, e)
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
if result.returncode != 0:
|
||||
logger.error(
|
||||
"ffmpeg rc=%s for %s: %s",
|
||||
result.returncode, src, (result.stderr or '').strip()[:500],
|
||||
)
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
try:
|
||||
os.replace(tmp, dst)
|
||||
except OSError as e:
|
||||
logger.error("failed to publish transcoded %s: %s", dst, e)
|
||||
return False
|
||||
return True
|
||||
@@ -1,15 +0,0 @@
|
||||
"""
|
||||
Celery tasks module
|
||||
"""
|
||||
from app.tasks.celery import celery_app
|
||||
from app.tasks.scan import scan_folder, scan_all_source_roots, watch_folders
|
||||
from app.tasks.thumbs import generate_thumbnails, regenerate_all_thumbnails
|
||||
|
||||
__all__ = [
|
||||
'celery_app',
|
||||
'scan_folder',
|
||||
'scan_all_source_roots',
|
||||
'watch_folders',
|
||||
'generate_thumbnails',
|
||||
'regenerate_all_thumbnails'
|
||||
]
|
||||
@@ -1,66 +0,0 @@
|
||||
"""
|
||||
Celery configuration and app initialization
|
||||
"""
|
||||
import logging
|
||||
|
||||
from celery import Celery
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
celery_app = Celery(
|
||||
'mulita',
|
||||
broker=settings.celery_broker_url,
|
||||
backend=settings.celery_result_backend,
|
||||
include=[
|
||||
'app.tasks.scan',
|
||||
'app.tasks.thumbs',
|
||||
'app.tasks.video',
|
||||
'app.services.metadata',
|
||||
]
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer='json',
|
||||
accept_content=['json'],
|
||||
result_serializer='json',
|
||||
timezone='UTC',
|
||||
enable_utc=True,
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
task_soft_time_limit=300,
|
||||
task_time_limit=600,
|
||||
task_routes={
|
||||
'generate_thumbnails': {'queue': 'high'},
|
||||
'regenerate_all_thumbnails': {'queue': 'high'},
|
||||
'backfill_phashes': {'queue': 'high'},
|
||||
'regroup_duplicates': {'queue': 'high'},
|
||||
'incremental_regroup_duplicates': {'queue': 'high'},
|
||||
'scan_folder': {'queue': 'low'},
|
||||
'scan_all_source_roots': {'queue': 'low'},
|
||||
'backfill_gps': {'queue': 'low'},
|
||||
# CPU-heavy but tolerant of the low-priority queue (doesn't block
|
||||
# any user-facing flow).
|
||||
'pretranscode_video': {'queue': 'low'},
|
||||
# `watch_folders` is retired (file events come from NC webhooks)
|
||||
# but the task definition still exists as a no-op shim for any
|
||||
# in-flight apply_async. Route it to the default queue so the
|
||||
# remaining worker actually drains it.
|
||||
'watch_folders': {'queue': 'default'},
|
||||
'discard_missing_photos_beat': {'queue': 'low'},
|
||||
},
|
||||
task_default_queue='default',
|
||||
task_default_exchange='default',
|
||||
task_default_exchange_type='direct',
|
||||
task_default_routing_key='default',
|
||||
broker_connection_retry_on_startup=True,
|
||||
# Periodic catch-up so external file deletions in Nextcloud get
|
||||
# reflected even when the real-time watcher missed the event
|
||||
# (worker restart window, mount transient, etc).
|
||||
beat_schedule={
|
||||
'discard-missing-photos-every-30min': {
|
||||
'task': 'discard_missing_photos_beat',
|
||||
'schedule': 30 * 60,
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -1,749 +0,0 @@
|
||||
"""
|
||||
Celery tasks for scanning folders and indexing photos
|
||||
"""
|
||||
import os
|
||||
import hashlib
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
import logging
|
||||
import json
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
from celery import shared_task
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
import aiofiles
|
||||
import redis
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo, Folder, SourceRoot
|
||||
from app.config import settings
|
||||
from app.tasks.thumbs import generate_thumbnails
|
||||
from app.tasks.video import pretranscode_video
|
||||
from app.services.metadata import extract_metadata
|
||||
from app.services.date_guess import has_date_warning
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Redis keys read by GET /api/v1/library/scan/status. The frontend
|
||||
# ScanProgress widget polls that endpoint, so anything we want to surface
|
||||
# in the UI lives here.
|
||||
REDIS_KEY_ACTIVE = 'scan:active'
|
||||
REDIS_KEY_CURRENT_FOLDER = 'scan:current_folder'
|
||||
REDIS_KEY_PROCESSED = 'scan:processed_files'
|
||||
REDIS_KEY_TOTAL = 'scan:total_files'
|
||||
REDIS_KEY_ERRORS = 'scan:errors'
|
||||
MAX_ERROR_ENTRIES = 50 # cap the errors list so a noisy scan doesn't blow Redis
|
||||
|
||||
|
||||
def _get_redis():
|
||||
"""Connect to the broker for progress writes. Returns None on failure
|
||||
so a Redis outage doesn't prevent the scan itself from running."""
|
||||
try:
|
||||
return redis.Redis.from_url(settings.celery_broker_url)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not reach Redis for scan progress: {e}")
|
||||
return None
|
||||
|
||||
# Supported file extensions
|
||||
PHOTO_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.tiff', '.tif', '.webp', '.bmp'}
|
||||
RAW_EXTENSIONS = {'.cr2', '.cr3', '.nef', '.arw', '.raf', '.dng', '.orf', '.rw2', '.pef', '.srw'}
|
||||
HEIC_EXTENSIONS = {'.heic', '.heif'}
|
||||
VIDEO_EXTENSIONS = {'.mp4', '.mov', '.avi', '.mkv', '.mts', '.m2ts', '.3gp', '.wmv', '.flv'}
|
||||
|
||||
SUPPORTED_EXTENSIONS = PHOTO_EXTENSIONS | RAW_EXTENSIONS | HEIC_EXTENSIONS | VIDEO_EXTENSIONS
|
||||
|
||||
def get_media_type(filepath: str) -> str:
|
||||
"""Determine media type from file extension"""
|
||||
ext = Path(filepath).suffix.lower()
|
||||
if ext in PHOTO_EXTENSIONS:
|
||||
return 'photo'
|
||||
elif ext in RAW_EXTENSIONS:
|
||||
return 'raw'
|
||||
elif ext in HEIC_EXTENSIONS:
|
||||
return 'heic'
|
||||
elif ext in VIDEO_EXTENSIONS:
|
||||
return 'video'
|
||||
return 'unknown'
|
||||
|
||||
async def calculate_file_hash(filepath: str) -> str:
|
||||
"""Calculate SHA-256 hash of a file"""
|
||||
hash_sha256 = hashlib.sha256()
|
||||
try:
|
||||
async with aiofiles.open(filepath, 'rb') as f:
|
||||
while chunk := await f.read(8192):
|
||||
hash_sha256.update(chunk)
|
||||
return hash_sha256.hexdigest()
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating hash for {filepath}: {e}")
|
||||
return ""
|
||||
|
||||
@shared_task(bind=True, name='scan_folder')
|
||||
def scan_folder(self, folder_path: str, source_root_id: Optional[str] = None):
|
||||
"""
|
||||
Scan a folder and index all photos/videos
|
||||
"""
|
||||
# Run async function in sync context
|
||||
return asyncio.run(_scan_folder_async(folder_path, source_root_id, self))
|
||||
|
||||
async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], task):
|
||||
"""Async implementation of folder scanning. Writes progress to Redis so
|
||||
GET /api/v1/library/scan/status can surface it to the frontend
|
||||
ScanProgress widget."""
|
||||
logger.info(f"Starting scan of folder: {folder_path}")
|
||||
|
||||
r = _get_redis()
|
||||
|
||||
PROGRESS_TTL = 3600 # 1 hour — auto-expire if scan crashes
|
||||
|
||||
def progress_set(key: str, value) -> None:
|
||||
if r is None:
|
||||
return
|
||||
try:
|
||||
r.set(key, str(value), ex=PROGRESS_TTL)
|
||||
except Exception as e:
|
||||
logger.debug(f"scan progress set failed: {e}")
|
||||
|
||||
def progress_push_error(message: str) -> None:
|
||||
if r is None:
|
||||
return
|
||||
try:
|
||||
r.lpush(REDIS_KEY_ERRORS, message)
|
||||
r.ltrim(REDIS_KEY_ERRORS, 0, MAX_ERROR_ENTRIES - 1)
|
||||
except Exception as e:
|
||||
logger.debug(f"scan progress push_error failed: {e}")
|
||||
|
||||
# Mark scan active immediately so the UI starts polling fast.
|
||||
progress_set(REDIS_KEY_ACTIVE, 'true')
|
||||
progress_set(REDIS_KEY_CURRENT_FOLDER, folder_path)
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# Get or create source root
|
||||
if not source_root_id:
|
||||
source_root = await get_or_create_source_root(session, folder_path)
|
||||
source_root_id = source_root.id
|
||||
else:
|
||||
source_root = (await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == source_root_id)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
# Inherit user_id from the source root's owner
|
||||
owner_user_id = source_root.user_id if source_root else None
|
||||
|
||||
# Per-scan memoization cache for "is this folder's effective
|
||||
# is_hidden true?" Populated on first lookup by walking the
|
||||
# parent_id chain up to the source root. Keyed by folder_id
|
||||
# so repeated photos in the same folder pay only one lookup.
|
||||
hidden_folder_cache: dict[str, bool] = {}
|
||||
|
||||
async def is_folder_effectively_hidden(folder_row: Folder) -> bool:
|
||||
if folder_row.id in hidden_folder_cache:
|
||||
return hidden_folder_cache[folder_row.id]
|
||||
# Walk parents. If the current folder is hidden, short-
|
||||
# circuit. Otherwise climb until we hit a root (no
|
||||
# parent_id) or a cached ancestor.
|
||||
if folder_row.is_hidden:
|
||||
hidden_folder_cache[folder_row.id] = True
|
||||
return True
|
||||
parent_id = folder_row.parent_id
|
||||
while parent_id is not None:
|
||||
if parent_id in hidden_folder_cache:
|
||||
hidden_folder_cache[folder_row.id] = hidden_folder_cache[parent_id]
|
||||
return hidden_folder_cache[folder_row.id]
|
||||
parent = (
|
||||
await session.execute(
|
||||
select(Folder).where(Folder.id == parent_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if parent is None:
|
||||
break
|
||||
if parent.is_hidden:
|
||||
hidden_folder_cache[folder_row.id] = True
|
||||
return True
|
||||
parent_id = parent.parent_id
|
||||
hidden_folder_cache[folder_row.id] = False
|
||||
return False
|
||||
|
||||
# Pre-walk to compute the total file count upfront. Without this
|
||||
# the progress bar would jump every time a new subfolder is
|
||||
# encountered because the running total kept growing.
|
||||
total_files = 0
|
||||
for _root, _dirs, files in os.walk(folder_path):
|
||||
total_files += sum(
|
||||
1 for f in files if Path(f).suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
)
|
||||
progress_set(REDIS_KEY_TOTAL, total_files)
|
||||
progress_set(REDIS_KEY_PROCESSED, 0)
|
||||
|
||||
processed_files = 0
|
||||
errors = []
|
||||
|
||||
for root, dirs, files in os.walk(folder_path):
|
||||
# Get or create folder entry
|
||||
folder = await get_or_create_folder(session, root, source_root_id, owner_user_id)
|
||||
progress_set(REDIS_KEY_CURRENT_FOLDER, root)
|
||||
|
||||
# Filter supported files
|
||||
supported_files = [f for f in files if Path(f).suffix.lower() in SUPPORTED_EXTENSIONS]
|
||||
|
||||
# Process files in batches
|
||||
batch_size = settings.scanner.batch_size
|
||||
for i in range(0, len(supported_files), batch_size):
|
||||
batch = supported_files[i:i + batch_size]
|
||||
# Defer task dispatch until AFTER commit so workers don't
|
||||
# query for rows that aren't visible to other sessions yet.
|
||||
pending_dispatch: list[str] = []
|
||||
pending_video_pretranscode: list[tuple[str, str]] = []
|
||||
|
||||
for filename in batch:
|
||||
filepath = os.path.join(root, filename)
|
||||
|
||||
try:
|
||||
# Check if file already exists in database
|
||||
existing = await session.execute(
|
||||
select(Photo).where(Photo.filepath == filepath)
|
||||
)
|
||||
existing_photo = existing.scalar_one_or_none()
|
||||
if existing_photo is not None:
|
||||
# Resurrect a previously-discarded row only
|
||||
# when the file's mtime is newer than
|
||||
# discarded_at. Bare existence on disk isn't
|
||||
# proof the user changed their mind: every
|
||||
# backend boot fires scan_all_source_roots,
|
||||
# which used to walk every file and silently
|
||||
# un-discard the lot. The mtime check still
|
||||
# covers the legitimate flows (WebDAV DELETE
|
||||
# + re-upload, trashbin restore via PUT-
|
||||
# overwrite, any "I removed it then put it
|
||||
# back") because those rewrite the file and
|
||||
# bump mtime past the discard time. Rows
|
||||
# with discarded_at IS NULL (legacy) are
|
||||
# left alone — preserve user intent over
|
||||
# best-effort cleanup.
|
||||
if existing_photo.is_discarded:
|
||||
discarded_at = existing_photo.discarded_at
|
||||
try:
|
||||
mtime = os.path.getmtime(filepath)
|
||||
except OSError:
|
||||
mtime = 0.0
|
||||
file_modified_after_discard = (
|
||||
discarded_at is not None
|
||||
and mtime
|
||||
> discarded_at.replace(
|
||||
tzinfo=timezone.utc
|
||||
).timestamp()
|
||||
)
|
||||
if file_modified_after_discard:
|
||||
existing_photo.is_discarded = False
|
||||
existing_photo.discarded_at = None
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"Resurrected discarded photo "
|
||||
f"(file modified after discard): "
|
||||
f"{filepath}"
|
||||
)
|
||||
extract_metadata.delay(existing_photo.id)
|
||||
else:
|
||||
logger.debug(
|
||||
f"Skipping discarded photo "
|
||||
f"(file unchanged since discard): "
|
||||
f"{filepath}"
|
||||
)
|
||||
else:
|
||||
logger.debug(f"File already indexed: {filepath}")
|
||||
processed_files += 1
|
||||
progress_set(REDIS_KEY_PROCESSED, processed_files)
|
||||
continue
|
||||
|
||||
# Get file stats
|
||||
stat = os.stat(filepath)
|
||||
|
||||
# Calculate file hash for duplicate detection
|
||||
file_hash = await calculate_file_hash(filepath)
|
||||
|
||||
# Check for duplicate by hash. We only care
|
||||
# whether *any* other photo shares this hash, so
|
||||
# use a count rather than scalar_one_or_none()
|
||||
# which raises "Multiple rows were found" the
|
||||
# moment the library has 2+ copies of the same
|
||||
# file (i.e. exactly the case we're trying to
|
||||
# flag).
|
||||
is_dup = False
|
||||
if file_hash:
|
||||
dup_count = (await session.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
Photo.file_hash == file_hash
|
||||
)
|
||||
)).scalar() or 0
|
||||
is_dup = dup_count > 0
|
||||
|
||||
# Inherit the effective-hidden flag from the
|
||||
# folder's ancestry. If any ancestor folder
|
||||
# has is_hidden=true, the new photo is
|
||||
# immediately marked hidden so it never
|
||||
# briefly appears in cross-cutting views
|
||||
# between scan and the next manual recompute.
|
||||
effective_hidden = await is_folder_effectively_hidden(folder)
|
||||
|
||||
# Create photo entry
|
||||
mtime_dt = datetime.fromtimestamp(stat.st_mtime)
|
||||
photo = Photo(
|
||||
filepath=filepath,
|
||||
filename=filename,
|
||||
folder_id=folder.id,
|
||||
user_id=owner_user_id,
|
||||
file_hash=file_hash,
|
||||
media_type=get_media_type(filepath),
|
||||
original_format=Path(filepath).suffix.upper()[1:],
|
||||
file_size=stat.st_size,
|
||||
taken_at=mtime_dt,
|
||||
taken_at_source='filesystem',
|
||||
# First-pass flag based on the filesystem mtime;
|
||||
# metadata.extract_metadata re-runs this once
|
||||
# EXIF has been parsed so a real DateTimeOriginal
|
||||
# can clear the warning.
|
||||
has_date_warning=has_date_warning(filepath, mtime_dt),
|
||||
is_duplicate=is_dup,
|
||||
is_hidden=effective_hidden,
|
||||
processing_status='pending'
|
||||
)
|
||||
|
||||
session.add(photo)
|
||||
await session.flush() # Assign defaults / FK ids
|
||||
|
||||
# Queue dispatch happens after the batch commit
|
||||
# below; otherwise the worker can race the writer
|
||||
# and see "Photo not found".
|
||||
pending_dispatch.append(photo.id)
|
||||
if photo.media_type == 'video':
|
||||
# Pre-transcode HEVC and other non-web-safe
|
||||
# videos at scan time so the user doesn't
|
||||
# pay the encode cost on first <video> click.
|
||||
pending_video_pretranscode.append(
|
||||
(photo.id, filepath)
|
||||
)
|
||||
|
||||
processed_files += 1
|
||||
progress_set(REDIS_KEY_PROCESSED, processed_files)
|
||||
|
||||
# Celery internal progress (used by celery tooling)
|
||||
if processed_files % 10 == 0:
|
||||
task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={
|
||||
'current': processed_files,
|
||||
'total': total_files,
|
||||
'folder': root,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing file {filepath}: {e}")
|
||||
errors.append({'file': filepath, 'error': str(e)})
|
||||
progress_push_error(f"{filepath}: {e}")
|
||||
continue
|
||||
|
||||
# Commit batch, then queue worker tasks. Dispatch order
|
||||
# matters: commit first so workers can find the rows.
|
||||
await session.commit()
|
||||
|
||||
for photo_id in pending_dispatch:
|
||||
generate_thumbnails.delay(photo_id)
|
||||
extract_metadata.delay(photo_id)
|
||||
for vid_photo_id, vid_path in pending_video_pretranscode:
|
||||
pretranscode_video.delay(vid_photo_id, vid_path)
|
||||
|
||||
# Update folder scan timestamp
|
||||
folder.last_scanned = datetime.utcnow()
|
||||
folder.photo_count = processed_files
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"Scan complete. Processed {processed_files}/{total_files} files. Errors: {len(errors)}")
|
||||
|
||||
return {
|
||||
'status': 'completed',
|
||||
'processed': processed_files,
|
||||
'total': total_files,
|
||||
'errors': errors,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Scan failed: {e}")
|
||||
progress_push_error(f"scan failed: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
# Always mark inactive on the way out so a crashed scan doesn't
|
||||
# leave the UI thinking we're still scanning.
|
||||
progress_set(REDIS_KEY_ACTIVE, 'false')
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
"""Canonicalise a filesystem path so we don't get duplicate DB rows for
|
||||
the same physical directory due to trailing slashes, redundant separators,
|
||||
or `.` segments. Symlinks are NOT resolved (we want to keep mount paths
|
||||
intact for cross-machine portability)."""
|
||||
return os.path.normpath(path)
|
||||
|
||||
|
||||
async def get_or_create_source_root(session: AsyncSession, path: str) -> SourceRoot:
|
||||
"""Get or create a source root entry, matching by normalized path."""
|
||||
from sqlalchemy import select
|
||||
|
||||
norm = _normalize_path(path)
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.path == norm)
|
||||
)
|
||||
source_root = result.scalar_one_or_none()
|
||||
|
||||
if not source_root:
|
||||
source_root = SourceRoot(
|
||||
name=Path(norm).name,
|
||||
path=norm,
|
||||
)
|
||||
session.add(source_root)
|
||||
await session.flush()
|
||||
|
||||
return source_root
|
||||
|
||||
|
||||
async def get_or_create_folder(
|
||||
session: AsyncSession, path: str, source_root_id: str, user_id: str = None
|
||||
) -> Folder:
|
||||
"""Get or create a folder entry, matching by normalized path."""
|
||||
from sqlalchemy import select
|
||||
|
||||
norm = _normalize_path(path)
|
||||
result = await session.execute(
|
||||
select(Folder).where(Folder.path == norm)
|
||||
)
|
||||
folder = result.scalar_one_or_none()
|
||||
|
||||
if not folder:
|
||||
parent_path = _normalize_path(str(Path(norm).parent))
|
||||
|
||||
if parent_path != norm: # Not the filesystem root
|
||||
parent_result = await session.execute(
|
||||
select(Folder).where(Folder.path == parent_path)
|
||||
)
|
||||
parent = parent_result.scalar_one_or_none()
|
||||
if parent:
|
||||
parent_id = parent.id
|
||||
else:
|
||||
# Recursively create parent
|
||||
parent = await get_or_create_folder(session, parent_path, source_root_id, user_id)
|
||||
parent_id = parent.id
|
||||
else:
|
||||
parent_id = None
|
||||
|
||||
folder = Folder(
|
||||
name=Path(norm).name,
|
||||
path=norm,
|
||||
parent_id=parent_id,
|
||||
source_root_id=source_root_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
session.add(folder)
|
||||
await session.flush()
|
||||
|
||||
return folder
|
||||
|
||||
@shared_task(name='scan_all_source_roots')
|
||||
def scan_all_source_roots():
|
||||
"""Scan every active source root currently registered in the DB."""
|
||||
# Clear stale per-scan progress before queuing new work so the UI sees
|
||||
# a clean slate even if a previous run crashed mid-flight.
|
||||
r = _get_redis()
|
||||
if r is not None:
|
||||
try:
|
||||
r.delete(REDIS_KEY_ERRORS)
|
||||
r.set(REDIS_KEY_PROCESSED, 0)
|
||||
r.set(REDIS_KEY_TOTAL, 0)
|
||||
except Exception as e:
|
||||
logger.debug(f"scan_all_source_roots redis reset failed: {e}")
|
||||
|
||||
return asyncio.run(_scan_all_source_roots_async())
|
||||
|
||||
|
||||
async def _scan_all_source_roots_async():
|
||||
"""Read every active SourceRoot from the DB and queue a scan_folder task
|
||||
for each. Source roots whose path no longer exists on disk are skipped
|
||||
with a warning (the cleanup service surfaces those at startup too).
|
||||
|
||||
After dispatching the scans, queue a delayed `regroup_duplicates`
|
||||
pass so duplicate clusters are recomputed once the new photos have
|
||||
finished thumbnailing (and therefore picked up phashes). The
|
||||
countdown is a best-effort hint — on a big library the user can
|
||||
still hit Settings → Re-detect duplicates to force a fresh pass.
|
||||
"""
|
||||
from app.tasks.thumbs import incremental_regroup_duplicates_task
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active == True) # noqa: E712
|
||||
)
|
||||
source_roots = result.scalars().all()
|
||||
dispatched = 0
|
||||
for sr in source_roots:
|
||||
if os.path.exists(sr.path):
|
||||
scan_folder.delay(sr.path, sr.id)
|
||||
dispatched += 1
|
||||
else:
|
||||
logger.warning(f"Source root path does not exist: {sr.path}")
|
||||
|
||||
if dispatched > 0:
|
||||
# 60s gives the thumbs worker a window to compute phashes for
|
||||
# the new photos before regrouping. The task is idempotent, so
|
||||
# firing too early just means the next manual run picks up the
|
||||
# late arrivals — no corrupted state.
|
||||
try:
|
||||
# Use incremental mode: only compare newly added photos
|
||||
# against the full library via CLIP HNSW + pHash.
|
||||
# O(new × log N) instead of O(N²).
|
||||
scan_start = datetime.now(timezone.utc).isoformat()
|
||||
incremental_regroup_duplicates_task.apply_async(
|
||||
kwargs={'since_iso': scan_start},
|
||||
countdown=60,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue post-scan regroup: {e}")
|
||||
|
||||
# NOTE: we used to auto-queue `backfill_gps` here so photos
|
||||
# scanned before the GPS-extraction fix would eventually get
|
||||
# their coordinates populated. That fix shipped a long time
|
||||
# ago, so on every modern restart it just re-ran
|
||||
# extract_metadata for every photo that legitimately has no
|
||||
# GPS in EXIF (screenshots, indoor shots, scans) — tens of
|
||||
# thousands of pointless tasks that saturated worker-light
|
||||
# for ~30 min after each deploy. Trigger manually via
|
||||
# POST /api/v1/library/backfill-gps if you ever need it
|
||||
# again (e.g. another extractor-logic fix lands).
|
||||
|
||||
|
||||
@shared_task(name='watch_folders', bind=True)
|
||||
def watch_folders(self):
|
||||
"""Retired: file events now arrive via NC webhook_listeners.
|
||||
|
||||
Kept as a no-op task so any in-flight queue items (a leftover
|
||||
apply_async from a restart before this commit, or an admin-button
|
||||
trigger) don't crash workers. Will be removed entirely once the
|
||||
queue drains.
|
||||
"""
|
||||
logger.info(
|
||||
"watch_folders task is retired; file events come from NC "
|
||||
"webhook_listeners. No-op."
|
||||
)
|
||||
return {'status': 'retired'}
|
||||
|
||||
async def handle_file_deletion(filepath: str):
|
||||
"""Handle deletion of a file from the filesystem"""
|
||||
from sqlalchemy import select
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(Photo).where(Photo.filepath == filepath)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
|
||||
if photo:
|
||||
# Mark as missing or delete from database
|
||||
photo.is_discarded = True
|
||||
photo.discarded_at = datetime.utcnow()
|
||||
await session.commit()
|
||||
logger.info(f"Marked photo as discarded: {filepath}")
|
||||
|
||||
|
||||
async def handle_directory_deletion(dirpath: str) -> int:
|
||||
"""Mark every Photo under `dirpath` as discarded — used when Nextcloud
|
||||
fires a NodeDeletedEvent on a folder. NC emits ONE event for the
|
||||
folder itself (not one per child file), so without this we'd never
|
||||
see the children disappear except via the 30-min reconcile sweep.
|
||||
|
||||
Returns the number of photos affected. Matches by `filepath LIKE
|
||||
dirpath + '/%'` (the trailing slash is important — we don't want
|
||||
`/photos/foo` to also match `/photos/foobar.jpg`).
|
||||
"""
|
||||
from sqlalchemy import update
|
||||
|
||||
prefix = dirpath.rstrip("/") + "/"
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
update(Photo)
|
||||
.where(
|
||||
Photo.filepath.like(prefix + "%"),
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
.values(is_discarded=True, discarded_at=datetime.utcnow())
|
||||
)
|
||||
await session.commit()
|
||||
n = result.rowcount or 0
|
||||
if n:
|
||||
logger.info(f"Marked {n} photos as discarded under {dirpath}")
|
||||
return n
|
||||
|
||||
|
||||
async def handle_directory_rename(old_dirpath: str, new_dirpath: str) -> dict:
|
||||
"""Reflect a Nextcloud-side folder rename in mule's DB.
|
||||
|
||||
NC emits a single NodeRenamedEvent on the directory — children
|
||||
don't get their own events. We mirror the same prefix-rewrite the
|
||||
PATCH /folders/{id} endpoint does inline, so heaps, tags, ratings,
|
||||
and other state keyed on Photo.id survive intact.
|
||||
|
||||
Same-source-root case (the common one): prefix-rewrite filepath /
|
||||
path on photos, folders, source_roots in one transaction.
|
||||
|
||||
Cross-source-root case (folder moved between two registered roots,
|
||||
e.g. Photos/x → Memories/x): discard the old subtree and rely on
|
||||
the scan_folder dispatched by a NodeWritten/NodeCreated event (or
|
||||
the 30-min reconcile sweep) to add fresh Photo rows under the new
|
||||
root. Mirrors the "different boundary, different identity" model
|
||||
mule has elsewhere.
|
||||
|
||||
Idempotent: re-running with the same args is a no-op because no
|
||||
row matches `LIKE old_prefix||'/%'` after the first pass. That
|
||||
makes the feedback loop (mule PATCH → WebDAV MOVE → NC webhook →
|
||||
handler) safe.
|
||||
"""
|
||||
from sqlalchemy import or_, text, update
|
||||
from app.models.folders import SourceRoot
|
||||
|
||||
old_prefix = old_dirpath.rstrip("/")
|
||||
new_prefix = new_dirpath.rstrip("/")
|
||||
if not old_prefix or not new_prefix or old_prefix == new_prefix:
|
||||
return {"status": "noop"}
|
||||
|
||||
async def _source_root_id_for(session, path: str) -> Optional[str]:
|
||||
"""Find the active SourceRoot whose path is a prefix of `path`."""
|
||||
roots = (await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active == True) # noqa: E712
|
||||
)).scalars().all()
|
||||
norm = os.path.normpath(path)
|
||||
for sr in roots:
|
||||
root = os.path.normpath(sr.path)
|
||||
if norm == root or norm.startswith(root + os.sep):
|
||||
return sr.id
|
||||
return None
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
old_root_id = await _source_root_id_for(session, old_prefix)
|
||||
new_root_id = await _source_root_id_for(session, new_prefix)
|
||||
|
||||
# Cross-root rename: discard old subtree; let webhook-dispatched
|
||||
# scan_folder add fresh rows under the new root.
|
||||
if old_root_id and new_root_id and old_root_id != new_root_id:
|
||||
result = await session.execute(
|
||||
update(Photo)
|
||||
.where(
|
||||
Photo.filepath.like(old_prefix + "/%"),
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
.values(is_discarded=True, discarded_at=datetime.utcnow())
|
||||
)
|
||||
await session.commit()
|
||||
n = result.rowcount or 0
|
||||
logger.info(
|
||||
f"Cross-root rename {old_prefix} -> {new_prefix}: "
|
||||
f"discarded {n} photos in old root"
|
||||
)
|
||||
return {"status": "cross_root", "discarded": n}
|
||||
|
||||
# Same-root: iterate the matching rows in Python and rewrite
|
||||
# the prefix attribute-side. We tried a single UPDATE … SET …
|
||||
# SUBSTRING(... FROM LENGTH(:old)+1) raw-SQL approach but
|
||||
# asyncpg miscategorises the LENGTH() result and rejects it
|
||||
# as "$2: int (expected str)". The PATCH /folders/{id}
|
||||
# endpoint already loops in Python for the same reason — match
|
||||
# its pattern. Folder renames are rare and typically span ≤1k
|
||||
# photos, so per-row UPDATEs are fine.
|
||||
old_pat = old_prefix + "/%"
|
||||
photos = (await session.execute(
|
||||
select(Photo).where(Photo.filepath.like(old_pat))
|
||||
)).scalars().all()
|
||||
for p in photos:
|
||||
p.filepath = new_prefix + p.filepath[len(old_prefix):]
|
||||
|
||||
folders = (await session.execute(
|
||||
select(Folder).where(
|
||||
or_(
|
||||
Folder.path == old_prefix,
|
||||
Folder.path.like(old_pat),
|
||||
)
|
||||
)
|
||||
)).scalars().all()
|
||||
# Path is load-bearing (FKs join on it implicitly via filepath),
|
||||
# name is purely display. The renamed folder itself gets its
|
||||
# leaf basename refreshed too so the sidebar tree doesn't show
|
||||
# stale text. Descendant folders keep their existing name
|
||||
# because the rename was on the ancestor — only the path changes.
|
||||
new_basename = os.path.basename(new_prefix)
|
||||
for f in folders:
|
||||
if f.path == old_prefix:
|
||||
f.path = new_prefix
|
||||
f.name = new_basename
|
||||
else:
|
||||
f.path = new_prefix + f.path[len(old_prefix):]
|
||||
|
||||
source_roots = (await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.path == old_prefix)
|
||||
)).scalars().all()
|
||||
for sr in source_roots:
|
||||
sr.path = new_prefix
|
||||
await session.commit()
|
||||
return {
|
||||
"status": "renamed",
|
||||
"photos": len(photos),
|
||||
"folders": len(folders),
|
||||
"source_roots": len(source_roots),
|
||||
}
|
||||
|
||||
|
||||
@shared_task(name='backfill_gps')
|
||||
def backfill_gps():
|
||||
"""Re-run metadata extraction on every non-discarded photo that is
|
||||
missing latitude/longitude. Used both as a one-shot kick-off after the
|
||||
GPS columns are added on an existing install (see app/database.py) and
|
||||
as a manual trigger from POST /api/v1/library/backfill-gps. Each
|
||||
extract_metadata call is itself a Celery task, so this just enqueues —
|
||||
it does not block on extraction completing."""
|
||||
return asyncio.run(_backfill_gps_async())
|
||||
|
||||
|
||||
async def _backfill_gps_async():
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Newest-first so the most recent photos get their GPS + EXIF
|
||||
# written before the worker climbs back through the archive.
|
||||
result = await session.execute(
|
||||
select(Photo.id)
|
||||
.where(
|
||||
Photo.latitude.is_(None),
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
)
|
||||
photo_ids = [row[0] for row in result.all()]
|
||||
|
||||
for pid in photo_ids:
|
||||
extract_metadata.delay(pid)
|
||||
|
||||
logger.info(f"backfill_gps: queued extract_metadata for {len(photo_ids)} photos")
|
||||
return {'queued': len(photo_ids)}
|
||||
|
||||
|
||||
@shared_task(name='discard_missing_photos_beat')
|
||||
def discard_missing_photos_beat():
|
||||
"""Periodic catch-up for filesystem deletions the watcher missed
|
||||
(e.g. while the worker was restarting). Walks every active source
|
||||
root that is currently mounted and present, and soft-discards any
|
||||
Photo whose file is gone. Hard-deletion stays manual via
|
||||
POST /api/v1/library/maintenance/prune-missing.
|
||||
|
||||
Wired to a 30-minute beat schedule in app/tasks/celery.py.
|
||||
"""
|
||||
from app.services.cleanup import discard_missing_photos
|
||||
return asyncio.run(discard_missing_photos())
|
||||
@@ -1,565 +0,0 @@
|
||||
"""
|
||||
Celery tasks for thumbnail generation
|
||||
"""
|
||||
import os
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import logging
|
||||
from typing import Tuple, Optional
|
||||
import json
|
||||
|
||||
from celery import shared_task
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from PIL import Image
|
||||
import imageio
|
||||
from pillow_heif import register_heif_opener
|
||||
import ffmpeg
|
||||
|
||||
# Try to import optional libraries
|
||||
try:
|
||||
import pyvips
|
||||
PYVIPS_AVAILABLE = True
|
||||
except ImportError:
|
||||
PYVIPS_AVAILABLE = False
|
||||
print("pyvips not available, using Pillow for image processing")
|
||||
|
||||
try:
|
||||
import rawpy
|
||||
RAWPY_AVAILABLE = True
|
||||
except ImportError:
|
||||
RAWPY_AVAILABLE = False
|
||||
print("rawpy not available, using exiftool for RAW preview extraction")
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo
|
||||
from app.config import settings
|
||||
|
||||
# Register HEIF opener with Pillow
|
||||
register_heif_opener()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thumbnail sizes configuration
|
||||
THUMB_SIZES = {
|
||||
'small': settings.thumbnails.small,
|
||||
'medium': settings.thumbnails.medium,
|
||||
'large': settings.thumbnails.large
|
||||
}
|
||||
|
||||
# Sizes the worker writes to /data/thumbs. Empty set since Phase 4 —
|
||||
# the API serves all sizes via Nextcloud's /core/preview proxy.
|
||||
# generate_thumbnails still runs the decode-and-pHash side-effect
|
||||
# (perceptual dedup is mule-only and needs the original-resolution
|
||||
# pixels) but no longer touches the disk.
|
||||
WORKER_THUMB_SIZES: set[str] = set()
|
||||
|
||||
def get_thumb_path(photo_id: str, size: str, user_id: str = None) -> str:
|
||||
"""Get the path for a thumbnail file.
|
||||
|
||||
When user_id is provided, thumbnails are stored under a user-specific
|
||||
subdirectory to enforce isolation between users.
|
||||
"""
|
||||
if user_id:
|
||||
thumb_dir = f"/data/thumbs/{user_id}/{photo_id}"
|
||||
else:
|
||||
thumb_dir = f"/data/thumbs/{photo_id}"
|
||||
os.makedirs(thumb_dir, exist_ok=True)
|
||||
return f"{thumb_dir}/{size}.{settings.thumbnails.format}"
|
||||
|
||||
def process_standard_image(filepath: str) -> Image.Image:
|
||||
"""Process standard image formats (JPEG, PNG, etc.)"""
|
||||
return Image.open(filepath)
|
||||
|
||||
def process_raw_image(filepath: str) -> Image.Image:
|
||||
"""Process RAW image formats"""
|
||||
if RAWPY_AVAILABLE:
|
||||
try:
|
||||
with rawpy.imread(filepath) as raw:
|
||||
# Use half_size for faster processing
|
||||
rgb = raw.postprocess(use_camera_wb=True, half_size=True)
|
||||
# Convert numpy array to PIL Image
|
||||
return Image.fromarray(rgb, 'RGB')
|
||||
except Exception as e:
|
||||
logger.warning(f"rawpy failed for {filepath}: {e}; trying embedded preview")
|
||||
preview = extract_raw_preview(filepath)
|
||||
if preview is not None:
|
||||
return preview
|
||||
# iPhone "Apple ProRAW" / Linear DNG has no embedded preview and
|
||||
# LibRaw rejects it as not-a-RAW. It IS a TIFF container with a
|
||||
# developed RGB image inside, so PIL opens it directly.
|
||||
try:
|
||||
logger.warning(f"embedded preview missing for {filepath}; trying PIL TIFF fallback")
|
||||
return Image.open(filepath)
|
||||
except Exception as e2:
|
||||
logger.error(f"PIL fallback also failed for {filepath}: {e2}")
|
||||
raise
|
||||
else:
|
||||
# Use exiftool to extract embedded preview
|
||||
return extract_raw_preview(filepath)
|
||||
|
||||
def extract_raw_preview(filepath: str) -> Optional[Image.Image]:
|
||||
"""Extract embedded JPEG preview from RAW file"""
|
||||
try:
|
||||
# Use exiftool to extract preview
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp:
|
||||
cmd = ['exiftool', '-b', '-PreviewImage', filepath]
|
||||
result = subprocess.run(cmd, capture_output=True)
|
||||
|
||||
if result.returncode == 0 and result.stdout:
|
||||
tmp.write(result.stdout)
|
||||
tmp.flush()
|
||||
return Image.open(tmp.name)
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting RAW preview from {filepath}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def process_heic_image(filepath: str) -> Image.Image:
|
||||
"""Process HEIC/HEIF image formats.
|
||||
|
||||
Tries pillow-heif first (fast, native). Falls back to ffmpeg for
|
||||
files that libheif rejects — e.g. iPhone photos with too many
|
||||
auxiliary image references (depth maps, gain maps).
|
||||
"""
|
||||
try:
|
||||
img = Image.open(filepath)
|
||||
if img.mode != 'RGB':
|
||||
img = img.convert('RGB')
|
||||
return img
|
||||
except Exception as e:
|
||||
logger.warning(f"pillow-heif failed for {filepath}: {e} — trying vips")
|
||||
|
||||
# vips fallback: handles tiled Apple HEIC files (bursts, HDR gain
|
||||
# maps, depth maps) that pillow-heif/libheif rejects due to too many
|
||||
# auxiliary image references.
|
||||
import subprocess, tempfile
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
result = subprocess.run(
|
||||
['vips', 'heifload', filepath, tmp_path],
|
||||
capture_output=True, timeout=60, stdin=subprocess.DEVNULL,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
img = Image.open(tmp_path).convert('RGB')
|
||||
os.unlink(tmp_path)
|
||||
return img
|
||||
logger.error(f"vips HEIC decode failed for {filepath}: {result.stderr.decode()[-200:]}")
|
||||
os.unlink(tmp_path)
|
||||
except Exception as e2:
|
||||
logger.error(f"vips fallback failed for {filepath}: {e2}")
|
||||
raise RuntimeError(f"Cannot decode HEIC: {filepath}")
|
||||
|
||||
def process_video_thumbnail(filepath: str) -> Image.Image:
|
||||
"""Extract a still frame from a video file as a PIL Image."""
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
|
||||
tmp_path: Optional[str] = None
|
||||
try:
|
||||
# Find a usable seek timestamp. Some camera MOVs only expose
|
||||
# duration at the format level, and stream 0 isn't always the
|
||||
# video stream — search explicitly and fall back to the format
|
||||
# duration, then to t=0 if neither is available.
|
||||
probe = ffmpeg.probe(filepath)
|
||||
duration: Optional[float] = None
|
||||
for stream_info in probe.get('streams', []):
|
||||
if stream_info.get('codec_type') != 'video':
|
||||
continue
|
||||
raw_duration = stream_info.get('duration')
|
||||
if raw_duration is not None:
|
||||
try:
|
||||
duration = float(raw_duration)
|
||||
break
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if duration is None:
|
||||
raw_duration = probe.get('format', {}).get('duration')
|
||||
if raw_duration is not None:
|
||||
try:
|
||||
duration = float(raw_duration)
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
|
||||
# Seek to 10% in for a representative frame; clamp very short
|
||||
# clips to t=0 so we don't seek past the end.
|
||||
timestamp = max(0.0, (duration or 0.0) * 0.1)
|
||||
|
||||
# NamedTemporaryFile creates the file on disk, so we MUST tell
|
||||
# ffmpeg to overwrite it (otherwise it prompts on stdin and the
|
||||
# call hangs/fails — which is why videos were getting the gray
|
||||
# placeholder). We close the handle immediately and clean up
|
||||
# in `finally` ourselves.
|
||||
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
stream = ffmpeg.input(filepath, ss=timestamp)
|
||||
stream = ffmpeg.output(
|
||||
stream,
|
||||
tmp_path,
|
||||
vframes=1,
|
||||
format='image2',
|
||||
vcodec='mjpeg',
|
||||
)
|
||||
ffmpeg.run(
|
||||
stream,
|
||||
capture_stdout=True,
|
||||
capture_stderr=True,
|
||||
overwrite_output=True,
|
||||
)
|
||||
|
||||
# Load the frame fully into memory so we can delete the temp
|
||||
# file immediately. Pillow's `Image.open` is lazy, which would
|
||||
# otherwise leave the file dangling.
|
||||
with open(tmp_path, 'rb') as fh:
|
||||
data = fh.read()
|
||||
if not data:
|
||||
raise RuntimeError("ffmpeg produced an empty frame")
|
||||
return Image.open(BytesIO(data)).copy()
|
||||
except ffmpeg.Error as e:
|
||||
stderr = (e.stderr or b'').decode('utf-8', errors='replace')
|
||||
logger.error(
|
||||
f"ffmpeg failed extracting video thumbnail from {filepath}: {stderr}"
|
||||
)
|
||||
return create_placeholder_thumbnail('video')
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting video thumbnail from {filepath}: {e}")
|
||||
return create_placeholder_thumbnail('video')
|
||||
finally:
|
||||
if tmp_path and os.path.exists(tmp_path):
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def create_placeholder_thumbnail(media_type: str) -> Image.Image:
|
||||
"""Create a placeholder thumbnail for failed processing"""
|
||||
# Create a simple gray placeholder
|
||||
img = Image.new('RGB', (640, 480), color=(128, 128, 128))
|
||||
return img
|
||||
|
||||
def auto_rotate_image(image: Image.Image) -> Image.Image:
|
||||
"""Auto-rotate image based on EXIF orientation"""
|
||||
try:
|
||||
# Get EXIF data
|
||||
exif = image._getexif()
|
||||
if exif:
|
||||
orientation = exif.get(274) # Orientation tag
|
||||
|
||||
rotation_map = {
|
||||
3: 180,
|
||||
6: 270, # Note: PIL uses different rotation values than vips
|
||||
8: 90
|
||||
}
|
||||
|
||||
if orientation in rotation_map:
|
||||
image = image.rotate(rotation_map[orientation], expand=True)
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
pass # No orientation data available
|
||||
|
||||
return image
|
||||
|
||||
def generate_thumbnail(image: Image.Image, size: int, output_path: str):
|
||||
"""Generate a thumbnail of the specified size.
|
||||
|
||||
Works on a copy so the caller's image is never mutated — this is
|
||||
critical because the thumbnail loop iterates multiple sizes and
|
||||
in-place shrinking would degrade later (larger) sizes.
|
||||
"""
|
||||
img = image.copy()
|
||||
img.thumbnail((size, size), Image.Resampling.LANCZOS)
|
||||
|
||||
img.save(
|
||||
output_path,
|
||||
'WEBP',
|
||||
quality=settings.thumbnails.quality,
|
||||
method=4 # Balance between speed and compression
|
||||
)
|
||||
img.close()
|
||||
|
||||
@shared_task(bind=True, name='generate_thumbnails')
|
||||
def generate_thumbnails(self, photo_id: str):
|
||||
"""Generate thumbnails for a photo"""
|
||||
return asyncio.run(_generate_thumbnails_async(photo_id, self))
|
||||
|
||||
async def _generate_thumbnails_async(photo_id: str, task):
|
||||
"""Async implementation of thumbnail generation"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Declared up front so the except block below can safely check it
|
||||
# even if the initial SELECT raises (e.g. asyncpg transport error).
|
||||
photo: Optional[Photo] = None
|
||||
try:
|
||||
# Get photo from database
|
||||
result = await session.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
|
||||
if not photo:
|
||||
logger.error(f"Photo not found: {photo_id}")
|
||||
return {'status': 'error', 'message': 'Photo not found'}
|
||||
|
||||
# Check if file exists
|
||||
if not os.path.exists(photo.filepath):
|
||||
logger.error(f"File not found: {photo.filepath}")
|
||||
photo.processing_status = 'failed'
|
||||
photo.processing_error = 'File not found'
|
||||
await session.commit()
|
||||
return {'status': 'error', 'message': 'File not found'}
|
||||
|
||||
# Update processing status
|
||||
photo.processing_status = 'processing'
|
||||
await session.commit()
|
||||
|
||||
# Load and process the image based on type
|
||||
image = None
|
||||
|
||||
if photo.media_type == 'photo':
|
||||
image = process_standard_image(photo.filepath)
|
||||
elif photo.media_type == 'raw':
|
||||
image = process_raw_image(photo.filepath)
|
||||
elif photo.media_type == 'heic':
|
||||
image = process_heic_image(photo.filepath)
|
||||
elif photo.media_type == 'video':
|
||||
image = process_video_thumbnail(photo.filepath)
|
||||
else:
|
||||
logger.error(f"Unsupported media type: {photo.media_type}")
|
||||
image = create_placeholder_thumbnail(photo.media_type)
|
||||
|
||||
# Fallback: some files wear a RAW/HEIC extension but are actually
|
||||
# plain JPEGs — e.g. iPhones that write ProRAW-style .DNG for
|
||||
# images where no RAW sensor data was captured, or re-exports
|
||||
# that kept the original suffix. Pillow can open them directly,
|
||||
# so before giving up, try reading the file as a standard image.
|
||||
if not image and photo.media_type in ('raw', 'heic'):
|
||||
try:
|
||||
image = process_standard_image(photo.filepath)
|
||||
if image is not None:
|
||||
logger.info(
|
||||
f"{photo.filepath}: {photo.media_type} decode failed "
|
||||
f"but file opens as a standard image — using fallback"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
f"Standard-image fallback failed for {photo.filepath}: {e}"
|
||||
)
|
||||
|
||||
if not image:
|
||||
raise Exception("Failed to process image")
|
||||
|
||||
# Auto-rotate based on EXIF
|
||||
image = auto_rotate_image(image)
|
||||
|
||||
# Store original dimensions
|
||||
photo.width = image.width
|
||||
photo.height = image.height
|
||||
|
||||
# Perceptual hash from the original-resolution decoded frame.
|
||||
# pHash is robust to resize/recompression but the thumbnail
|
||||
# loop below mutates `image` in place, so this MUST run before
|
||||
# the loop sees it. Failures are non-fatal — phash is a
|
||||
# nice-to-have, not a blocker for thumbnail generation.
|
||||
try:
|
||||
import imagehash
|
||||
photo.phash = str(imagehash.phash(image)) # 16-char hex
|
||||
except Exception as e:
|
||||
logger.warning(f"phash failed for {photo_id}: {e}")
|
||||
photo.phash = None
|
||||
|
||||
# Generate only the sizes the worker still owns on disk
|
||||
# (see WORKER_THUMB_SIZES above). The API serves the rest
|
||||
# via Nextcloud's preview endpoint.
|
||||
for size_name, size_value in THUMB_SIZES.items():
|
||||
if size_name not in WORKER_THUMB_SIZES:
|
||||
continue
|
||||
thumb_path = get_thumb_path(photo_id, size_name, photo.user_id)
|
||||
generate_thumbnail(image, size_value, thumb_path)
|
||||
|
||||
# Update database with thumbnail path
|
||||
setattr(photo, f'thumb_{size_name}', thumb_path)
|
||||
|
||||
# Update progress
|
||||
task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current_size': size_name, 'photo_id': photo_id}
|
||||
)
|
||||
|
||||
# Update processing status
|
||||
photo.processing_status = 'completed'
|
||||
photo.processing_error = None
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"Thumbnails generated for photo {photo_id}")
|
||||
|
||||
return {'status': 'success', 'photo_id': photo_id}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating thumbnails for {photo_id}: {e}")
|
||||
|
||||
# Update error status. If the session is in a bad state (e.g.
|
||||
# the original failure was a transport error) rollback first so
|
||||
# the status write has a clean transaction to commit into.
|
||||
try:
|
||||
await session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if photo is not None:
|
||||
try:
|
||||
photo.processing_status = 'failed'
|
||||
photo.processing_error = str(e)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
f"Could not mark photo {photo_id} as failed"
|
||||
)
|
||||
|
||||
return {'status': 'error', 'message': str(e)}
|
||||
|
||||
@shared_task(name='regenerate_all_thumbnails')
|
||||
def regenerate_all_thumbnails():
|
||||
"""Regenerate thumbnails for all photos"""
|
||||
return asyncio.run(_regenerate_all_thumbnails_async())
|
||||
|
||||
async def _regenerate_all_thumbnails_async():
|
||||
"""Async implementation of regenerating all thumbnails.
|
||||
|
||||
Queue order matters on first-boot and recovery runs: we dispatch
|
||||
newest-first (by EXIF taken_at, fallback added_at) so the user's
|
||||
most recent photos become fully-indexed before the 2012 archive even
|
||||
starts. Picking up the library in pipeline order means the grid,
|
||||
timeline and All Photos view populate top-down instead of the worker
|
||||
chewing through random insertion-order rows while the UI still
|
||||
shows grey placeholders.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Get all photos that need thumbnails, newest first.
|
||||
result = await session.execute(
|
||||
select(Photo)
|
||||
.where(Photo.processing_status.in_(['pending', 'failed']))
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
|
||||
logger.info(f"Regenerating thumbnails for {len(photos)} photos")
|
||||
|
||||
for photo in photos:
|
||||
generate_thumbnails.delay(photo.id)
|
||||
|
||||
return {'status': 'queued', 'count': len(photos)}
|
||||
|
||||
|
||||
# ── Perceptual hash backfill ────────────────────────────────────────────
|
||||
#
|
||||
# When phash was added post-launch, every existing photo has phash=NULL.
|
||||
# This task fills them in by reading the existing thumb_large (the cheap
|
||||
# option — pHash is robust to scale, and the thumb is already on local
|
||||
# disk so we avoid re-decoding the original RAW/HEIC). Falls back to the
|
||||
# original filepath if the thumb isn't available for some reason. Runs
|
||||
# in batches to keep memory bounded and to give the user incremental
|
||||
# progress visible in the worker logs.
|
||||
|
||||
@shared_task(name='backfill_phashes')
|
||||
def backfill_phashes():
|
||||
"""Compute and persist phash for every photo currently missing one."""
|
||||
return asyncio.run(_backfill_phashes_async())
|
||||
|
||||
|
||||
async def _backfill_phashes_async():
|
||||
import imagehash
|
||||
from PIL import Image as _PILImage
|
||||
|
||||
BATCH = 100
|
||||
total_done = 0
|
||||
total_failed = 0
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
while True:
|
||||
# Newest-first so the recent end of the library gets phashes
|
||||
# (and therefore duplicate detection) ahead of the archive.
|
||||
result = await session.execute(
|
||||
select(Photo)
|
||||
.where(Photo.phash.is_(None))
|
||||
.where(Photo.processing_status == 'completed')
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
.limit(BATCH)
|
||||
)
|
||||
batch = result.scalars().all()
|
||||
if not batch:
|
||||
break
|
||||
|
||||
for photo in batch:
|
||||
source = photo.thumb_large or photo.filepath
|
||||
try:
|
||||
if not source or not os.path.exists(source):
|
||||
photo.phash = None
|
||||
total_failed += 1
|
||||
continue
|
||||
with _PILImage.open(source) as im:
|
||||
photo.phash = str(imagehash.phash(im))
|
||||
total_done += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"phash backfill failed for {photo.id}: {e}")
|
||||
total_failed += 1
|
||||
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"Backfilled phashes: {total_done} done, {total_failed} failed"
|
||||
)
|
||||
|
||||
return {
|
||||
'status': 'success',
|
||||
'computed': total_done,
|
||||
'failed': total_failed,
|
||||
}
|
||||
|
||||
|
||||
@shared_task(
|
||||
name='regroup_duplicates',
|
||||
# Full regroup scales with O(N²) on phash plus one pgvector query per
|
||||
# embedded photo. On a 16k-photo library that's comfortably past the
|
||||
# default 5-minute soft limit — bump to 2h / 2h30m. (Passing None here
|
||||
# does NOT disable limits; Celery falls back to the worker default
|
||||
# of 300s/600s. An explicit number overrides.)
|
||||
soft_time_limit=7200,
|
||||
time_limit=9000,
|
||||
)
|
||||
def regroup_duplicates_task():
|
||||
"""Full recompute of duplicate groups (pHash + CLIP similarity).
|
||||
|
||||
Used by the Settings → Re-detect duplicates button."""
|
||||
from app.services.duplicates import regroup_duplicates
|
||||
return asyncio.run(regroup_duplicates())
|
||||
|
||||
|
||||
@shared_task(
|
||||
name='incremental_regroup_duplicates',
|
||||
# O(new × N); still cheaper than a full regroup but can easily exceed
|
||||
# the 5-minute default after a big batch import. Same caveat as
|
||||
# regroup_duplicates above — None would just re-inherit the worker
|
||||
# default, so we pass explicit values.
|
||||
soft_time_limit=3600,
|
||||
time_limit=4200,
|
||||
)
|
||||
def incremental_regroup_duplicates_task(since_iso: str | None = None):
|
||||
"""Incremental duplicate detection for newly added photos.
|
||||
|
||||
Compares only photos added after `since_iso` against the full library
|
||||
using CLIP vector similarity (O(new × log N) via HNSW) plus pHash.
|
||||
Default post-scan path — much faster than a full regroup."""
|
||||
from app.services.duplicates import incremental_regroup
|
||||
from datetime import datetime, timezone
|
||||
since = None
|
||||
if since_iso:
|
||||
since = datetime.fromisoformat(since_iso)
|
||||
return asyncio.run(incremental_regroup(since=since))
|
||||
@@ -1,62 +0,0 @@
|
||||
"""Background pre-transcode for video photos so /playback is a cache
|
||||
hit on first user click. Dispatched from scan_folder when a new video
|
||||
row is created, and from the backfill admin endpoint for the existing
|
||||
library."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from celery import shared_task
|
||||
|
||||
from app.services.video import (
|
||||
PLAYBACK_OK_EXTS,
|
||||
PLAYBACK_OK_VCODECS,
|
||||
cache_path_for,
|
||||
ffprobe_video_codec,
|
||||
transcode_to_h264_mp4,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task(
|
||||
name='pretranscode_video',
|
||||
# Override the 10-minute global task_time_limit. Long videos (30+ min
|
||||
# raw clips, the occasional .3gp from the 2016 archive) can legitimately
|
||||
# take half an hour to transcode on this CPU-only box.
|
||||
time_limit=1800,
|
||||
soft_time_limit=1740,
|
||||
)
|
||||
def pretranscode_video(photo_id: str, src_path: str):
|
||||
"""Idempotent: skip if cache exists and is newer than source, skip
|
||||
if source is already a passthrough-safe codec/container.
|
||||
|
||||
Reports the outcome as a status string so the admin backfill can
|
||||
summarise. The /playback endpoint also falls back to a sync
|
||||
transcode if the cache miss races a queued task."""
|
||||
if not os.path.exists(src_path):
|
||||
logger.debug("pretranscode skip — source missing: %s", src_path)
|
||||
return {'status': 'missing', 'photo_id': photo_id}
|
||||
|
||||
cache_path = cache_path_for(photo_id)
|
||||
if cache_path.exists():
|
||||
try:
|
||||
if os.path.getmtime(src_path) <= os.path.getmtime(cache_path):
|
||||
return {'status': 'cached', 'photo_id': photo_id}
|
||||
cache_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
ext = os.path.splitext(src_path)[1].lower()
|
||||
if ext in PLAYBACK_OK_EXTS:
|
||||
codec = ffprobe_video_codec(src_path)
|
||||
if codec in PLAYBACK_OK_VCODECS:
|
||||
return {'status': 'passthrough', 'photo_id': photo_id, 'codec': codec}
|
||||
|
||||
ok = transcode_to_h264_mp4(src_path, str(cache_path))
|
||||
return {
|
||||
'status': 'transcoded' if ok else 'failed',
|
||||
'photo_id': photo_id,
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
"""Post-init_db bootstrap: run or stamp Alembic migrations.
|
||||
|
||||
On a FRESH Postgres install, init_db's create_all has already built the
|
||||
full schema from the current models. Running `alembic upgrade head` would
|
||||
fail because the older migrations try ADD COLUMN on columns that already
|
||||
exist. So we detect the fresh-install case (alembic_version table is
|
||||
missing or empty) and `stamp head` instead.
|
||||
|
||||
On an EXISTING install, the alembic_version table has a revision and
|
||||
`upgrade head` applies only the new deltas.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from sqlalchemy import create_engine, text, inspect
|
||||
from app.config import settings
|
||||
|
||||
|
||||
def run():
|
||||
# Use a sync engine for this one-shot script.
|
||||
sync_url = settings.database_url.replace("+asyncpg", "").replace("+aiosqlite", "")
|
||||
engine = create_engine(sync_url)
|
||||
|
||||
with engine.connect() as conn:
|
||||
inspector = inspect(engine)
|
||||
tables = inspector.get_table_names()
|
||||
|
||||
if "alembic_version" not in tables:
|
||||
# Fresh install — create_all built everything. Stamp head.
|
||||
print("Fresh install detected — stamping alembic head")
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "stamp", "head"],
|
||||
check=True,
|
||||
)
|
||||
else:
|
||||
row = conn.execute(text("SELECT version_num FROM alembic_version")).first()
|
||||
if row is None:
|
||||
print("Empty alembic_version — stamping head")
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "stamp", "head"],
|
||||
check=True,
|
||||
)
|
||||
else:
|
||||
print(f"Existing install at revision {row[0]} — running alembic upgrade head")
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "upgrade", "head"],
|
||||
check=True,
|
||||
)
|
||||
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@@ -1,66 +0,0 @@
|
||||
# Core dependencies
|
||||
fastapi==0.109.0
|
||||
uvicorn[standard]==0.27.0
|
||||
python-multipart==0.0.6
|
||||
|
||||
# Database
|
||||
sqlalchemy[asyncio]==2.0.25
|
||||
aiosqlite==0.19.0 # SQLite escape hatch (docker-compose.sqlite.yml override)
|
||||
asyncpg==0.29.0 # async Postgres driver (default)
|
||||
psycopg2-binary==2.9.9 # sync Postgres driver, used by Alembic CLI
|
||||
alembic==1.13.1
|
||||
|
||||
# Redis and Celery
|
||||
redis==5.0.1
|
||||
celery==5.3.6
|
||||
flower==2.0.1
|
||||
|
||||
# Image processing
|
||||
# pyvips==2.2.1 # Optional - having compatibility issues, using Pillow as fallback
|
||||
rawpy==0.26.1 # RAW decoder (CR2/NEF/ARW/DNG/…). cp312 wheels
|
||||
# ship with libraw bundled; the older 0.19 pin
|
||||
# had numpy 2.x incompatibilities — 0.26 is fine
|
||||
# with our numpy 1.26. iPhone ProRAW-style DNGs
|
||||
# that aren't real RAW still fail here; thumbs.py
|
||||
# falls back to opening them as JPEG in that case.
|
||||
pillow==10.2.0
|
||||
pillow-heif==0.15.0
|
||||
imagehash==4.3.1 # perceptual hash for duplicate detection
|
||||
imageio==2.33.1
|
||||
imageio-ffmpeg==0.4.9
|
||||
|
||||
# Video processing
|
||||
ffmpeg-python==0.2.0
|
||||
|
||||
# Metadata extraction
|
||||
pyexiftool==0.5.6
|
||||
|
||||
# File watching
|
||||
watchfiles==0.21.0
|
||||
|
||||
# Transitive dep of rawpy 0.26.1, which requires numpy<2 (see comment above).
|
||||
numpy>=1.26.0,<2.0
|
||||
|
||||
# Utilities
|
||||
pyyaml==6.0.1
|
||||
pydantic==2.5.3
|
||||
pydantic-settings==2.1.0
|
||||
python-dotenv==1.0.0
|
||||
httpx[http2]==0.26.0
|
||||
aiofiles==23.2.1
|
||||
|
||||
# Security and authentication
|
||||
python-jose[cryptography]==3.3.0
|
||||
passlib[bcrypt]==1.7.4
|
||||
bcrypt==4.0.1
|
||||
# OIDC single sign-on (Authentik, etc.). Authlib drives the Auth Code +
|
||||
# PKCE flow; itsdangerous signs the short-lived Starlette session cookie
|
||||
# that holds the PKCE state during the IdP round-trip.
|
||||
authlib==1.3.1
|
||||
itsdangerous==2.1.2
|
||||
|
||||
# Development
|
||||
pytest==7.4.4
|
||||
pytest-asyncio==0.23.3
|
||||
black==23.12.1
|
||||
ruff==0.1.11
|
||||
@@ -1,140 +0,0 @@
|
||||
"""Backfill Photo.nextcloud_fileid for photos under Nextcloud-rooted paths.
|
||||
|
||||
The Phase-1 thumbnail proxy reads `Photo.nextcloud_fileid` to know which
|
||||
file to ask Nextcloud's /core/preview endpoint about. New photos pick it
|
||||
up at scan time; this script catches up the existing library.
|
||||
|
||||
Run inside the backend container, e.g.:
|
||||
|
||||
pct exec 120 -- docker exec mulita-backend python -m scripts.backfill_nextcloud_fileid
|
||||
|
||||
Idempotent: skips rows that already have nextcloud_fileid set, and any
|
||||
row whose path isn't under the Nextcloud bind mount. One PROPFIND per
|
||||
photo. At ~50ms each that's ~18 minutes for a 22k-row library — run
|
||||
during off-hours.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo
|
||||
from app.models.user import User
|
||||
from app.services.nextcloud_dav import fetch_fileid, is_nextcloud_path
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("backfill_nextcloud_fileid")
|
||||
|
||||
BATCH = 500
|
||||
|
||||
|
||||
async def _user_cache(session: AsyncSession) -> dict[str, User]:
|
||||
"""One SELECT per script run instead of per photo."""
|
||||
result = await session.execute(select(User))
|
||||
return {u.id: u for u in result.scalars().all()}
|
||||
|
||||
|
||||
async def run() -> None:
|
||||
async with AsyncSessionLocal() as session:
|
||||
users = await _user_cache(session)
|
||||
total = await session.scalar(
|
||||
select(func.count(Photo.id)).where(Photo.nextcloud_fileid.is_(None))
|
||||
)
|
||||
logger.info("photos with NULL nextcloud_fileid: %s", total)
|
||||
|
||||
done = 0
|
||||
skipped_no_user = 0
|
||||
skipped_not_nc = 0
|
||||
skipped_no_creds = 0
|
||||
filled = 0
|
||||
not_found = 0
|
||||
|
||||
# Keep selecting the next batch of NULL-fileid rows until the
|
||||
# set is empty. NO offset() — each batch's writes shrink the
|
||||
# `WHERE nextcloud_fileid IS NULL` set, so an offset would skip
|
||||
# over the rows that were just filled in by the previous batch.
|
||||
# Rows we couldn't resolve (skipped or not_found) stay in the
|
||||
# set; we track them in a "stuck ids" set so the loop terminates
|
||||
# instead of spinning on them forever.
|
||||
stuck: set[str] = set()
|
||||
while True:
|
||||
stmt = (
|
||||
select(Photo)
|
||||
.where(Photo.nextcloud_fileid.is_(None))
|
||||
.order_by(Photo.id)
|
||||
.limit(BATCH)
|
||||
)
|
||||
if stuck:
|
||||
stmt = stmt.where(Photo.id.notin_(stuck))
|
||||
result = await session.execute(stmt)
|
||||
rows = list(result.scalars().all())
|
||||
if not rows:
|
||||
break
|
||||
|
||||
batch_started_filled = filled
|
||||
for photo in rows:
|
||||
done += 1
|
||||
if not photo.user_id:
|
||||
skipped_no_user += 1
|
||||
stuck.add(photo.id)
|
||||
continue
|
||||
owner = users.get(photo.user_id)
|
||||
if owner is None:
|
||||
skipped_no_user += 1
|
||||
stuck.add(photo.id)
|
||||
continue
|
||||
if not photo.filepath or not is_nextcloud_path(photo.filepath):
|
||||
skipped_not_nc += 1
|
||||
stuck.add(photo.id)
|
||||
continue
|
||||
if not owner.nextcloud_app_password_enc:
|
||||
skipped_no_creds += 1
|
||||
stuck.add(photo.id)
|
||||
continue
|
||||
fid: Optional[int] = None
|
||||
try:
|
||||
fid = fetch_fileid(owner, photo.filepath)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"PROPFIND failed for photo %s (%s): %s",
|
||||
photo.id, photo.filepath, e,
|
||||
)
|
||||
if fid is None:
|
||||
not_found += 1
|
||||
stuck.add(photo.id)
|
||||
continue
|
||||
photo.nextcloud_fileid = fid
|
||||
filled += 1
|
||||
|
||||
await session.commit()
|
||||
# Safety: if a whole batch produced no new fills, every row
|
||||
# in it is already in `stuck` — break to avoid an infinite
|
||||
# loop on the same set.
|
||||
if filled == batch_started_filled and len(rows) < BATCH:
|
||||
break
|
||||
logger.info(
|
||||
"progress: scanned=%s filled=%s not_found=%s "
|
||||
"skipped(no_user=%s not_nc=%s no_creds=%s) of total=%s",
|
||||
done, filled, not_found,
|
||||
skipped_no_user, skipped_not_nc, skipped_no_creds,
|
||||
total,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"done: scanned=%s filled=%s not_found=%s "
|
||||
"skipped(no_user=%s not_nc=%s no_creds=%s)",
|
||||
done, filled, not_found,
|
||||
skipped_no_user, skipped_not_nc, skipped_no_creds,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run())
|
||||
@@ -1,115 +0,0 @@
|
||||
"""Bring the mule-image DB into 100% sync with Nextcloud + filesystem.
|
||||
|
||||
Multi-phase one-shot operation invoked via:
|
||||
|
||||
docker exec mulita-backend python scripts/full_refresh.py [--dry-run]
|
||||
|
||||
Phases:
|
||||
1. Data integrity (sync, ~1s): cleanup_data_integrity dedupes
|
||||
SourceRoots / Folders by normalized path and recomputes folder
|
||||
photo_count.
|
||||
2. Forward scan (async, minutes): walk every active SourceRoot on
|
||||
disk, create/update Photo rows for new files, resurrect any
|
||||
accidentally-discarded photos whose mtime advanced.
|
||||
3. Hard prune (sync, seconds): delete Photo + Folder rows for paths
|
||||
that no longer exist on disk under a *mounted* root. Skips
|
||||
unmounted roots — matches prune_missing_photos's existing
|
||||
refuse-when-empty behavior.
|
||||
4. Orphan thumbnail dirs (sync, seconds): remove
|
||||
/data/thumbs/{user_id}/{photo_id}/ for any photo_id that's no
|
||||
longer in the photos table.
|
||||
|
||||
Pass --dry-run to compute counts for phases 3+4 without making changes.
|
||||
Phases 1 and 2 always run for real — they're idempotent and additive.
|
||||
|
||||
Print a structured summary at the end. Exit non-zero on any phase
|
||||
error; partial completion still surfaces the counts gathered so far.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from app.services.cleanup import (
|
||||
cleanup_data_integrity,
|
||||
prune_missing_photos,
|
||||
prune_orphan_thumbnails,
|
||||
)
|
||||
from app.tasks.scan import _scan_folder_async
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.folders import SourceRoot
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("full_refresh")
|
||||
|
||||
|
||||
async def _scan_all_inline() -> int:
|
||||
"""Scan every active SourceRoot inline (not via celery). Returns the
|
||||
number of roots actually walked."""
|
||||
import os
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active.is_(True))
|
||||
)
|
||||
roots = result.scalars().all()
|
||||
|
||||
walked = 0
|
||||
for sr in roots:
|
||||
if not os.path.exists(sr.path):
|
||||
logger.warning("source root path missing, skipping: %s", sr.path)
|
||||
continue
|
||||
logger.info("scanning %s …", sr.path)
|
||||
await _scan_folder_async(sr.path, sr.id, task=None)
|
||||
walked += 1
|
||||
return walked
|
||||
|
||||
|
||||
async def main(dry_run: bool) -> dict:
|
||||
summary: dict = {"dry_run": dry_run}
|
||||
|
||||
logger.info("phase 1: cleanup_data_integrity")
|
||||
summary["phase1_cleanup"] = await cleanup_data_integrity()
|
||||
|
||||
logger.info("phase 2: scan_all_source_roots (inline)")
|
||||
summary["phase2_scan_roots_walked"] = await _scan_all_inline()
|
||||
|
||||
logger.info("phase 3: prune_missing_photos (dry_run=%s)", dry_run)
|
||||
summary["phase3_prune"] = await prune_missing_photos(dry_run=dry_run)
|
||||
|
||||
logger.info("phase 4: prune_orphan_thumbnails (dry_run=%s)", dry_run)
|
||||
summary["phase4_orphan_thumbs"] = await prune_orphan_thumbnails(
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def cli() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Phases 3+4 report counts without making changes",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
result = asyncio.run(main(dry_run=args.dry_run))
|
||||
except Exception:
|
||||
logger.exception("full_refresh failed")
|
||||
return 1
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(cli())
|
||||
@@ -1,173 +0,0 @@
|
||||
"""Register (or re-register) the four file-event webhooks against the
|
||||
Nextcloud webhook_listeners app, pointing them at mule's internal
|
||||
receiver.
|
||||
|
||||
Idempotent: deletes any existing webhooks whose URI matches the target
|
||||
mule URL before posting fresh ones. Run after a config change (target
|
||||
URL, secret) or after the NC stack is rebuilt fresh.
|
||||
|
||||
docker exec mulita-backend python -m scripts.register_nc_webhooks
|
||||
|
||||
Reads:
|
||||
- NEXTCLOUD_BASE_URL — already set for the DAV client
|
||||
- NEXTCLOUD_WEBHOOK_TARGET — http URL of mule's webhook endpoint
|
||||
(default: http://192.168.8.136:8001/api/v1/internal/nc-webhook)
|
||||
- NEXTCLOUD_WEBHOOK_SECRET — shared bearer secret; must match the
|
||||
backend env var of the same name
|
||||
|
||||
Registers as the first mule user with `is_admin=true` (or, failing
|
||||
that, the first user with NC credentials configured). The OCS endpoint
|
||||
itself only requires basic auth as that NC user.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.user import User
|
||||
from app.services.secrets import decrypt
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("register_nc_webhooks")
|
||||
|
||||
DEFAULT_TARGET = "http://192.168.8.136:8001/api/v1/internal/nc-webhook"
|
||||
|
||||
EVENTS = [
|
||||
"OCP\\Files\\Events\\Node\\NodeCreatedEvent",
|
||||
"OCP\\Files\\Events\\Node\\NodeWrittenEvent",
|
||||
"OCP\\Files\\Events\\Node\\NodeDeletedEvent",
|
||||
"OCP\\Files\\Events\\Node\\NodeRenamedEvent",
|
||||
]
|
||||
|
||||
|
||||
async def _pick_admin() -> User | None:
|
||||
"""Pick a mule user we can use to authenticate against NC's OCS
|
||||
API. Prefer admin role, fall back to any user with NC creds set."""
|
||||
async with AsyncSessionLocal() as s:
|
||||
# First try admins.
|
||||
r = await s.execute(
|
||||
select(User).where(
|
||||
User.role == "admin",
|
||||
User.nextcloud_app_password_enc.is_not(None),
|
||||
)
|
||||
)
|
||||
u = r.scalars().first()
|
||||
if u:
|
||||
return u
|
||||
# Fall back to any user with creds.
|
||||
r = await s.execute(
|
||||
select(User).where(User.nextcloud_app_password_enc.is_not(None))
|
||||
)
|
||||
return r.scalars().first()
|
||||
|
||||
|
||||
def _ocs(base: str, path: str) -> str:
|
||||
return f"{base.rstrip('/')}/ocs/v2.php/apps/webhook_listeners/api/v1{path}"
|
||||
|
||||
|
||||
def _ocs_headers() -> dict[str, str]:
|
||||
# OCS-APIRequest header is mandatory for OCS endpoints.
|
||||
return {
|
||||
"OCS-APIRequest": "true",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def _list_existing(c: httpx.Client, base: str) -> list[dict[str, Any]]:
|
||||
r = c.get(_ocs(base, "/webhooks"), headers=_ocs_headers())
|
||||
r.raise_for_status()
|
||||
body = r.json()
|
||||
return body.get("ocs", {}).get("data", []) or []
|
||||
|
||||
|
||||
def _delete(c: httpx.Client, base: str, webhook_id: str) -> None:
|
||||
r = c.delete(_ocs(base, f"/webhooks/{webhook_id}"), headers=_ocs_headers())
|
||||
if r.status_code not in (200, 204):
|
||||
logger.warning(
|
||||
"delete webhook %s returned %s: %s",
|
||||
webhook_id, r.status_code, r.text[:200],
|
||||
)
|
||||
|
||||
|
||||
def _register(
|
||||
c: httpx.Client,
|
||||
base: str,
|
||||
target: str,
|
||||
secret: str,
|
||||
event_class: str,
|
||||
) -> dict[str, Any]:
|
||||
body = {
|
||||
"uri": target,
|
||||
"httpMethod": "POST",
|
||||
"event": event_class,
|
||||
"authMethod": "header",
|
||||
"authData": {"Authorization": f"Bearer {secret}"},
|
||||
"headers": {"Content-Type": "application/json"},
|
||||
}
|
||||
r = c.post(_ocs(base, "/webhooks"), headers=_ocs_headers(), json=body)
|
||||
r.raise_for_status()
|
||||
return r.json().get("ocs", {}).get("data", {})
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
base = os.environ.get("NEXTCLOUD_BASE_URL", "").rstrip("/")
|
||||
target = os.environ.get("NEXTCLOUD_WEBHOOK_TARGET", DEFAULT_TARGET)
|
||||
secret = os.environ.get("NEXTCLOUD_WEBHOOK_SECRET", "")
|
||||
if not base:
|
||||
logger.error("NEXTCLOUD_BASE_URL is not set")
|
||||
return 2
|
||||
if not secret:
|
||||
logger.error("NEXTCLOUD_WEBHOOK_SECRET is not set")
|
||||
return 2
|
||||
|
||||
user = await _pick_admin()
|
||||
if user is None:
|
||||
logger.error(
|
||||
"no mule user has nextcloud_app_password_enc set; can't auth to OCS"
|
||||
)
|
||||
return 2
|
||||
|
||||
nc_user = user.nextcloud_username or user.username
|
||||
pw = decrypt(user.nextcloud_app_password_enc)
|
||||
if not nc_user or not pw:
|
||||
logger.error("user %s has incomplete NC credentials", user.username)
|
||||
return 2
|
||||
|
||||
auth = httpx.BasicAuth(nc_user, pw)
|
||||
with httpx.Client(timeout=30, auth=auth, follow_redirects=False) as c:
|
||||
existing = _list_existing(c, base)
|
||||
logger.info("found %d existing webhook(s)", len(existing))
|
||||
|
||||
# Delete any pointing at the same target URI — idempotent rerun.
|
||||
for w in existing:
|
||||
if w.get("uri") == target:
|
||||
logger.info(
|
||||
"removing existing webhook id=%s event=%s",
|
||||
w.get("id"), w.get("event"),
|
||||
)
|
||||
_delete(c, base, str(w["id"]))
|
||||
|
||||
# Register fresh.
|
||||
for event_class in EVENTS:
|
||||
data = _register(c, base, target, secret, event_class)
|
||||
logger.info(
|
||||
"registered: id=%s event=%s",
|
||||
data.get("id"), event_class,
|
||||
)
|
||||
|
||||
logger.info("done — %d webhooks registered against %s", len(EVENTS), target)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
24
docker-compose.gpu.yml
Normal file
24
docker-compose.gpu.yml
Normal file
@@ -0,0 +1,24 @@
|
||||
# Overlay for hosts with a VA-API-capable GPU passed through (Intel
|
||||
# QSV, AMD VCN/VCE, any VA-API driver). PhotoPrism's :latest image
|
||||
# ships VA-API-enabled ffmpeg; this file just wires the device + group
|
||||
# membership + encoder selection. Layered in by the deploy script on
|
||||
# hosts where /dev/dri/renderD128 exists.
|
||||
#
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
|
||||
|
||||
services:
|
||||
photoprism:
|
||||
devices:
|
||||
- /dev/dri/renderD128:/dev/dri/renderD128
|
||||
- /dev/dri/card0:/dev/dri/card0
|
||||
# Match host GIDs (render=992, video=44 on Debian). PhotoPrism's
|
||||
# container user (PP_UID:PP_GID, typically 33:10000) is not in
|
||||
# these groups by default; group_add grants access to the device
|
||||
# nodes without changing the primary user.
|
||||
group_add:
|
||||
- "992"
|
||||
- "44"
|
||||
environment:
|
||||
PHOTOPRISM_FFMPEG_ENCODER: ${PP_FFMPEG_ENCODER:-vaapi}
|
||||
PHOTOPRISM_FFMPEG_BITRATE: ${PP_FFMPEG_BITRATE:-32}
|
||||
@@ -1,176 +0,0 @@
|
||||
# PhotoPrism stack — replaces the legacy mule-image backend over the course of
|
||||
# milestones M0–M5 (see /home/dtoro/.claude/plans/i-want-you-to-twinkly-galaxy.md).
|
||||
#
|
||||
# This compose file is intentionally separate from docker-compose.yml so the
|
||||
# legacy mule-image stack keeps running side-by-side until M5 cutover, when
|
||||
# data is migrated and the legacy backend is retired.
|
||||
#
|
||||
# docker compose -f docker-compose.photoprism.yml up -d
|
||||
#
|
||||
# M0 scope: mariadb + photoprism only. Library mounted READ-ONLY so initial
|
||||
# indexing cannot mutate originals while we validate. Backwrite, sidecar
|
||||
# service, web frontend, and reverse proxy land in later milestones.
|
||||
|
||||
services:
|
||||
mariadb:
|
||||
# Fully-qualified for podman (which refuses short names by default).
|
||||
# Docker resolves the same digest.
|
||||
image: docker.io/library/mariadb:11
|
||||
container_name: pp-mariadb
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --innodb-buffer-pool-size=512M
|
||||
- --transaction-isolation=READ-COMMITTED
|
||||
- --character-set-server=utf8mb4
|
||||
- --collation-server=utf8mb4_unicode_ci
|
||||
- --max-connections=512
|
||||
- --innodb-rollback-on-timeout=OFF
|
||||
- --innodb-lock-wait-timeout=120
|
||||
environment:
|
||||
MARIADB_AUTO_UPGRADE: "1"
|
||||
MARIADB_INITDB_SKIP_TZINFO: "1"
|
||||
MARIADB_DATABASE: ${PP_DB_NAME:-photoprism}
|
||||
MARIADB_USER: ${PP_DB_USER:-photoprism}
|
||||
MARIADB_PASSWORD: ${PP_DB_PASSWORD:?set PP_DB_PASSWORD in .env.photoprism}
|
||||
MARIADB_ROOT_PASSWORD: ${PP_DB_ROOT_PASSWORD:?set PP_DB_ROOT_PASSWORD in .env.photoprism}
|
||||
# Loopback-only host port so the mule-sidecar (running as a host process
|
||||
# in M4) can reach `mule_sidecar.*` over TCP. Not exposed beyond
|
||||
# 127.0.0.1; the photoprism container still resolves mariadb by service
|
||||
# name on the photoprism-network bridge.
|
||||
ports:
|
||||
- "127.0.0.1:${PP_DB_PORT:-3306}:3306"
|
||||
volumes:
|
||||
- pp_mariadb_data:/var/lib/mysql
|
||||
# The init script creates the mule_sidecar database + user that the Go
|
||||
# sidecar service will use in M4. Idempotent; no-op on subsequent boots.
|
||||
# ":Z" is the SELinux private-relabel flag — needed on Fedora/RHEL hosts,
|
||||
# silently no-op on Debian/Ubuntu and macOS Docker Desktop.
|
||||
- ./mariadb/init:/docker-entrypoint-initdb.d:ro,Z
|
||||
healthcheck:
|
||||
test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 60s
|
||||
networks: [photoprism-network]
|
||||
|
||||
photoprism:
|
||||
image: docker.io/photoprism/photoprism:latest
|
||||
container_name: pp-app
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
mariadb:
|
||||
condition: service_healthy
|
||||
# PhotoPrism's container drops to a non-root user via PHOTOPRISM_UID /
|
||||
# PHOTOPRISM_GID. Match the host user that owns ${PHOTO_DIRS} so the
|
||||
# process can read originals (and later write sidecars).
|
||||
user: "${PP_UID:-1000}:${PP_GID:-1000}"
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
- apparmor:unconfined
|
||||
ports:
|
||||
- "${PP_PORT:-2342}:2342"
|
||||
environment:
|
||||
PHOTOPRISM_ADMIN_USER: ${PP_ADMIN_USER:-admin}
|
||||
PHOTOPRISM_ADMIN_PASSWORD: ${PP_ADMIN_PASSWORD:?set PP_ADMIN_PASSWORD in .env.photoprism}
|
||||
PHOTOPRISM_AUTH_MODE: ${PP_AUTH_MODE:-password}
|
||||
PHOTOPRISM_SITE_URL: ${PP_SITE_URL:-http://localhost:2342/}
|
||||
PHOTOPRISM_ORIGINALS_LIMIT: ${PP_ORIGINALS_LIMIT:-50000}
|
||||
PHOTOPRISM_HTTP_COMPRESSION: gzip
|
||||
PHOTOPRISM_LOG_LEVEL: ${PP_LOG_LEVEL:-info}
|
||||
# M0 safety: keep originals read-only. Flip to "false" in M2 when the
|
||||
# right-sidebar enables metadata edits and we want EXIF backwrite.
|
||||
PHOTOPRISM_READONLY: ${PP_READONLY:-true}
|
||||
PHOTOPRISM_EXPERIMENTAL: "false"
|
||||
PHOTOPRISM_DISABLE_CHOWN: "true"
|
||||
PHOTOPRISM_DISABLE_WEBDAV: ${PP_DISABLE_WEBDAV:-false}
|
||||
PHOTOPRISM_DISABLE_SETTINGS: "false"
|
||||
PHOTOPRISM_DISABLE_TLS: "true"
|
||||
PHOTOPRISM_DEFAULT_TLS: "false"
|
||||
# AI/vision pipeline back on — per plan we re-introduce TF labels + faces.
|
||||
PHOTOPRISM_TENSORFLOW_OFF: "false"
|
||||
PHOTOPRISM_DETECT_NSFW: "true"
|
||||
PHOTOPRISM_UPLOAD_NSFW: "true"
|
||||
# Database
|
||||
PHOTOPRISM_DATABASE_DRIVER: mysql
|
||||
PHOTOPRISM_DATABASE_SERVER: mariadb:3306
|
||||
PHOTOPRISM_DATABASE_NAME: ${PP_DB_NAME:-photoprism}
|
||||
PHOTOPRISM_DATABASE_USER: ${PP_DB_USER:-photoprism}
|
||||
PHOTOPRISM_DATABASE_PASSWORD: ${PP_DB_PASSWORD}
|
||||
# Sidecars next to originals — read by the migrator at M5.
|
||||
PHOTOPRISM_SIDECAR_PATH: ""
|
||||
PHOTOPRISM_SIDECAR_YAML: "true"
|
||||
# EXIF backwrite — disabled in M0 (READONLY blocks writes anyway).
|
||||
# Re-enable in M2 by overriding in .env.photoprism: PP_BACKUP_DATABASE=true.
|
||||
PHOTOPRISM_DISABLE_BACKUPS: "false"
|
||||
PHOTOPRISM_BACKUP_DATABASE: ${PP_BACKUP_DATABASE:-true}
|
||||
PHOTOPRISM_DISABLE_EXIFTOOL: "false"
|
||||
# OIDC — set in .env.photoprism when the IdP (Authentik) is wired up.
|
||||
# Empty values keep OIDC dormant; the username/password login still works.
|
||||
PHOTOPRISM_OIDC_PROVIDER_NAME: ${OIDC_PROVIDER_NAME:-}
|
||||
PHOTOPRISM_OIDC_ISSUER_URL: ${OIDC_ISSUER_URL:-}
|
||||
PHOTOPRISM_OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-}
|
||||
PHOTOPRISM_OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-}
|
||||
PHOTOPRISM_OIDC_REDIRECT_URI: ${OIDC_REDIRECT_URI:-}
|
||||
PHOTOPRISM_OIDC_SCOPES: ${OIDC_SCOPES:-openid profile email}
|
||||
PHOTOPRISM_OIDC_REGISTER: ${OIDC_REGISTER:-true}
|
||||
PHOTOPRISM_OIDC_ROLE: ${OIDC_ROLE:-user}
|
||||
working_dir: /photoprism
|
||||
volumes:
|
||||
# Existing photo library — mounted read-only in M0; flip to :rw in M2
|
||||
# when the right-sidebar starts saving edits. ",Z" relabels for SELinux
|
||||
# on Fedora/RHEL; silent no-op elsewhere.
|
||||
- "${PHOTO_DIRS:?set PHOTO_DIRS in .env.photoprism}:/photoprism/originals:${PP_ORIGINALS_MODE:-ro},Z"
|
||||
- "./pp/storage:/photoprism/storage:Z"
|
||||
- "./pp/import:/photoprism/import:Z"
|
||||
networks: [photoprism-network]
|
||||
|
||||
# mule-sidecar — Go + Gin + GORM service for endpoints PhotoPrism's API
|
||||
# does not expose (file rename, folder mutations, heap convert, duplicate
|
||||
# scan, per-photo marks). Same wire contract as the M3 Node prototype;
|
||||
# the SvelteKit dev server proxies /api/sidecar/* here.
|
||||
sidecar:
|
||||
build:
|
||||
context: ./sidecar
|
||||
container_name: pp-sidecar
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
mariadb:
|
||||
condition: service_healthy
|
||||
photoprism:
|
||||
condition: service_started
|
||||
# Match PhotoPrism's UID/GID so renames/folder mutations preserve the
|
||||
# ownership the indexer expects on the bind-mounted originals.
|
||||
user: "${PP_UID:-1000}:${PP_GID:-1000}"
|
||||
ports:
|
||||
# Loopback only — Vite (host) proxies /api/sidecar/* to this port.
|
||||
# Behind a reverse proxy in production; never published beyond the
|
||||
# host.
|
||||
- "127.0.0.1:${SIDECAR_PORT:-8000}:8000"
|
||||
environment:
|
||||
ORIGINALS_ROOT: /photoprism/originals
|
||||
PHOTOPRISM_BASE_URL: http://photoprism:2342
|
||||
# Bind on all interfaces inside the container so the host-side
|
||||
# 127.0.0.1:8000 port mapping can reach the listener. The Go
|
||||
# binary defaults to 127.0.0.1 for the host-mode dev loop.
|
||||
SIDECAR_LISTEN_ADDR: 0.0.0.0
|
||||
SIDECAR_PORT: "8000"
|
||||
SIDECAR_DB_HOST: mariadb
|
||||
SIDECAR_DB_PORT: "3306"
|
||||
SIDECAR_DB_USER: sidecar
|
||||
# Rotate before any non-local deployment. Provisioned by
|
||||
# mariadb/init/01-sidecar.sql on first boot of the mariadb volume.
|
||||
SIDECAR_DB_PASSWORD: ${SIDECAR_DB_PASSWORD:-replace-at-m4-bringup}
|
||||
SIDECAR_DB_NAME: mule_sidecar
|
||||
volumes:
|
||||
# Sidecar mutates originals (rename, folder mutations, heap
|
||||
# convert) — always rw regardless of PhotoPrism's mount mode.
|
||||
- "${PHOTO_DIRS:?set PHOTO_DIRS in .env.photoprism}:/photoprism/originals:rw,Z"
|
||||
networks: [photoprism-network]
|
||||
|
||||
networks:
|
||||
photoprism-network:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
pp_mariadb_data:
|
||||
@@ -1,9 +1,9 @@
|
||||
# Podman-rootless overlay for the PhotoPrism stack.
|
||||
#
|
||||
# Apply alongside the base compose file:
|
||||
# podman-compose --env-file .env.photoprism \
|
||||
# -f docker-compose.photoprism.yml \
|
||||
# -f docker-compose.photoprism.podman.yml \
|
||||
# podman-compose --env-file .env \
|
||||
# -f docker-compose.yml \
|
||||
# -f docker-compose.podman.yml \
|
||||
# up -d
|
||||
#
|
||||
# Adds the podman-specific bits that would break a vanilla docker compose run:
|
||||
@@ -1,50 +0,0 @@
|
||||
# SQLite escape hatch override.
|
||||
#
|
||||
# Usage (omit the `db` service from the up command):
|
||||
#
|
||||
# docker compose -f docker-compose.yml -f docker-compose.sqlite.yml \
|
||||
# up frontend backend worker redis
|
||||
#
|
||||
# This pins the backend and worker to the legacy SQLite database file at
|
||||
# /data/db/mulita.db (in the existing db_data volume), drops the dependency
|
||||
# on Postgres, and skips Alembic — the SQLite schema is still managed by
|
||||
# the inline ALTERs in app/database.py:init_db.
|
||||
#
|
||||
# Vision features that depend on pgvector (PR4 onward) will refuse to enable
|
||||
# in this mode; the search/embedding endpoints will return 503 with a clear
|
||||
# error pointing back at the default Postgres setup.
|
||||
|
||||
services:
|
||||
backend:
|
||||
command: sh -c "uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload"
|
||||
environment:
|
||||
- DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-*}
|
||||
- SECRET_KEY=${SECRET_KEY:-mulita-dev-secret-change-me}
|
||||
- ACCESS_TOKEN_EXPIRE_MINUTES=${ACCESS_TOKEN_EXPIRE_MINUTES:-60}
|
||||
- REFRESH_TOKEN_EXPIRE_DAYS=${REFRESH_TOKEN_EXPIRE_DAYS:-30}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
|
||||
worker:
|
||||
environment:
|
||||
- DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- CELERYD_CONCURRENCY=${CELERYD_CONCURRENCY:-4}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
backend:
|
||||
condition: service_started
|
||||
@@ -1,268 +1,209 @@
|
||||
# Compose stack for the PhotoPrism-backed photo app: mariadb + photoprism +
|
||||
# Go sidecar. The SvelteKit web/ frontend runs separately (Vite in dev,
|
||||
# static build in prod) and proxies /api/v1/* to photoprism and
|
||||
# /api/sidecar/* to the sidecar.
|
||||
#
|
||||
# podman-compose --env-file .env \
|
||||
# -f docker-compose.yml -f docker-compose.podman.yml up -d
|
||||
|
||||
services:
|
||||
frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
dockerfile: Dockerfile
|
||||
container_name: mulita-frontend
|
||||
ports:
|
||||
# Host port is configurable via FRONTEND_PORT in .env so multiple
|
||||
# instances / other services on the same host don't collide.
|
||||
- "${FRONTEND_PORT:-3000}:80"
|
||||
depends_on:
|
||||
- backend
|
||||
networks:
|
||||
- mulita-network
|
||||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||||
# docker DNS forwards the lookup to the host's resolver, which
|
||||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||||
# traefik exposure list, so TLS handshakes against it die with
|
||||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||||
extra_hosts:
|
||||
- "cloud.hubris.network:192.168.8.175"
|
||||
mariadb:
|
||||
# Fully-qualified for podman (which refuses short names by default).
|
||||
# Docker resolves the same digest.
|
||||
image: docker.io/library/mariadb:11
|
||||
container_name: pp-mariadb
|
||||
restart: unless-stopped
|
||||
|
||||
backend:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
container_name: mulita-backend
|
||||
ports:
|
||||
# Direct backend access on the host is rarely needed (the frontend
|
||||
# talks to it through the nginx /api proxy on the same network),
|
||||
# but it's exposed for debugging / curl. Override with BACKEND_PORT.
|
||||
- "${BACKEND_PORT:-8001}:8000"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
# The single host → container mount for your photo library. Set
|
||||
# PHOTO_DIRS in .env to your library root. Mounted :rw because file
|
||||
# operations (rename, move, empty discard pile) need to mutate the
|
||||
# filesystem; flip to :ro for a strict read-only library and the
|
||||
# write endpoints will return EROFS.
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
# Optional Nextcloud integration: mount the homecloud data dir so
|
||||
# users can register subfolders of their Nextcloud `files/` tree
|
||||
# as per-user SourceRoots. Reads use this path directly; mutations
|
||||
# (upload, delete, rename, move) dispatch via WebDAV against
|
||||
# NEXTCLOUD_BASE_URL so Nextcloud's oc_filecache stays in sync.
|
||||
# Leave NEXTCLOUD_USERS_HOST_PATH unset (or pointing at a no-op
|
||||
# path) to disable.
|
||||
- ${NEXTCLOUD_USERS_HOST_PATH:-./photos}:/nextcloud-users:rw
|
||||
- thumbs_data:/data/thumbs
|
||||
- proxies_data:/data/proxies
|
||||
- video_cache_data:/data/video-cache
|
||||
- db_data:/data/db # retained so the docker-compose.sqlite.yml override has somewhere to put mulita.db
|
||||
# Run Alembic migrations before starting uvicorn. On a fresh Postgres
|
||||
# the empty 0001 baseline is a no-op stamp; create_all in init_db then
|
||||
# builds the schema.
|
||||
# init_db creates all tables from models (idempotent create_all),
|
||||
# then Alembic runs migrations for existing installs. On fresh DBs
|
||||
# create_all already built the full schema, so bootstrap.py stamps
|
||||
# alembic head to skip redundant ALTER statements.
|
||||
command: sh -c "python -c 'import asyncio; from app.database import init_db; asyncio.run(init_db())' && python bootstrap.py && uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers 2 --proxy-headers"
|
||||
command:
|
||||
- --innodb-buffer-pool-size=512M
|
||||
- --transaction-isolation=READ-COMMITTED
|
||||
- --character-set-server=utf8mb4
|
||||
- --collation-server=utf8mb4_unicode_ci
|
||||
- --max-connections=512
|
||||
- --innodb-rollback-on-timeout=OFF
|
||||
- --innodb-lock-wait-timeout=120
|
||||
environment:
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=/photos
|
||||
- ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-*}
|
||||
- SECRET_KEY=${SECRET_KEY:-mulita-dev-secret-change-me}
|
||||
- ACCESS_TOKEN_EXPIRE_MINUTES=${ACCESS_TOKEN_EXPIRE_MINUTES:-60}
|
||||
- REFRESH_TOKEN_EXPIRE_DAYS=${REFRESH_TOKEN_EXPIRE_DAYS:-30}
|
||||
# Authentik / OIDC single sign-on. Leave OIDC_ENABLED=false to
|
||||
# hide the SSO button and stick with username/password. When
|
||||
# enabled, set OIDC_ISSUER to the Authentik provider URL (the one
|
||||
# that serves /.well-known/openid-configuration), and paste the
|
||||
# client id/secret from the Authentik application. OIDC_REDIRECT_URI
|
||||
# must match the one registered on the Authentik side exactly —
|
||||
# e.g. https://photovault.example.com/api/v1/auth/oidc/callback.
|
||||
- OIDC_ENABLED=${OIDC_ENABLED:-false}
|
||||
- OIDC_ISSUER=${OIDC_ISSUER:-}
|
||||
- OIDC_CLIENT_ID=${OIDC_CLIENT_ID:-}
|
||||
- OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET:-}
|
||||
- OIDC_REDIRECT_URI=${OIDC_REDIRECT_URI:-}
|
||||
- OIDC_SCOPES=${OIDC_SCOPES:-openid profile email}
|
||||
- OIDC_PROVIDER_LABEL=${OIDC_PROVIDER_LABEL:-Authentik}
|
||||
- OIDC_ALLOW_SIGNUP=${OIDC_ALLOW_SIGNUP:-true}
|
||||
- OIDC_ADMIN_GROUPS=${OIDC_ADMIN_GROUPS:-}
|
||||
- OIDC_LINK_BY_USERNAME=${OIDC_LINK_BY_USERNAME:-false}
|
||||
- SESSION_SECRET=${SESSION_SECRET:-}
|
||||
# Nextcloud integration. NEXTCLOUD_USERS_ROOT is the in-container
|
||||
# path that NEXTCLOUD_USERS_HOST_PATH binds to. NEXTCLOUD_BASE_URL
|
||||
# is the public-facing Nextcloud URL used for outgoing WebDAV
|
||||
# calls (must be reachable from the backend container; e.g.
|
||||
# https://cloud.example.com or http://nextcloud:80 if you put it
|
||||
# on the same docker network). Leave NEXTCLOUD_BASE_URL unset to
|
||||
# keep the integration off — the router endpoints stay registered
|
||||
# but mutating endpoints fail with a clear error.
|
||||
- NEXTCLOUD_USERS_ROOT=${NEXTCLOUD_USERS_ROOT:-/nextcloud-users}
|
||||
- NEXTCLOUD_BASE_URL=${NEXTCLOUD_BASE_URL:-}
|
||||
- NEXTCLOUD_WEBHOOK_SECRET=${NEXTCLOUD_WEBHOOK_SECRET:-}
|
||||
- SECRET_KEY=${SECRET_KEY:-mulita-dev-secret-change-me}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- mulita-network
|
||||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||||
# docker DNS forwards the lookup to the host's resolver, which
|
||||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||||
# traefik exposure list, so TLS handshakes against it die with
|
||||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||||
extra_hosts:
|
||||
- "cloud.hubris.network:192.168.8.175"
|
||||
restart: unless-stopped
|
||||
|
||||
# ── Celery workers ─────────────────────────────────────────────────────
|
||||
#
|
||||
# The ingestion pipeline is split across two worker services so CPU-heavy
|
||||
# vision tasks (embed / detect / OCR / faces / classify) cannot starve
|
||||
# the fast IO-bound tasks (scan / thumbnails / EXIF / phash / duplicates).
|
||||
#
|
||||
# worker-light listens on default,high,low — IO-bound, cheap
|
||||
# worker-vision listens on vision — CPU-bound, loads ONNX
|
||||
#
|
||||
# Both share the same image, photo volume, and model cache, so there's
|
||||
# no disk duplication and model weights are loaded lazily only by
|
||||
# worker-vision. Each service has its own concurrency knob; both
|
||||
# workers ship their heartbeat to the same Redis broker so the
|
||||
# Settings > Workers panel lists them side-by-side.
|
||||
#
|
||||
# Sizing defaults target a 6-core / 16 GB host:
|
||||
# CELERY_LIGHT_CONCURRENCY=2 (enough for parallel thumbnail + EXIF)
|
||||
# CELERY_VISION_CONCURRENCY=5 (5 × ~2GB ONNX = ~10GB RAM, 5/6 cores)
|
||||
# Raise these in .env and run `docker compose up -d worker-light worker-vision`
|
||||
# to scale. Keep light under ~4 and vision under your physical core
|
||||
# count; more just thrashes.
|
||||
worker-light:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
image: mule-image-worker
|
||||
container_name: mulita-worker-light
|
||||
command: sh -c "celery -A app.tasks.celery worker --beat --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_LIGHT_CONCURRENCY:-2} -Q default,high,low -n light@%h"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
- ${NEXTCLOUD_USERS_HOST_PATH:-./photos}:/nextcloud-users:rw
|
||||
- thumbs_data:/data/thumbs
|
||||
- proxies_data:/data/proxies
|
||||
- video_cache_data:/data/video-cache
|
||||
- db_data:/data/db
|
||||
environment:
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=/photos
|
||||
- NEXTCLOUD_USERS_ROOT=${NEXTCLOUD_USERS_ROOT:-/nextcloud-users}
|
||||
- NEXTCLOUD_BASE_URL=${NEXTCLOUD_BASE_URL:-}
|
||||
- NEXTCLOUD_WEBHOOK_SECRET=${NEXTCLOUD_WEBHOOK_SECRET:-}
|
||||
- SECRET_KEY=${SECRET_KEY:-mulita-dev-secret-change-me}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
# NullPool — see app/database.py for rationale.
|
||||
- MULITA_CELERY_WORKER=1
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
backend:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "celery -A app.tasks.celery inspect ping -d light@$$HOSTNAME 2>/dev/null | grep -q OK"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 120s
|
||||
networks:
|
||||
- mulita-network
|
||||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||||
# docker DNS forwards the lookup to the host's resolver, which
|
||||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||||
# traefik exposure list, so TLS handshakes against it die with
|
||||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||||
extra_hosts:
|
||||
- "cloud.hubris.network:192.168.8.175"
|
||||
restart: unless-stopped
|
||||
|
||||
# worker-watcher used to live here — it ran the long-lived
|
||||
# watchfiles-based `watch_folders` task plus celery `--beat`. Both
|
||||
# responsibilities moved on Phase 2:
|
||||
# * file events now come from NC's webhook_listeners → POST
|
||||
# /api/v1/internal/nc-webhook (see backend/app/routers/nc_webhook.py)
|
||||
# * `--beat` was folded into worker-light's command so the
|
||||
# periodic discard_missing_photos_beat job still fires.
|
||||
|
||||
db:
|
||||
image: postgres:16
|
||||
container_name: mulita-db
|
||||
environment:
|
||||
POSTGRES_USER: mulita
|
||||
POSTGRES_PASSWORD: mulita
|
||||
POSTGRES_DB: mulita
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- mulita-network
|
||||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||||
# docker DNS forwards the lookup to the host's resolver, which
|
||||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||||
# traefik exposure list, so TLS handshakes against it die with
|
||||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||||
extra_hosts:
|
||||
- "cloud.hubris.network:192.168.8.175"
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U mulita -d mulita"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: mulita-redis
|
||||
# Host port exposed only for local debugging; the backend / worker
|
||||
# reach Redis via the internal mulita-network on its container name.
|
||||
MARIADB_AUTO_UPGRADE: "1"
|
||||
MARIADB_INITDB_SKIP_TZINFO: "1"
|
||||
MARIADB_DATABASE: ${PP_DB_NAME:-photoprism}
|
||||
MARIADB_USER: ${PP_DB_USER:-photoprism}
|
||||
MARIADB_PASSWORD: ${PP_DB_PASSWORD:?set PP_DB_PASSWORD in .env}
|
||||
MARIADB_ROOT_PASSWORD: ${PP_DB_ROOT_PASSWORD:?set PP_DB_ROOT_PASSWORD in .env}
|
||||
# Loopback-only host port so the mule-sidecar (running as a host process
|
||||
# in M4) can reach `mule_sidecar.*` over TCP. Not exposed beyond
|
||||
# 127.0.0.1; the photoprism container still resolves mariadb by service
|
||||
# name on the photoprism-network bridge.
|
||||
ports:
|
||||
- "${REDIS_PORT:-6379}:6379"
|
||||
- "127.0.0.1:${PP_DB_PORT:-3306}:3306"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
networks:
|
||||
- mulita-network
|
||||
# Pin cloud.hubris.network to the LAN caddy IP. Without this, the
|
||||
# docker DNS forwards the lookup to the host's resolver, which
|
||||
# returns the public IONOS VPS IP — but cloud isn't in the VPS
|
||||
# traefik exposure list, so TLS handshakes against it die with
|
||||
# "unexpected eof while reading". Caddy on 192.168.8.175 holds the
|
||||
# cloud.hubris.network cert and proxies to the Nextcloud LXC.
|
||||
extra_hosts:
|
||||
- "cloud.hubris.network:192.168.8.175"
|
||||
restart: unless-stopped
|
||||
command: redis-server --appendonly yes
|
||||
- pp_mariadb_data:/var/lib/mysql
|
||||
# The init script creates the mule_sidecar database + user that the Go
|
||||
# sidecar service will use in M4. Idempotent; no-op on subsequent boots.
|
||||
# ":Z" is the SELinux private-relabel flag — needed on Fedora/RHEL hosts,
|
||||
# silently no-op on Debian/Ubuntu and macOS Docker Desktop.
|
||||
- ./mariadb/init:/docker-entrypoint-initdb.d:ro,Z
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
test: ["CMD", "/usr/bin/mariadb-admin", "ping", "-h", "127.0.0.1", "--silent"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
retries: 12
|
||||
start_period: 60s
|
||||
networks: [photoprism-network]
|
||||
|
||||
photoprism:
|
||||
image: docker.io/photoprism/photoprism:latest
|
||||
container_name: pp-app
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
mariadb:
|
||||
condition: service_healthy
|
||||
# PhotoPrism's container drops to a non-root user via PHOTOPRISM_UID /
|
||||
# PHOTOPRISM_GID. Match the host user that owns ${PHOTO_DIRS} so the
|
||||
# process can read originals (and later write sidecars).
|
||||
user: "${PP_UID:-1000}:${PP_GID:-1000}"
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
- apparmor:unconfined
|
||||
ports:
|
||||
# Loopback only — the SvelteKit web/ app (Vite dev or built bundle)
|
||||
# is the user-facing surface; PhotoPrism's own UI stays off the
|
||||
# public interface. Vite proxies /api/v1/* here, and the host-mode
|
||||
# sidecar reaches PHOTOPRISM_BASE_URL=http://localhost:2342. Admin
|
||||
# access to PP's UI is via SSH tunnel only.
|
||||
- "127.0.0.1:${PP_PORT:-2342}:2342"
|
||||
environment:
|
||||
PHOTOPRISM_ADMIN_USER: ${PP_ADMIN_USER:-admin}
|
||||
PHOTOPRISM_ADMIN_PASSWORD: ${PP_ADMIN_PASSWORD:?set PP_ADMIN_PASSWORD in .env}
|
||||
PHOTOPRISM_AUTH_MODE: ${PP_AUTH_MODE:-password}
|
||||
PHOTOPRISM_SITE_URL: ${PP_SITE_URL:-http://localhost:2342/}
|
||||
PHOTOPRISM_ORIGINALS_LIMIT: ${PP_ORIGINALS_LIMIT:-50000}
|
||||
PHOTOPRISM_HTTP_COMPRESSION: gzip
|
||||
PHOTOPRISM_LOG_LEVEL: ${PP_LOG_LEVEL:-info}
|
||||
# Indexer concurrency. Defaults to NumCPU/2 (= 3 on a 6-core LXC),
|
||||
# but each worker forks TF + ffmpeg + libvips so effective load is
|
||||
# much higher — a fresh index of 1.2k photos on M0 pushed the LXC
|
||||
# load to 50+ and starved sibling containers. Pin to a low value
|
||||
# for shared hosts; raise on dedicated machines.
|
||||
PHOTOPRISM_WORKERS: ${PP_WORKERS:-2}
|
||||
# podman-compose doesn't expand nested ${A:-${B:-…}}, so keep this
|
||||
# one-level. Override both PP_WORKERS and PP_INDEX_WORKERS if you
|
||||
# want them to differ.
|
||||
PHOTOPRISM_INDEX_WORKERS: ${PP_INDEX_WORKERS:-2}
|
||||
# M0 safety: keep originals read-only. Flip to "false" in M2 when the
|
||||
# right-sidebar enables metadata edits and we want EXIF backwrite.
|
||||
PHOTOPRISM_READONLY: ${PP_READONLY:-true}
|
||||
PHOTOPRISM_EXPERIMENTAL: "false"
|
||||
PHOTOPRISM_DISABLE_CHOWN: "true"
|
||||
PHOTOPRISM_DISABLE_WEBDAV: ${PP_DISABLE_WEBDAV:-false}
|
||||
PHOTOPRISM_DISABLE_SETTINGS: "false"
|
||||
PHOTOPRISM_DISABLE_TLS: "true"
|
||||
PHOTOPRISM_DEFAULT_TLS: "false"
|
||||
# AI/vision pipeline back on — per plan we re-introduce TF labels + faces.
|
||||
PHOTOPRISM_TENSORFLOW_OFF: "false"
|
||||
PHOTOPRISM_DETECT_NSFW: "true"
|
||||
PHOTOPRISM_UPLOAD_NSFW: "true"
|
||||
# Database
|
||||
PHOTOPRISM_DATABASE_DRIVER: mysql
|
||||
PHOTOPRISM_DATABASE_SERVER: mariadb:3306
|
||||
PHOTOPRISM_DATABASE_NAME: ${PP_DB_NAME:-photoprism}
|
||||
PHOTOPRISM_DATABASE_USER: ${PP_DB_USER:-photoprism}
|
||||
PHOTOPRISM_DATABASE_PASSWORD: ${PP_DB_PASSWORD}
|
||||
# Sidecars next to originals — read by the migrator at M5.
|
||||
PHOTOPRISM_SIDECAR_PATH: ""
|
||||
PHOTOPRISM_SIDECAR_YAML: "true"
|
||||
# EXIF backwrite — disabled in M0 (READONLY blocks writes anyway).
|
||||
# Override in .env: PP_BACKUP_DATABASE=true.
|
||||
PHOTOPRISM_DISABLE_BACKUPS: "false"
|
||||
PHOTOPRISM_BACKUP_DATABASE: ${PP_BACKUP_DATABASE:-true}
|
||||
PHOTOPRISM_DISABLE_EXIFTOOL: "false"
|
||||
# OIDC — set in .env when the IdP (Authentik) is wired up.
|
||||
# Empty values keep OIDC dormant; the username/password login still works.
|
||||
# PhotoPrism's CLI flags are --oidc-uri / --oidc-client / --oidc-secret
|
||||
# / --oidc-provider, so the env-var names it actually reads are
|
||||
# PHOTOPRISM_OIDC_URI / _CLIENT / _SECRET / _PROVIDER (NOT _ISSUER_URL
|
||||
# / _CLIENT_ID / _CLIENT_SECRET / _PROVIDER_NAME — those are silently
|
||||
# ignored, OIDC stays dormant, and `photoprism show config` reports
|
||||
# blank oidc-uri / oidc-client). PHOTOPRISM_OIDC_REDIRECT is a bool
|
||||
# (auto-redirect-from-/library/login), not a URL — PhotoPrism builds
|
||||
# the callback from PHOTOPRISM_SITE_URL.
|
||||
PHOTOPRISM_OIDC_PROVIDER: ${OIDC_PROVIDER_NAME:-${OIDC_PROVIDER:-}}
|
||||
PHOTOPRISM_OIDC_URI: ${OIDC_ISSUER_URL:-${OIDC_URI:-}}
|
||||
PHOTOPRISM_OIDC_CLIENT: ${OIDC_CLIENT_ID:-${OIDC_CLIENT:-}}
|
||||
PHOTOPRISM_OIDC_SECRET: ${OIDC_CLIENT_SECRET:-${OIDC_SECRET:-}}
|
||||
PHOTOPRISM_OIDC_SCOPES: ${OIDC_SCOPES:-openid profile email}
|
||||
PHOTOPRISM_OIDC_REGISTER: ${OIDC_REGISTER:-true}
|
||||
PHOTOPRISM_OIDC_ROLE: ${OIDC_ROLE:-user}
|
||||
PHOTOPRISM_OIDC_REDIRECT: ${OIDC_REDIRECT:-false}
|
||||
working_dir: /photoprism
|
||||
volumes:
|
||||
# Existing photo library — mounted read-only in M0; flip to :rw in M2
|
||||
# when the right-sidebar starts saving edits. ",Z" relabels for SELinux
|
||||
# on Fedora/RHEL; silent no-op elsewhere.
|
||||
- "${PHOTO_DIRS:?set PHOTO_DIRS in .env}:/photoprism/originals:${PP_ORIGINALS_MODE:-ro},Z"
|
||||
- "./pp/storage:/photoprism/storage:Z"
|
||||
- "./pp/import:/photoprism/import:Z"
|
||||
networks: [photoprism-network]
|
||||
|
||||
# mule-sidecar — Go + Gin + GORM service for endpoints PhotoPrism's API
|
||||
# does not expose (file rename, folder mutations, heap convert, duplicate
|
||||
# scan, per-photo marks). Same wire contract as the M3 Node prototype;
|
||||
# the SvelteKit dev server proxies /api/sidecar/* here.
|
||||
sidecar:
|
||||
build:
|
||||
context: ./sidecar
|
||||
container_name: pp-sidecar
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
mariadb:
|
||||
condition: service_healthy
|
||||
photoprism:
|
||||
condition: service_started
|
||||
# Match PhotoPrism's UID/GID so renames/folder mutations preserve the
|
||||
# ownership the indexer expects on the bind-mounted originals.
|
||||
user: "${PP_UID:-1000}:${PP_GID:-1000}"
|
||||
ports:
|
||||
# Loopback only — Vite (host) proxies /api/sidecar/* to this port.
|
||||
# Behind a reverse proxy in production; never published beyond the
|
||||
# host.
|
||||
- "127.0.0.1:${SIDECAR_PORT:-8000}:8000"
|
||||
environment:
|
||||
ORIGINALS_ROOT: /photoprism/originals
|
||||
PHOTOPRISM_BASE_URL: http://photoprism:2342
|
||||
# Bind on all interfaces inside the container so the host-side
|
||||
# 127.0.0.1:8000 port mapping can reach the listener. The Go
|
||||
# binary defaults to 127.0.0.1 for the host-mode dev loop.
|
||||
SIDECAR_LISTEN_ADDR: 0.0.0.0
|
||||
SIDECAR_PORT: "8000"
|
||||
SIDECAR_DB_HOST: mariadb
|
||||
SIDECAR_DB_PORT: "3306"
|
||||
SIDECAR_DB_USER: sidecar
|
||||
# Rotate before any non-local deployment. Provisioned by
|
||||
# mariadb/init/01-sidecar.sql on first boot of the mariadb volume.
|
||||
SIDECAR_DB_PASSWORD: ${SIDECAR_DB_PASSWORD:-replace-at-m4-bringup}
|
||||
SIDECAR_DB_NAME: mule_sidecar
|
||||
# Second DB connection for poking PhotoPrism's own schema (only
|
||||
# used by the user-basepath reconciler today). Stays inert if
|
||||
# PP_DB_PASSWORD is empty — the reconciler then silently no-ops.
|
||||
PP_DB_HOST: mariadb
|
||||
PP_DB_PORT: "3306"
|
||||
PP_DB_USER: ${PP_DB_USER:-photoprism}
|
||||
PP_DB_PASSWORD: ${PP_DB_PASSWORD:-}
|
||||
PP_DB_NAME: ${PP_DB_NAME:-photoprism}
|
||||
# Declarative username → originals-relative BasePath mapping.
|
||||
# Format: comma-separated `user:path` pairs. Sidecar applies it
|
||||
# to auth_users on boot and every 60s, and `mkdir -p`s each
|
||||
# target subdirectory so PhotoPrism's ACL filter has somewhere to
|
||||
# point. Leave empty to disable.
|
||||
# USER_BASEPATHS="test:test, alice:family/alice"
|
||||
USER_BASEPATHS: ${USER_BASEPATHS:-}
|
||||
volumes:
|
||||
# Sidecar mutates originals (rename, folder mutations, heap
|
||||
# convert) — always rw regardless of PhotoPrism's mount mode.
|
||||
- "${PHOTO_DIRS:?set PHOTO_DIRS in .env}:/photoprism/originals:rw,Z"
|
||||
networks: [photoprism-network]
|
||||
|
||||
networks:
|
||||
mulita-network:
|
||||
photoprism-network:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
thumbs_data:
|
||||
proxies_data:
|
||||
video_cache_data:
|
||||
db_data:
|
||||
redis_data:
|
||||
pg_data:
|
||||
pp_mariadb_data:
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
# Build stage
|
||||
FROM node:18-alpine as build
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files
|
||||
COPY package*.json ./
|
||||
|
||||
# Install dependencies
|
||||
RUN npm ci
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the application
|
||||
RUN npm run build
|
||||
|
||||
# Production stage
|
||||
FROM nginx:alpine
|
||||
|
||||
# Copy built assets from build stage
|
||||
COPY --from=build /app/dist /usr/share/nginx/html
|
||||
|
||||
# Copy nginx configuration
|
||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
|
||||
# Expose port
|
||||
EXPOSE 80
|
||||
|
||||
# Start nginx
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"$schema": "https://ui.shadcn.com/schema.json",
|
||||
"style": "default",
|
||||
"rsc": false,
|
||||
"tsx": true,
|
||||
"tailwind": {
|
||||
"config": "tailwind.config.js",
|
||||
"css": "src/index.css",
|
||||
"baseColor": "zinc",
|
||||
"cssVariables": false,
|
||||
"prefix": ""
|
||||
},
|
||||
"aliases": {
|
||||
"components": "@/components",
|
||||
"utils": "@/lib/utils",
|
||||
"ui": "@/components/ui",
|
||||
"hooks": "@/hooks",
|
||||
"lib": "@/lib"
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en" class="dark">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/png" href="/favicon.png" />
|
||||
<link rel="apple-touch-icon" href="/favicon.png" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="theme-color" content="#0f0f0f" />
|
||||
<title>Mulimago</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,54 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Enable gzip
|
||||
gzip on;
|
||||
gzip_vary on;
|
||||
gzip_min_length 1024;
|
||||
gzip_types text/plain text/css text/xml text/javascript application/javascript application/xml+rss application/json;
|
||||
|
||||
# API proxy
|
||||
location /api/ {
|
||||
proxy_pass http://backend:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket support for real-time updates
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
|
||||
# Thumbnail serving with X-Accel-Redirect
|
||||
location /internal_thumbs/ {
|
||||
internal;
|
||||
alias /data/thumbs/;
|
||||
}
|
||||
|
||||
# SPA routing - serve index.html for all routes
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# Never cache index.html (or any HTML). The asset filenames are
|
||||
# content-hashed by Vite, so a fresh index.html is the only thing
|
||||
# that tells the browser to fetch the new bundle. Without this the
|
||||
# browser happily serves a stale index.html → stale bundle hash →
|
||||
# users see the old build until they hard-reload.
|
||||
location = /index.html {
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache";
|
||||
expires 0;
|
||||
}
|
||||
|
||||
# Cache static assets (filenames are content-hashed, so 1y is safe)
|
||||
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
}
|
||||
6288
frontend/package-lock.json
generated
6288
frontend/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,73 +0,0 @@
|
||||
{
|
||||
"name": "mulita-frontend",
|
||||
"private": true,
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"preview": "vite preview",
|
||||
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@radix-ui/react-accordion": "^1.1.2",
|
||||
"@radix-ui/react-alert-dialog": "^1.0.5",
|
||||
"@radix-ui/react-checkbox": "^1.0.4",
|
||||
"@radix-ui/react-collapsible": "^1.1.12",
|
||||
"@radix-ui/react-context-menu": "^2.1.5",
|
||||
"@radix-ui/react-dialog": "^1.0.5",
|
||||
"@radix-ui/react-dropdown-menu": "^2.0.6",
|
||||
"@radix-ui/react-label": "^2.0.2",
|
||||
"@radix-ui/react-popover": "^1.0.7",
|
||||
"@radix-ui/react-radio-group": "^1.3.8",
|
||||
"@radix-ui/react-scroll-area": "^1.0.5",
|
||||
"@radix-ui/react-select": "^2.0.0",
|
||||
"@radix-ui/react-separator": "^1.0.3",
|
||||
"@radix-ui/react-slider": "^1.1.2",
|
||||
"@radix-ui/react-slot": "^1.2.4",
|
||||
"@radix-ui/react-switch": "^1.0.3",
|
||||
"@radix-ui/react-tabs": "^1.0.4",
|
||||
"@radix-ui/react-toast": "^1.1.5",
|
||||
"@radix-ui/react-toggle": "^1.1.10",
|
||||
"@radix-ui/react-toggle-group": "^1.1.11",
|
||||
"@radix-ui/react-tooltip": "^1.0.7",
|
||||
"@tanstack/react-query": "^5.17.0",
|
||||
"@tanstack/react-virtual": "^3.0.1",
|
||||
"axios": "^1.6.5",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.0",
|
||||
"cmdk": "^1.1.1",
|
||||
"date-fns": "^3.2.0",
|
||||
"framer-motion": "^10.18.0",
|
||||
"leaflet": "^1.9.4",
|
||||
"lucide-react": "^0.303.0",
|
||||
"react": "^18.2.0",
|
||||
"react-day-picker": "^8.10.1",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-hotkeys-hook": "^4.4.3",
|
||||
"react-intersection-observer": "^9.5.3",
|
||||
"react-leaflet": "^4.2.1",
|
||||
"react-leaflet-cluster": "^2.1.0",
|
||||
"sonner": "^2.0.7",
|
||||
"tailwind-merge": "^2.2.0",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"zustand": "^4.4.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tanstack/react-query-devtools": "^5.96.2",
|
||||
"@types/leaflet": "^1.9.8",
|
||||
"@types/react": "^18.2.46",
|
||||
"@types/react-dom": "^18.2.18",
|
||||
"@typescript-eslint/eslint-plugin": "^6.17.0",
|
||||
"@typescript-eslint/parser": "^6.17.0",
|
||||
"@vitejs/plugin-react": "^4.2.1",
|
||||
"autoprefixer": "^10.4.16",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-plugin-react-hooks": "^4.6.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.5",
|
||||
"postcss": "^8.4.33",
|
||||
"tailwindcss": "^3.4.0",
|
||||
"typescript": "^5.3.3",
|
||||
"vite": "^5.0.10"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 821 KiB |
@@ -1,209 +0,0 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { Timeline } from './components/timeline/Timeline'
|
||||
import { DuplicatesView } from './components/duplicates/DuplicatesView'
|
||||
import { MapView } from './components/map/MapView'
|
||||
import { MemoriesView } from './components/memories/MemoriesView'
|
||||
import { TagsView } from './components/tags/TagsView'
|
||||
import { ColorsView } from './components/colors/ColorsView'
|
||||
import { RatedView } from './components/rated/RatedView'
|
||||
import { LeftSidebar } from './components/layout/LeftSidebar'
|
||||
import { RightSidebar } from './components/layout/RightSidebar'
|
||||
import { TopBar } from './components/layout/TopBar'
|
||||
import { ScanProgress } from './components/ScanProgress'
|
||||
import { ToastContainer } from './components/ToastContainer'
|
||||
import { KeyboardHints } from './components/KeyboardHints'
|
||||
import { PreviewView } from './components/preview/PreviewView'
|
||||
import { FilterBar } from './components/filter/FilterBar'
|
||||
import { DiscardActionBar } from './components/discard/DiscardActionBar'
|
||||
import { SettingsPage } from './components/dialogs/SettingsDialog'
|
||||
import { usePhotoStore } from './store/photoStore'
|
||||
import { useFilterStore } from './store/filterStore'
|
||||
import { useKeyboardShortcuts } from './hooks/useKeyboardShortcuts'
|
||||
import { useFilterUrlSync } from './hooks/useFilterUrlSync'
|
||||
import { usePhotosQuery } from './hooks/usePhotosQuery'
|
||||
import { AuthProvider, useAuth } from './contexts/AuthContext'
|
||||
import { LoginPage } from './components/auth/LoginPage'
|
||||
import { SetupPage } from './components/auth/SetupPage'
|
||||
import { OidcCallback } from './components/auth/OidcCallback'
|
||||
import { TooltipProvider } from '@/components/ui/tooltip'
|
||||
|
||||
function MainApp() {
|
||||
const [leftSidebarOpen, setLeftSidebarOpen] = useState(true)
|
||||
const [rightSidebarOpen, setRightSidebarOpen] = useState(true)
|
||||
// Respect the user's manual collapse of the metadata panel. Once they
|
||||
// close it explicitly (via `i` hotkey or the sidebar toggle button),
|
||||
// selecting a new photo should NOT force it back open. Cleared when
|
||||
// they open it manually again.
|
||||
const rightCollapsedByUser = useRef(false)
|
||||
const viewMode = usePhotoStore((state) => state.viewMode)
|
||||
const activePhotoId = usePhotoStore((state) => state.activePhotoId)
|
||||
const currentSection = useFilterStore((s) => s.currentSection)
|
||||
|
||||
// Close the metadata panel when the user switches between sections so
|
||||
// it doesn't carry over a now-irrelevant selection. It re-opens once a
|
||||
// photo gains focus in the new section (effect below).
|
||||
const prevSectionRef = useRef(currentSection)
|
||||
useEffect(() => {
|
||||
if (prevSectionRef.current !== currentSection) {
|
||||
prevSectionRef.current = currentSection
|
||||
setRightSidebarOpen(false)
|
||||
}
|
||||
}, [currentSection])
|
||||
|
||||
useEffect(() => {
|
||||
if (!activePhotoId) {
|
||||
setRightSidebarOpen(false)
|
||||
return
|
||||
}
|
||||
// User explicitly collapsed the panel — don't undo that just because
|
||||
// they picked a different photo.
|
||||
if (rightCollapsedByUser.current) return
|
||||
setRightSidebarOpen(true)
|
||||
}, [activePhotoId])
|
||||
|
||||
const toggleRightSidebar = () => {
|
||||
setRightSidebarOpen((prev) => {
|
||||
const next = !prev
|
||||
rightCollapsedByUser.current = !next
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
// Bidirectional sync of filter store with URL query params.
|
||||
useFilterUrlSync()
|
||||
|
||||
// Subscribe to the same photos query the Timeline uses, so the keyboard
|
||||
// "open preview on first photo" path can read from the live cache regardless
|
||||
// of what filter key it's stored under.
|
||||
const { data: allPhotos } = usePhotosQuery()
|
||||
|
||||
// Set up global keyboard shortcuts. Prefer the Timeline's published
|
||||
// visible sequence (which respects per-month ordering) over the raw
|
||||
// backend list — otherwise "Space on a blank selection" would open
|
||||
// the globally first photo, which isn't what the user sees at the
|
||||
// top-left of the grid.
|
||||
useKeyboardShortcuts({
|
||||
onToggleLeftSidebar: () => setLeftSidebarOpen(!leftSidebarOpen),
|
||||
onToggleRightSidebar: toggleRightSidebar,
|
||||
getFirstPhotoId: () =>
|
||||
usePhotoStore.getState().visiblePhotoIds[0] ??
|
||||
allPhotos?.[0]?.id ??
|
||||
null,
|
||||
})
|
||||
|
||||
// Settings page is a full-page section — hide filter bar, right sidebar,
|
||||
// and keyboard hints when it's active.
|
||||
const isSettings = currentSection === 'settings'
|
||||
|
||||
// Right sidebar stays open by default and shows whatever's selected
|
||||
// (or an empty state if nothing is). User can still toggle it manually.
|
||||
const showRightSidebar = rightSidebarOpen && !isSettings
|
||||
|
||||
return (
|
||||
<TooltipProvider delayDuration={300}>
|
||||
<div className="flex flex-col h-screen bg-bg text-text">
|
||||
<TopBar />
|
||||
|
||||
<div className="flex flex-1 overflow-hidden">
|
||||
{/* Left Sidebar */}
|
||||
<div
|
||||
className={`transition-all duration-200 ${
|
||||
leftSidebarOpen ? 'w-60' : 'w-0'
|
||||
} overflow-hidden border-r border-border bg-surface`}
|
||||
>
|
||||
<LeftSidebar />
|
||||
</div>
|
||||
|
||||
{/* Main column — filter bar, discard bar, timeline. Lives to the
|
||||
* right of the left sidebar so the filter row doesn't bleed
|
||||
* across the sidebar. relative so the KeyboardHints overlay
|
||||
* centers against this column, not the viewport. */}
|
||||
<div className="relative flex min-w-0 flex-1 flex-col">
|
||||
{!isSettings && (
|
||||
<FilterBar
|
||||
leftSidebarOpen={leftSidebarOpen}
|
||||
rightSidebarOpen={showRightSidebar}
|
||||
onToggleLeftSidebar={() => setLeftSidebarOpen(!leftSidebarOpen)}
|
||||
onToggleRightSidebar={toggleRightSidebar}
|
||||
/>
|
||||
)}
|
||||
{!isSettings && <DiscardActionBar />}
|
||||
<div className="flex-1 overflow-auto">
|
||||
{currentSection === 'settings' ? (
|
||||
<SettingsPage />
|
||||
) : currentSection === 'map' ? (
|
||||
<MapView />
|
||||
) : currentSection === 'memories' ? (
|
||||
<MemoriesView />
|
||||
) : currentSection === 'duplicates' ? (
|
||||
<DuplicatesView />
|
||||
) : currentSection === 'tags' ? (
|
||||
<TagsView />
|
||||
) : currentSection === 'colors' ? (
|
||||
<ColorsView />
|
||||
) : currentSection === 'rated' ? (
|
||||
<RatedView />
|
||||
) : (
|
||||
<Timeline />
|
||||
)}
|
||||
</div>
|
||||
{!isSettings && viewMode !== 'preview' && <KeyboardHints />}
|
||||
</div>
|
||||
|
||||
{/* Right Sidebar */}
|
||||
<div
|
||||
className={`transition-all duration-200 ${
|
||||
showRightSidebar ? 'w-72' : 'w-0'
|
||||
} overflow-hidden border-l border-border bg-surface`}
|
||||
>
|
||||
<RightSidebar />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Scan Progress Indicator */}
|
||||
<ScanProgress />
|
||||
|
||||
{/* Toast Notifications */}
|
||||
<ToastContainer />
|
||||
|
||||
{/* Preview overlay — covers TopBar when active */}
|
||||
{viewMode === 'preview' && <PreviewView />}
|
||||
|
||||
</div>
|
||||
</TooltipProvider>
|
||||
)
|
||||
}
|
||||
|
||||
/** Auth-gated shell: shows setup, login, or the main app. */
|
||||
function App() {
|
||||
return (
|
||||
<AuthProvider>
|
||||
<AuthGate />
|
||||
</AuthProvider>
|
||||
)
|
||||
}
|
||||
|
||||
function AuthGate() {
|
||||
const { user, isLoading, needsSetup } = useAuth()
|
||||
|
||||
// OIDC callback lands on /auth/callback — handle it even while
|
||||
// isLoading, so the callback page can adopt tokens and transition
|
||||
// straight to MainApp without flashing the login screen.
|
||||
if (window.location.pathname.startsWith('/auth/callback')) {
|
||||
return <OidcCallback />
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex min-h-screen items-center justify-center bg-bg">
|
||||
<div className="text-text-muted">Loading…</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (needsSetup) return <SetupPage />
|
||||
if (!user) return <LoginPage />
|
||||
return <MainApp />
|
||||
}
|
||||
|
||||
export default App
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 646 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 374 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 821 KiB |
@@ -1,136 +0,0 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { useHotkeys } from 'react-hotkeys-hook'
|
||||
import { ChevronDown, ChevronUp } from 'lucide-react'
|
||||
import { usePhotoStore } from '../store/photoStore'
|
||||
import { useFilterStore } from '../store/filterStore'
|
||||
|
||||
const STORAGE_KEY = 'keyboard-hints-collapsed'
|
||||
|
||||
interface Hint {
|
||||
key: string
|
||||
action: string
|
||||
}
|
||||
|
||||
/** Build the hint list for the current context. Returns an empty array
|
||||
* when no shortcuts apply, which lets the caller hide the panel
|
||||
* entirely instead of rendering an empty pill. */
|
||||
function getHints(opts: {
|
||||
selectedCount: number
|
||||
currentSection: string
|
||||
viewMode: string
|
||||
}): Hint[] {
|
||||
const { selectedCount, currentSection, viewMode } = opts
|
||||
|
||||
// Preview mode: culling shortcuts apply to the photo on screen, plus
|
||||
// arrow nav between photos and Esc to close.
|
||||
if (viewMode === 'preview') {
|
||||
const preview: Hint[] = [
|
||||
{ key: '←→', action: 'Navigate' },
|
||||
{ key: '1-5', action: 'Rate' },
|
||||
{ key: 'S', action: 'Select → heap' },
|
||||
]
|
||||
if (currentSection === 'discarded') {
|
||||
preview.push({ key: 'U', action: 'Restore' })
|
||||
} else {
|
||||
preview.push({ key: 'X', action: 'Discard' })
|
||||
}
|
||||
preview.push(
|
||||
{ key: 'I', action: 'Info panel' },
|
||||
{ key: 'Space', action: 'Close' },
|
||||
{ key: 'Esc', action: 'Close' }
|
||||
)
|
||||
return preview
|
||||
}
|
||||
|
||||
if (selectedCount > 0) {
|
||||
const base: Hint[] = [
|
||||
{ key: '1-5', action: 'Rate' },
|
||||
{ key: 'S', action: 'Select → heap' },
|
||||
]
|
||||
if (currentSection === 'discarded') {
|
||||
base.push({ key: 'U', action: 'Restore' })
|
||||
} else {
|
||||
base.push({ key: 'X', action: 'Discard' })
|
||||
}
|
||||
base.push(
|
||||
{ key: 'Space', action: 'Preview' },
|
||||
{ key: 'I', action: 'Info panel' },
|
||||
{ key: 'Esc', action: 'Deselect' }
|
||||
)
|
||||
return base
|
||||
}
|
||||
|
||||
return [
|
||||
{ key: '↑↓←→', action: 'Navigate' },
|
||||
{ key: 'Space', action: 'Preview' },
|
||||
{ key: 'Tab', action: 'Library panel' },
|
||||
{ key: 'I', action: 'Info panel' },
|
||||
]
|
||||
}
|
||||
|
||||
export function KeyboardHints() {
|
||||
const selectedCount = usePhotoStore((s) => s.selectedPhotos.length)
|
||||
const viewMode = usePhotoStore((s) => s.viewMode)
|
||||
const currentSection = useFilterStore((s) => s.currentSection)
|
||||
|
||||
const [collapsed, setCollapsed] = useState(
|
||||
() => typeof window !== 'undefined' && localStorage.getItem(STORAGE_KEY) === '1'
|
||||
)
|
||||
useEffect(() => {
|
||||
localStorage.setItem(STORAGE_KEY, collapsed ? '1' : '0')
|
||||
}, [collapsed])
|
||||
|
||||
// `H` toggles the panel.
|
||||
useHotkeys('h', () => setCollapsed((c) => !c), { preventDefault: true })
|
||||
|
||||
const hints = getHints({ selectedCount, currentSection, viewMode })
|
||||
|
||||
// Nothing relevant to show — hide entirely.
|
||||
if (hints.length === 0) return null
|
||||
|
||||
return (
|
||||
<div className="pointer-events-none absolute bottom-0 left-1/2 z-30 -translate-x-1/2 pb-4">
|
||||
{collapsed ? (
|
||||
// Collapsed handle: a small pill peeking from the bottom so the
|
||||
// user can re-open the panel without remembering the shortcut.
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setCollapsed(false)}
|
||||
className="pointer-events-auto flex items-center gap-1.5 rounded-full border border-white/15 bg-black/80 px-3 py-1 text-[11px] text-white/80 shadow-xl backdrop-blur-md transition-colors hover:bg-black/90 hover:text-white"
|
||||
title="Show shortcuts (H)"
|
||||
>
|
||||
<ChevronUp className="h-3 w-3" />
|
||||
Shortcuts
|
||||
<kbd className="rounded bg-white/15 px-1 py-0.5 text-[10px] font-medium text-white">
|
||||
H
|
||||
</kbd>
|
||||
</button>
|
||||
) : (
|
||||
<div className="pointer-events-auto flex items-center gap-3 whitespace-nowrap rounded-full border border-white/15 bg-black/80 px-4 py-1.5 shadow-xl ring-1 ring-black/40 backdrop-blur-md">
|
||||
{hints.map((hint, i) => (
|
||||
<div key={i} className="flex items-center gap-1.5">
|
||||
<kbd className="rounded bg-white/15 px-1.5 py-0.5 text-[11px] font-medium text-white shadow-sm">
|
||||
{hint.key}
|
||||
</kbd>
|
||||
<span className="whitespace-nowrap text-xs text-white/85">
|
||||
{hint.action}
|
||||
</span>
|
||||
<span className="ml-1 text-white/30">•</span>
|
||||
</div>
|
||||
))}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setCollapsed(true)}
|
||||
className="-mr-1 flex items-center gap-1 rounded-full px-1.5 py-0.5 text-[11px] text-white/60 transition-colors hover:bg-white/10 hover:text-white"
|
||||
title="Hide shortcuts (H)"
|
||||
>
|
||||
<kbd className="rounded bg-white/15 px-1 py-0.5 text-[10px] font-medium text-white">
|
||||
H
|
||||
</kbd>
|
||||
<ChevronDown className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
import { useEffect, useRef } from 'react'
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { library, WorkerStatus } from '../services/api'
|
||||
|
||||
interface ScanStatus {
|
||||
is_scanning: boolean
|
||||
current_folder?: string
|
||||
processed_files: number
|
||||
total_files: number
|
||||
errors: string[]
|
||||
}
|
||||
|
||||
type Phase = 'idle' | 'scanning' | 'processing' | 'done'
|
||||
|
||||
/**
|
||||
* Headless background-activity orchestrator. Polls scan + worker status
|
||||
* and invalidates affected query caches when a scan/processing pass
|
||||
* completes. The visible status indicator now lives inline in the
|
||||
* LeftSidebar (small spinner next to the FOLDERS header / specific
|
||||
* folder rows) — see useScanActivity.
|
||||
*/
|
||||
export function ScanProgress() {
|
||||
const queryClient = useQueryClient()
|
||||
const wasScanningRef = useRef(false)
|
||||
const wasProcessingRef = useRef(false)
|
||||
|
||||
const { data: scanStatus } = useQuery<ScanStatus>({
|
||||
queryKey: ['scan-status'],
|
||||
queryFn: () => library.scanStatus(),
|
||||
refetchInterval: (query) =>
|
||||
query.state.data?.is_scanning ? 2000 : 30000,
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
const isScanning = scanStatus?.is_scanning ?? false
|
||||
|
||||
// Poll worker status to track vision queue activity.
|
||||
// Fast polling (3s) while processing, slow (30s) otherwise.
|
||||
const { data: workerStatus } = useQuery<WorkerStatus>({
|
||||
queryKey: ['worker-status-progress'],
|
||||
queryFn: () => library.maintenance.workerStatus(),
|
||||
refetchInterval: (query) => {
|
||||
const q = totalQueued(query.state.data)
|
||||
return q > 0 ? 3000 : 30000
|
||||
},
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
const totalActive = totalQueued(workerStatus)
|
||||
|
||||
const phase: Phase = isScanning
|
||||
? 'scanning'
|
||||
: totalActive > 0
|
||||
? 'processing'
|
||||
: 'idle'
|
||||
|
||||
useEffect(() => {
|
||||
if (phase === 'scanning') {
|
||||
wasScanningRef.current = true
|
||||
wasProcessingRef.current = false
|
||||
} else if (phase === 'processing') {
|
||||
wasProcessingRef.current = true
|
||||
if (wasScanningRef.current) {
|
||||
wasScanningRef.current = false
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders', 'tree'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['heaps'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['tags'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['library', 'stats'] })
|
||||
}
|
||||
} else if (phase === 'idle') {
|
||||
if (wasScanningRef.current) {
|
||||
wasScanningRef.current = false
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders', 'tree'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['heaps'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['tags'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['library', 'stats'] })
|
||||
}
|
||||
if (wasProcessingRef.current) {
|
||||
wasProcessingRef.current = false
|
||||
queryClient.invalidateQueries({ queryKey: ['tags'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['library', 'stats'] })
|
||||
}
|
||||
}
|
||||
}, [phase, queryClient])
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function totalQueued(ws: WorkerStatus | undefined): number {
|
||||
if (!ws) return 0
|
||||
const queued = Object.values(ws.queues ?? {}).reduce((a, b) => a + b, 0)
|
||||
const active = ws.workers?.reduce(
|
||||
(sum, w) => sum + (w.active ?? 0) + (w.reserved ?? 0), 0
|
||||
) ?? 0
|
||||
return queued + active
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user