Compare commits
160 Commits
b7aa2aed3d
...
feat/gpu-c
| Author | SHA1 | Date | |
|---|---|---|---|
| 6c96c22b33 | |||
| 70dc1b6bdf | |||
| e3d4f6d92e | |||
| 9fc650fb12 | |||
| 29f7ad7073 | |||
| c134afe023 | |||
| a54d90a2d9 | |||
| 97f51a05c4 | |||
| d1ddc48f81 | |||
| 55c870c155 | |||
| 981328faff | |||
| e1707c314d | |||
| 64c0da794d | |||
| 0f4e2e0b8f | |||
| ea1803ec2f | |||
| fc5f30fad1 | |||
| a7b8a60473 | |||
| 24dfa996b3 | |||
| e364e4128f | |||
| d2a76fa58c | |||
| 0d5f380948 | |||
| 680fa90cbe | |||
| 9bba097d91 | |||
| 155e9bb126 | |||
| c783f129cc | |||
| 24c449f475 | |||
| e36f1939c6 | |||
|
|
2a75896274 | ||
|
|
85847848c4 | ||
|
|
b0c8c06b2b | ||
|
|
986dab7334 | ||
| 829d7bed83 | |||
| d70244f17e | |||
| a38c3c6e9b | |||
|
|
8ac406ac1f | ||
| 70de4b65ec | |||
| 79a9ef49d4 | |||
| bd39d310ab | |||
|
|
f9f276a986 | ||
|
|
ca9f6e6bd2 | ||
| 9d955d6b94 | |||
| af96922e89 | |||
| ccbc1050de | |||
|
|
cfd85a1fe8 | ||
| aa63d4c11d | |||
| 79ec511482 | |||
|
|
505fef5dfc | ||
|
|
8083328f2d | ||
| e669e80a91 | |||
|
|
3d8e050af4 | ||
|
|
cce1d876c3 | ||
| 7e10f0b462 | |||
| 6b8c7abc20 | |||
|
|
9a3ad3e579 | ||
|
|
4abe6d758c | ||
| cb5bc120dc | |||
| a72619e3d1 | |||
| 84e433ff63 | |||
| d35de8a2a9 | |||
| d5e4f23c0f | |||
| 5153aeebec | |||
| 4362e475a7 | |||
| 032dce6c85 | |||
| 0766b47bb2 | |||
| 17df1ecd09 | |||
| 8c2526d982 | |||
|
|
423a73a8a6 | ||
|
|
a27267f7ad | ||
|
|
6915c30911 | ||
|
|
4e1af0f356 | ||
|
|
5e2823ae94 | ||
|
|
99edd7d395 | ||
|
|
ab3c55dd96 | ||
|
|
c30b387dc3 | ||
|
|
347f58b4f3 | ||
|
|
68bbe6f024 | ||
|
|
c4df92720b | ||
|
|
2a5270d399 | ||
|
|
1b6ff45726 | ||
|
|
09c12ea35b | ||
|
|
abe5c1ec6b | ||
|
|
ea08d7e3e8 | ||
|
|
356062ead3 | ||
|
|
7a1c6b618b | ||
|
|
5a67ed7e7b | ||
|
|
f4618ddf97 | ||
|
|
2a5759cc8d | ||
|
|
f27f3cb820 | ||
|
|
f4a03b63f4 | ||
|
|
94088253f8 | ||
|
|
f657e2c0ba | ||
|
|
362fbc6d83 | ||
|
|
d24c64e0a0 | ||
|
|
18dce33fa3 | ||
|
|
28738acb56 | ||
|
|
576b0c236d | ||
|
|
9e9b1ba224 | ||
|
|
11202a92e7 | ||
|
|
a0d275490b | ||
|
|
6311412fc0 | ||
|
|
611d445d92 | ||
|
|
f14ea69223 | ||
|
|
f290784bf3 | ||
|
|
f743733edd | ||
|
|
c69322a89d | ||
|
|
f63daf16a8 | ||
|
|
d580796dc8 | ||
|
|
7b153f0d28 | ||
|
|
76551d898b | ||
|
|
89f99d220a | ||
|
|
63dd39d172 | ||
|
|
09a00f7419 | ||
|
|
99d504842e | ||
|
|
eeeb16a0f1 | ||
|
|
1695eae226 | ||
|
|
172f869e15 | ||
|
|
1408ec3fa3 | ||
|
|
b1c3ee68dd | ||
|
|
4bb2c959a8 | ||
|
|
758fda619e | ||
|
|
0eee0cecde | ||
|
|
4b137989c6 | ||
|
|
f811aae641 | ||
|
|
e4127f1e04 | ||
|
|
4c7e981daf | ||
|
|
65f6c14487 | ||
|
|
bc0bb44c05 | ||
|
|
80dd9d0a8b | ||
| e8e1adcf37 | |||
| 319be20389 | |||
| 11343c17dc | |||
| 3c022cef68 | |||
| b6be24c357 | |||
| b518a293cd | |||
| b7f2eb7832 | |||
| 96624bf853 | |||
| b9916866b3 | |||
| 68d8a6d064 | |||
| e6ca78881f | |||
|
|
c5582ffc65 | ||
|
|
eac005109c | ||
|
|
45f1649979 | ||
|
|
66b3bc5e1f | ||
|
|
d72a218b46 | ||
| 744a7fa0c3 | |||
| 967cf23b82 | |||
| a073ee7fb9 | |||
| e65e798021 | |||
| 7efac4354e | |||
| 8529771122 | |||
| a6eb406052 | |||
| 574d71371f | |||
|
|
5c531f11da | ||
|
|
800ee447ad | ||
|
|
7c68e1400b | ||
|
|
ecd8bbe61d | ||
|
|
2adaaf18a1 | ||
|
|
edd569d095 | ||
|
|
f090a809a9 | ||
|
|
e974ffbfd2 |
22
.env
22
.env
@@ -1,22 +0,0 @@
|
||||
# Mulita / PhotoVault local environment.
|
||||
# See .env.example for the full list of knobs and their docs.
|
||||
|
||||
# REQUIRED — host path to your photo library.
|
||||
PHOTO_DIRS=/mnt/library/homecloud/admin/files/
|
||||
|
||||
# Ports — change if 3000 / 8001 collide with other services on the host.
|
||||
FRONTEND_PORT=3000
|
||||
BACKEND_PORT=8001
|
||||
REDIS_PORT=6379
|
||||
|
||||
# CORS — wildcard for local dev. Lock down for real deployments.
|
||||
ALLOWED_ORIGINS=*
|
||||
|
||||
# Logging + timezone.
|
||||
LOG_LEVEL=INFO
|
||||
TZ=UTC
|
||||
|
||||
# Celery worker pools — split worker-light (IO) and worker-vision (CPU).
|
||||
# Defaults target a 6-core / 16 GB host.
|
||||
CELERY_LIGHT_CONCURRENCY=2
|
||||
CELERY_VISION_CONCURRENCY=5
|
||||
165
.env.example
165
.env.example
@@ -1,113 +1,70 @@
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Mulita / PhotoVault — example environment file
|
||||
# Example environment file. Copy to `.env` and adjust.
|
||||
#
|
||||
# Copy this file to `.env` and adjust the values for your setup. Every key
|
||||
# below has a sensible default in docker-compose.yml, so you only need to
|
||||
# uncomment the ones you actually want to change.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# podman-compose --env-file .env \
|
||||
# -f docker-compose.yml -f docker-compose.podman.yml up -d
|
||||
|
||||
|
||||
# ── REQUIRED ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# Host path to your photo library. The compose file mounts this at /photos
|
||||
# inside the backend + worker containers. The backend creates a default
|
||||
# source root pointing at /photos on first boot, so once this is set the
|
||||
# library is scanned with zero further configuration.
|
||||
# Host path to your photo library. PhotoPrism reads this in place and
|
||||
# writes EXIF backwrites next to originals (when PP_ORIGINALS_MODE=rw).
|
||||
PHOTO_DIRS=/mnt/library/homecloud/admin/files/
|
||||
|
||||
# Bootstrap admin password. The first PhotoPrism boot creates an `admin`
|
||||
# account with this password. Rotate after first login from the UI.
|
||||
PP_ADMIN_PASSWORD=please-change-me
|
||||
|
||||
# MariaDB passwords. Generate with `openssl rand -hex 24`.
|
||||
PP_DB_PASSWORD=please-change-me
|
||||
PP_DB_ROOT_PASSWORD=please-change-me
|
||||
|
||||
|
||||
# ── OPTIONAL ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# Loopback host port for PhotoPrism's API (and UI, if you tunnel to it).
|
||||
# Vite proxies /api/v1/* here and the host-mode sidecar reaches it on
|
||||
# localhost. Not published on the public interface.
|
||||
PP_PORT=2342
|
||||
|
||||
# Site URL — used for share links, OIDC redirect URI, and reverse-proxy aware
|
||||
# URL generation. Set to the public hostname once the proxy is in front.
|
||||
PP_SITE_URL=http://localhost:2342/
|
||||
|
||||
# Auth mode — "password" for username/password (default), "public" for an
|
||||
# unauthenticated kiosk mode (don't use this on a multi-user library).
|
||||
PP_AUTH_MODE=password
|
||||
|
||||
# Library mount mode. "rw" allows rename / folder mutations / EXIF backwrite;
|
||||
# "ro" is safe-for-archives but disables those sidecar endpoints. Set in
|
||||
# lockstep with PP_READONLY below.
|
||||
PP_ORIGINALS_MODE=rw
|
||||
PP_READONLY=false
|
||||
|
||||
# UID/GID inside the PhotoPrism container. Set these to the host UID/GID that
|
||||
# owns ${PHOTO_DIRS}. `id -u` and `id -g`.
|
||||
PP_UID=1000
|
||||
PP_GID=1000
|
||||
|
||||
|
||||
# ── OIDC SSO (Authentik or equivalent) ───────────────────────────────────────
|
||||
# Leave blank to keep OIDC dormant. Fill in to enable the "Sign in with OIDC"
|
||||
# button on the login page; OIDC_REGISTER=true auto-creates accounts at role
|
||||
# `user` (override to `admin` to grant full access on first SSO login).
|
||||
#
|
||||
# Examples:
|
||||
# macOS / Linux: PHOTO_DIRS=/Users/you/Pictures
|
||||
# Network share: PHOTO_DIRS=/mnt/nas/photos
|
||||
# Windows (WSL): PHOTO_DIRS=/mnt/c/Users/you/Pictures
|
||||
PHOTO_DIRS=./photos
|
||||
# The compose file reads these and maps them to PhotoPrism's actual env-var
|
||||
# names (PHOTOPRISM_OIDC_URI / _CLIENT / _SECRET / _PROVIDER) — see the
|
||||
# comment in docker-compose.yml. The PhotoPrism callback URI is auto-derived
|
||||
# from PP_SITE_URL; do not set it manually.
|
||||
|
||||
# OIDC_PROVIDER_NAME=Authentik
|
||||
# OIDC_ISSUER_URL=https://auth.example.com/application/o/photoprism/
|
||||
# OIDC_CLIENT_ID=...
|
||||
# OIDC_CLIENT_SECRET=...
|
||||
# OIDC_SCOPES=openid profile email
|
||||
# OIDC_REGISTER=true
|
||||
# OIDC_ROLE=user
|
||||
|
||||
|
||||
# ── PORTS ────────────────────────────────────────────────────────────────────
|
||||
# ── LOGGING ──────────────────────────────────────────────────────────────────
|
||||
|
||||
# Host port the SPA is served on. Browse to http://<host>:<FRONTEND_PORT>/.
|
||||
FRONTEND_PORT=3000
|
||||
|
||||
# Host port for the backend API. Almost never needed directly — the frontend
|
||||
# nginx proxies /api/ to the backend over the internal compose network. Kept
|
||||
# exposed for debugging / curl.
|
||||
BACKEND_PORT=8001
|
||||
|
||||
# Redis host port. Internal services reach Redis on its container name; this
|
||||
# is just for local debugging.
|
||||
REDIS_PORT=6379
|
||||
|
||||
|
||||
# ── AUTH ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Secret key used to sign JWT tokens. Generate a strong random value for
|
||||
# production (e.g. `openssl rand -base64 32`). The default is a deterministic
|
||||
# placeholder acceptable only for local/homelab use.
|
||||
# SECRET_KEY=change-me-to-a-random-string
|
||||
|
||||
# How long access and refresh tokens stay valid. Access tokens are short-lived
|
||||
# and silently refreshed by the frontend; refresh tokens let a session survive
|
||||
# across browser restarts.
|
||||
# ACCESS_TOKEN_EXPIRE_MINUTES=60
|
||||
# REFRESH_TOKEN_EXPIRE_DAYS=30
|
||||
|
||||
|
||||
# ── CORS ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Comma-separated list of allowed origins for direct browser access to the
|
||||
# backend. Same-origin requests through the nginx / vite proxy never trip
|
||||
# CORS, so this only matters when something hits the backend port directly
|
||||
# from a different origin (e.g. another machine, dev tools, a reverse proxy
|
||||
# under a different hostname).
|
||||
#
|
||||
# Default "*" is permissive, fine for a single-user homelab. Lock it down in
|
||||
# real deployments:
|
||||
# ALLOWED_ORIGINS=https://photos.example.com
|
||||
# ALLOWED_ORIGINS=https://photos.example.com,http://192.168.1.10:3000
|
||||
ALLOWED_ORIGINS=*
|
||||
|
||||
|
||||
# ── LOGGING / TIMEZONE ───────────────────────────────────────────────────────
|
||||
|
||||
# Python log level for the backend and Celery worker. Bump to DEBUG when
|
||||
# chasing scan / thumbnail issues.
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# Container timezone. Affects the timestamps in logs and the "added at"
|
||||
# field on newly imported photos. Defaults to UTC.
|
||||
# TZ=Europe/Berlin
|
||||
# TZ=America/New_York
|
||||
TZ=UTC
|
||||
|
||||
|
||||
# ── WORKER CONCURRENCY ───────────────────────────────────────────────────────
|
||||
#
|
||||
# The ingestion pipeline runs on two Celery worker services with separate
|
||||
# concurrency knobs so heavy vision tasks can't starve cheap IO tasks:
|
||||
#
|
||||
# worker-light (default / high / low queues)
|
||||
# Runs: scan, thumbnails, EXIF, pHash, duplicate regrouping.
|
||||
# Mostly IO-bound — 2 prefork children keep a library streaming in.
|
||||
#
|
||||
# worker-vision (vision queue)
|
||||
# Runs: embeddings, object detection, OCR, face extraction, content
|
||||
# classification. Each prefork child loads ~2 GB of ONNX model weights,
|
||||
# so set this to roughly (physical_cores − 1) and watch RAM.
|
||||
#
|
||||
# Defaults target a ~6 core / 16 GB host. Raise these, then
|
||||
# docker compose up -d worker-light worker-vision
|
||||
# to pick them up. Lower for a Pi; go higher on a workstation.
|
||||
#
|
||||
# The old `CELERYD_CONCURRENCY=N` single-worker variable is no longer
|
||||
# read — delete it from your .env if it's set.
|
||||
CELERY_LIGHT_CONCURRENCY=2
|
||||
CELERY_VISION_CONCURRENCY=5
|
||||
|
||||
|
||||
# ── INTERNAL (rarely overridden) ─────────────────────────────────────────────
|
||||
|
||||
# These point at the in-compose Redis and the bind-mounted SQLite db. Override
|
||||
# only if you're running Mulita without docker-compose or against an external
|
||||
# Redis.
|
||||
# REDIS_URL=redis://redis:6379
|
||||
# CELERY_BROKER_URL=redis://redis:6379
|
||||
# CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
# DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
PP_LOG_LEVEL=info
|
||||
|
||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -34,6 +34,7 @@ dist-ssr/
|
||||
.DS_Store
|
||||
|
||||
# Environment
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
@@ -60,9 +61,18 @@ build/
|
||||
# Docker
|
||||
docker-compose.override.yml
|
||||
|
||||
# PhotoPrism state (sidecars, cache, thumbs, db backups) — regenerable.
|
||||
/pp/storage/
|
||||
/pp/import/
|
||||
|
||||
# Sidecar runtime state (per-user marks etc.) — generated, not seed data.
|
||||
/sidecar/data/
|
||||
|
||||
# Sidecar Go build output.
|
||||
/sidecar/mule-sidecar
|
||||
|
||||
# Photos (for development)
|
||||
/photos/
|
||||
|
||||
# Thumbnails
|
||||
/thumbs/
|
||||
/trash/backend/yolov8n.pt
|
||||
|
||||
330
README.md
330
README.md
@@ -1,230 +1,140 @@
|
||||
# Mulita - Self-Hosted Photo Management Application
|
||||
# mule-image
|
||||
|
||||
A self-hosted, Docker-deployed photo management application inspired by Lightroom's workflow. Mulita provides a fast, keyboard-driven interface to browse, organize, tag, and manage your photo library.
|
||||
|
||||
## Features
|
||||
|
||||
- **Photo Organization**: Browse photos in a timeline view with virtual scrolling for performance
|
||||
- **Thumbnail Generation**: Automatic thumbnail generation for all photo formats including RAW
|
||||
- **Metadata Extraction**: Full EXIF/XMP metadata extraction and GPS mapping
|
||||
- **Keyboard Shortcuts**: Lightroom-style keyboard navigation and actions
|
||||
- **File Support**: JPEG, PNG, RAW formats (CR2, CR3, NEF, ARW, etc.), HEIC/HEIF, and videos
|
||||
- **Heaps**: Temporary collections for organizing photos
|
||||
- **Tags & Ratings**: Organize with tags, star ratings, and color labels — each with a card-grid browse view that drills into a full Timeline detail
|
||||
- **Dark Mode**: Photography-optimized dark interface
|
||||
- **Vision Pipeline**: YOLO object detection, OCR text extraction, CLIP embeddings for semantic search, InsightFace face detection and clustering
|
||||
- **People View**: Browse identified people as cards, click to see all photos of a person
|
||||
- **Map View**: Browse GPS-tagged photos on an interactive Leaflet map
|
||||
- **Duplicate Detection**: Perceptual hash-based duplicate grouping with best-pick UI
|
||||
- **Semantic Search**: Natural-language photo search powered by CLIP embeddings
|
||||
|
||||
## Tech Stack
|
||||
|
||||
### Backend
|
||||
- Python 3.12 with FastAPI
|
||||
- PostgreSQL + pgvector with SQLAlchemy (async) and Alembic migrations
|
||||
- Celery + Redis for background tasks
|
||||
- pyvips for fast thumbnail generation
|
||||
- ExifTool for metadata extraction
|
||||
- ONNX Runtime for vision models (YOLO, CLIP, InsightFace)
|
||||
|
||||
### Frontend
|
||||
- React 18 with TypeScript
|
||||
- Vite for fast development
|
||||
- TanStack Query for data fetching
|
||||
- TanStack Virtual for virtualized scrolling
|
||||
- Tailwind CSS for styling
|
||||
- Zustand for state management
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
- Docker and Docker Compose
|
||||
|
||||
### Setup (one variable)
|
||||
|
||||
1. Clone the repo:
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd muleimage
|
||||
```
|
||||
|
||||
2. Copy the example env file and set **one** variable — the **host**
|
||||
directory that contains your photo library. Whatever you point at
|
||||
will become your library inside Mulita.
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# then edit .env and set PHOTO_DIRS:
|
||||
# macOS / Linux: PHOTO_DIRS=/Users/you/Pictures
|
||||
# Network share: PHOTO_DIRS=/mnt/nas/photos
|
||||
# Windows (WSL): PHOTO_DIRS=/mnt/c/Users/you/Pictures
|
||||
```
|
||||
|
||||
3. Start the stack:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
4. Open `http://localhost:3000`. On first boot Mulita will:
|
||||
- Mount your `PHOTO_DIRS` at `/photos` inside the container
|
||||
- Auto-create a source root called **Library** pointing at `/photos`
|
||||
- Queue an initial scan, generate thumbnails, and start serving them
|
||||
|
||||
You don't need to touch `mulita.yml` or the API to get started.
|
||||
|
||||
### Configuration knobs
|
||||
|
||||
Everything is environment-driven. `PHOTO_DIRS` is the only required
|
||||
value; the rest have sensible defaults documented in `.env.example`:
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|----------------------|---------|----------------------------------------------------|
|
||||
| `PHOTO_DIRS` | — | **Required.** Host path mounted at `/photos`. |
|
||||
| `FRONTEND_PORT` | `3000` | SPA host port. Bump if `3000` is taken. |
|
||||
| `BACKEND_PORT` | `8001` | Direct backend port (debug only — frontend uses internal nginx proxy). |
|
||||
| `REDIS_PORT` | `6379` | Redis host port (internal services don't need it). |
|
||||
| `ALLOWED_ORIGINS` | `*` | Comma-separated CORS origins for direct backend access. Lock down for prod, e.g. `https://photos.example.com`. |
|
||||
| `LOG_LEVEL` | `INFO` | Backend + worker log level. `DEBUG` for chasing scan issues. |
|
||||
| `TZ` | `UTC` | Container timezone. Affects log timestamps and "added at". |
|
||||
| `CELERYD_CONCURRENCY`| `4` | Parallel worker processes (scans, thumbs, metadata). Lower on a Pi, higher on a beefy host. |
|
||||
|
||||
### Accessing from another machine
|
||||
|
||||
The frontend talks to the backend through its bundled nginx, which
|
||||
proxies `/api/` to the backend on the internal compose network. That
|
||||
means requests are always **same-origin** as the page, so accessing
|
||||
Mulita from another host works without any CORS dance:
|
||||
|
||||
```
|
||||
http://<your-server-ip>:3000
|
||||
```
|
||||
|
||||
If you want to put it behind a reverse proxy at e.g.
|
||||
`https://photos.your.tld`, set `ALLOWED_ORIGINS` to that host so the
|
||||
backend's direct port (`BACKEND_PORT`) also accepts cross-origin
|
||||
requests if anything bypasses the proxy.
|
||||
|
||||
### How libraries are managed
|
||||
|
||||
Mulita is **config-driven**: the host directory you mount via
|
||||
`PHOTO_DIRS` becomes your library, and the backend automatically
|
||||
registers it as a source root on startup. There is no UI for adding
|
||||
or removing source roots — to change what Mulita scans, edit `.env`
|
||||
(or `docker-compose.yml` for multi-mount setups) and restart the
|
||||
stack.
|
||||
|
||||
This keeps the model simple: **the docker mount IS the library**.
|
||||
No two layers, no confusion about which view to use.
|
||||
|
||||
### Changing or adding libraries
|
||||
|
||||
To point at a different library:
|
||||
1. Edit `PHOTO_DIRS` in `.env`
|
||||
2. `docker compose down`
|
||||
3. (Optional, for a clean slate) `docker volume rm muleimage_db_data muleimage_thumbs_data muleimage_proxies_data`
|
||||
4. `docker compose up -d`
|
||||
|
||||
The new library shows up automatically. Without step 3 the old
|
||||
library's metadata stays in the DB and you'll see a warning at
|
||||
startup that the old source root's path is missing on disk —
|
||||
that's a hint to clean up.
|
||||
|
||||
For multiple libraries, edit `docker-compose.yml` and add additional
|
||||
mount lines:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- ${PHOTO_DIRS}:/photos:rw
|
||||
- /Volumes/Archive:/archive:rw # additional library
|
||||
```
|
||||
|
||||
Each mounted directory will need a corresponding source root row in
|
||||
the DB; today that means `POST /api/v1/folders` via curl, or wait
|
||||
for the multi-mount auto-registration that's on the roadmap.
|
||||
|
||||
### Read-only libraries
|
||||
|
||||
The default mount is `:rw` because file operations (rename, move,
|
||||
empty discard pile) need to mutate the filesystem. If you want a
|
||||
strict read-only library — pointing at a network share, an
|
||||
authoritative archive, etc. — flip `:rw` to `:ro` in
|
||||
`docker-compose.yml`. Mulita will keep working for browsing, rating,
|
||||
color labels, picks, heaps, and the (soft) discard flag, but the
|
||||
following will return an OS error:
|
||||
|
||||
- `PATCH /photos/{id}` with a new `filename` (rename)
|
||||
- `POST /photos/move` (bulk move)
|
||||
- `DELETE /discard/empty` (file unlinks)
|
||||
|
||||
**Heads up**: with `:rw`, Mulita has full write access to whatever
|
||||
host directory you mount. Treat the same way you would Lightroom's
|
||||
catalog folder.
|
||||
Self-hosted photo management built on top of [PhotoPrism][pp]. A SvelteKit
|
||||
frontend ([`web/`](web/)) plus a small Go service ([`sidecar/`](sidecar/))
|
||||
fill in the keyboard-driven UI and the file/folder/mark endpoints
|
||||
PhotoPrism's REST API does not expose. PhotoPrism itself handles
|
||||
indexing, originals, thumbnails, and the database; we never re-implement
|
||||
those.
|
||||
|
||||
## Architecture
|
||||
|
||||
The application consists of 5 Docker services:
|
||||
|
||||
- **frontend**: React SPA served by Nginx
|
||||
- **backend**: FastAPI REST API
|
||||
- **worker**: Celery workers for background tasks (thumbnails, metadata, vision pipeline)
|
||||
- **redis**: Message broker for Celery
|
||||
- **db**: PostgreSQL with pgvector extension (for CLIP/face embeddings)
|
||||
|
||||
## Keyboard Shortcuts
|
||||
|
||||
| Key | Action |
|
||||
|-----|--------|
|
||||
| `←` `→` `↑` `↓` | Navigate photos |
|
||||
| `Space` | Quick preview |
|
||||
| `Enter` | Open loupe view |
|
||||
| `T` | Add to active heap |
|
||||
| `1-5` | Set star rating |
|
||||
| `Tab` | Toggle left sidebar |
|
||||
| `I` | Toggle metadata panel |
|
||||
| `G` | Grid view |
|
||||
| `E` | Loupe view |
|
||||
| `Delete` | Move to trash |
|
||||
|
||||
## Development
|
||||
|
||||
### Backend Development
|
||||
```bash
|
||||
cd backend
|
||||
pip install -r requirements.txt
|
||||
uvicorn app.main:app --reload
|
||||
```text
|
||||
┌──────────────────┐ /api/v1/* ┌──────────────┐
|
||||
│ SvelteKit web/ │ ───────────────▶ │ photoprism │ ──▶ mariadb
|
||||
│ (Vite : 5173) │ /api/sidecar/* │ :2342 │
|
||||
│ │ ─────────┐ └──────────────┘
|
||||
└──────────────────┘ ▼
|
||||
┌──────────────┐
|
||||
│ sidecar │ ──▶ mariadb (mule_sidecar.*)
|
||||
│ :8000 │ ──▶ originals FS (rename / folders / dups)
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
### Frontend Development
|
||||
Three compose services — `mariadb`, `photoprism`, `sidecar` — plus the
|
||||
SvelteKit `web/` app served separately. PhotoPrism's port `2342` is
|
||||
**bound to `127.0.0.1` only**; it isn't a user-facing surface. The
|
||||
SvelteKit app is.
|
||||
|
||||
What the sidecar adds on top of PhotoPrism (full list in
|
||||
[`sidecar/README.md`](sidecar/README.md)):
|
||||
|
||||
- Per-photo marks (rating + color) persisted to `mule_sidecar.marks`
|
||||
- File rename + folder create/rename/delete with PhotoPrism reindex
|
||||
- Heap (album) → folder conversion
|
||||
- Perceptual-hash duplicate scan + archive
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
cp .env.example .env
|
||||
# edit .env: set PHOTO_DIRS to the host path holding your library
|
||||
# rotate PP_ADMIN_PASSWORD, PP_DB_PASSWORD, PP_DB_ROOT_PASSWORD
|
||||
# before any non-local deployment.
|
||||
|
||||
podman-compose --env-file .env \
|
||||
-f docker-compose.yml \
|
||||
-f docker-compose.podman.yml \
|
||||
up -d
|
||||
```
|
||||
|
||||
Then serve the frontend. For local use the simplest path is the Vite
|
||||
dev server:
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npm install
|
||||
npm run dev
|
||||
# open http://localhost:5173
|
||||
```
|
||||
|
||||
For a static deployment, `npm run build` produces a bundle under
|
||||
`web/build/` that any static file host (nginx, Caddy, GitHub Pages-style)
|
||||
can serve. Reverse-proxy `/api/v1/*` to `http://127.0.0.1:2342` and
|
||||
`/api/sidecar/*` to `http://127.0.0.1:8000`.
|
||||
|
||||
PhotoPrism's own UI is still reachable from the host at
|
||||
`http://127.0.0.1:2342` if you need admin features (user management,
|
||||
settings) — set up an SSH tunnel from your laptop if the server is
|
||||
remote.
|
||||
|
||||
## Configuration
|
||||
|
||||
Source roots are managed by the UI / API (the database owns them). Edit
|
||||
`mulita.yml` to configure operational settings only:
|
||||
All knobs live in [`.env.example`](.env.example). The required ones:
|
||||
|
||||
- Thumbnail sizes, quality, and format
|
||||
- Scanner behaviour (watch, batch size, initial scan)
|
||||
- Performance tuning (concurrency, cache TTLs, DB pool)
|
||||
| Variable | Notes |
|
||||
|----------------------|-----------------------------------------------------------------------------------------------|
|
||||
| `PHOTO_DIRS` | Host path mounted at `/photoprism/originals`. The library. |
|
||||
| `PP_ADMIN_PASSWORD` | First-boot admin password. Rotate. |
|
||||
| `PP_DB_PASSWORD` | MariaDB password for the `photoprism` user. Rotate. |
|
||||
| `PP_DB_ROOT_PASSWORD`| MariaDB root password. Rotate. |
|
||||
| `PP_UID` / `PP_GID` | Host UID/GID that owns `PHOTO_DIRS`. PhotoPrism + sidecar drop to this user inside. |
|
||||
| `PP_PORT` | Loopback host port for PhotoPrism (default `2342`). |
|
||||
| `PP_ORIGINALS_MODE` | `rw` (default) or `ro` — see [Read-only libraries](#read-only-libraries). |
|
||||
| `SIDECAR_PORT` | Loopback host port for the sidecar (default `8000`). |
|
||||
|
||||
## Performance
|
||||
Sidecar-specific env (DB DSN, `USER_BASEPATHS`, etc.) is documented in
|
||||
[`sidecar/README.md`](sidecar/README.md).
|
||||
|
||||
- Handles 100,000+ photos efficiently
|
||||
- Virtual scrolling for smooth timeline navigation
|
||||
- Thumbnail generation at 10+ photos/second
|
||||
- PostgreSQL full-text search with tsvector indexing
|
||||
- pgvector for fast nearest-neighbor embedding search
|
||||
## Read-only libraries
|
||||
|
||||
## Future Features
|
||||
The default originals mount is `:rw` because file operations (rename,
|
||||
folder mutations, duplicate archive, heap convert) need to mutate the
|
||||
filesystem. To run against a read-only archive, set
|
||||
`PP_ORIGINALS_MODE=ro` in `.env`. Browsing, marks, ratings, and color
|
||||
labels still work; the following sidecar endpoints return an OS error:
|
||||
|
||||
- Smart albums (auto-populated by saved filters)
|
||||
- Export presets
|
||||
- Multi-user support
|
||||
- `POST /api/sidecar/files/:uid/rename`
|
||||
- `POST /api/sidecar/folders` / `:rel/rename` / `DELETE /:rel`
|
||||
- `POST /api/sidecar/albums/:uid/convert`
|
||||
- `POST /api/sidecar/duplicates/archive`
|
||||
|
||||
## License
|
||||
PhotoPrism's `PHOTOPRISM_READONLY` is controlled separately by
|
||||
`PP_READONLY` and gates its own backwrite / import paths.
|
||||
|
||||
MIT
|
||||
## Dev iteration loop
|
||||
|
||||
For fast iteration on the sidecar without rebuilding its image on every
|
||||
change, run it as a host process — bring up just `mariadb` and
|
||||
`photoprism` from compose, then build and run the Go binary locally.
|
||||
Full instructions in [`sidecar/README.md`](sidecar/README.md#dev-iteration-loop-host-build).
|
||||
|
||||
## Layout
|
||||
|
||||
```text
|
||||
.
|
||||
├── docker-compose.yml base stack: mariadb + photoprism + sidecar
|
||||
├── docker-compose.podman.yml rootless-podman overlay (keep-id mapping)
|
||||
├── docker-compose.gpu.yml opt-in VA-API GPU passthrough overlay
|
||||
├── .env.example required env vars (copy to .env)
|
||||
├── mariadb/init/ first-boot SQL: creates mule_sidecar DB + user
|
||||
├── pp/ PhotoPrism bind-mounted state (storage, import)
|
||||
├── sidecar/ Go service — see sidecar/README.md
|
||||
└── web/ SvelteKit frontend
|
||||
```
|
||||
|
||||
## GPU video acceleration (optional)
|
||||
|
||||
Hosts with a VA-API-capable GPU (Intel iGPU, AMD APU, etc.) can layer
|
||||
[`docker-compose.gpu.yml`](docker-compose.gpu.yml) to hand `/dev/dri/*`
|
||||
to PhotoPrism and switch ffmpeg to hardware encode/decode — a large
|
||||
perf win for video thumbnails and HEVC→H.264 transcodes:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
|
||||
```
|
||||
|
||||
Set `PP_FFMPEG_ENCODER=vaapi` in `.env` (default for the overlay). Verify
|
||||
with `docker exec pp-app photoprism show config | grep -i ffmpeg`.
|
||||
|
||||
[pp]: https://photoprism.app/
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
# Build dependencies
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
# Image processing libraries
|
||||
libvips42 \
|
||||
libvips-dev \
|
||||
# ExifTool for metadata extraction
|
||||
libimage-exiftool-perl \
|
||||
# FFmpeg for video processing
|
||||
ffmpeg \
|
||||
# Git for some Python packages
|
||||
git \
|
||||
# PostgreSQL client (for potential future use)
|
||||
postgresql-client \
|
||||
# Clean up
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy requirements first for better caching
|
||||
COPY requirements.txt .
|
||||
# Install PyTorch CPU-only FIRST so open-clip-torch doesn't pull the full
|
||||
# CUDA build (~7 GB). CPU inference is all we need — the heavy lifting
|
||||
# happens through ONNX Runtime.
|
||||
RUN pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu \
|
||||
&& pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p /data/thumbs /data/db /data/proxies /data/models /app/config
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8000
|
||||
|
||||
# Run the application
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
|
||||
@@ -1,48 +0,0 @@
|
||||
# Alembic configuration for PhotoVault.
|
||||
#
|
||||
# The actual database URL is loaded at runtime by alembic/env.py from the
|
||||
# DATABASE_URL environment variable (with the async driver suffix stripped).
|
||||
# The placeholder below is only used for `alembic revision --autogenerate`
|
||||
# when no env var is set.
|
||||
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
version_path_separator = os
|
||||
sqlalchemy.url = postgresql+psycopg2://mulita:mulita@localhost:5432/mulita
|
||||
|
||||
[post_write_hooks]
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -1,96 +0,0 @@
|
||||
"""
|
||||
Alembic environment for PhotoVault.
|
||||
|
||||
Pulls DATABASE_URL from the environment so the same migrations work in
|
||||
docker compose and locally. Strips the async driver suffix because Alembic
|
||||
runs synchronously via psycopg2.
|
||||
|
||||
Future-migration note
|
||||
---------------------
|
||||
Fresh installs run `Base.metadata.create_all` in `app.database.init_db`
|
||||
*before* migrations would normally apply, so any migration that adds a
|
||||
column / index / table to an object the model already declares will see
|
||||
that object already present. Write migrations defensively:
|
||||
|
||||
op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS new_col TEXT")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_foo ON foo(bar)")
|
||||
|
||||
For brand-new tables that the model also declares, the same applies — use
|
||||
`op.execute("CREATE TABLE IF NOT EXISTS ...")` or check first.
|
||||
"""
|
||||
from logging.config import fileConfig
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
from alembic import context
|
||||
|
||||
# Make `app` importable from this script.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.database import Base # noqa: E402
|
||||
# Import all models so they're registered on Base.metadata for autogenerate.
|
||||
from app.models import ( # noqa: E402, F401
|
||||
Photo,
|
||||
Folder,
|
||||
SourceRoot,
|
||||
Tag,
|
||||
Heap,
|
||||
HeapPhoto,
|
||||
Embedding,
|
||||
)
|
||||
|
||||
config = context.config
|
||||
|
||||
# Resolve DATABASE_URL from env. Strip async driver suffixes — Alembic
|
||||
# uses sync drivers.
|
||||
db_url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url")
|
||||
if db_url:
|
||||
if "+asyncpg" in db_url:
|
||||
db_url = db_url.replace("+asyncpg", "+psycopg2")
|
||||
elif db_url.startswith("postgresql://"):
|
||||
db_url = db_url.replace("postgresql://", "postgresql+psycopg2://", 1)
|
||||
elif "+aiosqlite" in db_url:
|
||||
db_url = db_url.replace("+aiosqlite", "")
|
||||
config.set_main_option("sqlalchemy.url", db_url)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode (emit SQL only)."""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations against a live database."""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -1,26 +0,0 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -1,27 +0,0 @@
|
||||
"""baseline (empty)
|
||||
|
||||
Revision ID: 0001_baseline
|
||||
Revises:
|
||||
Create Date: 2026-04-10
|
||||
|
||||
The current schema is created by SQLAlchemy `Base.metadata.create_all` in
|
||||
`app.database.init_db()` on first boot. Alembic only owns deltas from
|
||||
PR3 onward. This baseline is intentionally empty so `alembic upgrade head`
|
||||
on a fresh DB simply creates the `alembic_version` table and stamps it.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "0001_baseline"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -1,85 +0,0 @@
|
||||
"""extend tags for vision pipeline
|
||||
|
||||
Revision ID: 0002_extend_tags
|
||||
Revises: 0001_baseline
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Add kind, source, representative_photo_id to tags table.
|
||||
Add confidence, bbox, source to photo_tags association.
|
||||
Switch uniqueness from (name) to (name, kind).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision: str = "0002_extend_tags"
|
||||
down_revision: Union[str, None] = "0001_baseline"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ── tags table ────────────────────────────────────────────────────
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS kind VARCHAR NOT NULL DEFAULT 'user'")
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS source VARCHAR")
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS representative_photo_id VARCHAR REFERENCES photos(id) ON DELETE SET NULL")
|
||||
|
||||
# Create index on kind for filtering
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_tags_kind ON tags(kind)")
|
||||
|
||||
# Drop old unique constraint on name (if it exists) and add (name, kind).
|
||||
# SQLAlchemy create_all may have created either — handle both cases.
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Drop the old single-column unique index/constraint if present.
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_indexes
|
||||
WHERE tablename = 'tags' AND indexname = 'ix_tags_name'
|
||||
) THEN
|
||||
DROP INDEX ix_tags_name;
|
||||
END IF;
|
||||
|
||||
-- Some SQLAlchemy versions create a unique constraint directly.
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM information_schema.table_constraints
|
||||
WHERE table_name = 'tags' AND constraint_name = 'tags_name_key'
|
||||
) THEN
|
||||
ALTER TABLE tags DROP CONSTRAINT tags_name_key;
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'uq_tags_name_kind'
|
||||
) THEN
|
||||
ALTER TABLE tags ADD CONSTRAINT uq_tags_name_kind UNIQUE (name, kind);
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
# ── photo_tags table ──────────────────────────────────────────────
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS confidence FLOAT")
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS bbox JSONB")
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS source VARCHAR")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# photo_tags columns
|
||||
op.drop_column("photo_tags", "source")
|
||||
op.drop_column("photo_tags", "bbox")
|
||||
op.drop_column("photo_tags", "confidence")
|
||||
|
||||
# tags: restore old unique constraint
|
||||
op.execute("ALTER TABLE tags DROP CONSTRAINT IF EXISTS uq_tags_name_kind")
|
||||
op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tags_name ON tags(name)")
|
||||
|
||||
# tags columns
|
||||
op.drop_column("tags", "representative_photo_id")
|
||||
op.drop_column("tags", "source")
|
||||
op.drop_column("tags", "kind")
|
||||
@@ -1,52 +0,0 @@
|
||||
"""pgvector embeddings
|
||||
|
||||
Revision ID: 0003_pgvector_embeddings
|
||||
Revises: 0002_extend_tags
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Rewrite the embeddings table to use pgvector Vector(512) instead of
|
||||
LargeBinary. Add composite PK (photo_id, model), created_at, and
|
||||
HNSW index on vector column.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0003_pgvector_embeddings"
|
||||
down_revision: Union[str, None] = "0002_extend_tags"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Drop the old placeholder table and recreate with pgvector types.
|
||||
# No data to preserve — it was never populated.
|
||||
op.execute("DROP TABLE IF EXISTS embeddings")
|
||||
op.execute("""
|
||||
CREATE TABLE embeddings (
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
model VARCHAR(64) NOT NULL,
|
||||
vector vector(512),
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
PRIMARY KEY (photo_id, model)
|
||||
)
|
||||
""")
|
||||
# HNSW index for cosine similarity search.
|
||||
# Defer creation on large backfills — drop and recreate afterward.
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
||||
ON embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS embeddings")
|
||||
op.execute("""
|
||||
CREATE TABLE embeddings (
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
model VARCHAR,
|
||||
vector BYTEA,
|
||||
PRIMARY KEY (photo_id)
|
||||
)
|
||||
""")
|
||||
@@ -1,82 +0,0 @@
|
||||
"""ocr_text table and Postgres FTS
|
||||
|
||||
Revision ID: 0004_ocr_fts
|
||||
Revises: 0003_pgvector_embeddings
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Create ocr_text table for storing OCR results. Add a tsvector column
|
||||
to photos for unified full-text search (filename + user_title +
|
||||
user_notes) with a GIN index. OCR text is rolled up into a materialized
|
||||
view or joined at query time.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0004_ocr_fts"
|
||||
down_revision: Union[str, None] = "0003_pgvector_embeddings"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ── ocr_text table ────────────────────────────────────────────────
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ocr_text (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
language VARCHAR(8) DEFAULT '',
|
||||
confidence FLOAT,
|
||||
bbox JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT now()
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_ocr_text_photo_id ON ocr_text(photo_id)")
|
||||
|
||||
# ── tsvector column on photos ─────────────────────────────────────
|
||||
op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS search_vector tsvector")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_photos_search_vector ON photos USING GIN (search_vector)")
|
||||
|
||||
# Trigger to auto-update search_vector on INSERT/UPDATE
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION photos_search_vector_update() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_vector :=
|
||||
setweight(to_tsvector('english', coalesce(NEW.filename, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.user_title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.user_notes, '')), 'B');
|
||||
RETURN NEW;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_trigger WHERE tgname = 'photos_search_vector_trigger'
|
||||
) THEN
|
||||
CREATE TRIGGER photos_search_vector_trigger
|
||||
BEFORE INSERT OR UPDATE OF filename, user_title, user_notes
|
||||
ON photos
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION photos_search_vector_update();
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
# Backfill existing rows
|
||||
op.execute("""
|
||||
UPDATE photos SET search_vector =
|
||||
setweight(to_tsvector('english', coalesce(filename, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(user_title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(user_notes, '')), 'B')
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TRIGGER IF EXISTS photos_search_vector_trigger ON photos")
|
||||
op.execute("DROP FUNCTION IF EXISTS photos_search_vector_update()")
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_search_vector")
|
||||
op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS search_vector")
|
||||
op.execute("DROP TABLE IF EXISTS ocr_text")
|
||||
@@ -1,41 +0,0 @@
|
||||
"""face_embeddings table
|
||||
|
||||
Revision ID: 0005_face_embeddings
|
||||
Revises: 0004_ocr_fts
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Create face_embeddings table with pgvector Vector(128) for SFace
|
||||
recognition embeddings and HNSW index.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0005_face_embeddings"
|
||||
down_revision: Union[str, None] = "0004_ocr_fts"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS face_embeddings (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
bbox JSONB,
|
||||
vector vector(128),
|
||||
cluster_id VARCHAR REFERENCES tags(id) ON DELETE SET NULL,
|
||||
quality FLOAT,
|
||||
created_at TIMESTAMPTZ DEFAULT now()
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_face_embeddings_photo_id ON face_embeddings(photo_id)")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_face_embeddings_cluster_id ON face_embeddings(cluster_id)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS face_embeddings")
|
||||
@@ -1,39 +0,0 @@
|
||||
"""face_embeddings vector 128 -> 512
|
||||
|
||||
Revision ID: 0006_face_512d
|
||||
Revises: 0005_face_embeddings
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Resize face_embeddings.vector from Vector(128) to Vector(512) for
|
||||
ArcFace embeddings (InsightFace). Drops existing data and HNSW index,
|
||||
recreates both.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0006_face_512d"
|
||||
down_revision: Union[str, None] = "0005_face_embeddings"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Drop index, truncate (old 128-d vectors are incompatible), resize
|
||||
op.execute("DROP INDEX IF EXISTS ix_face_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM face_embeddings")
|
||||
op.execute("ALTER TABLE face_embeddings ALTER COLUMN vector TYPE vector(512)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_face_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM face_embeddings")
|
||||
op.execute("ALTER TABLE face_embeddings ALTER COLUMN vector TYPE vector(128)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
@@ -1,67 +0,0 @@
|
||||
"""folders + photos is_hidden flag
|
||||
|
||||
Revision ID: 0007_folder_hidden
|
||||
Revises: 0006_face_512d
|
||||
Create Date: 2026-04-11
|
||||
|
||||
Adds an "exclude from cross-cutting views" flag:
|
||||
|
||||
folders.is_hidden — user-toggled on a folder or source root. When
|
||||
true, photos in that subtree are hidden from
|
||||
library-wide views (All Photos, Map, Tags,
|
||||
People, Search, Duplicates, sidebar counts) but
|
||||
remain indexed and visible when the user
|
||||
navigates into the folder directly.
|
||||
|
||||
photos.is_hidden — denormalized: true iff any ancestor folder in
|
||||
the photo's folder chain has is_hidden=true.
|
||||
Kept as a real column (rather than a recursive
|
||||
query per read) because the filter runs on
|
||||
essentially every photo query in the app, and
|
||||
the toggle operation that recomputes it is
|
||||
rare. Indexed so `WHERE NOT is_hidden` doesn't
|
||||
fall off the rating/taken_at indexes.
|
||||
|
||||
Both columns default to false so existing rows need no backfill.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0007_folder_hidden"
|
||||
down_revision: Union[str, None] = "0006_face_512d"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"folders",
|
||||
sa.Column(
|
||||
"is_hidden",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"photos",
|
||||
sa.Column(
|
||||
"is_hidden",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_photos_is_hidden",
|
||||
"photos",
|
||||
["is_hidden"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_photos_is_hidden", table_name="photos")
|
||||
op.drop_column("photos", "is_hidden")
|
||||
op.drop_column("folders", "is_hidden")
|
||||
@@ -1,49 +0,0 @@
|
||||
"""photos has_date_warning flag
|
||||
|
||||
Revision ID: 0008_photos_date_warning
|
||||
Revises: 0007_folder_hidden
|
||||
Create Date: 2026-04-11
|
||||
|
||||
Adds `photos.has_date_warning` — a denormalized boolean that's true when
|
||||
the scanner's folder/filename date guesser disagrees with the stored
|
||||
taken_at by more than 24h (or taken_at is missing and the path would
|
||||
provide a date). Surfacing this as a real column means the filter bar
|
||||
can restrict the timeline to suspicious photos without the client
|
||||
recomputing the heuristic for every row.
|
||||
|
||||
Indexed because the filter is meant to run on top of the existing
|
||||
taken_at / folder queries that dominate the timeline, and we want the
|
||||
partial `WHERE has_date_warning` scan to stay cheap as the library
|
||||
grows.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0008_photos_date_warning"
|
||||
down_revision: Union[str, None] = "0007_folder_hidden"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"photos",
|
||||
sa.Column(
|
||||
"has_date_warning",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_photos_has_date_warning",
|
||||
"photos",
|
||||
["has_date_warning"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_photos_has_date_warning", table_name="photos")
|
||||
op.drop_column("photos", "has_date_warning")
|
||||
@@ -1,144 +0,0 @@
|
||||
"""users table and user_id foreign keys
|
||||
|
||||
Revision ID: 0009_users_and_auth
|
||||
Revises: 0008_photos_date_warning
|
||||
Create Date: 2026-04-12
|
||||
|
||||
Introduces multi-user support:
|
||||
1. Creates the `users` table.
|
||||
2. Adds `user_id` FK columns to photos, folders, source_roots, heaps, tags.
|
||||
3. For existing installs: creates a default admin user and assigns all
|
||||
existing rows to that user. The generated password is printed to the
|
||||
backend logs — the admin should change it on first login.
|
||||
4. Replaces the unique constraint on tags (name, kind) with
|
||||
(name, kind, user_id) so each user can have their own tags.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
import uuid
|
||||
import secrets
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0009_users_and_auth"
|
||||
down_revision: Union[str, None] = "0008_photos_date_warning"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# 1. Create users table (IF NOT EXISTS — safe on fresh installs where
|
||||
# init_db's create_all has already laid down the schema).
|
||||
conn.execute(sa.text("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id VARCHAR NOT NULL PRIMARY KEY,
|
||||
username VARCHAR(50) NOT NULL UNIQUE,
|
||||
email VARCHAR UNIQUE,
|
||||
hashed_password VARCHAR NOT NULL,
|
||||
role VARCHAR NOT NULL DEFAULT 'user',
|
||||
is_active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT now(),
|
||||
media_path VARCHAR NOT NULL
|
||||
)
|
||||
"""))
|
||||
conn.execute(sa.text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_users_username ON users (username)"
|
||||
))
|
||||
|
||||
# 2. Add user_id columns (nullable initially for the data migration)
|
||||
for table in ("photos", "folders", "source_roots", "heaps", "tags"):
|
||||
conn.execute(sa.text(
|
||||
f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS user_id VARCHAR"
|
||||
))
|
||||
conn.execute(sa.text(
|
||||
f"CREATE INDEX IF NOT EXISTS ix_{table}_user_id ON {table} (user_id)"
|
||||
))
|
||||
# FK — check if it already exists before adding
|
||||
fk_name = f"fk_{table}_user_id"
|
||||
fk_exists = conn.execute(sa.text(
|
||||
"SELECT 1 FROM information_schema.table_constraints "
|
||||
"WHERE constraint_name = :name AND table_name = :tbl"
|
||||
), {"name": fk_name, "tbl": table}).scalar()
|
||||
if not fk_exists:
|
||||
conn.execute(sa.text(
|
||||
f"ALTER TABLE {table} ADD CONSTRAINT {fk_name} "
|
||||
f"FOREIGN KEY (user_id) REFERENCES users(id)"
|
||||
))
|
||||
|
||||
# 3. Data migration: if rows exist, create a default admin and assign
|
||||
conn = op.get_bind()
|
||||
photo_count = conn.execute(sa.text("SELECT COUNT(*) FROM photos")).scalar()
|
||||
|
||||
if photo_count > 0:
|
||||
admin_id = str(uuid.uuid4())
|
||||
generated_password = secrets.token_urlsafe(16)
|
||||
|
||||
# Hash the password using passlib at migration time
|
||||
from passlib.context import CryptContext
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
hashed = pwd_context.hash(generated_password)
|
||||
|
||||
# Every user gets a subfolder — including the migrated admin.
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"INSERT INTO users (id, username, hashed_password, role, media_path) "
|
||||
"VALUES (:id, :username, :hashed, :role, :media_path)"
|
||||
),
|
||||
{
|
||||
"id": admin_id,
|
||||
"username": "admin",
|
||||
"hashed": hashed,
|
||||
"role": "admin",
|
||||
"media_path": "/photos/admin",
|
||||
},
|
||||
)
|
||||
|
||||
# Assign all existing rows to the default admin
|
||||
for table in ("photos", "folders", "source_roots", "heaps", "tags"):
|
||||
conn.execute(
|
||||
sa.text(f"UPDATE {table} SET user_id = :uid WHERE user_id IS NULL"),
|
||||
{"uid": admin_id},
|
||||
)
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger("alembic.migration")
|
||||
logger.warning(
|
||||
f"=== MIGRATION 0009 === Default admin created. "
|
||||
f"Username: admin | Password: {generated_password} | "
|
||||
f"Change this password on first login!"
|
||||
)
|
||||
|
||||
# 4. Replace tag unique constraint to include user_id
|
||||
# Check whether the old constraint exists before trying to drop it
|
||||
# (on fresh installs create_all creates the new constraint directly).
|
||||
old_uq_exists = conn.execute(sa.text(
|
||||
"SELECT 1 FROM information_schema.table_constraints "
|
||||
"WHERE constraint_name = 'uq_tags_name_kind' AND table_name = 'tags'"
|
||||
)).scalar()
|
||||
if old_uq_exists:
|
||||
op.drop_constraint("uq_tags_name_kind", "tags", type_="unique")
|
||||
|
||||
new_uq_exists = conn.execute(sa.text(
|
||||
"SELECT 1 FROM information_schema.table_constraints "
|
||||
"WHERE constraint_name = 'uq_tags_name_kind_user' AND table_name = 'tags'"
|
||||
)).scalar()
|
||||
if not new_uq_exists:
|
||||
op.create_unique_constraint("uq_tags_name_kind_user", "tags", ["name", "kind", "user_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Reverse the tag constraint
|
||||
op.drop_constraint("uq_tags_name_kind_user", "tags", type_="unique")
|
||||
op.create_unique_constraint("uq_tags_name_kind", "tags", ["name", "kind"])
|
||||
|
||||
# Drop user_id columns and FKs
|
||||
for table in ("photos", "folders", "source_roots", "heaps", "tags"):
|
||||
op.drop_constraint(f"fk_{table}_user_id", table, type_="foreignkey")
|
||||
op.drop_index(f"ix_{table}_user_id", table_name=table)
|
||||
op.drop_column(table, "user_id")
|
||||
|
||||
# Drop users table
|
||||
op.drop_index("ix_users_username", table_name="users")
|
||||
op.drop_table("users")
|
||||
@@ -1,39 +0,0 @@
|
||||
"""embeddings vector 512 -> 768
|
||||
|
||||
Revision ID: 0010_embeddings_768d
|
||||
Revises: 0009_users_and_auth
|
||||
Create Date: 2026-04-12
|
||||
|
||||
Resize embeddings.vector from Vector(512) to Vector(768) for
|
||||
SigLIP2 ViT-B/16 embeddings. Drops existing data and HNSW index,
|
||||
recreates with the new dimension. Existing embeddings will be
|
||||
regenerated by the vision backfill task.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0010_embeddings_768d"
|
||||
down_revision: Union[str, None] = "0009_users_and_auth"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM embeddings")
|
||||
op.execute("ALTER TABLE embeddings ALTER COLUMN vector TYPE vector(768)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
||||
ON embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM embeddings")
|
||||
op.execute("ALTER TABLE embeddings ALTER COLUMN vector TYPE vector(512)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
||||
ON embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
@@ -1,47 +0,0 @@
|
||||
"""
|
||||
Authentication utilities — password hashing and JWT token management.
|
||||
"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from jose import jwt, JWTError
|
||||
from passlib.context import CryptContext
|
||||
|
||||
from app.config import settings
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
|
||||
ALGORITHM = "HS256"
|
||||
|
||||
|
||||
def hash_password(plain: str) -> str:
|
||||
return pwd_context.hash(plain)
|
||||
|
||||
|
||||
def verify_password(plain: str, hashed: str) -> bool:
|
||||
return pwd_context.verify(plain, hashed)
|
||||
|
||||
|
||||
def create_access_token(user_id: str, role: str) -> str:
|
||||
expire = datetime.now(timezone.utc) + timedelta(minutes=settings.access_token_expire_minutes)
|
||||
payload = {
|
||||
"sub": user_id,
|
||||
"role": role,
|
||||
"exp": expire,
|
||||
"type": "access",
|
||||
}
|
||||
return jwt.encode(payload, settings.secret_key, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def create_refresh_token(user_id: str) -> str:
|
||||
expire = datetime.now(timezone.utc) + timedelta(days=settings.refresh_token_expire_days)
|
||||
payload = {
|
||||
"sub": user_id,
|
||||
"exp": expire,
|
||||
"type": "refresh",
|
||||
}
|
||||
return jwt.encode(payload, settings.secret_key, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def decode_token(token: str) -> dict:
|
||||
"""Decode and validate a JWT. Raises JWTError on any problem."""
|
||||
return jwt.decode(token, settings.secret_key, algorithms=[ALGORITHM])
|
||||
@@ -1,213 +0,0 @@
|
||||
"""
|
||||
Application configuration using Pydantic Settings
|
||||
"""
|
||||
from pydantic_settings import BaseSettings
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
|
||||
class ThumbnailSettings(BaseModel):
|
||||
"""Thumbnail generation settings"""
|
||||
small: int = 240
|
||||
medium: int = 640
|
||||
large: int = 1280
|
||||
quality: int = 85
|
||||
format: str = "webp"
|
||||
|
||||
class ScannerSettings(BaseModel):
|
||||
"""File scanner settings"""
|
||||
watch: bool = True
|
||||
initial_scan_on_start: bool = True
|
||||
batch_size: int = 100
|
||||
concurrent_workers: int = 4
|
||||
|
||||
class PerformanceSettings(BaseModel):
|
||||
"""Performance tuning settings"""
|
||||
max_concurrent_thumbnails: int = 10
|
||||
cache_ttl: int = 3600
|
||||
db_pool_size: int = 5
|
||||
db_pool_max_overflow: int = 5
|
||||
db_pool_recycle: int = 3600
|
||||
|
||||
class EmbedderSettings(BaseModel):
|
||||
"""CLIP / SigLIP embedding model settings.
|
||||
Supported: "openclip_vitb32" (512-d), "siglip2_vitb16" (768-d, default)."""
|
||||
name: str = "openclip_vitb32"
|
||||
batch_size: int = 8
|
||||
|
||||
class OCRSettings(BaseModel):
|
||||
"""PaddleOCR / rapidocr settings"""
|
||||
enabled: bool = True
|
||||
languages: list[str] = ["en"]
|
||||
min_confidence: float = 0.5
|
||||
|
||||
class DetectorSettings(BaseModel):
|
||||
"""YOLOv8n object detection settings"""
|
||||
enabled: bool = True
|
||||
min_confidence: float = 0.35
|
||||
max_detections: int = 50
|
||||
|
||||
class FacesSettings(BaseModel):
|
||||
"""YuNet + SFace face detection/recognition settings"""
|
||||
enabled: bool = True
|
||||
min_face_size: int = 40
|
||||
recognition_threshold: float = 0.65
|
||||
cluster_eps: float = 0.5
|
||||
|
||||
class ClassifierSettings(BaseModel):
|
||||
"""CLIP zero-shot content classification settings"""
|
||||
enabled: bool = True
|
||||
min_confidence: float = 0.3
|
||||
|
||||
class VisionSettings(BaseModel):
|
||||
"""AI vision pipeline settings. Disabled when running on SQLite
|
||||
(pgvector is required for embedding storage)."""
|
||||
enabled: bool = True
|
||||
backend: str = "onnx" # "onnx" | "rocm" (future)
|
||||
models_dir: str = "/data/models"
|
||||
# ONNX Runtime execution providers in priority order.
|
||||
# Auto-detected at startup; falls back to CPU if GPU is unavailable.
|
||||
# Options: "CUDAExecutionProvider", "ROCMExecutionProvider",
|
||||
# "OpenVINOExecutionProvider", "CPUExecutionProvider"
|
||||
execution_providers: list[str] = ["CPUExecutionProvider"]
|
||||
embedder: EmbedderSettings = EmbedderSettings()
|
||||
ocr: OCRSettings = OCRSettings()
|
||||
detector: DetectorSettings = DetectorSettings()
|
||||
faces: FacesSettings = FacesSettings()
|
||||
classifier: ClassifierSettings = ClassifierSettings()
|
||||
worker_concurrency: int = 2
|
||||
|
||||
class MulitaConfig(BaseModel):
|
||||
"""Main configuration from YAML file. Source roots and the discard
|
||||
workflow are owned by the database now — only operational settings
|
||||
live here."""
|
||||
thumbnails: ThumbnailSettings = ThumbnailSettings()
|
||||
scanner: ScannerSettings = ScannerSettings()
|
||||
performance: PerformanceSettings = PerformanceSettings()
|
||||
vision: VisionSettings = VisionSettings()
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings"""
|
||||
# Database — Postgres + pgvector by default. The SQLite escape hatch
|
||||
# remains supported via the docker-compose.sqlite.yml override and by
|
||||
# setting DATABASE_URL=sqlite+aiosqlite:///... in .env for local dev.
|
||||
database_url: str = Field(
|
||||
default="postgresql+asyncpg://mulita:mulita@db:5432/mulita",
|
||||
env="DATABASE_URL"
|
||||
)
|
||||
|
||||
# Redis
|
||||
redis_url: str = Field(
|
||||
default="redis://localhost:6379",
|
||||
env="REDIS_URL"
|
||||
)
|
||||
|
||||
# Celery
|
||||
celery_broker_url: str = Field(
|
||||
default="redis://localhost:6379",
|
||||
env="CELERY_BROKER_URL"
|
||||
)
|
||||
celery_result_backend: str = Field(
|
||||
default="redis://localhost:6379",
|
||||
env="CELERY_RESULT_BACKEND"
|
||||
)
|
||||
|
||||
# Photo directories
|
||||
photo_dirs: str = Field(
|
||||
default="/photos",
|
||||
env="PHOTO_DIRS"
|
||||
)
|
||||
|
||||
# API settings
|
||||
api_host: str = Field(default="0.0.0.0", env="API_HOST")
|
||||
api_port: int = Field(default=8000, env="API_PORT")
|
||||
|
||||
# CORS — comma-separated list of allowed origins, or "*" for any.
|
||||
# Same-origin requests (the normal case behind nginx / vite proxy)
|
||||
# never trip CORS, so this is only for direct browser access from
|
||||
# other origins (LAN IP, reverse proxy, dev tools).
|
||||
allowed_origins: str = Field(default="*", env="ALLOWED_ORIGINS")
|
||||
|
||||
# Logging — accepts standard python levels (DEBUG, INFO, WARNING,
|
||||
# ERROR, CRITICAL). Bumped from INFO when chasing a problem.
|
||||
log_level: str = Field(default="INFO", env="LOG_LEVEL")
|
||||
|
||||
# Auth — JWT signing key. Set SECRET_KEY in .env for production.
|
||||
# If unset, a deterministic fallback is used (acceptable for
|
||||
# single-machine homelab deploys, but set a real key if the instance
|
||||
# is network-exposed).
|
||||
secret_key: str = Field(
|
||||
default="mulita-dev-secret-change-me",
|
||||
env="SECRET_KEY",
|
||||
)
|
||||
access_token_expire_minutes: int = Field(default=60, env="ACCESS_TOKEN_EXPIRE_MINUTES")
|
||||
refresh_token_expire_days: int = Field(default=30, env="REFRESH_TOKEN_EXPIRE_DAYS")
|
||||
|
||||
@property
|
||||
def cors_origins(self) -> list[str]:
|
||||
"""Parse the ALLOWED_ORIGINS env var into a list. Accepts:
|
||||
- "*" → wildcard (single-element list ["*"])
|
||||
- "http://a.com,http://b.com" → split + strip
|
||||
Empty entries are dropped.
|
||||
"""
|
||||
raw = (self.allowed_origins or "").strip()
|
||||
if not raw or raw == "*":
|
||||
return ["*"]
|
||||
return [o.strip() for o in raw.split(",") if o.strip()]
|
||||
|
||||
# App configuration from YAML
|
||||
_config: Optional[MulitaConfig] = None
|
||||
|
||||
@property
|
||||
def config(self) -> MulitaConfig:
|
||||
"""Load configuration from YAML file"""
|
||||
if self._config is None:
|
||||
config_path = Path("/app/config/mulita.yml")
|
||||
if not config_path.exists():
|
||||
config_path = Path("mulita.yml")
|
||||
|
||||
if config_path.exists():
|
||||
with open(config_path, "r") as f:
|
||||
config_data = yaml.safe_load(f)
|
||||
self._config = MulitaConfig(**config_data)
|
||||
else:
|
||||
self._config = MulitaConfig()
|
||||
|
||||
return self._config
|
||||
|
||||
@property
|
||||
def thumbnails(self) -> ThumbnailSettings:
|
||||
return self.config.thumbnails
|
||||
|
||||
@property
|
||||
def scanner(self) -> ScannerSettings:
|
||||
return self.config.scanner
|
||||
|
||||
@property
|
||||
def performance(self) -> PerformanceSettings:
|
||||
return self.config.performance
|
||||
|
||||
# ONNX Runtime execution providers, overridable via env var.
|
||||
# Comma-separated: "CUDAExecutionProvider,CPUExecutionProvider"
|
||||
# or "auto" for GPU auto-detection.
|
||||
vision_execution_providers: str = Field(
|
||||
default="CPUExecutionProvider",
|
||||
env="VISION_EXECUTION_PROVIDERS",
|
||||
)
|
||||
|
||||
@property
|
||||
def vision(self) -> VisionSettings:
|
||||
v = self.config.vision
|
||||
# Override execution_providers from env if set.
|
||||
providers = [p.strip() for p in self.vision_execution_providers.split(",") if p.strip()]
|
||||
if providers:
|
||||
v.execution_providers = providers
|
||||
return v
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
case_sensitive = False
|
||||
|
||||
# Global settings instance
|
||||
settings = Settings()
|
||||
@@ -1,199 +0,0 @@
|
||||
"""
|
||||
Database configuration and session management.
|
||||
|
||||
Schema management strategy
|
||||
--------------------------
|
||||
Postgres (default): Alembic owns schema deltas. `alembic upgrade head` is
|
||||
run before the app starts (in the container CMD). `init_db()` calls
|
||||
`create_all` afterward as the source of truth for fresh installs — it is
|
||||
idempotent for existing tables and creates any tables defined on
|
||||
`Base.metadata` that don't yet exist. Future Alembic migrations should be
|
||||
written defensively (`IF NOT EXISTS` etc.) so they remain safe to run on a
|
||||
fresh DB where `create_all` has already laid down the same objects.
|
||||
|
||||
SQLite (escape hatch via docker-compose.sqlite.yml): no Alembic. The
|
||||
historical inline ALTER TABLE block stays in place so existing dev
|
||||
installs keep upgrading.
|
||||
"""
|
||||
import os
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy.pool import NullPool
|
||||
from sqlalchemy import text
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_is_sqlite = settings.database_url.startswith("sqlite")
|
||||
_is_postgres = settings.database_url.startswith("postgresql")
|
||||
|
||||
# When running inside a Celery worker we use NullPool rather than the
|
||||
# default connection pool. The reasons stack up:
|
||||
#
|
||||
# 1. Celery's prefork model forks the master *after* imports, so every
|
||||
# child inherits the same asyncpg Connection objects — they share
|
||||
# a socket, and two children using one concurrently raises
|
||||
# "another operation is in progress".
|
||||
#
|
||||
# 2. Task bodies run under `asyncio.run()`, which spins up a fresh
|
||||
# event loop per invocation. A pooled asyncpg Connection created
|
||||
# on loop A, returned to the pool, and checked out on loop B
|
||||
# raises "Future attached to a different loop".
|
||||
#
|
||||
# NullPool dodges both: every session checkout opens a brand-new
|
||||
# connection on the *current* loop and the connection is closed at
|
||||
# session end. Connection setup is cheap compared to task cost, so this
|
||||
# is the right default for the worker. The FastAPI backend keeps the
|
||||
# normal pool because it serves many short requests on a single long-
|
||||
# lived event loop, where pooling is a clear win.
|
||||
_is_celery_worker = os.environ.get("MULITA_CELERY_WORKER") == "1"
|
||||
|
||||
if _is_sqlite:
|
||||
db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", ""))
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False,
|
||||
connect_args={
|
||||
"check_same_thread": False,
|
||||
"timeout": 30,
|
||||
},
|
||||
)
|
||||
elif _is_celery_worker:
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False,
|
||||
poolclass=NullPool,
|
||||
)
|
||||
else:
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False,
|
||||
pool_size=settings.performance.db_pool_size,
|
||||
max_overflow=settings.performance.db_pool_max_overflow,
|
||||
pool_recycle=settings.performance.db_pool_recycle,
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
|
||||
# Create async session factory
|
||||
AsyncSessionLocal = async_sessionmaker(
|
||||
engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False
|
||||
)
|
||||
|
||||
# Base class for models
|
||||
Base = declarative_base()
|
||||
|
||||
async def get_db() -> AsyncSession:
|
||||
"""Dependency to get database session"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
async def init_db():
|
||||
"""Initialize database, create tables if they don't exist"""
|
||||
async with engine.begin() as conn:
|
||||
# Import all models to register them with Base
|
||||
from app.models import User, Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto, Embedding
|
||||
|
||||
# Postgres: ensure pgvector is available before create_all touches
|
||||
# any Vector columns (added in later PRs but the extension is cheap
|
||||
# and idempotent to create now).
|
||||
if _is_postgres:
|
||||
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
|
||||
|
||||
# Create all tables. Note: create_all only creates *missing* tables —
|
||||
# it does NOT add new columns to existing tables when the model gains
|
||||
# them. On Postgres, Alembic handles deltas; on SQLite, the inline
|
||||
# ALTER block below is the legacy fallback.
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
gps_columns_added = False
|
||||
|
||||
if _is_sqlite:
|
||||
# WAL mode for better concurrency.
|
||||
await conn.execute(text("PRAGMA journal_mode=WAL"))
|
||||
await conn.execute(text("PRAGMA synchronous=NORMAL"))
|
||||
await conn.execute(text("PRAGMA cache_size=10000"))
|
||||
await conn.execute(text("PRAGMA temp_store=MEMORY"))
|
||||
|
||||
# ── Idempotent column adds (SQLite only) ─────────────────────
|
||||
# SQLite supports ADD COLUMN but not "IF NOT EXISTS" for
|
||||
# columns, so introspect via PRAGMA first. Each entry is
|
||||
# (column_name, ALTER statement). Add new columns at the
|
||||
# bottom. On Postgres these live in Alembic migrations.
|
||||
existing_cols = {
|
||||
row[1]
|
||||
for row in (
|
||||
await conn.execute(text("PRAGMA table_info(photos)"))
|
||||
).fetchall()
|
||||
}
|
||||
pending_alters: list[tuple[str, str]] = [
|
||||
("phash", "ALTER TABLE photos ADD COLUMN phash VARCHAR(16)"),
|
||||
(
|
||||
"duplicate_group_id",
|
||||
"ALTER TABLE photos ADD COLUMN duplicate_group_id VARCHAR",
|
||||
),
|
||||
("latitude", "ALTER TABLE photos ADD COLUMN latitude REAL"),
|
||||
("longitude", "ALTER TABLE photos ADD COLUMN longitude REAL"),
|
||||
]
|
||||
for col_name, alter_sql in pending_alters:
|
||||
if col_name not in existing_cols:
|
||||
logger.info(f"Adding photos.{col_name} column")
|
||||
await conn.execute(text(alter_sql))
|
||||
if col_name in ("latitude", "longitude"):
|
||||
gps_columns_added = True
|
||||
await conn.execute(
|
||||
text("CREATE INDEX IF NOT EXISTS ix_photos_phash ON photos(phash)")
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_photos_duplicate_group_id "
|
||||
"ON photos(duplicate_group_id)"
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_photos_lat_lon "
|
||||
"ON photos(latitude, longitude)"
|
||||
)
|
||||
)
|
||||
|
||||
logger.info("Database initialized successfully")
|
||||
|
||||
# If we just introduced the GPS columns on an existing SQLite
|
||||
# install, kick off a one-shot backfill so the Map view is
|
||||
# populated without a manual full re-scan. Postgres installs are
|
||||
# always fresh (no SQLite→PG migration path), so this code path
|
||||
# is SQLite-only.
|
||||
if _is_sqlite and gps_columns_added:
|
||||
try:
|
||||
from app.tasks.scan import backfill_gps
|
||||
backfill_gps.delay()
|
||||
logger.info("Queued one-shot backfill_gps task after column add")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue backfill_gps task: {e}")
|
||||
|
||||
async def create_fts_table():
|
||||
"""Create Full-Text Search table for SQLite. On Postgres this is
|
||||
replaced by a tsvector column on the photos table (added in PR5)."""
|
||||
if _is_sqlite:
|
||||
async with engine.begin() as conn:
|
||||
# Create FTS5 virtual table for full-text search
|
||||
await conn.execute(text("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS photos_fts USING fts5(
|
||||
photo_id UNINDEXED,
|
||||
filename,
|
||||
user_title,
|
||||
user_notes,
|
||||
exif_text,
|
||||
tokenize='unicode61'
|
||||
)
|
||||
"""))
|
||||
logger.info("FTS5 table created successfully")
|
||||
@@ -1,182 +0,0 @@
|
||||
"""
|
||||
FastAPI dependencies for authentication and user-scoped data access.
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, HTTPException, Query, Request, status
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
from jose import JWTError
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.auth import decode_token
|
||||
from app.database import get_db
|
||||
from app.models.user import User
|
||||
from app.models.photos import Photo
|
||||
from app.models.folders import Folder, SourceRoot
|
||||
from app.models.heaps import Heap
|
||||
from app.models.tags import Tag
|
||||
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/login")
|
||||
|
||||
|
||||
async def get_current_user(
|
||||
token: str = Depends(oauth2_scheme),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User:
|
||||
"""Decode JWT, look up user, raise 401 if invalid or inactive."""
|
||||
credentials_exception = HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or expired token",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
try:
|
||||
payload = decode_token(token)
|
||||
user_id: str = payload.get("sub")
|
||||
token_type: str = payload.get("type")
|
||||
if user_id is None or token_type != "access":
|
||||
raise credentials_exception
|
||||
except JWTError:
|
||||
raise credentials_exception
|
||||
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None or not user.is_active:
|
||||
raise credentials_exception
|
||||
return user
|
||||
|
||||
|
||||
async def get_current_user_media(
|
||||
request: Request,
|
||||
token: Optional[str] = Query(None, alias="token"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User:
|
||||
"""Authenticate via Authorization header OR ?token= query parameter.
|
||||
|
||||
Used for media endpoints (thumbnails, originals, proxies) where the
|
||||
URL is set as an <img src> or <video src> and the browser can't
|
||||
attach an Authorization header. The frontend appends ?token=JWT to
|
||||
media URLs so they pass auth without custom fetch logic.
|
||||
"""
|
||||
# Try Authorization header first.
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
jwt_token = None
|
||||
if auth_header.startswith("Bearer "):
|
||||
jwt_token = auth_header[7:]
|
||||
elif token:
|
||||
jwt_token = token
|
||||
|
||||
if not jwt_token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Missing token",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
credentials_exception = HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or expired token",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
try:
|
||||
payload = decode_token(jwt_token)
|
||||
user_id: str = payload.get("sub")
|
||||
token_type: str = payload.get("type")
|
||||
if user_id is None or token_type != "access":
|
||||
raise credentials_exception
|
||||
except JWTError:
|
||||
raise credentials_exception
|
||||
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None or not user.is_active:
|
||||
raise credentials_exception
|
||||
return user
|
||||
|
||||
|
||||
async def require_admin(
|
||||
user: User = Depends(get_current_user),
|
||||
) -> User:
|
||||
"""Raise 403 if user is not an admin."""
|
||||
if user.role != "admin":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Admin privileges required",
|
||||
)
|
||||
return user
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User-scoped query helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def user_photos_query(user: User):
|
||||
"""Base select for photos owned by user, with tags eager-loaded."""
|
||||
return (
|
||||
select(Photo)
|
||||
.options(selectinload(Photo.tags))
|
||||
.where(Photo.user_id == user.id)
|
||||
)
|
||||
|
||||
|
||||
async def get_user_photo(
|
||||
photo_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> Photo:
|
||||
"""Fetch a single photo by ID, scoped to the user. Raises 404."""
|
||||
result = await db.execute(
|
||||
select(Photo)
|
||||
.options(selectinload(Photo.tags))
|
||||
.where(Photo.id == photo_id, Photo.user_id == user.id)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
if photo is None:
|
||||
raise HTTPException(status_code=404, detail="Photo not found")
|
||||
return photo
|
||||
|
||||
|
||||
async def get_user_folder(
|
||||
folder_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> Folder:
|
||||
"""Fetch a single folder by ID, scoped to the user. Raises 404."""
|
||||
result = await db.execute(
|
||||
select(Folder).where(Folder.id == folder_id, Folder.user_id == user.id)
|
||||
)
|
||||
folder = result.scalar_one_or_none()
|
||||
if folder is None:
|
||||
raise HTTPException(status_code=404, detail="Folder not found")
|
||||
return folder
|
||||
|
||||
|
||||
async def get_user_heap(
|
||||
heap_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> Heap:
|
||||
"""Fetch a single heap by ID, scoped to the user. Raises 404."""
|
||||
result = await db.execute(
|
||||
select(Heap).where(Heap.id == heap_id, Heap.user_id == user.id)
|
||||
)
|
||||
heap = result.scalar_one_or_none()
|
||||
if heap is None:
|
||||
raise HTTPException(status_code=404, detail="Heap not found")
|
||||
return heap
|
||||
|
||||
|
||||
async def get_user_tag(
|
||||
tag_id: str,
|
||||
user: User,
|
||||
db: AsyncSession,
|
||||
) -> Tag:
|
||||
"""Fetch a single tag by ID, scoped to the user. Raises 404."""
|
||||
result = await db.execute(
|
||||
select(Tag).where(Tag.id == tag_id, Tag.user_id == user.id)
|
||||
)
|
||||
tag = result.scalar_one_or_none()
|
||||
if tag is None:
|
||||
raise HTTPException(status_code=404, detail="Tag not found")
|
||||
return tag
|
||||
@@ -1,110 +0,0 @@
|
||||
"""
|
||||
Mulita - Photo Management Application
|
||||
Main FastAPI application entry point
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
import logging
|
||||
import os
|
||||
|
||||
from app.config import settings
|
||||
from app.database import init_db
|
||||
from app.routers import photos, folders, heaps, tags, discard, library, search, auth, admin
|
||||
from app.services.scanner import start_initial_scan, bootstrap_default_source_root
|
||||
from app.services.cleanup import cleanup_data_integrity
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Manage application lifecycle"""
|
||||
logger.info("Starting Mulita application...")
|
||||
|
||||
# Initialize database
|
||||
await init_db()
|
||||
|
||||
# First-boot convenience: if there are no source roots in the DB yet,
|
||||
# create one for the default /photos mount so the user sees their
|
||||
# library immediately without configuring anything in the UI.
|
||||
try:
|
||||
await bootstrap_default_source_root()
|
||||
except Exception as e:
|
||||
logger.error(f"Bootstrap source root failed (continuing): {e}")
|
||||
|
||||
# One-shot cleanup of duplicate source_roots / folders left over from
|
||||
# earlier scanner versions that didn't normalize paths. Idempotent.
|
||||
try:
|
||||
await cleanup_data_integrity()
|
||||
except Exception as e:
|
||||
logger.error(f"Startup cleanup failed (continuing): {e}")
|
||||
|
||||
# Start initial scan if configured
|
||||
if settings.scanner.initial_scan_on_start:
|
||||
logger.info("Starting initial library scan...")
|
||||
await start_initial_scan()
|
||||
|
||||
yield
|
||||
|
||||
logger.info("Shutting down Mulita application...")
|
||||
|
||||
# Create FastAPI app
|
||||
app = FastAPI(
|
||||
title="Mulita Photo Management API",
|
||||
description="Self-hosted photo management application inspired by Lightroom",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan
|
||||
)
|
||||
|
||||
# Configure CORS. The frontend normally talks to the backend through the
|
||||
# nginx (prod) or vite (dev) proxy, so requests are same-origin and never
|
||||
# trip CORS. ALLOWED_ORIGINS in .env controls the fallback for direct
|
||||
# browser access from other origins (LAN IP, reverse proxy under a
|
||||
# different host). Defaults to "*" since this is a single-user homelab
|
||||
# tool; lock it down by setting e.g. ALLOWED_ORIGINS=https://photos.your.tld
|
||||
# in production deployments.
|
||||
_origins = settings.cors_origins
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=_origins,
|
||||
# Wildcard origins can't be combined with credentials per the CORS
|
||||
# spec, so credentials get auto-disabled in that case.
|
||||
allow_credentials=_origins != ["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Mount static files for serving thumbnails (with X-Accel-Redirect support)
|
||||
if os.path.exists("/data/thumbs"):
|
||||
app.mount("/thumbs", StaticFiles(directory="/data/thumbs"), name="thumbs")
|
||||
|
||||
# Include routers
|
||||
app.include_router(auth.router, prefix="/api/v1/auth", tags=["auth"])
|
||||
app.include_router(admin.router, prefix="/api/v1/admin", tags=["admin"])
|
||||
app.include_router(photos.router, prefix="/api/v1/photos", tags=["photos"])
|
||||
app.include_router(folders.router, prefix="/api/v1/folders", tags=["folders"])
|
||||
app.include_router(heaps.router, prefix="/api/v1/heaps", tags=["heaps"])
|
||||
app.include_router(tags.router, prefix="/api/v1/tags", tags=["tags"])
|
||||
app.include_router(discard.router, prefix="/api/v1/discard", tags=["discard"])
|
||||
app.include_router(library.router, prefix="/api/v1/library", tags=["library"])
|
||||
app.include_router(search.router, prefix="/api/v1/photos/search", tags=["search"])
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
"""Root endpoint"""
|
||||
return {
|
||||
"name": "Mulita Photo Management API",
|
||||
"version": "1.0.0",
|
||||
"status": "running"
|
||||
}
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
"""Health check endpoint for Docker"""
|
||||
return {"status": "healthy"}
|
||||
@@ -1,25 +0,0 @@
|
||||
"""
|
||||
Database models for Mulita
|
||||
"""
|
||||
from app.models.user import User
|
||||
from app.models.photos import Photo
|
||||
from app.models.folders import Folder, SourceRoot
|
||||
from app.models.tags import Tag, PhotoTag
|
||||
from app.models.heaps import Heap, HeapPhoto
|
||||
from app.models.embeddings import Embedding
|
||||
from app.models.ocr_text import OCRText
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
|
||||
__all__ = [
|
||||
'User',
|
||||
'Photo',
|
||||
'Folder',
|
||||
'SourceRoot',
|
||||
'Tag',
|
||||
'PhotoTag',
|
||||
'Heap',
|
||||
'HeapPhoto',
|
||||
'Embedding',
|
||||
'OCRText',
|
||||
'FaceEmbedding',
|
||||
]
|
||||
@@ -1,24 +0,0 @@
|
||||
"""
|
||||
Embedding model — stores CLIP/SigLIP image embeddings via pgvector.
|
||||
|
||||
Composite PK (photo_id, model) allows re-embedding with newer models
|
||||
without clobbering old vectors.
|
||||
|
||||
Vector dimension is 768 to support SigLIP2 ViT-B/16 (the default).
|
||||
OpenCLIP ViT-B/32 (512-d) embeddings are zero-padded on insert so
|
||||
both models coexist in the same column. The padding is invisible to
|
||||
cosine similarity (zeros don't affect the angle).
|
||||
"""
|
||||
from sqlalchemy import Column, String, ForeignKey, DateTime, func
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Embedding(Base):
|
||||
__tablename__ = 'embeddings'
|
||||
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True)
|
||||
model = Column(String(64), primary_key=True) # e.g. 'siglip2_vitb16'
|
||||
vector = Column(Vector(512)) # OpenCLIP ViT-B/32 → 512-d
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -1,24 +0,0 @@
|
||||
"""
|
||||
Face embedding model — stores per-face detection + recognition vectors.
|
||||
|
||||
cluster_id FKs to tags.id where kind='face_cluster'. Null means
|
||||
unclustered (will be assigned by recluster_faces).
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, DateTime, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from pgvector.sqlalchemy import Vector
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class FaceEmbedding(Base):
|
||||
__tablename__ = 'face_embeddings'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
bbox = Column(JSONB) # [x1, y1, x2, y2] normalized 0-1
|
||||
vector = Column(Vector(512)) # ArcFace → 512-d
|
||||
cluster_id = Column(String, ForeignKey('tags.id', ondelete='SET NULL'), nullable=True, index=True)
|
||||
quality = Column(Float)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -1,58 +0,0 @@
|
||||
"""
|
||||
Folder and SourceRoot model definitions
|
||||
"""
|
||||
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey, Index
|
||||
from sqlalchemy.sql import func
|
||||
from sqlalchemy.orm import relationship
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
class SourceRoot(Base):
|
||||
__tablename__ = 'source_roots'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False)
|
||||
path = Column(String, unique=True, nullable=False)
|
||||
is_active = Column(Boolean, default=True)
|
||||
added_at = Column(DateTime, server_default=func.now())
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
|
||||
# Relationships
|
||||
folders = relationship("Folder", back_populates="source_root")
|
||||
|
||||
class Folder(Base):
|
||||
__tablename__ = 'folders'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False)
|
||||
path = Column(String, unique=True, nullable=False)
|
||||
parent_id = Column(String, ForeignKey('folders.id'))
|
||||
source_root_id = Column(String, ForeignKey('source_roots.id'))
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
photo_count = Column(Integer, default=0)
|
||||
last_scanned = Column(DateTime)
|
||||
|
||||
# "Hide from views" — when true, photos in this folder (and every
|
||||
# descendant folder) are excluded from cross-cutting views like
|
||||
# All Photos, Map, Tags, People, Search and the sidebar counts.
|
||||
# Photos are still scanned, thumbnailed and indexed — they just
|
||||
# stop showing up unless the user navigates directly to a folder
|
||||
# inside the hidden subtree. The effective flag is materialized
|
||||
# onto Photo.is_hidden so queries don't have to walk parent_id.
|
||||
is_hidden = Column(Boolean, nullable=False, default=False, server_default='false')
|
||||
|
||||
# Relationships
|
||||
source_root = relationship("SourceRoot", back_populates="folders")
|
||||
photos = relationship("Photo", backref="folder")
|
||||
|
||||
# Indexes
|
||||
__table_args__ = (
|
||||
Index('ix_folders_path', 'path'),
|
||||
Index('ix_folders_parent_id', 'parent_id'),
|
||||
Index('ix_folders_source_root_id', 'source_root_id'),
|
||||
)
|
||||
@@ -1,40 +0,0 @@
|
||||
"""
|
||||
Heap model definitions
|
||||
"""
|
||||
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey, Table, Index
|
||||
from sqlalchemy.sql import func
|
||||
from sqlalchemy.orm import relationship
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
# Association table for many-to-many relationship with additional fields
|
||||
heap_photos = Table(
|
||||
'heap_photos',
|
||||
Base.metadata,
|
||||
Column('heap_id', String, ForeignKey('heaps.id', ondelete='CASCADE'), primary_key=True),
|
||||
Column('photo_id', String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True),
|
||||
Column('added_at', DateTime, server_default=func.now()),
|
||||
Column('sort_order', Integer, default=0),
|
||||
Index('ix_heap_photos_heap_id', 'heap_id'),
|
||||
Index('ix_heap_photos_photo_id', 'photo_id'),
|
||||
)
|
||||
|
||||
class Heap(Base):
|
||||
__tablename__ = 'heaps'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False)
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
updated_at = Column(DateTime, onupdate=func.now())
|
||||
is_active = Column(Boolean, default=False) # For active heap feature
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
|
||||
# Relationships
|
||||
photos = relationship("Photo", secondary=heap_photos, backref="heaps")
|
||||
|
||||
class HeapPhoto:
|
||||
"""Helper class for heap-photo associations (not a table model)"""
|
||||
pass
|
||||
@@ -1,20 +0,0 @@
|
||||
"""
|
||||
OCR text model — stores text regions extracted from photos via rapidocr.
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, Text, DateTime, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class OCRText(Base):
|
||||
__tablename__ = 'ocr_text'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
text = Column(Text, nullable=False)
|
||||
language = Column(String(8), default='')
|
||||
confidence = Column(Float)
|
||||
bbox = Column(JSONB) # [x1, y1, x2, y2] normalized 0-1
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -1,121 +0,0 @@
|
||||
"""
|
||||
Photo model definition
|
||||
"""
|
||||
from sqlalchemy import Column, String, Integer, Float, Boolean, DateTime, ForeignKey, Text, Index
|
||||
from sqlalchemy.sql import func
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
class Photo(Base):
|
||||
__tablename__ = 'photos'
|
||||
|
||||
# Primary key
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
|
||||
# File information
|
||||
filepath = Column(String, unique=True, nullable=False)
|
||||
filename = Column(String, nullable=False)
|
||||
folder_id = Column(String, ForeignKey('folders.id'))
|
||||
file_hash = Column(String, index=True) # SHA-256 hash for duplicate detection
|
||||
|
||||
# Media information
|
||||
media_type = Column(String, nullable=False) # 'photo' | 'video' | 'raw' | 'heic'
|
||||
original_format = Column(String) # 'CR3', 'NEF', 'HEIC', 'MP4', 'JPEG', etc.
|
||||
width = Column(Integer)
|
||||
height = Column(Integer)
|
||||
file_size = Column(Integer)
|
||||
|
||||
# Timestamps
|
||||
taken_at = Column(DateTime) # from EXIF DateTimeOriginal, fallback to file mtime
|
||||
taken_at_source = Column(String) # 'exif' | 'filesystem' | 'manual'
|
||||
added_at = Column(DateTime, server_default=func.now())
|
||||
updated_at = Column(DateTime, onupdate=func.now())
|
||||
|
||||
# Discard status. The DB column names stay is_trashed/trashed_at to avoid
|
||||
# a migration; only the Python attribute name reflects the rename.
|
||||
is_discarded = Column('is_trashed', Boolean, default=False)
|
||||
discarded_at = Column('trashed_at', DateTime)
|
||||
|
||||
# "Hidden from views" — materialized from Folder.is_hidden walking
|
||||
# the ancestry chain. True iff any ancestor folder (including the
|
||||
# photo's direct folder) is hidden. Cross-cutting queries filter
|
||||
# `AND NOT is_hidden`; per-folder browses ignore the flag so the
|
||||
# user can still open a hidden folder and see its contents. The
|
||||
# column is maintained by two places: the scanner sets it on new
|
||||
# rows, and POST /folders/{id}/hide recomputes it on toggle.
|
||||
is_hidden = Column(Boolean, nullable=False, default=False, server_default='false', index=True)
|
||||
|
||||
# "Capture date is probably wrong" — denormalized from the folder/filename
|
||||
# date-guesser. Set at scan time and recomputed on every taken_at edit so
|
||||
# the filter bar can query it directly. See services/date_guess.py for
|
||||
# the heuristic; kept as a stored column because recomputing on every
|
||||
# list query would mean running the regex stack across thousands of rows.
|
||||
has_date_warning = Column(Boolean, nullable=False, default=False, server_default='false', index=True)
|
||||
|
||||
# Thumbnail paths
|
||||
thumb_small = Column(String) # path to 240px thumb
|
||||
thumb_medium = Column(String) # path to 640px thumb
|
||||
thumb_large = Column(String) # path to 1280px thumb
|
||||
|
||||
# Processing status
|
||||
processing_status = Column(String, default='pending') # 'pending' | 'processing' | 'completed' | 'failed'
|
||||
processing_error = Column(Text)
|
||||
|
||||
# Metadata
|
||||
exif_json = Column(Text) # full EXIF/XMP blob as JSON
|
||||
|
||||
# GPS coordinates extracted from EXIF, in signed decimal degrees
|
||||
# (S latitude / W longitude are negative). Stored as first-class columns
|
||||
# so the Map view and any future location filters can query/index them
|
||||
# without parsing exif_json on every request.
|
||||
latitude = Column(Float)
|
||||
longitude = Column(Float)
|
||||
|
||||
# User-editable fields
|
||||
user_title = Column(String)
|
||||
user_notes = Column(Text)
|
||||
rating = Column(Integer, default=0) # 0-5 stars
|
||||
color_label = Column(String) # 'red'|'orange'|'yellow'|'green'|'blue'|'purple'|NULL
|
||||
# Note: is_rejected was merged into is_discarded (a single soft "discarded"
|
||||
# concept). is_picked was unified with active-heap membership — picking a
|
||||
# photo just means adding it to the active heap. Both DB columns may still
|
||||
# exist on legacy installs but are no longer read or written.
|
||||
|
||||
# Duplicate detection.
|
||||
#
|
||||
# - file_hash (above): SHA-256 of the raw bytes. Catches byte-identical
|
||||
# copies but not visually-identical re-encodes / resizes / screenshots.
|
||||
# - phash: 16-char hex of a 64-bit perceptual hash, computed by the
|
||||
# thumbs worker from the decoded original frame. Robust to resize and
|
||||
# re-compression — this is what actually identifies "the same photo
|
||||
# saved twice with different JPEG quality".
|
||||
# - duplicate_group_id: shared by every photo in the same duplicate
|
||||
# cluster. Maintained by app.services.duplicates.regroup_duplicates,
|
||||
# not on individual writes — recomputed in batches after scans / on
|
||||
# demand from the Settings panel.
|
||||
# - is_duplicate: derived boolean (group_id IS NOT NULL). Kept as a real
|
||||
# column so the existing PhotoThumbnail badge and /library/stats
|
||||
# duplicates count don't have to change.
|
||||
is_duplicate = Column(Boolean, default=False)
|
||||
phash = Column(String(16), index=True)
|
||||
duplicate_group_id = Column(String, index=True)
|
||||
|
||||
# Live photo support
|
||||
live_photo_video_id = Column(String, ForeignKey('photos.id'))
|
||||
|
||||
# Indexes for performance
|
||||
__table_args__ = (
|
||||
Index('ix_photos_taken_at', 'taken_at'),
|
||||
Index('ix_photos_folder_id', 'folder_id'),
|
||||
Index('ix_photos_is_trashed', 'is_trashed'),
|
||||
Index('ix_photos_rating', 'rating'),
|
||||
Index('ix_photos_color_label', 'color_label'),
|
||||
Index('ix_photos_media_type', 'media_type'),
|
||||
Index('ix_photos_processing_status', 'processing_status'),
|
||||
Index('ix_photos_lat_lon', 'latitude', 'longitude'),
|
||||
)
|
||||
@@ -1,62 +0,0 @@
|
||||
"""
|
||||
Tag model definitions.
|
||||
|
||||
Tags are unified across user-created tags, ML-detected objects, scene
|
||||
labels, and face clusters via the `kind` column. The `photo_tags`
|
||||
association carries per-photo ML metadata (confidence, bounding box,
|
||||
source model).
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, Table, Index, UniqueConstraint
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
# Association table for many-to-many relationship
|
||||
photo_tags = Table(
|
||||
'photo_tags',
|
||||
Base.metadata,
|
||||
Column('photo_id', String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True),
|
||||
Column('tag_id', String, ForeignKey('tags.id', ondelete='CASCADE'), primary_key=True),
|
||||
# ML metadata — null for user-applied tags
|
||||
Column('confidence', Float, nullable=True),
|
||||
Column('bbox', JSONB, nullable=True), # [x1, y1, x2, y2] normalized 0-1
|
||||
Column('source', String, nullable=True), # e.g. "vision:yolov8n", "vision:sface"
|
||||
Index('ix_photo_tags_photo_id', 'photo_id'),
|
||||
Index('ix_photo_tags_tag_id', 'tag_id'),
|
||||
)
|
||||
|
||||
class Tag(Base):
|
||||
__tablename__ = 'tags'
|
||||
__table_args__ = (
|
||||
UniqueConstraint('name', 'kind', 'user_id', name='uq_tags_name_kind_user'),
|
||||
)
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False, index=True)
|
||||
color = Column(String) # Hex color code for UI display
|
||||
|
||||
# Owner
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=True, index=True)
|
||||
|
||||
# Tag classification
|
||||
kind = Column(String, nullable=False, default='user', index=True)
|
||||
# kind values: 'user' | 'object' | 'scene' | 'face_cluster'
|
||||
|
||||
# Which model produced this tag (null for user-created)
|
||||
source = Column(String, nullable=True)
|
||||
# e.g. "vision:yolov8n", "vision:sface", null
|
||||
|
||||
# For face clusters: the photo used as the cluster representative thumbnail
|
||||
representative_photo_id = Column(
|
||||
String, ForeignKey('photos.id', ondelete='SET NULL'), nullable=True
|
||||
)
|
||||
|
||||
# Relationships
|
||||
photos = relationship("Photo", secondary=photo_tags, backref="tags")
|
||||
|
||||
|
||||
class PhotoTag:
|
||||
"""Helper class for photo-tag associations (not a table model)"""
|
||||
pass
|
||||
@@ -1,23 +0,0 @@
|
||||
"""
|
||||
User model definition
|
||||
"""
|
||||
from sqlalchemy import Column, String, Boolean, DateTime
|
||||
from sqlalchemy.sql import func
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = 'users'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
username = Column(String(50), unique=True, nullable=False, index=True)
|
||||
email = Column(String, unique=True, nullable=True)
|
||||
hashed_password = Column(String, nullable=False)
|
||||
role = Column(String, nullable=False, default='user') # 'admin' | 'user'
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
|
||||
# Absolute path to this user's photo directory (e.g., "/photos/daniel")
|
||||
media_path = Column(String, nullable=False)
|
||||
@@ -1,260 +0,0 @@
|
||||
"""
|
||||
Admin router — user management and app configuration.
|
||||
All endpoints require admin role.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from typing import Optional, List
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func as sa_func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth import hash_password
|
||||
from app.database import get_db
|
||||
from app.dependencies import require_admin
|
||||
from app.models.user import User
|
||||
from app.models.photos import Photo
|
||||
from app.models.folders import SourceRoot
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class CreateUserRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
role: str = "user" # 'admin' | 'user'
|
||||
|
||||
class UpdateUserRequest(BaseModel):
|
||||
role: Optional[str] = None
|
||||
is_active: Optional[bool] = None
|
||||
new_password: Optional[str] = None
|
||||
|
||||
class UserDetailResponse(BaseModel):
|
||||
id: str
|
||||
username: str
|
||||
email: Optional[str]
|
||||
role: str
|
||||
is_active: bool
|
||||
media_path: str
|
||||
created_at: Optional[str]
|
||||
photo_count: int = 0
|
||||
|
||||
class UserListResponse(BaseModel):
|
||||
users: List[UserDetailResponse]
|
||||
total: int
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/users", response_model=UserListResponse)
|
||||
async def list_users(
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all users with their photo counts."""
|
||||
result = await db.execute(select(User).order_by(User.created_at))
|
||||
users = result.scalars().all()
|
||||
|
||||
user_list = []
|
||||
for u in users:
|
||||
count_result = await db.execute(
|
||||
select(sa_func.count(Photo.id)).where(Photo.user_id == u.id)
|
||||
)
|
||||
photo_count = count_result.scalar() or 0
|
||||
|
||||
user_list.append(UserDetailResponse(
|
||||
id=u.id,
|
||||
username=u.username,
|
||||
email=u.email,
|
||||
role=u.role,
|
||||
is_active=u.is_active,
|
||||
media_path=u.media_path,
|
||||
created_at=u.created_at.isoformat() if u.created_at else None,
|
||||
photo_count=photo_count,
|
||||
))
|
||||
|
||||
return UserListResponse(users=user_list, total=len(user_list))
|
||||
|
||||
|
||||
@router.post("/users", status_code=201, response_model=UserDetailResponse)
|
||||
async def create_user(
|
||||
body: CreateUserRequest,
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Create a new user. Creates their media directory and source root."""
|
||||
if body.role not in ("admin", "user"):
|
||||
raise HTTPException(status_code=400, detail="Role must be 'admin' or 'user'")
|
||||
if len(body.username.strip()) < 2:
|
||||
raise HTTPException(status_code=400, detail="Username must be at least 2 characters")
|
||||
if len(body.password) < 6:
|
||||
raise HTTPException(status_code=400, detail="Password must be at least 6 characters")
|
||||
|
||||
# Check for duplicate username
|
||||
existing = await db.execute(
|
||||
select(User).where(User.username == body.username.strip())
|
||||
)
|
||||
if existing.scalar_one_or_none() is not None:
|
||||
raise HTTPException(status_code=409, detail="Username already taken")
|
||||
|
||||
media_path = os.path.join(settings.photo_dirs, body.username.strip())
|
||||
os.makedirs(media_path, exist_ok=True)
|
||||
|
||||
user = User(
|
||||
username=body.username.strip(),
|
||||
hashed_password=hash_password(body.password),
|
||||
role=body.role,
|
||||
media_path=media_path,
|
||||
)
|
||||
db.add(user)
|
||||
await db.flush() # get user.id before creating source root
|
||||
|
||||
source_root = SourceRoot(
|
||||
name=f"{user.username}'s Library",
|
||||
path=media_path,
|
||||
user_id=user.id,
|
||||
)
|
||||
db.add(source_root)
|
||||
await db.commit()
|
||||
|
||||
logger.info(f"Admin '{admin.username}' created user '{user.username}' (role={user.role})")
|
||||
|
||||
return UserDetailResponse(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
email=user.email,
|
||||
role=user.role,
|
||||
is_active=user.is_active,
|
||||
media_path=user.media_path,
|
||||
created_at=user.created_at.isoformat() if user.created_at else None,
|
||||
photo_count=0,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/users/{user_id}", response_model=UserDetailResponse)
|
||||
async def get_user(
|
||||
user_id: str,
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get a single user's details."""
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
|
||||
count_result = await db.execute(
|
||||
select(sa_func.count(Photo.id)).where(Photo.user_id == user.id)
|
||||
)
|
||||
photo_count = count_result.scalar() or 0
|
||||
|
||||
return UserDetailResponse(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
email=user.email,
|
||||
role=user.role,
|
||||
is_active=user.is_active,
|
||||
media_path=user.media_path,
|
||||
created_at=user.created_at.isoformat() if user.created_at else None,
|
||||
photo_count=photo_count,
|
||||
)
|
||||
|
||||
|
||||
@router.patch("/users/{user_id}", response_model=UserDetailResponse)
|
||||
async def update_user(
|
||||
user_id: str,
|
||||
body: UpdateUserRequest,
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update a user's role, active status, or password."""
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
|
||||
if body.role is not None:
|
||||
if body.role not in ("admin", "user"):
|
||||
raise HTTPException(status_code=400, detail="Role must be 'admin' or 'user'")
|
||||
# Prevent demoting the last admin
|
||||
if user.role == "admin" and body.role == "user":
|
||||
admin_count = (await db.execute(
|
||||
select(sa_func.count(User.id)).where(User.role == "admin", User.is_active == True)
|
||||
)).scalar()
|
||||
if admin_count <= 1:
|
||||
raise HTTPException(status_code=400, detail="Cannot demote the last admin")
|
||||
user.role = body.role
|
||||
|
||||
if body.is_active is not None:
|
||||
# Prevent deactivating the last admin
|
||||
if user.role == "admin" and not body.is_active:
|
||||
admin_count = (await db.execute(
|
||||
select(sa_func.count(User.id)).where(User.role == "admin", User.is_active == True)
|
||||
)).scalar()
|
||||
if admin_count <= 1:
|
||||
raise HTTPException(status_code=400, detail="Cannot deactivate the last admin")
|
||||
user.is_active = body.is_active
|
||||
|
||||
if body.new_password is not None:
|
||||
if len(body.new_password) < 6:
|
||||
raise HTTPException(status_code=400, detail="Password must be at least 6 characters")
|
||||
user.hashed_password = hash_password(body.new_password)
|
||||
|
||||
await db.commit()
|
||||
|
||||
count_result = await db.execute(
|
||||
select(sa_func.count(Photo.id)).where(Photo.user_id == user.id)
|
||||
)
|
||||
photo_count = count_result.scalar() or 0
|
||||
|
||||
return UserDetailResponse(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
email=user.email,
|
||||
role=user.role,
|
||||
is_active=user.is_active,
|
||||
media_path=user.media_path,
|
||||
created_at=user.created_at.isoformat() if user.created_at else None,
|
||||
photo_count=photo_count,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/users/{user_id}")
|
||||
async def delete_user(
|
||||
user_id: str,
|
||||
admin: User = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Soft-delete a user by deactivating them. Media is preserved."""
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
|
||||
if user.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="Cannot delete yourself")
|
||||
|
||||
# Prevent deleting the last admin
|
||||
if user.role == "admin":
|
||||
admin_count = (await db.execute(
|
||||
select(sa_func.count(User.id)).where(User.role == "admin", User.is_active == True)
|
||||
)).scalar()
|
||||
if admin_count <= 1:
|
||||
raise HTTPException(status_code=400, detail="Cannot delete the last admin")
|
||||
|
||||
user.is_active = False
|
||||
await db.commit()
|
||||
|
||||
logger.info(f"Admin '{admin.username}' deactivated user '{user.username}'")
|
||||
return {"status": "ok", "detail": f"User '{user.username}' deactivated"}
|
||||
@@ -1,202 +0,0 @@
|
||||
"""
|
||||
Authentication router — login, token refresh, profile, first-run setup.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func as sa_func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth import hash_password, verify_password, create_access_token, create_refresh_token, decode_token
|
||||
from app.database import get_db
|
||||
from app.dependencies import get_current_user
|
||||
from app.models.user import User
|
||||
from app.models.folders import SourceRoot
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / response schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
access_token: str
|
||||
refresh_token: str
|
||||
token_type: str = "bearer"
|
||||
|
||||
class RefreshRequest(BaseModel):
|
||||
refresh_token: str
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
id: str
|
||||
username: str
|
||||
email: Optional[str]
|
||||
role: str
|
||||
is_active: bool
|
||||
created_at: Optional[str]
|
||||
|
||||
class SetupRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
current_password: str
|
||||
new_password: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/login", response_model=TokenResponse)
|
||||
async def login(body: LoginRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""Authenticate with username + password, receive JWT tokens."""
|
||||
result = await db.execute(
|
||||
select(User).where(User.username == body.username)
|
||||
)
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if user is None or not verify_password(body.password, user.hashed_password):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid username or password",
|
||||
)
|
||||
if not user.is_active:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Account is deactivated",
|
||||
)
|
||||
|
||||
return TokenResponse(
|
||||
access_token=create_access_token(user.id, user.role),
|
||||
refresh_token=create_refresh_token(user.id),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/refresh", response_model=TokenResponse)
|
||||
async def refresh_token(body: RefreshRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""Exchange a valid refresh token for a new access + refresh pair."""
|
||||
try:
|
||||
payload = decode_token(body.refresh_token)
|
||||
if payload.get("type") != "refresh":
|
||||
raise ValueError("not a refresh token")
|
||||
user_id = payload["sub"]
|
||||
except Exception:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or expired refresh token",
|
||||
)
|
||||
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None or not user.is_active:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="User not found or deactivated",
|
||||
)
|
||||
|
||||
return TokenResponse(
|
||||
access_token=create_access_token(user.id, user.role),
|
||||
refresh_token=create_refresh_token(user.id),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserResponse)
|
||||
async def get_me(current_user: User = Depends(get_current_user)):
|
||||
"""Return the authenticated user's profile."""
|
||||
return UserResponse(
|
||||
id=current_user.id,
|
||||
username=current_user.username,
|
||||
email=current_user.email,
|
||||
role=current_user.role,
|
||||
is_active=current_user.is_active,
|
||||
created_at=current_user.created_at.isoformat() if current_user.created_at else None,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/change-password")
|
||||
async def change_password(
|
||||
body: ChangePasswordRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Change the authenticated user's password."""
|
||||
if not verify_password(body.current_password, current_user.hashed_password):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Current password is incorrect",
|
||||
)
|
||||
current_user.hashed_password = hash_password(body.new_password)
|
||||
await db.commit()
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.post("/setup", response_model=TokenResponse, status_code=201)
|
||||
async def setup(body: SetupRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""First-run only: create the initial admin account.
|
||||
|
||||
Returns 409 if any user already exists. This endpoint is
|
||||
unauthenticated by design — it can only run once.
|
||||
"""
|
||||
count = (await db.execute(select(sa_func.count(User.id)))).scalar()
|
||||
if count > 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="Setup already completed — users exist",
|
||||
)
|
||||
|
||||
if len(body.username.strip()) < 2:
|
||||
raise HTTPException(status_code=400, detail="Username must be at least 2 characters")
|
||||
if len(body.password) < 6:
|
||||
raise HTTPException(status_code=400, detail="Password must be at least 6 characters")
|
||||
|
||||
# Every user — including the initial admin — gets their own subfolder
|
||||
# under the photo mount root. Nobody owns the root directory itself.
|
||||
media_path = os.path.join(settings.photo_dirs, body.username.strip())
|
||||
os.makedirs(media_path, exist_ok=True)
|
||||
|
||||
user = User(
|
||||
username=body.username.strip(),
|
||||
hashed_password=hash_password(body.password),
|
||||
role="admin",
|
||||
media_path=media_path,
|
||||
)
|
||||
db.add(user)
|
||||
await db.flush() # get user.id before creating source root
|
||||
|
||||
source_root = SourceRoot(
|
||||
name=f"{user.username}'s Library",
|
||||
path=media_path,
|
||||
user_id=user.id,
|
||||
)
|
||||
db.add(source_root)
|
||||
await db.commit()
|
||||
|
||||
logger.info(f"Initial admin account created: {user.username}")
|
||||
|
||||
return TokenResponse(
|
||||
access_token=create_access_token(user.id, user.role),
|
||||
refresh_token=create_refresh_token(user.id),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def auth_status(db: AsyncSession = Depends(get_db)):
|
||||
"""Public endpoint: returns whether setup has been completed.
|
||||
|
||||
The frontend calls this to decide whether to show the setup page
|
||||
or the login page.
|
||||
"""
|
||||
count = (await db.execute(select(sa_func.count(User.id)))).scalar()
|
||||
return {"setup_completed": count > 0}
|
||||
@@ -1,97 +0,0 @@
|
||||
"""
|
||||
Discard API router
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from fastapi import APIRouter, Depends, HTTPException, Body
|
||||
from sqlalchemy import select, and_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("")
|
||||
async def list_discarded(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""List discarded photos"""
|
||||
result = await db.execute(
|
||||
select(Photo).where(Photo.is_discarded == True, Photo.user_id == current_user.id)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
return photos
|
||||
|
||||
@router.post("/restore")
|
||||
async def restore_photos(photo_ids: list[str], db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Restore photos from the discard pile"""
|
||||
result = await db.execute(
|
||||
select(Photo).where(and_(Photo.id.in_(photo_ids), Photo.is_discarded == True, Photo.user_id == current_user.id))
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
|
||||
for photo in photos:
|
||||
photo.is_discarded = False
|
||||
photo.discarded_at = None
|
||||
|
||||
await db.commit()
|
||||
return {"status": "success", "restored": len(photos)}
|
||||
|
||||
@router.delete("/empty")
|
||||
async def empty_discard(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Permanently delete all discarded photos and unlink their files from
|
||||
disk. Failures on individual files are logged but don't abort the batch.
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(Photo).where(Photo.is_discarded == True, Photo.user_id == current_user.id)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
return await _permanently_delete(db, photos)
|
||||
|
||||
|
||||
@router.delete("")
|
||||
async def delete_discarded(
|
||||
photo_ids: list[str] = Body(..., embed=True),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Permanently delete a specific subset of discarded photos. The photos
|
||||
must already be in the discard pile — non-discarded ids are skipped so
|
||||
this can never bypass the soft-delete safety net.
|
||||
"""
|
||||
if not photo_ids:
|
||||
return {"status": "success", "deleted": 0, "file_errors": 0}
|
||||
result = await db.execute(
|
||||
select(Photo).where(
|
||||
and_(Photo.id.in_(photo_ids), Photo.is_discarded == True, Photo.user_id == current_user.id)
|
||||
)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
return await _permanently_delete(db, photos)
|
||||
|
||||
|
||||
async def _permanently_delete(db: AsyncSession, photos: list[Photo]) -> dict:
|
||||
"""Shared helper: unlink files for the given photos and delete their
|
||||
rows. Per-file errors are counted but don't abort the batch.
|
||||
"""
|
||||
deleted = 0
|
||||
file_errors = 0
|
||||
for photo in photos:
|
||||
try:
|
||||
if photo.filepath and os.path.exists(photo.filepath):
|
||||
os.unlink(photo.filepath)
|
||||
except OSError as e:
|
||||
file_errors += 1
|
||||
logger.error(f"Failed to unlink {photo.filepath}: {e}")
|
||||
await db.delete(photo)
|
||||
deleted += 1
|
||||
|
||||
await db.commit()
|
||||
return {
|
||||
"status": "success",
|
||||
"deleted": deleted,
|
||||
"file_errors": file_errors,
|
||||
}
|
||||
@@ -1,553 +0,0 @@
|
||||
"""
|
||||
Folders API router. Source roots themselves are config-driven (PHOTO_DIRS
|
||||
in .env → backend bootstrap on startup) — adding or removing one is a
|
||||
docker-compose change. Sub-folders inside a source root can be created,
|
||||
renamed, and deleted from the UI; those changes are mirrored to disk.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from typing import Literal, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func, update as sql_update, delete as sql_delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Folder, SourceRoot, Photo
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user, get_user_folder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class FolderRename(BaseModel):
|
||||
name: str
|
||||
|
||||
|
||||
class FolderCreate(BaseModel):
|
||||
name: str
|
||||
parent_id: str # Folder.id (NOT a SourceRoot id)
|
||||
|
||||
|
||||
class FolderHide(BaseModel):
|
||||
hidden: bool
|
||||
|
||||
|
||||
def _validate_folder_name(name: str) -> str:
|
||||
"""Trim + sanity-check a folder name. Rejects names that contain a
|
||||
path separator or that resolve to a parent traversal — those would
|
||||
let the user escape the parent directory through this endpoint.
|
||||
"""
|
||||
name = (name or '').strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Name cannot be empty")
|
||||
if '/' in name or '\\' in name or name in ('.', '..'):
|
||||
raise HTTPException(status_code=400, detail="Invalid folder name")
|
||||
return name
|
||||
|
||||
@router.get("")
|
||||
async def get_folders(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Get all source folders"""
|
||||
# Get source roots instead of regular folders
|
||||
result = await db.execute(select(SourceRoot).where(SourceRoot.is_active == True, SourceRoot.user_id == current_user.id))
|
||||
source_roots = result.scalars().all()
|
||||
|
||||
folders_list = []
|
||||
for root in source_roots:
|
||||
# Get photo count for this source root
|
||||
folder_result = await db.execute(
|
||||
select(Folder).where(Folder.source_root_id == root.id)
|
||||
)
|
||||
folders = folder_result.scalars().all()
|
||||
photo_count = sum(f.photo_count for f in folders)
|
||||
|
||||
folders_list.append({
|
||||
"id": root.id,
|
||||
"name": root.name or os.path.basename(root.path),
|
||||
"path": root.path,
|
||||
"photo_count": photo_count
|
||||
})
|
||||
|
||||
return {"folders": folders_list}
|
||||
|
||||
@router.get("/tree")
|
||||
async def get_folder_tree(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Recursive folder tree, one root per active SourceRoot. The tree
|
||||
starts at the Folder row matching the SourceRoot.path (the scanner
|
||||
creates one for every walked directory), with the SourceRoot's
|
||||
display name overlaid so the top-level entry reads as "Library"
|
||||
instead of "/photos".
|
||||
|
||||
Returns a list of root nodes; each node has:
|
||||
{ id, name, path, photo_count, children: [...] }
|
||||
|
||||
photo_count is **recursive** — every node reports the total non-
|
||||
discarded photos in its own subtree, so the badge matches what the
|
||||
user sees when they click the row (which also filters recursively).
|
||||
|
||||
The stored Folder.photo_count column is intentionally NOT trusted;
|
||||
the scanner's bookkeeping for that field has historically been
|
||||
wrong (it leaks the global total into whichever folder os.walk
|
||||
visited last). We compute counts here from the photos table.
|
||||
|
||||
Sub-folders that physically belong to the same source root but
|
||||
weren't created on disk (e.g. the / row the scanner sometimes
|
||||
creates as a parent walk) are skipped via path-prefix filtering.
|
||||
"""
|
||||
sr_result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active == True, SourceRoot.user_id == current_user.id) # noqa: E712
|
||||
)
|
||||
source_roots = sr_result.scalars().all()
|
||||
|
||||
out = []
|
||||
for sr in source_roots:
|
||||
# Folders physically inside this source root, by path prefix.
|
||||
prefix = os.path.normpath(sr.path).rstrip(os.sep)
|
||||
f_result = await db.execute(
|
||||
select(Folder).where(
|
||||
Folder.source_root_id == sr.id,
|
||||
# Either the folder IS the source root, or it sits beneath it.
|
||||
(Folder.path == prefix) | (Folder.path.like(prefix + os.sep + '%'))
|
||||
)
|
||||
)
|
||||
folders = f_result.scalars().all()
|
||||
if not folders:
|
||||
continue
|
||||
|
||||
# Direct (non-recursive) photo counts per folder, computed from
|
||||
# the photos table. Excludes discarded AND hidden photos so the
|
||||
# sidebar badge matches the "All Photos"-style cross-cutting
|
||||
# views. Users can still click into a hidden folder and see its
|
||||
# contents; the badge count simply won't reflect those photos.
|
||||
folder_ids = [f.id for f in folders]
|
||||
direct_counts: dict[str, int] = {}
|
||||
if folder_ids:
|
||||
count_result = await db.execute(
|
||||
select(Photo.folder_id, func.count(Photo.id))
|
||||
.where(
|
||||
Photo.is_discarded == False, # noqa: E712
|
||||
Photo.is_hidden == False, # noqa: E712
|
||||
Photo.folder_id.in_(folder_ids),
|
||||
)
|
||||
.group_by(Photo.folder_id)
|
||||
)
|
||||
direct_counts = {row[0]: int(row[1]) for row in count_result.all()}
|
||||
|
||||
# Build a path → node map so we can attach children regardless of
|
||||
# parent_id consistency. We populate photo_count with the direct
|
||||
# count first, then accumulate descendants in a post-order pass.
|
||||
# `is_hidden` on each node carries the user-set folder flag (NOT
|
||||
# the effective ancestry flag) so the frontend can render the
|
||||
# hidden icon on the exact folder the user toggled.
|
||||
nodes = {
|
||||
f.path: {
|
||||
"id": f.id,
|
||||
"name": f.name or os.path.basename(f.path),
|
||||
"path": f.path,
|
||||
"photo_count": direct_counts.get(f.id, 0),
|
||||
"is_hidden": bool(f.is_hidden),
|
||||
"children": [],
|
||||
}
|
||||
for f in folders
|
||||
}
|
||||
|
||||
root_node = None
|
||||
for f in folders:
|
||||
node = nodes[f.path]
|
||||
if f.path == prefix:
|
||||
root_node = node
|
||||
# Override the display name with the source root's label.
|
||||
node["name"] = sr.name or node["name"]
|
||||
continue
|
||||
parent_path = os.path.normpath(os.path.dirname(f.path))
|
||||
parent = nodes.get(parent_path)
|
||||
if parent is not None:
|
||||
parent["children"].append(node)
|
||||
# If parent isn't in the set (orphan from a partial scan), drop
|
||||
# the node — it can't be rendered consistently.
|
||||
|
||||
if root_node is not None:
|
||||
# Sort children alphabetically at every level.
|
||||
def sort_recursive(n):
|
||||
n["children"].sort(key=lambda c: c["name"].lower())
|
||||
for c in n["children"]:
|
||||
sort_recursive(c)
|
||||
sort_recursive(root_node)
|
||||
|
||||
# Post-order: each node's recursive count is its own direct
|
||||
# count plus the sum of every descendant's recursive count.
|
||||
def accumulate(n) -> int:
|
||||
total = n["photo_count"]
|
||||
for c in n["children"]:
|
||||
total += accumulate(c)
|
||||
n["photo_count"] = total
|
||||
return total
|
||||
accumulate(root_node)
|
||||
|
||||
out.append(root_node)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
@router.patch("/{folder_id}")
|
||||
async def rename_folder(
|
||||
folder_id: str,
|
||||
body: FolderRename,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Rename a folder. Two cases:
|
||||
|
||||
- SourceRoot id → just change the display label. The on-disk path
|
||||
is owned by the docker mount and never moves.
|
||||
- Folder id → rename the directory on disk AND update every
|
||||
descendant Folder.path + Photo.filepath that
|
||||
lived under the old prefix. Refuses to rename
|
||||
the source-root folder itself (= the row that
|
||||
matches the SourceRoot.path) because that would
|
||||
require renaming the docker mount.
|
||||
"""
|
||||
name = _validate_folder_name(body.name)
|
||||
|
||||
# Try SourceRoot first (display-only rename).
|
||||
sr_result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
source_root = sr_result.scalar_one_or_none()
|
||||
if source_root:
|
||||
source_root.name = name
|
||||
await db.commit()
|
||||
return {
|
||||
"id": source_root.id,
|
||||
"name": source_root.name,
|
||||
"path": source_root.path,
|
||||
}
|
||||
|
||||
# Otherwise it's a Folder row.
|
||||
folder = await get_user_folder(folder_id, current_user, db)
|
||||
|
||||
# Refuse to rename the bare source root mount through here.
|
||||
sr_check = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder.source_root_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
sr = sr_check.scalar_one_or_none()
|
||||
if sr and os.path.normpath(folder.path) == os.path.normpath(sr.path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot rename the source root mount; rename the docker mount instead.",
|
||||
)
|
||||
|
||||
old_path = os.path.normpath(folder.path).rstrip(os.sep)
|
||||
parent_dir = os.path.dirname(old_path)
|
||||
new_path = os.path.join(parent_dir, name)
|
||||
|
||||
if os.path.exists(new_path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"A folder named '{name}' already exists here",
|
||||
)
|
||||
|
||||
try:
|
||||
shutil.move(old_path, new_path)
|
||||
except OSError as e:
|
||||
raise HTTPException(status_code=500, detail=f"Rename failed: {e}")
|
||||
|
||||
# Update folder paths: this row + every descendant. SQLite REPLACE
|
||||
# rewrites the prefix; we use the trailing separator on the LIKE
|
||||
# pattern so a folder named "foo" doesn't accidentally match "foobar".
|
||||
await db.execute(
|
||||
sql_update(Folder)
|
||||
.where(Folder.id == folder.id)
|
||||
.values(path=new_path, name=name)
|
||||
)
|
||||
descendant_prefix = old_path + os.sep
|
||||
descendants = await db.execute(
|
||||
select(Folder).where(Folder.path.like(descendant_prefix + '%'))
|
||||
)
|
||||
for d in descendants.scalars().all():
|
||||
d.path = new_path + d.path[len(old_path):]
|
||||
|
||||
# Update every photo whose filepath lives under the old prefix.
|
||||
photos_result = await db.execute(
|
||||
select(Photo).where(Photo.filepath.like(descendant_prefix + '%'))
|
||||
)
|
||||
for p in photos_result.scalars().all():
|
||||
p.filepath = new_path + p.filepath[len(old_path):]
|
||||
# Photos directly inside this folder (not in a subdir) won't match
|
||||
# the descendant_prefix LIKE if their old path was old_path + '/file'
|
||||
# — actually they DO match, since 'oldpath/file' starts with
|
||||
# 'oldpath/'. So the loop above already covers them.
|
||||
|
||||
await db.commit()
|
||||
return {
|
||||
"id": folder.id,
|
||||
"name": folder.name,
|
||||
"path": folder.path,
|
||||
}
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_folder(body: FolderCreate, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Create a new sub-folder under an existing Folder. Mirrors the
|
||||
create to disk so the next scan sees it. Body: { name, parent_id }.
|
||||
parent_id MUST be an existing Folder row id (any descendant of a
|
||||
source root); creating a brand-new top-level mount is a docker
|
||||
operation, not a UI one.
|
||||
"""
|
||||
name = _validate_folder_name(body.name)
|
||||
|
||||
parent = await get_user_folder(body.parent_id, current_user, db)
|
||||
|
||||
new_path = os.path.join(parent.path, name)
|
||||
if os.path.exists(new_path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"A folder named '{name}' already exists here",
|
||||
)
|
||||
|
||||
try:
|
||||
os.makedirs(new_path, exist_ok=False)
|
||||
except OSError as e:
|
||||
raise HTTPException(status_code=500, detail=f"Create failed: {e}")
|
||||
|
||||
new_folder = Folder(
|
||||
name=name,
|
||||
path=new_path,
|
||||
source_root_id=parent.source_root_id,
|
||||
user_id=current_user.id,
|
||||
photo_count=0,
|
||||
)
|
||||
db.add(new_folder)
|
||||
await db.commit()
|
||||
await db.refresh(new_folder)
|
||||
return {
|
||||
"id": new_folder.id,
|
||||
"name": new_folder.name,
|
||||
"path": new_folder.path,
|
||||
"parent_id": parent.id,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{folder_id}")
|
||||
async def delete_folder(
|
||||
folder_id: str,
|
||||
mode: Literal['discard', 'permanent'] = Query('discard'),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Delete a folder. Behavior depends on mode:
|
||||
|
||||
- mode=discard (default): mark every photo whose filepath lives
|
||||
under this folder as is_discarded=true. The folder row, its
|
||||
descendant rows, and the on-disk directory are LEFT INTACT —
|
||||
the user can still recover photos from the discard pile, and
|
||||
a re-scan won't double-import them.
|
||||
|
||||
- mode=permanent: unlink every photo file under this folder,
|
||||
remove the photo + folder rows from the DB, and rmtree the
|
||||
on-disk directory. Irreversible.
|
||||
|
||||
Refuses to delete the bare source-root mount in either mode (deleting
|
||||
the docker mount through the UI would be a footgun).
|
||||
"""
|
||||
folder = await get_user_folder(folder_id, current_user, db)
|
||||
|
||||
sr_check = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder.source_root_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
sr = sr_check.scalar_one_or_none()
|
||||
if sr and os.path.normpath(folder.path) == os.path.normpath(sr.path):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot delete the source root mount through the UI",
|
||||
)
|
||||
|
||||
folder_path = os.path.normpath(folder.path).rstrip(os.sep)
|
||||
descendant_prefix = folder_path + os.sep
|
||||
|
||||
# Collect every photo under this folder OR any descendant. We match
|
||||
# by filepath prefix instead of folder_id because that catches photos
|
||||
# in nested subfolders without a recursive folder walk.
|
||||
photos_result = await db.execute(
|
||||
select(Photo).where(
|
||||
(Photo.filepath == folder_path)
|
||||
| (Photo.filepath.like(descendant_prefix + '%'))
|
||||
)
|
||||
)
|
||||
photos = photos_result.scalars().all()
|
||||
|
||||
if mode == 'discard':
|
||||
from datetime import datetime
|
||||
now = datetime.utcnow()
|
||||
for p in photos:
|
||||
p.is_discarded = True
|
||||
p.discarded_at = now
|
||||
await db.commit()
|
||||
return {
|
||||
"status": "success",
|
||||
"mode": "discard",
|
||||
"discarded": len(photos),
|
||||
}
|
||||
|
||||
# mode == 'permanent'
|
||||
file_errors = 0
|
||||
for p in photos:
|
||||
try:
|
||||
if p.filepath and os.path.exists(p.filepath):
|
||||
os.unlink(p.filepath)
|
||||
except OSError as e:
|
||||
file_errors += 1
|
||||
logger.error(f"Failed to unlink {p.filepath}: {e}")
|
||||
await db.delete(p)
|
||||
|
||||
# Delete this folder + every descendant Folder row.
|
||||
await db.execute(
|
||||
sql_delete(Folder).where(
|
||||
(Folder.id == folder.id)
|
||||
| (Folder.path.like(descendant_prefix + '%'))
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
if os.path.isdir(folder_path):
|
||||
shutil.rmtree(folder_path)
|
||||
except OSError as e:
|
||||
logger.error(f"Failed to rmtree {folder_path}: {e}")
|
||||
# Don't raise — DB rows are already gone, leaving an orphan
|
||||
# directory is the lesser evil.
|
||||
|
||||
await db.commit()
|
||||
return {
|
||||
"status": "success",
|
||||
"mode": "permanent",
|
||||
"deleted_photos": len(photos),
|
||||
"file_errors": file_errors,
|
||||
}
|
||||
|
||||
|
||||
async def _recompute_photo_hidden_flags(db: AsyncSession) -> None:
|
||||
"""Rematerialize photos.is_hidden from the full folder ancestry.
|
||||
|
||||
`photos.is_hidden` is true iff any ancestor folder in the photo's
|
||||
folder chain (including the folder the photo is directly in) has
|
||||
`folders.is_hidden = true`. Rather than do a recursive walk in
|
||||
Python, we lean on Postgres's WITH RECURSIVE to compute each
|
||||
folder's effective hidden state in a single query, then join on
|
||||
photos to bulk-update the flag.
|
||||
|
||||
Called after any folders.is_hidden toggle AND after moving photos
|
||||
between folders, since the photo's effective-hidden state can
|
||||
change even when no folder flag changes. Cheap — one O(folders)
|
||||
CTE + one O(photos) UPDATE. On a 13k-photo library this runs in
|
||||
under 50ms.
|
||||
"""
|
||||
from sqlalchemy import text as _text
|
||||
|
||||
await db.execute(
|
||||
_text("""
|
||||
WITH RECURSIVE folder_chain AS (
|
||||
-- Base: source-root folders (no parent_id). Their own
|
||||
-- is_hidden is the starting effective value.
|
||||
SELECT id, is_hidden AS effective_hidden
|
||||
FROM folders
|
||||
WHERE parent_id IS NULL
|
||||
UNION ALL
|
||||
-- Step: a child folder inherits from its parent. The
|
||||
-- effective flag is true if the parent's effective flag
|
||||
-- is true OR the child's own flag is true. Short-circuit
|
||||
-- would be nice but a plain OR does the job.
|
||||
SELECT f.id, (f.is_hidden OR fc.effective_hidden) AS effective_hidden
|
||||
FROM folders f
|
||||
JOIN folder_chain fc ON f.parent_id = fc.id
|
||||
)
|
||||
UPDATE photos p
|
||||
SET is_hidden = fc.effective_hidden
|
||||
FROM folder_chain fc
|
||||
WHERE p.folder_id = fc.id
|
||||
AND p.is_hidden IS DISTINCT FROM fc.effective_hidden
|
||||
""")
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{folder_id}/hide")
|
||||
async def set_folder_hidden(
|
||||
folder_id: str,
|
||||
body: FolderHide,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Toggle the "hide from views" flag on a folder or source root.
|
||||
|
||||
A hidden folder's photos are excluded from every cross-cutting view
|
||||
(All Photos, Map, Tags, People, Search, sidebar counts, duplicates)
|
||||
but remain fully indexed and visible when the user navigates
|
||||
directly into the folder. The flag cascades to every descendant
|
||||
folder via the photos.is_hidden recompute — the child folder's own
|
||||
`is_hidden` column stays where the user set it, but a photo under a
|
||||
hidden ancestor will still be marked hidden.
|
||||
|
||||
Accepts both Folder ids and SourceRoot ids. For a SourceRoot, we
|
||||
look up the root Folder row (the one matching source_root.path) and
|
||||
flip that — source roots themselves don't carry the column because
|
||||
the whole subtree lives on a single Folder row anyway.
|
||||
"""
|
||||
# SourceRoot path — resolve to the Folder row at the mount point.
|
||||
sr_result = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == current_user.id)
|
||||
)
|
||||
source_root = sr_result.scalar_one_or_none()
|
||||
|
||||
folder: Optional[Folder]
|
||||
if source_root:
|
||||
root_folder_result = await db.execute(
|
||||
select(Folder).where(
|
||||
Folder.source_root_id == source_root.id,
|
||||
Folder.user_id == current_user.id,
|
||||
Folder.path == os.path.normpath(source_root.path),
|
||||
)
|
||||
)
|
||||
folder = root_folder_result.scalar_one_or_none()
|
||||
if folder is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Source root has no indexed Folder row yet; scan first.",
|
||||
)
|
||||
else:
|
||||
folder = await get_user_folder(folder_id, current_user, db)
|
||||
|
||||
folder.is_hidden = bool(body.hidden)
|
||||
await db.flush()
|
||||
|
||||
# Rematerialize photos.is_hidden across the whole tree. Cheap
|
||||
# enough (tens of ms on a typical library) that we don't need to
|
||||
# scope the update to just this folder's subtree — doing it
|
||||
# globally also fixes any drift introduced by earlier moves.
|
||||
await _recompute_photo_hidden_flags(db)
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"id": folder.id,
|
||||
"name": folder.name,
|
||||
"path": folder.path,
|
||||
"is_hidden": folder.is_hidden,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{folder_id}/scan")
|
||||
async def scan_folder(folder_id: str, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Trigger manual re-scan of source root folder"""
|
||||
from app.tasks.celery import celery_app
|
||||
|
||||
result = await db.execute(select(SourceRoot).where(SourceRoot.id == folder_id, SourceRoot.user_id == current_user.id))
|
||||
source_root = result.scalar_one_or_none()
|
||||
|
||||
if not source_root:
|
||||
raise HTTPException(status_code=404, detail="Source folder not found")
|
||||
|
||||
# Queue scan task using the task name defined in the decorator
|
||||
task = celery_app.send_task('scan_folder', args=[source_root.path, source_root.id])
|
||||
return {"status": "success", "message": f"Scan queued for {source_root.path}", "task_id": task.id}
|
||||
@@ -1,442 +0,0 @@
|
||||
"""
|
||||
Heaps API router
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import logging
|
||||
from typing import Optional, Literal
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func, update, insert, delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Heap, Photo, Folder
|
||||
from app.models.folders import SourceRoot
|
||||
from app.models.heaps import heap_photos
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user, get_user_heap
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Schemas ───────────────────────────────────────────────────────────────
|
||||
|
||||
class HeapCreate(BaseModel):
|
||||
name: str
|
||||
|
||||
|
||||
class HeapUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
is_active: Optional[bool] = None
|
||||
|
||||
|
||||
class HeapPhotosBody(BaseModel):
|
||||
photo_ids: list[str]
|
||||
|
||||
|
||||
class HeapConvertBody(BaseModel):
|
||||
target_id: str # folder id OR source root id
|
||||
mode: Literal['move', 'copy'] = 'move'
|
||||
delete_heap: bool = False
|
||||
# Optional subfolder name to create inside the target. If provided, the
|
||||
# actual destination is target_dir/subfolder_name (created if missing).
|
||||
# Path separators and dot-segments are rejected.
|
||||
subfolder_name: Optional[str] = None
|
||||
|
||||
|
||||
# ── Endpoints ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("")
|
||||
async def list_heaps(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List all heaps with photo counts."""
|
||||
# LEFT JOIN heap_photos and group so we can return counts in one query.
|
||||
count_subq = (
|
||||
select(
|
||||
heap_photos.c.heap_id,
|
||||
func.count(heap_photos.c.photo_id).label("photo_count"),
|
||||
)
|
||||
.group_by(heap_photos.c.heap_id)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
stmt = (
|
||||
select(Heap, count_subq.c.photo_count)
|
||||
.outerjoin(count_subq, Heap.id == count_subq.c.heap_id)
|
||||
.where(Heap.user_id == current_user.id)
|
||||
.order_by(Heap.created_at.asc())
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
rows = result.all()
|
||||
|
||||
return [
|
||||
{
|
||||
"id": h.id,
|
||||
"name": h.name,
|
||||
"is_active": bool(h.is_active),
|
||||
"created_at": h.created_at,
|
||||
"updated_at": h.updated_at,
|
||||
"photo_count": int(count or 0),
|
||||
}
|
||||
for h, count in rows
|
||||
]
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_heap(
|
||||
body: HeapCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Create a new heap."""
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Heap name is required")
|
||||
heap = Heap(name=name, user_id=current_user.id)
|
||||
db.add(heap)
|
||||
await db.commit()
|
||||
await db.refresh(heap)
|
||||
return {
|
||||
"id": heap.id,
|
||||
"name": heap.name,
|
||||
"is_active": bool(heap.is_active),
|
||||
"created_at": heap.created_at,
|
||||
"updated_at": heap.updated_at,
|
||||
"photo_count": 0,
|
||||
}
|
||||
|
||||
|
||||
@router.patch("/{heap_id}")
|
||||
async def update_heap(
|
||||
heap_id: str,
|
||||
body: HeapUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Rename a heap and/or toggle active state. Setting is_active=true on
|
||||
one heap deactivates all others (single-active invariant)."""
|
||||
heap = await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
if body.name is not None:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Heap name is required")
|
||||
heap.name = name
|
||||
|
||||
if body.is_active is not None:
|
||||
if body.is_active:
|
||||
# Clear active flag on all other heaps for this user
|
||||
await db.execute(
|
||||
update(Heap)
|
||||
.where(Heap.user_id == current_user.id)
|
||||
.values(is_active=False)
|
||||
)
|
||||
heap.is_active = True
|
||||
else:
|
||||
heap.is_active = False
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(heap)
|
||||
return {
|
||||
"id": heap.id,
|
||||
"name": heap.name,
|
||||
"is_active": bool(heap.is_active),
|
||||
"created_at": heap.created_at,
|
||||
"updated_at": heap.updated_at,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{heap_id}/duplicate", status_code=201)
|
||||
async def duplicate_heap(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Create a new heap with the same membership as an existing one. The
|
||||
new heap is named "{original} (copy)" and is never the active target —
|
||||
duplicating shouldn't quietly steal the user's T-key destination.
|
||||
"""
|
||||
source = await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
new_heap = Heap(name=f"{source.name} (copy)", is_active=False, user_id=current_user.id)
|
||||
db.add(new_heap)
|
||||
await db.flush() # populate new_heap.id without committing yet
|
||||
|
||||
# Bulk-copy the membership rows. SELECT photo_id FROM heap_photos WHERE
|
||||
# heap_id = :src — INSERT each into the new heap. Done as a single
|
||||
# INSERT...SELECT to avoid round-tripping ids through Python.
|
||||
member_rows = await db.execute(
|
||||
select(heap_photos.c.photo_id).where(heap_photos.c.heap_id == heap_id)
|
||||
)
|
||||
photo_ids = [row[0] for row in member_rows.all()]
|
||||
if photo_ids:
|
||||
await db.execute(
|
||||
insert(heap_photos),
|
||||
[{"heap_id": new_heap.id, "photo_id": pid} for pid in photo_ids],
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(new_heap)
|
||||
return {
|
||||
"id": new_heap.id,
|
||||
"name": new_heap.name,
|
||||
"is_active": False,
|
||||
"photo_count": len(photo_ids),
|
||||
"created_at": new_heap.created_at,
|
||||
"updated_at": new_heap.updated_at,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{heap_id}", status_code=204)
|
||||
async def delete_heap(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Delete a heap. Photos themselves are unaffected — only the membership
|
||||
rows in heap_photos cascade-delete."""
|
||||
heap = await get_user_heap(heap_id, current_user, db)
|
||||
await db.delete(heap)
|
||||
await db.commit()
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/{heap_id}/photo_ids")
|
||||
async def get_heap_photo_ids(
|
||||
heap_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Return just the photo ids belonging to a heap. Used by the frontend
|
||||
to maintain a fast client-side membership lookup for the active heap
|
||||
(for the basket affordance on thumbnails) without fetching full photo
|
||||
records."""
|
||||
await get_user_heap(heap_id, current_user, db)
|
||||
result = await db.execute(
|
||||
select(heap_photos.c.photo_id).where(heap_photos.c.heap_id == heap_id)
|
||||
)
|
||||
return [row[0] for row in result.all()]
|
||||
|
||||
|
||||
@router.post("/{heap_id}/photos")
|
||||
async def add_photos_to_heap(
|
||||
heap_id: str,
|
||||
body: HeapPhotosBody,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Add photos to a heap. Idempotent: re-adding existing members is a
|
||||
no-op (handled by an INSERT OR IGNORE-style filter on duplicates)."""
|
||||
await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
if not body.photo_ids:
|
||||
return {"status": "success", "added": 0}
|
||||
|
||||
# Find which ids are already members so we don't violate the PK.
|
||||
existing = await db.execute(
|
||||
select(heap_photos.c.photo_id).where(
|
||||
heap_photos.c.heap_id == heap_id,
|
||||
heap_photos.c.photo_id.in_(body.photo_ids),
|
||||
)
|
||||
)
|
||||
existing_ids = {row[0] for row in existing.all()}
|
||||
new_ids = [pid for pid in body.photo_ids if pid not in existing_ids]
|
||||
|
||||
if new_ids:
|
||||
await db.execute(
|
||||
insert(heap_photos),
|
||||
[{"heap_id": heap_id, "photo_id": pid} for pid in new_ids],
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
return {"status": "success", "added": len(new_ids), "already_present": len(existing_ids)}
|
||||
|
||||
|
||||
@router.post("/{heap_id}/convert")
|
||||
async def convert_heap_to_folder(
|
||||
heap_id: str,
|
||||
body: HeapConvertBody,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Convert a heap into a folder by moving (or copying) every member
|
||||
photo into the target directory. Optionally deletes the heap row at
|
||||
the end.
|
||||
|
||||
target_id may be a Folder id or a SourceRoot id (matches the
|
||||
/photos/move convention so the same dropdown can populate it).
|
||||
"""
|
||||
heap = await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
# Resolve target_id → (target_dir, target_folder)
|
||||
sr_check = await db.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == body.target_id)
|
||||
)
|
||||
source_root = sr_check.scalar_one_or_none()
|
||||
|
||||
if source_root is not None:
|
||||
parent_dir = source_root.path
|
||||
parent_source_root_id = source_root.id
|
||||
else:
|
||||
folder_check = await db.execute(
|
||||
select(Folder).where(Folder.id == body.target_id)
|
||||
)
|
||||
parent_folder = folder_check.scalar_one_or_none()
|
||||
if parent_folder is None:
|
||||
raise HTTPException(status_code=404, detail="Target folder not found")
|
||||
parent_dir = parent_folder.path
|
||||
parent_source_root_id = parent_folder.source_root_id
|
||||
|
||||
if not os.path.isdir(parent_dir):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Target parent does not exist: {parent_dir}",
|
||||
)
|
||||
|
||||
# Resolve target_dir, creating an optional subfolder if requested.
|
||||
if body.subfolder_name is not None:
|
||||
sub = body.subfolder_name.strip()
|
||||
if not sub:
|
||||
raise HTTPException(status_code=400, detail="Subfolder name cannot be empty")
|
||||
if '/' in sub or '\\' in sub or sub in ('.', '..'):
|
||||
raise HTTPException(status_code=400, detail="Invalid subfolder name")
|
||||
target_dir = os.path.join(parent_dir, sub)
|
||||
if not os.path.exists(target_dir):
|
||||
try:
|
||||
os.makedirs(target_dir)
|
||||
except OSError as e:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to create subfolder: {e}",
|
||||
)
|
||||
elif not os.path.isdir(target_dir):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"{target_dir} exists but is not a directory",
|
||||
)
|
||||
else:
|
||||
target_dir = parent_dir
|
||||
|
||||
# Ensure a Folder row for the target, reusing the scanner helper so
|
||||
# path normalization + dedupe stay consistent.
|
||||
from app.tasks.scan import get_or_create_folder
|
||||
target_folder = await get_or_create_folder(db, target_dir, parent_source_root_id)
|
||||
|
||||
# Fetch the heap's photos via the join table.
|
||||
photo_result = await db.execute(
|
||||
select(Photo)
|
||||
.join(heap_photos, Photo.id == heap_photos.c.photo_id)
|
||||
.where(heap_photos.c.heap_id == heap_id)
|
||||
)
|
||||
photos = photo_result.scalars().all()
|
||||
|
||||
moved = 0
|
||||
copied = 0
|
||||
errors: list[dict] = []
|
||||
|
||||
def _unique_target_name(directory: str, filename: str) -> Optional[str]:
|
||||
if not os.path.exists(os.path.join(directory, filename)):
|
||||
return filename
|
||||
stem, ext = os.path.splitext(filename)
|
||||
for i in range(1, 100):
|
||||
suffix = '' if i == 1 else f' {i}'
|
||||
candidate = f"{stem} (copy{suffix}){ext}"
|
||||
if not os.path.exists(os.path.join(directory, candidate)):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
for photo in photos:
|
||||
if not os.path.exists(photo.filepath):
|
||||
errors.append({"id": photo.id, "error": "source file missing"})
|
||||
continue
|
||||
|
||||
if body.mode == 'move':
|
||||
if photo.folder_id == target_folder.id:
|
||||
continue # already there
|
||||
new_path = os.path.join(target_dir, photo.filename)
|
||||
if os.path.exists(new_path):
|
||||
errors.append({"id": photo.id, "error": f"name collision: {photo.filename}"})
|
||||
continue
|
||||
try:
|
||||
shutil.move(photo.filepath, new_path)
|
||||
except OSError as e:
|
||||
errors.append({"id": photo.id, "error": str(e)})
|
||||
continue
|
||||
photo.filepath = new_path
|
||||
photo.folder_id = target_folder.id
|
||||
moved += 1
|
||||
else: # copy
|
||||
new_name = _unique_target_name(target_dir, photo.filename)
|
||||
if new_name is None:
|
||||
errors.append({"id": photo.id, "error": "too many name collisions"})
|
||||
continue
|
||||
new_path = os.path.join(target_dir, new_name)
|
||||
try:
|
||||
shutil.copy2(photo.filepath, new_path)
|
||||
except OSError as e:
|
||||
errors.append({"id": photo.id, "error": str(e)})
|
||||
continue
|
||||
new_photo = Photo(
|
||||
filepath=new_path,
|
||||
filename=new_name,
|
||||
folder_id=target_folder.id,
|
||||
file_hash=photo.file_hash,
|
||||
media_type=photo.media_type,
|
||||
original_format=photo.original_format,
|
||||
width=photo.width,
|
||||
height=photo.height,
|
||||
file_size=photo.file_size,
|
||||
taken_at=photo.taken_at,
|
||||
taken_at_source=photo.taken_at_source,
|
||||
user_title=photo.user_title,
|
||||
user_notes=photo.user_notes,
|
||||
rating=photo.rating,
|
||||
color_label=photo.color_label,
|
||||
exif_json=photo.exif_json,
|
||||
is_duplicate=True,
|
||||
processing_status='pending',
|
||||
)
|
||||
db.add(new_photo)
|
||||
copied += 1
|
||||
|
||||
if body.delete_heap:
|
||||
await db.delete(heap)
|
||||
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"mode": body.mode,
|
||||
"moved": moved,
|
||||
"copied": copied,
|
||||
"errors": errors,
|
||||
"heap_deleted": body.delete_heap,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{heap_id}/photos")
|
||||
async def remove_photos_from_heap(
|
||||
heap_id: str,
|
||||
body: HeapPhotosBody,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Remove photos from a heap. Removing a non-member is a no-op."""
|
||||
await get_user_heap(heap_id, current_user, db)
|
||||
|
||||
if not body.photo_ids:
|
||||
return {"status": "success", "removed": 0}
|
||||
|
||||
res = await db.execute(
|
||||
delete(heap_photos).where(
|
||||
heap_photos.c.heap_id == heap_id,
|
||||
heap_photos.c.photo_id.in_(body.photo_ids),
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
return {"status": "success", "removed": res.rowcount or 0}
|
||||
@@ -1,856 +0,0 @@
|
||||
"""
|
||||
Library API router for stats, scanning, and maintenance.
|
||||
|
||||
The /maintenance/* endpoints are surfaced through the frontend Settings
|
||||
panel. They're intentionally idempotent and operate by re-queueing the
|
||||
existing Celery tasks rather than doing any heavy lifting in the
|
||||
request thread.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select, func, update, true as sa_true
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _owner_filter(user: User, scope: str | None):
|
||||
"""Return a column expression scoping photos to the current user,
|
||||
or a pass-through true() when an admin requests global scope."""
|
||||
if scope == "global" and user.role == "admin":
|
||||
return sa_true()
|
||||
return Photo.user_id == user.id
|
||||
|
||||
# Media types we accept in the regenerate-thumbnails request body. Mirrors
|
||||
# the values produced by `app.tasks.scan.get_media_type`.
|
||||
_VALID_MEDIA_TYPES = {'photo', 'raw', 'heic', 'video'}
|
||||
|
||||
@router.get("/stats")
|
||||
async def get_library_stats(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Get library statistics. Pass ?scope=global (admin only) for
|
||||
cross-user totals (used by the Settings page)."""
|
||||
owner = _owner_filter(current_user, scope)
|
||||
visible = owner & (Photo.is_discarded.is_(False)) & (Photo.is_hidden.is_(False))
|
||||
|
||||
all_photos_count = (
|
||||
await db.execute(select(func.count(Photo.id)).where(visible))
|
||||
).scalar() or 0
|
||||
|
||||
rated_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(visible, Photo.rating >= 1)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
colored_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
visible, Photo.color_label.is_not(None)
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
with_gps_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
visible, Photo.latitude.is_not(None)
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
duplicates_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
visible, Photo.is_duplicate.is_(True)
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
discarded_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(owner, Photo.is_discarded.is_(True))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
# Legacy split (kept for the existing /stats consumers).
|
||||
photo_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
owner,
|
||||
Photo.media_type.in_(['photo', 'heic', 'raw'])
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
video_count = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(owner, Photo.media_type == 'video')
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
size = (await db.execute(select(func.sum(Photo.file_size)).where(owner))).scalar() or 0
|
||||
|
||||
return {
|
||||
"all_photos": all_photos_count,
|
||||
"rated": rated_count,
|
||||
"colored": colored_count,
|
||||
"with_gps": with_gps_count,
|
||||
"duplicates": duplicates_count,
|
||||
"discarded": discarded_count,
|
||||
"total_photos": photo_count,
|
||||
"total_videos": video_count,
|
||||
"total_size": size,
|
||||
"total_size_gb": round(size / (1024**3), 2) if size else 0,
|
||||
}
|
||||
|
||||
@router.post("/scan")
|
||||
async def trigger_scan(current_user: User = Depends(get_current_user)):
|
||||
"""Trigger full library re-scan"""
|
||||
from app.tasks.scan import scan_all_source_roots
|
||||
|
||||
scan_all_source_roots.delay()
|
||||
|
||||
return {"status": "success", "message": "Library scan started"}
|
||||
|
||||
|
||||
@router.post("/backfill-gps")
|
||||
async def trigger_backfill_gps(current_user: User = Depends(get_current_user)):
|
||||
"""Re-run EXIF metadata extraction on every photo that's still missing
|
||||
GPS coordinates. Useful after fixing the EXIF parser, or any time the
|
||||
Map view looks emptier than expected. Returns immediately — work runs
|
||||
on the Celery worker."""
|
||||
from app.tasks.scan import backfill_gps
|
||||
|
||||
backfill_gps.delay()
|
||||
return {"status": "success", "message": "GPS backfill queued"}
|
||||
|
||||
@router.get("/scan/status")
|
||||
async def get_scan_status(db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Get current scan status"""
|
||||
import redis
|
||||
from app.config import settings
|
||||
|
||||
# Connect to Redis to get scan status
|
||||
r = redis.Redis.from_url(settings.redis_url)
|
||||
|
||||
# Get scan status from Redis (set by worker tasks)
|
||||
is_scanning = r.get('scan:active') == b'true'
|
||||
current_folder = r.get('scan:current_folder')
|
||||
processed_files = int(r.get('scan:processed_files') or 0)
|
||||
total_files = int(r.get('scan:total_files') or 0)
|
||||
errors = r.lrange('scan:errors', 0, -1)
|
||||
|
||||
return {
|
||||
"is_scanning": is_scanning,
|
||||
"current_folder": current_folder.decode() if current_folder else None,
|
||||
"processed_files": processed_files,
|
||||
"total_files": total_files,
|
||||
"errors": [e.decode() for e in errors] if errors else []
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Maintenance endpoints — surfaced via the Settings panel.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class RegenerateThumbnailsRequest(BaseModel):
|
||||
"""Optional filters narrowing which photos get re-queued. With both
|
||||
fields omitted the request resets every photo in the library."""
|
||||
media_types: Optional[List[str]] = Field(
|
||||
default=None,
|
||||
description="Restrict to these media_type values (photo/raw/heic/video).",
|
||||
)
|
||||
only_failed: bool = Field(
|
||||
default=False,
|
||||
description="If true, only re-queue photos whose processing_status is 'failed'.",
|
||||
)
|
||||
only_pending: bool = Field(
|
||||
default=False,
|
||||
description="If true, only (re-)queue photos whose processing_status is 'pending'. "
|
||||
"Useful for kicking rows that were created by a scan but never had "
|
||||
"their thumbnail task picked up.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/maintenance/thumbnail-stats")
|
||||
async def get_thumbnail_stats(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Counts of photos by processing_status, plus a media-type breakdown
|
||||
so the Settings panel can show the user what's outstanding."""
|
||||
owner = _owner_filter(current_user, scope)
|
||||
status_rows = (
|
||||
await db.execute(
|
||||
select(Photo.processing_status, func.count(Photo.id))
|
||||
.where(owner)
|
||||
.group_by(Photo.processing_status)
|
||||
)
|
||||
).all()
|
||||
|
||||
media_rows = (
|
||||
await db.execute(
|
||||
select(Photo.media_type, func.count(Photo.id))
|
||||
.where(owner)
|
||||
.group_by(Photo.media_type)
|
||||
)
|
||||
).all()
|
||||
|
||||
by_status = {status or 'unknown': count for status, count in status_rows}
|
||||
by_media_type = {media or 'unknown': count for media, count in media_rows}
|
||||
total = sum(by_status.values())
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"pending": by_status.get('pending', 0),
|
||||
"processing": by_status.get('processing', 0),
|
||||
"completed": by_status.get('completed', 0),
|
||||
"failed": by_status.get('failed', 0),
|
||||
"by_media_type": by_media_type,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/maintenance/regenerate-thumbnails")
|
||||
async def regenerate_thumbnails(
|
||||
body: RegenerateThumbnailsRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Reset matching photos' on-disk thumbnail directories and re-queue
|
||||
Celery thumbnail generation. Used by the Settings panel for the
|
||||
'regenerate video thumbnails' / 'regenerate failed' buttons.
|
||||
|
||||
Files on disk are removed under /data/thumbs/<photo_id>/ so the next
|
||||
request to /photos/{id}/thumb/{size} actually re-generates instead of
|
||||
serving the stale placeholder.
|
||||
"""
|
||||
from app.tasks.thumbs import generate_thumbnails
|
||||
|
||||
owner = _owner_filter(current_user, scope)
|
||||
|
||||
# Validate media_types early so a typo can't silently match nothing.
|
||||
media_types = body.media_types
|
||||
if media_types is not None:
|
||||
invalid = [m for m in media_types if m not in _VALID_MEDIA_TYPES]
|
||||
if invalid:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Invalid media_types: {invalid}. "
|
||||
f"Allowed: {sorted(_VALID_MEDIA_TYPES)}",
|
||||
}
|
||||
|
||||
query = select(Photo).where(owner)
|
||||
if media_types:
|
||||
query = query.where(Photo.media_type.in_(media_types))
|
||||
if body.only_failed:
|
||||
query = query.where(Photo.processing_status == 'failed')
|
||||
if body.only_pending:
|
||||
query = query.where(Photo.processing_status == 'pending')
|
||||
|
||||
photos = (await db.execute(query)).scalars().all()
|
||||
|
||||
cleared_dirs = 0
|
||||
file_errors = 0
|
||||
for photo in photos:
|
||||
thumb_dir = f"/data/thumbs/{photo.id}"
|
||||
if os.path.isdir(thumb_dir):
|
||||
try:
|
||||
shutil.rmtree(thumb_dir)
|
||||
cleared_dirs += 1
|
||||
except OSError as e:
|
||||
file_errors += 1
|
||||
logger.warning(f"Could not clear thumb dir {thumb_dir}: {e}")
|
||||
photo.processing_status = 'pending'
|
||||
photo.processing_error = None
|
||||
photo.thumb_small = None
|
||||
photo.thumb_medium = None
|
||||
photo.thumb_large = None
|
||||
|
||||
await db.commit()
|
||||
|
||||
# Queue celery tasks AFTER the commit so the worker sees the reset
|
||||
# state when it picks the job up.
|
||||
queued = 0
|
||||
for photo in photos:
|
||||
try:
|
||||
generate_thumbnails.delay(photo.id)
|
||||
queued += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue thumbnail job for {photo.id}: {e}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"matched": len(photos),
|
||||
"queued": queued,
|
||||
"cleared_dirs": cleared_dirs,
|
||||
"file_errors": file_errors,
|
||||
"filters": {
|
||||
"media_types": media_types,
|
||||
"only_failed": body.only_failed,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/worker-status")
|
||||
async def get_worker_status(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Diagnostics for the Celery worker fleet + recent task failures.
|
||||
|
||||
Surfaced in the Settings panel so the user can spot a stuck queue or
|
||||
a worker that's gone away without tailing container logs. Returns:
|
||||
|
||||
- workers: list of {name, status, active, concurrency, queues}
|
||||
derived from celery_app.control.inspect(). `status` is 'online'
|
||||
when ping succeeds, 'unreachable' otherwise. Empty list means no
|
||||
workers are responding at all (broker down, container crashed,
|
||||
wrong queue routing, etc.).
|
||||
- queues: per-queue depth read from Redis (LLEN of each queue key
|
||||
used by celery.kombu). Mirrors what tasks are waiting to be
|
||||
picked up.
|
||||
- failures: aggregate count of photos with processing_status='failed'
|
||||
plus the most recent N error messages so the user can see *why*
|
||||
things failed without opening the DB.
|
||||
- broker_ok: bool — could we even reach Redis?
|
||||
"""
|
||||
owner = _owner_filter(current_user, scope)
|
||||
from app.tasks.celery import celery_app
|
||||
from app.config import settings
|
||||
import redis as _redis
|
||||
|
||||
# ----- Celery inspect (workers + active tasks) -------------------------
|
||||
# Each inspect.* call is a separate broadcast-and-wait with its own
|
||||
# timeout, so running them serially multiplies the wait. Fan them out
|
||||
# to threads and gather, collapsing 6 × timeout into ~1 × timeout.
|
||||
# Timeout dropped to 0.5s — a responsive worker answers within a few
|
||||
# ms; anything past that is effectively "not responding" for the
|
||||
# purposes of a settings dashboard.
|
||||
import asyncio
|
||||
workers: list[dict] = []
|
||||
inspect_error: Optional[str] = None
|
||||
try:
|
||||
inspect = celery_app.control.inspect(timeout=0.5)
|
||||
ping, active, reserved, scheduled, stats, active_queues = await asyncio.gather(
|
||||
asyncio.to_thread(inspect.ping),
|
||||
asyncio.to_thread(inspect.active),
|
||||
asyncio.to_thread(inspect.reserved),
|
||||
asyncio.to_thread(inspect.scheduled),
|
||||
asyncio.to_thread(inspect.stats),
|
||||
asyncio.to_thread(inspect.active_queues),
|
||||
)
|
||||
ping = ping or {}
|
||||
active = active or {}
|
||||
reserved = reserved or {}
|
||||
scheduled = scheduled or {}
|
||||
stats = stats or {}
|
||||
active_queues = active_queues or {}
|
||||
|
||||
worker_names = set(ping) | set(active) | set(stats)
|
||||
for name in sorted(worker_names):
|
||||
wstats = stats.get(name) or {}
|
||||
pool = wstats.get('pool') or {}
|
||||
workers.append({
|
||||
"name": name,
|
||||
"status": "online" if name in ping else "unreachable",
|
||||
"active": len(active.get(name, []) or []),
|
||||
"reserved": len(reserved.get(name, []) or []),
|
||||
"scheduled": len(scheduled.get(name, []) or []),
|
||||
"concurrency": pool.get('max-concurrency'),
|
||||
"processed": (wstats.get('total') or {}),
|
||||
"queues": [q.get('name') for q in (active_queues.get(name) or [])],
|
||||
"active_tasks": [
|
||||
{
|
||||
"id": t.get('id'),
|
||||
"name": t.get('name'),
|
||||
"args": t.get('args'),
|
||||
"time_start": t.get('time_start'),
|
||||
}
|
||||
for t in (active.get(name) or [])[:10]
|
||||
],
|
||||
})
|
||||
except Exception as e:
|
||||
inspect_error = str(e)
|
||||
logger.warning(f"Celery inspect failed: {e}")
|
||||
|
||||
# ----- Broker / queue depth --------------------------------------------
|
||||
broker_ok = False
|
||||
queue_depths: dict[str, int] = {}
|
||||
broker_error: Optional[str] = None
|
||||
try:
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
r.ping()
|
||||
broker_ok = True
|
||||
# `vision` is the big one — embed / classify / detect / ocr /
|
||||
# extract_faces all land here, so it's where backlogs actually
|
||||
# pile up. Leaving it off the dashboard made it look like the
|
||||
# queue was always empty while the worker was clearly busy.
|
||||
for q in ('default', 'high', 'low', 'vision'):
|
||||
try:
|
||||
queue_depths[q] = int(r.llen(q) or 0)
|
||||
except Exception:
|
||||
queue_depths[q] = 0
|
||||
except Exception as e:
|
||||
broker_error = str(e)
|
||||
logger.warning(f"Redis broker unreachable: {e}")
|
||||
|
||||
# ----- Recent task failures from the photos table ----------------------
|
||||
failed_total = (
|
||||
await db.execute(
|
||||
select(func.count(Photo.id)).where(owner, Photo.processing_status == 'failed')
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
recent_failed_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Photo.id,
|
||||
Photo.filename,
|
||||
Photo.media_type,
|
||||
Photo.processing_error,
|
||||
Photo.updated_at,
|
||||
)
|
||||
.where(owner, Photo.processing_status == 'failed')
|
||||
.order_by(Photo.updated_at.desc().nullslast())
|
||||
.limit(20)
|
||||
)
|
||||
).all()
|
||||
|
||||
recent_failures = [
|
||||
{
|
||||
"photo_id": row[0],
|
||||
"filename": row[1],
|
||||
"media_type": row[2],
|
||||
"error": (row[3] or '')[:500],
|
||||
"updated_at": row[4].isoformat() if row[4] else None,
|
||||
}
|
||||
for row in recent_failed_rows
|
||||
]
|
||||
|
||||
# ----- Most recent scan errors (Redis list) ----------------------------
|
||||
scan_errors: list[str] = []
|
||||
try:
|
||||
if broker_ok:
|
||||
r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
|
||||
raw = r.lrange('scan:errors', 0, 19) or []
|
||||
scan_errors = [e.decode(errors='replace') for e in raw]
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not read scan:errors: {e}")
|
||||
|
||||
return {
|
||||
"broker_ok": broker_ok,
|
||||
"broker_error": broker_error,
|
||||
"inspect_error": inspect_error,
|
||||
"workers": workers,
|
||||
"worker_count": len(workers),
|
||||
"queues": queue_depths,
|
||||
"failures": {
|
||||
"total": failed_total,
|
||||
"recent": recent_failures,
|
||||
},
|
||||
"scan_errors": scan_errors,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/pipeline-stats")
|
||||
async def get_pipeline_stats(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Per-stage progress across the ingestion pipeline.
|
||||
|
||||
Returns a `{stage_key: {done, total, label}}` map so the Settings
|
||||
panel can render one progress bar per stage. `total` is the number
|
||||
of non-discarded photos the stage is *expected* to run on — which is
|
||||
every non-discarded photo for most stages, or a narrower subset when
|
||||
a stage is image-only (e.g. embeddings don't run on videos).
|
||||
|
||||
Keep the shape flat + serialisable; the frontend turns it straight
|
||||
into a list of rows without needing to know about the models.
|
||||
"""
|
||||
from app.config import settings as _settings
|
||||
from app.models import Embedding, FaceEmbedding, OCRText
|
||||
from app.models.tags import photo_tags # association Table, not a model
|
||||
|
||||
owner = _owner_filter(current_user, scope)
|
||||
not_discarded = owner & Photo.is_discarded.is_(False)
|
||||
|
||||
async def scalar_count(query):
|
||||
return (await db.execute(query)).scalar() or 0
|
||||
|
||||
# Total non-discarded photos — the denominator for most stages.
|
||||
total_photos = await scalar_count(
|
||||
select(func.count(Photo.id)).where(not_discarded)
|
||||
)
|
||||
|
||||
# Image-only denominator (embeddings, tags, faces, OCR, phash). We
|
||||
# exclude videos because those stages either don't apply or run off
|
||||
# the extracted video frame which is treated separately.
|
||||
total_images = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.media_type != 'video'
|
||||
)
|
||||
)
|
||||
|
||||
completed = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.processing_status == 'completed'
|
||||
)
|
||||
)
|
||||
with_exif = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.exif_json.is_not(None)
|
||||
)
|
||||
)
|
||||
with_gps = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded,
|
||||
Photo.latitude.is_not(None),
|
||||
Photo.longitude.is_not(None),
|
||||
)
|
||||
)
|
||||
with_phash = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.phash.is_not(None)
|
||||
)
|
||||
)
|
||||
|
||||
# Embeddings: count distinct photos that have a row for the currently
|
||||
# configured embedder model. A photo can have multiple model rows
|
||||
# (historical re-embeds) so COUNT(DISTINCT) is the right thing here.
|
||||
embedder_model = _settings.vision.embedder.name
|
||||
embeddings_done = await scalar_count(
|
||||
select(func.count(func.distinct(Embedding.photo_id)))
|
||||
.select_from(Embedding)
|
||||
.join(Photo, Photo.id == Embedding.photo_id)
|
||||
.where(not_discarded, Embedding.model == embedder_model)
|
||||
)
|
||||
|
||||
tagged_photos = await scalar_count(
|
||||
select(func.count(func.distinct(photo_tags.c.photo_id)))
|
||||
.select_from(photo_tags)
|
||||
.join(Photo, Photo.id == photo_tags.c.photo_id)
|
||||
.where(not_discarded)
|
||||
)
|
||||
ocr_done = await scalar_count(
|
||||
select(func.count(func.distinct(OCRText.photo_id)))
|
||||
.select_from(OCRText)
|
||||
.join(Photo, Photo.id == OCRText.photo_id)
|
||||
.where(not_discarded)
|
||||
)
|
||||
|
||||
# Faces: photos that have at least one face_embeddings row. A photo
|
||||
# with no faces legitimately finishes face extraction with zero rows,
|
||||
# so this undercounts by exactly "images with no visible people". We
|
||||
# surface the photo-with-faces count rather than "images scanned for
|
||||
# faces" because the latter isn't tracked anywhere.
|
||||
faces_photos = await scalar_count(
|
||||
select(func.count(func.distinct(FaceEmbedding.photo_id)))
|
||||
.select_from(FaceEmbedding)
|
||||
.join(Photo, Photo.id == FaceEmbedding.photo_id)
|
||||
.where(not_discarded)
|
||||
)
|
||||
face_rows = await scalar_count(select(func.count(FaceEmbedding.id)))
|
||||
face_clusters = await scalar_count(
|
||||
select(func.count(func.distinct(FaceEmbedding.cluster_id)))
|
||||
.where(FaceEmbedding.cluster_id.is_not(None))
|
||||
)
|
||||
|
||||
duplicate_groups = await scalar_count(
|
||||
select(func.count(func.distinct(Photo.duplicate_group_id))).where(
|
||||
not_discarded, Photo.duplicate_group_id.is_not(None)
|
||||
)
|
||||
)
|
||||
duplicate_members = await scalar_count(
|
||||
select(func.count(Photo.id)).where(
|
||||
not_discarded, Photo.duplicate_group_id.is_not(None)
|
||||
)
|
||||
)
|
||||
|
||||
# Ordered list so the frontend renders stages in pipeline order
|
||||
# without needing to know the sequence itself.
|
||||
stages = [
|
||||
{
|
||||
"key": "thumbnails",
|
||||
"label": "Thumbnails & pHash",
|
||||
"done": completed,
|
||||
"total": total_photos,
|
||||
"hint": "Generated on scan. Unlocks every downstream stage.",
|
||||
},
|
||||
{
|
||||
"key": "exif",
|
||||
"label": "EXIF metadata",
|
||||
"done": with_exif,
|
||||
"total": total_photos,
|
||||
"hint": "Camera, lens, capture time. Required for GPS + taken_at.",
|
||||
},
|
||||
{
|
||||
"key": "gps",
|
||||
"label": "GPS coordinates",
|
||||
"done": with_gps,
|
||||
"total": total_photos,
|
||||
"hint": "Subset of EXIF. Drives the map view; many photos legitimately have none.",
|
||||
"partial": True, # not every photo is expected to have GPS
|
||||
},
|
||||
{
|
||||
"key": "phash",
|
||||
"label": "Perceptual hashes",
|
||||
"done": with_phash,
|
||||
"total": total_images,
|
||||
"hint": "Feeds duplicate detection.",
|
||||
},
|
||||
{
|
||||
"key": "embeddings",
|
||||
"label": f"Embeddings ({embedder_model})",
|
||||
"done": embeddings_done,
|
||||
"total": total_images,
|
||||
"hint": "Semantic search + content classification.",
|
||||
},
|
||||
{
|
||||
"key": "tags",
|
||||
"label": "Object tags (YOLO)",
|
||||
"done": tagged_photos,
|
||||
"total": total_images,
|
||||
"hint": "Auto-generated object labels. Not every photo has a detectable object.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "ocr",
|
||||
"label": "OCR text",
|
||||
"done": ocr_done,
|
||||
"total": total_images,
|
||||
"hint": "Extracted text from screenshots / documents. Many photos have none.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "faces",
|
||||
"label": "Face detection",
|
||||
"done": faces_photos,
|
||||
"total": total_images,
|
||||
"hint": f"{face_rows} face rows detected across {faces_photos} photos.",
|
||||
"partial": True,
|
||||
},
|
||||
{
|
||||
"key": "face_clusters",
|
||||
"label": "Face clusters",
|
||||
"done": face_clusters,
|
||||
"total": face_clusters, # no meaningful "total" — it's just the current count
|
||||
"hint": "Built by recluster_faces. Run it after backfill to populate the People view.",
|
||||
"standalone": True,
|
||||
},
|
||||
{
|
||||
"key": "duplicates",
|
||||
"label": "Duplicate groups",
|
||||
"done": duplicate_groups,
|
||||
"total": duplicate_groups, # same — current count, not a progress ratio
|
||||
"hint": f"{duplicate_members} photos in {duplicate_groups} groups. Run regroup_duplicates after new imports.",
|
||||
"standalone": True,
|
||||
},
|
||||
]
|
||||
|
||||
return {
|
||||
"total_photos": total_photos,
|
||||
"total_images": total_images,
|
||||
"embedder_model": embedder_model,
|
||||
"stages": stages,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance/missing-stats")
|
||||
async def get_missing_stats(current_user: User = Depends(get_current_user)):
|
||||
"""Count photos whose files no longer exist on disk under a mounted
|
||||
source root. Surfaced in Settings so the user can see a number before
|
||||
pulling the trigger on prune-missing. Cheap enough to call freely."""
|
||||
from app.services.cleanup import prune_missing_photos
|
||||
return await prune_missing_photos(dry_run=True)
|
||||
|
||||
|
||||
@router.post("/maintenance/prune-missing")
|
||||
async def run_prune_missing(current_user: User = Depends(get_current_user)):
|
||||
"""Actually delete the orphaned photo rows reported by /missing-stats.
|
||||
Common cause: PHOTO_DIRS in .env was repointed at a different library
|
||||
leaving every old row dangling. Skips any photo whose source root
|
||||
isn't currently mounted (almost always means an unmounted drive)."""
|
||||
from app.services.cleanup import prune_missing_photos
|
||||
try:
|
||||
return {"status": "success", **(await prune_missing_photos(dry_run=False))}
|
||||
except Exception as e:
|
||||
logger.error(f"Prune missing failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@router.post("/maintenance/cleanup")
|
||||
async def run_data_integrity_cleanup(current_user: User = Depends(get_current_user)):
|
||||
"""Re-run the source-roots / folders / photos data-integrity cleanup
|
||||
that normally only runs on backend startup. Idempotent."""
|
||||
from app.services.cleanup import cleanup_data_integrity
|
||||
|
||||
try:
|
||||
await cleanup_data_integrity()
|
||||
return {"status": "success"}
|
||||
except Exception as e:
|
||||
logger.error(f"Manual cleanup failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Duplicate detection
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/duplicates/groups")
|
||||
async def get_duplicate_groups(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
scope: str | None = Query(None),
|
||||
):
|
||||
"""Return every duplicate group with its members.
|
||||
|
||||
Drives the frontend grouped grid view in the Duplicates section. One
|
||||
SQL query, bucketed in Python — no N+1, no per-member fetch. Groups
|
||||
are sorted by member_count DESC then earliest taken_at DESC so the
|
||||
biggest / most recent clusters bubble to the top.
|
||||
|
||||
Each group also carries a `reason` field:
|
||||
* "exact" — every member shares the same SHA-256 (true byte
|
||||
duplicates that the perceptual hash trivially caught)
|
||||
* "similar" — members differ at the byte level but match perceptually
|
||||
"""
|
||||
owner = _owner_filter(current_user, scope)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Photo.id,
|
||||
Photo.filename,
|
||||
Photo.taken_at,
|
||||
Photo.file_size,
|
||||
Photo.width,
|
||||
Photo.height,
|
||||
Photo.thumb_small,
|
||||
Photo.file_hash,
|
||||
Photo.folder_id,
|
||||
Photo.media_type,
|
||||
Photo.duplicate_group_id,
|
||||
)
|
||||
.where(owner)
|
||||
.where(Photo.duplicate_group_id.is_not(None))
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.where(Photo.is_hidden.is_(False))
|
||||
.order_by(Photo.duplicate_group_id)
|
||||
)
|
||||
).all()
|
||||
|
||||
# Bucket members by group_id.
|
||||
groups: dict[str, list[dict]] = {}
|
||||
for row in rows:
|
||||
member = {
|
||||
"id": row[0],
|
||||
"filename": row[1],
|
||||
"taken_at": row[2].isoformat() if row[2] else None,
|
||||
"file_size": row[3],
|
||||
"width": row[4],
|
||||
"height": row[5],
|
||||
"thumb_small": row[6],
|
||||
"file_hash": row[7],
|
||||
"folder_id": row[8],
|
||||
"media_type": row[9],
|
||||
}
|
||||
groups.setdefault(row[10], []).append(member)
|
||||
|
||||
def earliest(g: list[dict]) -> str:
|
||||
# Used as a secondary sort key. Photos with no taken_at sort last
|
||||
# by returning a far-future sentinel.
|
||||
taken = [m["taken_at"] for m in g if m["taken_at"]]
|
||||
return min(taken) if taken else "9999"
|
||||
|
||||
out = []
|
||||
for group_id, members in groups.items():
|
||||
if len(members) < 2:
|
||||
# Defensive: a regroup race could leave a singleton briefly.
|
||||
# Skip it so the UI never shows a "group of 1".
|
||||
continue
|
||||
# exact iff every member shares the same non-null file_hash
|
||||
# (true byte-identical copies that pHash also caught). Anything
|
||||
# else — different hashes, missing hashes — counts as "similar".
|
||||
all_hashes = [m["file_hash"] for m in members]
|
||||
reason = (
|
||||
"exact"
|
||||
if len(set(all_hashes)) == 1 and all_hashes[0] is not None
|
||||
else "similar"
|
||||
)
|
||||
out.append({
|
||||
"group_id": group_id,
|
||||
"member_count": len(members),
|
||||
"reason": reason,
|
||||
"members": members,
|
||||
})
|
||||
|
||||
out.sort(key=lambda g: (-g["member_count"], earliest(g["members"])))
|
||||
return {
|
||||
"groups": out,
|
||||
"total_groups": len(out),
|
||||
"total_members": sum(g["member_count"] for g in out),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/maintenance/regroup-duplicates")
|
||||
async def trigger_regroup_duplicates(current_user: User = Depends(get_current_user)):
|
||||
"""Recompute duplicate groups from current perceptual hashes.
|
||||
|
||||
Fires the celery `regroup_duplicates` task which walks every photo's
|
||||
phash, clusters by Hamming distance, and rewrites duplicate_group_id /
|
||||
is_duplicate columns. Idempotent."""
|
||||
from app.tasks.thumbs import regroup_duplicates_task
|
||||
try:
|
||||
regroup_duplicates_task.delay()
|
||||
return {"status": "queued"}
|
||||
except Exception as e:
|
||||
logger.error(f"Regroup queue failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@router.post("/maintenance/backfill-phashes")
|
||||
async def trigger_backfill_phashes(current_user: User = Depends(get_current_user)):
|
||||
"""Compute perceptual hashes for every photo currently missing one.
|
||||
|
||||
One-shot recovery path for libraries that existed before the phash
|
||||
column was added — the thumbs worker computes phash for everything
|
||||
new, but old rows need a backfill pass."""
|
||||
from app.tasks.thumbs import backfill_phashes
|
||||
try:
|
||||
backfill_phashes.delay()
|
||||
return {"status": "queued"}
|
||||
except Exception as e:
|
||||
logger.error(f"Backfill queue failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@router.post("/maintenance/start-watcher")
|
||||
async def start_file_watcher(current_user: User = Depends(get_current_user)):
|
||||
"""Start the filesystem watcher. Uses a Redis lock so only one
|
||||
instance runs at a time — safe to call repeatedly."""
|
||||
from app.tasks.scan import watch_folders
|
||||
try:
|
||||
watch_folders.apply_async(countdown=2)
|
||||
return {"status": "queued"}
|
||||
except Exception as e:
|
||||
logger.error(f"Watcher queue failed: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,76 +0,0 @@
|
||||
"""
|
||||
Search API router — unified hybrid search endpoint.
|
||||
"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo
|
||||
from app.services.search import hybrid_search
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
q: Optional[str] = None
|
||||
filters: Optional[dict] = None
|
||||
limit: int = 50
|
||||
offset: int = 0
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def search_photos(body: SearchRequest, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Unified search endpoint. Every query runs hybrid (FTS + semantic)
|
||||
by default — the user never picks a mode.
|
||||
|
||||
Filters:
|
||||
- tag_ids: list of tag IDs (any kind: user, object, face_cluster)
|
||||
- date_from / date_to: ISO date strings
|
||||
"""
|
||||
filters = body.filters or {}
|
||||
|
||||
results = await hybrid_search(
|
||||
db=db,
|
||||
q=body.q,
|
||||
tag_ids=filters.get("tag_ids"),
|
||||
date_from=filters.get("date_from"),
|
||||
date_to=filters.get("date_to"),
|
||||
limit=body.limit,
|
||||
offset=body.offset,
|
||||
)
|
||||
|
||||
if not results:
|
||||
return {"results": [], "total": 0}
|
||||
|
||||
# Hydrate with photo data
|
||||
photo_ids = [r["photo_id"] for r in results]
|
||||
stmt = select(Photo).where(Photo.id.in_(photo_ids), Photo.user_id == current_user.id)
|
||||
rows = (await db.execute(stmt)).scalars().all()
|
||||
photo_map = {p.id: p for p in rows}
|
||||
|
||||
hydrated = []
|
||||
for r in results:
|
||||
photo = photo_map.get(r["photo_id"])
|
||||
if not photo:
|
||||
continue
|
||||
hydrated.append({
|
||||
"id": photo.id,
|
||||
"filename": photo.filename,
|
||||
"filepath": photo.filepath,
|
||||
"media_type": photo.media_type,
|
||||
"width": photo.width,
|
||||
"height": photo.height,
|
||||
"taken_at": photo.taken_at.isoformat() if photo.taken_at else None,
|
||||
"rating": photo.rating,
|
||||
"color_label": photo.color_label,
|
||||
"thumb_small": photo.thumb_small,
|
||||
"thumb_medium": photo.thumb_medium,
|
||||
"score": r["score"],
|
||||
})
|
||||
|
||||
return {"results": hydrated, "total": len(hydrated)}
|
||||
@@ -1,207 +0,0 @@
|
||||
"""
|
||||
Tags API router.
|
||||
|
||||
Unified across user tags, ML-detected objects, and face clusters via
|
||||
the `kind` query parameter. Default behaviour (no kind filter) returns
|
||||
all tags — the frontend's "Hide auto-generated tags" toggle filters
|
||||
client-side or passes `kind=user`.
|
||||
"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo, Tag
|
||||
from app.models.tags import photo_tags
|
||||
from app.models.user import User
|
||||
from app.dependencies import get_current_user
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Schemas ───────────────────────────────────────────────────────────────
|
||||
|
||||
class TagCreate(BaseModel):
|
||||
name: str
|
||||
color: Optional[str] = None
|
||||
kind: str = "user"
|
||||
|
||||
|
||||
class TagUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
color: Optional[str] = None
|
||||
|
||||
|
||||
class TagMerge(BaseModel):
|
||||
target_id: str # tag to merge INTO
|
||||
|
||||
|
||||
# ── Endpoints ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("")
|
||||
async def list_tags(
|
||||
kind: Optional[str] = Query(None, description="Filter by kind: user, object, scene, face_cluster"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List all tags with their photo counts, optionally filtered by kind.
|
||||
|
||||
Photo counts here drive the Tags / People sidebar badges, so they
|
||||
exclude discarded + hidden-folder photos to match the rest of the
|
||||
cross-cutting views. A tag that only appears on hidden-folder
|
||||
photos will still show up with count=0 — we don't drop empty tags
|
||||
because the user may want to see them in the management UI.
|
||||
"""
|
||||
count_subq = (
|
||||
select(
|
||||
photo_tags.c.tag_id,
|
||||
func.count(photo_tags.c.photo_id).label("photo_count"),
|
||||
func.min(photo_tags.c.photo_id).label("first_photo_id"),
|
||||
)
|
||||
.select_from(
|
||||
photo_tags.join(Photo, Photo.id == photo_tags.c.photo_id)
|
||||
)
|
||||
.where(
|
||||
Photo.user_id == current_user.id,
|
||||
Photo.is_discarded.is_(False),
|
||||
Photo.is_hidden.is_(False),
|
||||
)
|
||||
.group_by(photo_tags.c.tag_id)
|
||||
.subquery()
|
||||
)
|
||||
stmt = (
|
||||
select(Tag, count_subq.c.photo_count, count_subq.c.first_photo_id)
|
||||
.outerjoin(count_subq, Tag.id == count_subq.c.tag_id)
|
||||
.where(Tag.user_id == current_user.id)
|
||||
)
|
||||
if kind:
|
||||
stmt = stmt.where(Tag.kind == kind)
|
||||
stmt = stmt.order_by(Tag.name.asc())
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = result.all()
|
||||
|
||||
return [
|
||||
{
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"color": tag.color,
|
||||
"kind": tag.kind,
|
||||
"source": tag.source,
|
||||
"representative_photo_id": tag.representative_photo_id or first_photo_id,
|
||||
"photo_count": int(count or 0),
|
||||
}
|
||||
for tag, count, first_photo_id in rows
|
||||
]
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_tag(body: TagCreate, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Create a new tag. The (name, kind) pair is unique — re-creating an
|
||||
existing pair returns the existing row (idempotent for autocomplete)."""
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Tag name is required")
|
||||
|
||||
existing = await db.execute(
|
||||
select(Tag).where(Tag.name == name, Tag.kind == body.kind, Tag.user_id == current_user.id)
|
||||
)
|
||||
found = existing.scalar_one_or_none()
|
||||
if found:
|
||||
return {
|
||||
"id": found.id, "name": found.name, "color": found.color,
|
||||
"kind": found.kind, "photo_count": 0,
|
||||
}
|
||||
|
||||
tag = Tag(name=name, color=body.color, kind=body.kind, user_id=current_user.id)
|
||||
db.add(tag)
|
||||
await db.commit()
|
||||
await db.refresh(tag)
|
||||
return {
|
||||
"id": tag.id, "name": tag.name, "color": tag.color,
|
||||
"kind": tag.kind, "photo_count": 0,
|
||||
}
|
||||
|
||||
|
||||
@router.patch("/{tag_id}")
|
||||
async def update_tag(
|
||||
tag_id: str, body: TagUpdate, db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Rename or recolor a tag (works for any kind — user, object, face_cluster)."""
|
||||
result = await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id))
|
||||
tag = result.scalar_one_or_none()
|
||||
if not tag:
|
||||
raise HTTPException(status_code=404, detail="Tag not found")
|
||||
|
||||
if body.name is not None:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Tag name is required")
|
||||
tag.name = name
|
||||
if body.color is not None:
|
||||
tag.color = body.color or None
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(tag)
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "kind": tag.kind}
|
||||
|
||||
|
||||
@router.post("/{tag_id}/merge")
|
||||
async def merge_tag(
|
||||
tag_id: str, body: TagMerge, db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Merge tag_id INTO target_id. All photo associations from the source
|
||||
tag are moved to the target, then the source tag is deleted.
|
||||
|
||||
Useful for merging auto-detected face clusters (e.g. "Person 3" → "Alice")
|
||||
or merging duplicate object labels."""
|
||||
if tag_id == body.target_id:
|
||||
raise HTTPException(status_code=400, detail="Cannot merge a tag into itself")
|
||||
|
||||
source = (await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id))).scalar_one_or_none()
|
||||
target = (await db.execute(select(Tag).where(Tag.id == body.target_id, Tag.user_id == current_user.id))).scalar_one_or_none()
|
||||
if not source:
|
||||
raise HTTPException(status_code=404, detail="Source tag not found")
|
||||
if not target:
|
||||
raise HTTPException(status_code=404, detail="Target tag not found")
|
||||
|
||||
# Move photo associations: update tag_id from source → target.
|
||||
# Skip any that would violate the PK (photo already tagged with target).
|
||||
existing_target_photos = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.tag_id == body.target_id
|
||||
)
|
||||
await db.execute(
|
||||
update(photo_tags)
|
||||
.where(
|
||||
photo_tags.c.tag_id == tag_id,
|
||||
photo_tags.c.photo_id.notin_(existing_target_photos),
|
||||
)
|
||||
.values(tag_id=body.target_id)
|
||||
)
|
||||
# Delete remaining source associations (duplicates that couldn't move)
|
||||
from sqlalchemy import delete as sa_delete
|
||||
await db.execute(
|
||||
sa_delete(photo_tags).where(photo_tags.c.tag_id == tag_id)
|
||||
)
|
||||
|
||||
# Delete source tag
|
||||
await db.delete(source)
|
||||
await db.commit()
|
||||
|
||||
return {"merged_into": target.id, "target_name": target.name}
|
||||
|
||||
|
||||
@router.delete("/{tag_id}", status_code=204)
|
||||
async def delete_tag(tag_id: str, db: AsyncSession = Depends(get_db), current_user: User = Depends(get_current_user)):
|
||||
"""Delete a tag. Photo associations cascade-delete via the FK."""
|
||||
result = await db.execute(select(Tag).where(Tag.id == tag_id, Tag.user_id == current_user.id))
|
||||
tag = result.scalar_one_or_none()
|
||||
if not tag:
|
||||
raise HTTPException(status_code=404, detail="Tag not found")
|
||||
await db.delete(tag)
|
||||
await db.commit()
|
||||
return None
|
||||
@@ -1,72 +0,0 @@
|
||||
"""
|
||||
Pydantic schemas for photos
|
||||
"""
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
class PhotoBase(BaseModel):
|
||||
"""Base photo schema"""
|
||||
filename: str
|
||||
media_type: str
|
||||
original_format: Optional[str] = None
|
||||
width: Optional[int] = None
|
||||
height: Optional[int] = None
|
||||
file_size: Optional[int] = None
|
||||
taken_at: Optional[datetime] = None
|
||||
taken_at_source: Optional[str] = None
|
||||
user_title: Optional[str] = None
|
||||
user_notes: Optional[str] = None
|
||||
rating: int = 0
|
||||
color_label: Optional[str] = None
|
||||
|
||||
class PhotoResponse(PhotoBase):
|
||||
"""Photo response schema"""
|
||||
id: str
|
||||
filepath: str
|
||||
folder_id: Optional[str] = None
|
||||
file_hash: Optional[str] = None
|
||||
added_at: datetime
|
||||
updated_at: Optional[datetime] = None
|
||||
is_discarded: bool = False
|
||||
discarded_at: Optional[datetime] = None
|
||||
thumb_small: Optional[str] = None
|
||||
thumb_medium: Optional[str] = None
|
||||
thumb_large: Optional[str] = None
|
||||
processing_status: str = 'pending'
|
||||
processing_error: Optional[str] = None
|
||||
exif_json: Optional[str] = None
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
is_duplicate: bool = False
|
||||
has_date_warning: bool = False
|
||||
live_photo_video_id: Optional[str] = None
|
||||
# tags: List[Dict[str, Any]] = [] # TODO: Enable when using eager loading
|
||||
|
||||
class Config:
|
||||
orm_mode = True
|
||||
from_attributes = True
|
||||
|
||||
class PhotoUpdate(BaseModel):
|
||||
"""Photo update schema"""
|
||||
filename: Optional[str] = None
|
||||
user_title: Optional[str] = None
|
||||
user_notes: Optional[str] = None
|
||||
rating: Optional[int] = Field(None, ge=0, le=5)
|
||||
color_label: Optional[str] = None
|
||||
is_discarded: Optional[bool] = None
|
||||
taken_at: Optional[datetime] = None
|
||||
|
||||
class PhotoListResponse(BaseModel):
|
||||
"""Photo list response with pagination"""
|
||||
photos: List[PhotoResponse]
|
||||
total: int
|
||||
page: int
|
||||
per_page: int
|
||||
pages: int
|
||||
|
||||
class BulkAction(BaseModel):
|
||||
"""Bulk action on photos"""
|
||||
ids: List[str]
|
||||
action: str # 'discard', 'restore', 'delete_permanent', 'move', 'copy', 'add_tag', 'remove_tag', 'set_rating', 'set_color'
|
||||
value: Optional[Any] = None # For actions that need a value (rating, color, tag_id, folder_id)
|
||||
@@ -1,275 +0,0 @@
|
||||
"""
|
||||
One-shot data integrity cleanup for source_roots / folders / photos.
|
||||
|
||||
Earlier versions of the scanner stored paths verbatim, so trailing slashes
|
||||
and redundant separators produced duplicate SourceRoot and Folder rows for
|
||||
the same physical directory. The watcher also auto-created source roots
|
||||
when fired with a parent dir. This module merges the duplicates and
|
||||
re-points photos to the canonical folder so the data lines up with the
|
||||
post-fix scanner.
|
||||
|
||||
Idempotent: safe to run on every backend startup.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from sqlalchemy import select, update, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo, Folder, SourceRoot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
return os.path.normpath(path)
|
||||
|
||||
|
||||
async def _dedupe_source_roots(session: AsyncSession) -> int:
|
||||
"""Group source roots by normalized path and merge duplicates. Returns
|
||||
the number of rows deleted."""
|
||||
result = await session.execute(select(SourceRoot))
|
||||
rows = result.scalars().all()
|
||||
|
||||
groups: dict[str, list[SourceRoot]] = {}
|
||||
for sr in rows:
|
||||
norm = _normalize_path(sr.path)
|
||||
groups.setdefault(norm, []).append(sr)
|
||||
|
||||
deleted = 0
|
||||
for norm, srs in groups.items():
|
||||
if len(srs) == 1:
|
||||
# Make sure the canonical row's path is normalized too.
|
||||
if srs[0].path != norm:
|
||||
srs[0].path = norm
|
||||
continue
|
||||
# Pick the canonical row: prefer one with a non-empty name and the
|
||||
# earliest added_at (most likely the original).
|
||||
canonical = sorted(
|
||||
srs,
|
||||
key=lambda s: (not bool(s.name), s.added_at or datetime.max),
|
||||
)[0]
|
||||
canonical.path = norm
|
||||
for sr in srs:
|
||||
if sr.id == canonical.id:
|
||||
continue
|
||||
# Re-point folders that referenced the duplicate root.
|
||||
await session.execute(
|
||||
update(Folder)
|
||||
.where(Folder.source_root_id == sr.id)
|
||||
.values(source_root_id=canonical.id)
|
||||
)
|
||||
await session.delete(sr)
|
||||
deleted += 1
|
||||
|
||||
return deleted
|
||||
|
||||
|
||||
async def _dedupe_folders(session: AsyncSession) -> int:
|
||||
"""Group folders by normalized path and merge duplicates. Returns the
|
||||
number of rows deleted."""
|
||||
result = await session.execute(select(Folder))
|
||||
rows = result.scalars().all()
|
||||
|
||||
groups: dict[str, list[Folder]] = {}
|
||||
for f in rows:
|
||||
norm = _normalize_path(f.path)
|
||||
groups.setdefault(norm, []).append(f)
|
||||
|
||||
deleted = 0
|
||||
for norm, folders in groups.items():
|
||||
if len(folders) == 1:
|
||||
if folders[0].path != norm:
|
||||
folders[0].path = norm
|
||||
continue
|
||||
# Canonical = the one with the most photos already attached, then
|
||||
# the lowest-id (deterministic tiebreaker).
|
||||
canonical = sorted(
|
||||
folders,
|
||||
key=lambda f: (-(f.photo_count or 0), f.id),
|
||||
)[0]
|
||||
canonical.path = norm
|
||||
for f in folders:
|
||||
if f.id == canonical.id:
|
||||
continue
|
||||
# Re-point photos to the canonical folder.
|
||||
await session.execute(
|
||||
update(Photo)
|
||||
.where(Photo.folder_id == f.id)
|
||||
.values(folder_id=canonical.id)
|
||||
)
|
||||
await session.delete(f)
|
||||
deleted += 1
|
||||
|
||||
return deleted
|
||||
|
||||
|
||||
async def _recompute_folder_counts(session: AsyncSession) -> None:
|
||||
"""Set folder.photo_count to the actual non-discarded photo count."""
|
||||
result = await session.execute(select(Folder))
|
||||
folders = result.scalars().all()
|
||||
for f in folders:
|
||||
count_result = await session.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
Photo.folder_id == f.id,
|
||||
Photo.is_discarded == False, # noqa: E712
|
||||
)
|
||||
)
|
||||
f.photo_count = int(count_result.scalar() or 0)
|
||||
|
||||
|
||||
async def _warn_stale_source_roots(session: AsyncSession) -> int:
|
||||
"""Log a warning for any active source root whose path no longer exists
|
||||
on disk. Doesn't delete — a missing path could be a temporarily
|
||||
unmounted drive, and silently dropping user data is worse than
|
||||
surfacing a noisy log line.
|
||||
"""
|
||||
result = await session.execute(select(SourceRoot))
|
||||
rows = result.scalars().all()
|
||||
stale = 0
|
||||
for sr in rows:
|
||||
if not os.path.isdir(sr.path):
|
||||
stale += 1
|
||||
logger.warning(
|
||||
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
||||
f"— is the docker mount still in place? "
|
||||
f"(Edit docker-compose.yml or PHOTO_DIRS in .env to fix.)"
|
||||
)
|
||||
return stale
|
||||
|
||||
|
||||
async def find_missing(
|
||||
session: AsyncSession,
|
||||
) -> tuple[list[str], list[str], list[str]]:
|
||||
"""Walk every non-discarded photo + every folder and check whether
|
||||
they still resolve on disk. Returns
|
||||
(deletable_photo_ids, deletable_folder_ids, skipped_photo_ids).
|
||||
|
||||
Skipped rows are photos/folders whose owning source_root path itself
|
||||
doesn't resolve — that's almost always an unmounted drive, and
|
||||
silently deleting those rows would be data loss. The caller can
|
||||
surface the skip count separately so the user knows the cleanup
|
||||
wasn't a no-op by accident.
|
||||
"""
|
||||
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
||||
sr_mounted: dict[str, bool] = {sr.id: os.path.isdir(sr.path) for sr in sr_rows}
|
||||
|
||||
photos = (await session.execute(
|
||||
select(Photo.id, Photo.filepath, Photo.folder_id)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
)).all()
|
||||
|
||||
folders = (await session.execute(
|
||||
select(Folder.id, Folder.path, Folder.source_root_id)
|
||||
)).all()
|
||||
folder_to_sr = {fid: srid for fid, _path, srid in folders}
|
||||
|
||||
deletable_photos: list[str] = []
|
||||
skipped: list[str] = []
|
||||
for pid, fp, folder_id in photos:
|
||||
sr_id = folder_to_sr.get(folder_id)
|
||||
if sr_id is None or not sr_mounted.get(sr_id, False):
|
||||
skipped.append(pid)
|
||||
continue
|
||||
if not os.path.exists(fp):
|
||||
deletable_photos.append(pid)
|
||||
|
||||
deletable_folders: list[str] = []
|
||||
for fid, fpath, sr_id in folders:
|
||||
if sr_id is None or not sr_mounted.get(sr_id, False):
|
||||
continue
|
||||
if not os.path.isdir(fpath):
|
||||
deletable_folders.append(fid)
|
||||
|
||||
return deletable_photos, deletable_folders, skipped
|
||||
|
||||
|
||||
async def prune_missing_photos(dry_run: bool = True) -> dict:
|
||||
"""Delete photo + folder rows whose paths are no longer on disk *and*
|
||||
whose source root is currently mounted. Common cause: PHOTO_DIRS in
|
||||
.env was repointed at a different library, leaving every old row
|
||||
orphaned.
|
||||
|
||||
Set dry_run=False to actually delete. The default is intentionally
|
||||
safe so the matching count can be surfaced in the UI before the
|
||||
user commits to it.
|
||||
|
||||
Function name kept for backwards compatibility — it now also prunes
|
||||
folders, not just photos.
|
||||
"""
|
||||
from sqlalchemy import delete
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
deletable_photos, deletable_folders, skipped = await find_missing(session)
|
||||
if not dry_run:
|
||||
CHUNK = 500
|
||||
# Photos first (folders may FK from them via folder_id).
|
||||
for i in range(0, len(deletable_photos), CHUNK):
|
||||
await session.execute(
|
||||
delete(Photo).where(
|
||||
Photo.id.in_(deletable_photos[i:i + CHUNK])
|
||||
)
|
||||
)
|
||||
# Then drop folders that ALSO no longer have any photos
|
||||
# pointing at them. We re-check after the photo delete so
|
||||
# we don't strand a folder that legitimately exists on
|
||||
# disk but happened to match the orphan list.
|
||||
if deletable_folders:
|
||||
for i in range(0, len(deletable_folders), CHUNK):
|
||||
chunk = deletable_folders[i:i + CHUNK]
|
||||
# Only delete folders that now have zero photos
|
||||
# left attached (defensive — should always be 0
|
||||
# if the path is gone, but a concurrent scan
|
||||
# could re-create rows).
|
||||
still_used = (await session.execute(
|
||||
select(Photo.folder_id)
|
||||
.where(Photo.folder_id.in_(chunk))
|
||||
.distinct()
|
||||
)).scalars().all()
|
||||
safe = [f for f in chunk if f not in set(still_used)]
|
||||
if safe:
|
||||
await session.execute(
|
||||
delete(Folder).where(Folder.id.in_(safe))
|
||||
)
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"Pruned {len(deletable_photos)} photo rows + "
|
||||
f"{len(deletable_folders)} folder rows"
|
||||
)
|
||||
key_p = "would_delete" if dry_run else "deleted"
|
||||
key_f = "would_delete_folders" if dry_run else "deleted_folders"
|
||||
return {
|
||||
key_p: len(deletable_photos),
|
||||
key_f: len(deletable_folders),
|
||||
"skipped_unmounted": len(skipped),
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"prune_missing_photos failed: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def cleanup_data_integrity() -> dict:
|
||||
"""Top-level entry point. Runs the dedupe + count refresh in a single
|
||||
transaction. Returns a small summary dict for logging."""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
sr_deleted = await _dedupe_source_roots(session)
|
||||
f_deleted = await _dedupe_folders(session)
|
||||
await _recompute_folder_counts(session)
|
||||
stale = await _warn_stale_source_roots(session)
|
||||
await session.commit()
|
||||
summary = {
|
||||
"source_roots_merged": sr_deleted,
|
||||
"folders_merged": f_deleted,
|
||||
"source_roots_stale": stale,
|
||||
}
|
||||
if sr_deleted or f_deleted:
|
||||
logger.info(f"Cleanup merged duplicates: {summary}")
|
||||
return summary
|
||||
except Exception as e:
|
||||
logger.error(f"Cleanup failed: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
@@ -1,214 +0,0 @@
|
||||
"""
|
||||
Folder/filename-based date guessing and "taken_at looks wrong" detection.
|
||||
|
||||
Direct Python port of `frontend/src/lib/guessDateFromPath.ts` — the logic
|
||||
must stay in sync because the frontend renders the suggestion hint in the
|
||||
info panel while the backend owns the `has_date_warning` flag that the
|
||||
filter bar queries. Any heuristic change has to be applied to both files.
|
||||
|
||||
The guesser walks a filepath, tries the filename first as the source of
|
||||
truth, then falls back to folder segments (deepest first) and multi-
|
||||
segment layouts. Returns ``None`` when no recognisable date can be
|
||||
extracted. `has_date_warning()` compares the guess to a stored `taken_at`
|
||||
and reports whether the difference is large enough to flag.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Literal, Optional
|
||||
|
||||
|
||||
Confidence = Literal["high", "medium", "low"]
|
||||
Source = Literal["folder", "filename"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DateGuess:
|
||||
date: datetime
|
||||
confidence: Confidence
|
||||
matched: str
|
||||
source: Source
|
||||
|
||||
|
||||
_MIN_YEAR = 1970
|
||||
# Bump the ceiling annually via `datetime.now()` rather than a literal so
|
||||
# we don't ship a time bomb. `+1` allows near-future timestamps (cameras
|
||||
# with a slightly advanced clock at year end) without opening the door to
|
||||
# 4-digit serial numbers that happen to start with "30xx".
|
||||
def _max_year() -> int:
|
||||
return datetime.now().year + 1
|
||||
|
||||
|
||||
def _valid_year(y: int) -> bool:
|
||||
return _MIN_YEAR <= y <= _max_year()
|
||||
|
||||
|
||||
def _make_date(y: int, m: int, d: int) -> Optional[datetime]:
|
||||
if not _valid_year(y):
|
||||
return None
|
||||
if not (1 <= m <= 12):
|
||||
return None
|
||||
if not (1 <= d <= 31):
|
||||
return None
|
||||
try:
|
||||
# Noon local so downstream day-bucketing is stable across timezone
|
||||
# rounding. The frontend mirrors this.
|
||||
return datetime(y, m, d, 12, 0, 0)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _segments(filepath: str) -> list[str]:
|
||||
return [s for s in re.split(r"[\\/]+", filepath) if s]
|
||||
|
||||
|
||||
_COMPACT_RE = re.compile(r"(?<!\d)(\d{4})(\d{2})(\d{2})(?!\d)")
|
||||
_DASHED_RE = re.compile(r"(?<!\d)(\d{4})[-_.](\d{1,2})[-_.](\d{1,2})(?!\d)")
|
||||
_MONTH_RE = re.compile(r"(?<!\d)(\d{4})[-_.](\d{1,2})(?!\d)")
|
||||
_YEAR_RE = re.compile(r"(?<!\d)(\d{4})(?!\d)")
|
||||
_FOUR_DIGITS = re.compile(r"^\d{4}$")
|
||||
_ONE_OR_TWO = re.compile(r"^\d{1,2}$")
|
||||
|
||||
|
||||
def _guess_from_string(
|
||||
input: str,
|
||||
source: Source,
|
||||
allow_year_only: bool,
|
||||
) -> Optional[DateGuess]:
|
||||
if not input:
|
||||
return None
|
||||
|
||||
m = _COMPACT_RE.search(input)
|
||||
if m:
|
||||
d = _make_date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="high",
|
||||
matched=f"{m.group(1)}-{m.group(2)}-{m.group(3)}",
|
||||
source=source,
|
||||
)
|
||||
|
||||
m = _DASHED_RE.search(input)
|
||||
if m:
|
||||
d = _make_date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="high",
|
||||
matched=f"{m.group(1)}-{m.group(2)}-{m.group(3)}",
|
||||
source=source,
|
||||
)
|
||||
|
||||
m = _MONTH_RE.search(input)
|
||||
if m:
|
||||
d = _make_date(int(m.group(1)), int(m.group(2)), 15)
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="medium",
|
||||
matched=f"{m.group(1)}-{m.group(2)}",
|
||||
source=source,
|
||||
)
|
||||
|
||||
if allow_year_only:
|
||||
m = _YEAR_RE.search(input)
|
||||
if m:
|
||||
d = _make_date(int(m.group(1)), 7, 1)
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="low",
|
||||
matched=m.group(1),
|
||||
source=source,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _guess_from_folder_layout(folders: list[str]) -> Optional[DateGuess]:
|
||||
# YYYY / MM / DD
|
||||
for i in range(len(folders) - 2):
|
||||
a, b, c = folders[i], folders[i + 1], folders[i + 2]
|
||||
if _FOUR_DIGITS.match(a) and _ONE_OR_TWO.match(b) and _ONE_OR_TWO.match(c):
|
||||
d = _make_date(int(a), int(b), int(c))
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="high",
|
||||
matched=f"{a}/{b}/{c}",
|
||||
source="folder",
|
||||
)
|
||||
# YYYY / MM
|
||||
for i in range(len(folders) - 1):
|
||||
a, b = folders[i], folders[i + 1]
|
||||
if _FOUR_DIGITS.match(a) and _ONE_OR_TWO.match(b):
|
||||
d = _make_date(int(a), int(b), 15)
|
||||
if d:
|
||||
return DateGuess(
|
||||
date=d,
|
||||
confidence="medium",
|
||||
matched=f"{a}/{b}",
|
||||
source="folder",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
_CONFIDENCE_RANK: dict[Confidence, int] = {"high": 3, "medium": 2, "low": 1}
|
||||
|
||||
|
||||
def guess_date_from_path(filepath: str) -> Optional[DateGuess]:
|
||||
"""Filename wins when it has any viable match; otherwise walk folder
|
||||
segments deepest-first and pick the strongest hit."""
|
||||
if not filepath:
|
||||
return None
|
||||
|
||||
segs = _segments(filepath)
|
||||
if not segs:
|
||||
return None
|
||||
filename = segs[-1]
|
||||
folders = segs[:-1]
|
||||
|
||||
from_filename = _guess_from_string(filename, "filename", allow_year_only=False)
|
||||
if from_filename:
|
||||
return from_filename
|
||||
|
||||
best: Optional[DateGuess] = None
|
||||
for seg in reversed(folders):
|
||||
hit = _guess_from_string(seg, "folder", allow_year_only=True)
|
||||
if not hit:
|
||||
continue
|
||||
if not best or _CONFIDENCE_RANK[hit.confidence] > _CONFIDENCE_RANK[best.confidence]:
|
||||
best = hit
|
||||
if hit.confidence == "high":
|
||||
break
|
||||
|
||||
from_layout = _guess_from_folder_layout(folders)
|
||||
if from_layout and (
|
||||
not best or _CONFIDENCE_RANK[from_layout.confidence] > _CONFIDENCE_RANK[best.confidence]
|
||||
):
|
||||
best = from_layout
|
||||
|
||||
return best
|
||||
|
||||
|
||||
_ONE_DAY = 24 * 60 * 60
|
||||
|
||||
|
||||
def has_date_warning(filepath: str, taken_at: Optional[datetime]) -> bool:
|
||||
"""True when the path-based guess disagrees with ``taken_at`` by more
|
||||
than 24h, or when ``taken_at`` is missing and the path would supply
|
||||
one. This is the authoritative flag stored on `photos.has_date_warning`
|
||||
and queried by the timeline filter."""
|
||||
guess = guess_date_from_path(filepath)
|
||||
if not guess:
|
||||
return False
|
||||
if taken_at is None:
|
||||
return True
|
||||
try:
|
||||
diff = abs((taken_at - guess.date).total_seconds())
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return diff > _ONE_DAY
|
||||
@@ -1,382 +0,0 @@
|
||||
"""
|
||||
Duplicate detection: group photos by perceptual-hash + CLIP similarity.
|
||||
|
||||
Strategy
|
||||
--------
|
||||
Two complementary signals are fused into a single grouping:
|
||||
|
||||
1. **Perceptual hash (pHash)** — 16-char hex hash from the thumbnail
|
||||
worker. Catches byte-identical copies and mild re-encodes via
|
||||
Hamming distance (threshold ≤ 6 bits out of 64).
|
||||
|
||||
2. **CLIP embedding similarity** — cosine distance over 512-d vectors
|
||||
stored in the `embeddings` table with an HNSW index. Catches
|
||||
visually similar photos even when pHash diverges (e.g. crops,
|
||||
different formats, screenshots of the same content).
|
||||
|
||||
Both signals feed a union-find structure that merges overlapping matches
|
||||
into connected components.
|
||||
|
||||
Incremental mode (default post-scan)
|
||||
-------------------------------------
|
||||
`incremental_regroup` only compares *newly added* photos (those whose
|
||||
`added_at` > watermark) against the entire library. Each new photo does:
|
||||
|
||||
- An HNSW vector similarity query: O(log N) via the index.
|
||||
- A pHash comparison against a small candidate set (same group members
|
||||
or nearby CLIP results) rather than the full N² sweep.
|
||||
|
||||
This makes the post-scan cost O(new × log N) instead of O(N²).
|
||||
|
||||
Full regroup
|
||||
------------
|
||||
`regroup_duplicates` still performs the full pairwise pHash pass +
|
||||
CLIP sweep, used for initial setup and manual re-detection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select, update, text
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models.photos import Photo
|
||||
from app.models.embeddings import Embedding
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# pHash Hamming distance threshold (6 out of 64 bits).
|
||||
DEFAULT_PHASH_THRESHOLD = 6
|
||||
|
||||
# CLIP cosine distance threshold. CLIP embeddings are L2-normalized,
|
||||
# so cosine distance = 1 - dot(a, b). A threshold of 0.08 catches
|
||||
# visually near-identical shots; 0.15 catches similar compositions.
|
||||
DEFAULT_CLIP_THRESHOLD = 0.10
|
||||
|
||||
|
||||
def _hex_to_int(h: str) -> int:
|
||||
"""Parse a 16-char hex pHash to a Python int. Returns -1 on bad input
|
||||
so the pairwise loop can skip the row without raising."""
|
||||
try:
|
||||
return int(h, 16)
|
||||
except (TypeError, ValueError):
|
||||
return -1
|
||||
|
||||
|
||||
def _hamming(a: int, b: int) -> int:
|
||||
"""Population count of XOR — the canonical hash distance metric."""
|
||||
x = a ^ b
|
||||
try:
|
||||
return x.bit_count() # type: ignore[attr-defined]
|
||||
except AttributeError:
|
||||
return bin(x).count('1')
|
||||
|
||||
|
||||
class _UnionFind:
|
||||
"""Tiny union-find / disjoint-set used to merge similar photos into
|
||||
connected components."""
|
||||
|
||||
def __init__(self, keys: list[str]) -> None:
|
||||
self._index = {k: i for i, k in enumerate(keys)}
|
||||
n = len(keys)
|
||||
self.parent = list(range(n))
|
||||
self.rank = [0] * n
|
||||
|
||||
def find(self, x: int) -> int:
|
||||
while self.parent[x] != x:
|
||||
self.parent[x] = self.parent[self.parent[x]]
|
||||
x = self.parent[x]
|
||||
return x
|
||||
|
||||
def union_by_key(self, key_a: str, key_b: str) -> None:
|
||||
ia, ib = self._index.get(key_a), self._index.get(key_b)
|
||||
if ia is None or ib is None:
|
||||
return
|
||||
ra, rb = self.find(ia), self.find(ib)
|
||||
if ra == rb:
|
||||
return
|
||||
if self.rank[ra] < self.rank[rb]:
|
||||
ra, rb = rb, ra
|
||||
self.parent[rb] = ra
|
||||
if self.rank[ra] == self.rank[rb]:
|
||||
self.rank[ra] += 1
|
||||
|
||||
def components(self, keys: list[str]) -> dict[int, list[str]]:
|
||||
"""Return {root_idx: [photo_ids...]} for groups of size >= 2."""
|
||||
groups: dict[int, list[str]] = {}
|
||||
for key in keys:
|
||||
idx = self._index[key]
|
||||
root = self.find(idx)
|
||||
groups.setdefault(root, []).append(key)
|
||||
return {r: members for r, members in groups.items() if len(members) >= 2}
|
||||
|
||||
|
||||
async def regroup_duplicates(
|
||||
phash_threshold: int = DEFAULT_PHASH_THRESHOLD,
|
||||
clip_threshold: float = DEFAULT_CLIP_THRESHOLD,
|
||||
) -> dict:
|
||||
"""Full recompute of duplicate groups using pHash + CLIP similarity.
|
||||
|
||||
Idempotent — safe to call as often as you like. Returns a summary dict.
|
||||
"""
|
||||
embedder_model = settings.vision.embedder.name
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Pull all visible photos with a phash or embedding.
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Photo.id, Photo.phash)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.where(Photo.is_hidden.is_(False))
|
||||
)
|
||||
).all()
|
||||
|
||||
if not rows:
|
||||
await _clear_all_groups(session)
|
||||
await session.commit()
|
||||
return {'photos_considered': 0, 'groups': 0, 'members': 0}
|
||||
|
||||
ids = [row[0] for row in rows]
|
||||
phash_map = {row[0]: _hex_to_int(row[1]) for row in rows if row[1]}
|
||||
|
||||
uf = _UnionFind(ids)
|
||||
|
||||
# ── Phase 1: pHash pairwise (O(N²) on photos with phash) ──
|
||||
phash_ids = [pid for pid in ids if pid in phash_map]
|
||||
phash_vals = [phash_map[pid] for pid in phash_ids]
|
||||
n = len(phash_ids)
|
||||
for i in range(n):
|
||||
hi = phash_vals[i]
|
||||
if hi < 0:
|
||||
continue
|
||||
for j in range(i + 1, n):
|
||||
hj = phash_vals[j]
|
||||
if hj < 0:
|
||||
continue
|
||||
if _hamming(hi, hj) <= phash_threshold:
|
||||
uf.union_by_key(phash_ids[i], phash_ids[j])
|
||||
|
||||
# ── Phase 2: CLIP similarity via pgvector ──
|
||||
# For each photo with an embedding, find its nearest neighbors
|
||||
# within the cosine distance threshold using the HNSW index.
|
||||
clip_matches = await _clip_neighbor_scan(
|
||||
session, ids, embedder_model, clip_threshold
|
||||
)
|
||||
for photo_id, neighbor_id in clip_matches:
|
||||
uf.union_by_key(photo_id, neighbor_id)
|
||||
|
||||
# ── Write results ──
|
||||
await _clear_all_groups(session)
|
||||
|
||||
groups = uf.components(ids)
|
||||
groups_created = 0
|
||||
members_total = 0
|
||||
for member_ids in groups.values():
|
||||
group_id = str(uuid.uuid4())
|
||||
await session.execute(
|
||||
update(Photo)
|
||||
.where(Photo.id.in_(member_ids))
|
||||
.values(duplicate_group_id=group_id, is_duplicate=True)
|
||||
)
|
||||
groups_created += 1
|
||||
members_total += len(member_ids)
|
||||
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"regroup_duplicates: {len(ids)} photos, "
|
||||
f"{groups_created} group(s), {members_total} member(s)"
|
||||
)
|
||||
return {
|
||||
'photos_considered': len(ids),
|
||||
'groups': groups_created,
|
||||
'members': members_total,
|
||||
}
|
||||
|
||||
|
||||
async def incremental_regroup(
|
||||
since: Optional[datetime] = None,
|
||||
phash_threshold: int = DEFAULT_PHASH_THRESHOLD,
|
||||
clip_threshold: float = DEFAULT_CLIP_THRESHOLD,
|
||||
) -> dict:
|
||||
"""Incremental duplicate detection for newly added photos.
|
||||
|
||||
Only photos added after `since` are compared against the full library.
|
||||
Much faster than a full regroup for post-scan updates:
|
||||
O(new × log N) via HNSW instead of O(N²).
|
||||
"""
|
||||
embedder_model = settings.vision.embedder.name
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
# If no watermark, fall back to full regroup.
|
||||
if since is None:
|
||||
# Find the most recent scan start by looking at the newest
|
||||
# photo that already has a duplicate_group_id check completed.
|
||||
# As a simple heuristic, use photos added in the last hour.
|
||||
from datetime import timedelta
|
||||
since = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
|
||||
# Get newly added photos (the "new" set).
|
||||
new_rows = (
|
||||
await session.execute(
|
||||
select(Photo.id, Photo.phash)
|
||||
.where(Photo.added_at >= since)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.where(Photo.is_hidden.is_(False))
|
||||
)
|
||||
).all()
|
||||
|
||||
if not new_rows:
|
||||
return {'photos_considered': 0, 'new_photos': 0, 'groups_updated': 0, 'members_added': 0}
|
||||
|
||||
new_ids = [r[0] for r in new_rows]
|
||||
new_phash = {r[0]: _hex_to_int(r[1]) for r in new_rows if r[1]}
|
||||
|
||||
# Get ALL existing photos for union-find (we need to merge into
|
||||
# existing groups).
|
||||
all_rows = (
|
||||
await session.execute(
|
||||
select(Photo.id, Photo.phash, Photo.duplicate_group_id)
|
||||
.where(Photo.is_discarded.is_(False))
|
||||
.where(Photo.is_hidden.is_(False))
|
||||
)
|
||||
).all()
|
||||
|
||||
all_ids = [r[0] for r in all_rows]
|
||||
all_phash = {r[0]: _hex_to_int(r[1]) for r in all_rows if r[1]}
|
||||
existing_groups: dict[str, str] = {
|
||||
r[0]: r[2] for r in all_rows if r[2]
|
||||
}
|
||||
|
||||
uf = _UnionFind(all_ids)
|
||||
|
||||
# Pre-seed existing groups into the union-find so we merge into
|
||||
# them rather than creating parallel groups.
|
||||
group_to_members: dict[str, list[str]] = {}
|
||||
for pid, gid in existing_groups.items():
|
||||
group_to_members.setdefault(gid, []).append(pid)
|
||||
for members in group_to_members.values():
|
||||
for i in range(1, len(members)):
|
||||
uf.union_by_key(members[0], members[i])
|
||||
|
||||
# ── Phase 1: pHash — compare each new photo against ALL photos ──
|
||||
for new_id in new_ids:
|
||||
nh = new_phash.get(new_id, -1)
|
||||
if nh < 0:
|
||||
continue
|
||||
for existing_id, eh in all_phash.items():
|
||||
if existing_id == new_id or eh < 0:
|
||||
continue
|
||||
if _hamming(nh, eh) <= phash_threshold:
|
||||
uf.union_by_key(new_id, existing_id)
|
||||
|
||||
# ── Phase 2: CLIP — vector similarity for new photos only ──
|
||||
clip_matches = await _clip_neighbor_scan(
|
||||
session, new_ids, embedder_model, clip_threshold
|
||||
)
|
||||
for photo_id, neighbor_id in clip_matches:
|
||||
uf.union_by_key(photo_id, neighbor_id)
|
||||
|
||||
# ── Write results ──
|
||||
# Only update groups that contain at least one new photo.
|
||||
# Clear all groups first, then rewrite.
|
||||
await _clear_all_groups(session)
|
||||
|
||||
groups = uf.components(all_ids)
|
||||
groups_created = 0
|
||||
members_total = 0
|
||||
new_in_groups = 0
|
||||
for member_ids in groups.values():
|
||||
group_id = str(uuid.uuid4())
|
||||
await session.execute(
|
||||
update(Photo)
|
||||
.where(Photo.id.in_(member_ids))
|
||||
.values(duplicate_group_id=group_id, is_duplicate=True)
|
||||
)
|
||||
groups_created += 1
|
||||
members_total += len(member_ids)
|
||||
if any(m in new_ids for m in member_ids):
|
||||
new_in_groups += len([m for m in member_ids if m in new_ids])
|
||||
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"incremental_regroup: {len(new_ids)} new photos, "
|
||||
f"{groups_created} group(s), {new_in_groups} new member(s) grouped"
|
||||
)
|
||||
return {
|
||||
'photos_considered': len(all_ids),
|
||||
'new_photos': len(new_ids),
|
||||
'groups_updated': groups_created,
|
||||
'members_added': new_in_groups,
|
||||
}
|
||||
|
||||
|
||||
async def _clip_neighbor_scan(
|
||||
session,
|
||||
photo_ids: list[str],
|
||||
embedder_model: str,
|
||||
threshold: float,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""For each photo in `photo_ids` that has a CLIP embedding, find
|
||||
neighbors within cosine distance `threshold` using pgvector HNSW.
|
||||
|
||||
Returns a list of (photo_id, neighbor_id) pairs.
|
||||
"""
|
||||
matches: list[tuple[str, str]] = []
|
||||
|
||||
if not photo_ids:
|
||||
return matches
|
||||
|
||||
# Batch: get all embeddings for the target photos.
|
||||
target_embeddings = (
|
||||
await session.execute(
|
||||
select(Embedding.photo_id, Embedding.vector)
|
||||
.where(Embedding.photo_id.in_(photo_ids))
|
||||
.where(Embedding.model == embedder_model)
|
||||
)
|
||||
).all()
|
||||
|
||||
if not target_embeddings:
|
||||
return matches
|
||||
|
||||
# For each target, query nearest neighbors via pgvector.
|
||||
# We use raw SQL for the <=> cosine distance operator.
|
||||
for photo_id, vector in target_embeddings:
|
||||
# pgvector cosine distance: <=> operator
|
||||
# Find top 20 nearest neighbors within threshold.
|
||||
result = await session.execute(
|
||||
text("""
|
||||
SELECT e.photo_id, (e.vector <=> :vec) AS distance
|
||||
FROM embeddings e
|
||||
JOIN photos p ON p.id = e.photo_id
|
||||
WHERE e.model = :model
|
||||
AND e.photo_id != :pid
|
||||
AND p.is_discarded = false
|
||||
AND p.is_hidden = false
|
||||
AND (e.vector <=> :vec) < :threshold
|
||||
ORDER BY e.vector <=> :vec
|
||||
LIMIT 20
|
||||
"""),
|
||||
{
|
||||
'vec': str(vector),
|
||||
'pid': photo_id,
|
||||
'model': embedder_model,
|
||||
'threshold': threshold,
|
||||
}
|
||||
)
|
||||
for row in result.all():
|
||||
matches.append((photo_id, row[0]))
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
async def _clear_all_groups(session) -> None:
|
||||
"""Reset duplicate_group_id / is_duplicate on every photo."""
|
||||
await session.execute(
|
||||
update(Photo).values(duplicate_group_id=None, is_duplicate=False)
|
||||
)
|
||||
@@ -1,72 +0,0 @@
|
||||
"""
|
||||
EXIF write-back helpers.
|
||||
|
||||
The rest of the app reads EXIF at scan time and stashes the result in Postgres
|
||||
(see `services/metadata.py`). This module handles the reverse direction: when
|
||||
the user corrects a date in the UI we also rewrite the relevant EXIF tags on
|
||||
disk so a later rescan won't clobber the fix and external tools see the same
|
||||
truth the DB does.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EXIFTOOL_TIMEOUT_SECONDS = 30
|
||||
|
||||
|
||||
class ExifWriteError(RuntimeError):
|
||||
"""Raised when exiftool fails to write tags to a file."""
|
||||
|
||||
|
||||
def _format_exif_dt(dt: datetime) -> str:
|
||||
return dt.strftime("%Y:%m:%d %H:%M:%S")
|
||||
|
||||
|
||||
async def write_taken_at(filepath: str, dt: datetime) -> None:
|
||||
"""Rewrite DateTimeOriginal / CreateDate / ModifyDate on the file.
|
||||
|
||||
- ``-overwrite_original`` so we don't litter the library with
|
||||
``<name>_original`` sidecars.
|
||||
- ``-P`` preserves the file's mtime so the scanner's mtime-based
|
||||
change detection stays quiet.
|
||||
- We set all three common date tags together because different viewers
|
||||
read different ones; keeping them in lockstep avoids confusing
|
||||
downstream tools and our own re-extraction pass.
|
||||
"""
|
||||
if not Path(filepath).exists():
|
||||
raise ExifWriteError(f"File not found: {filepath}")
|
||||
|
||||
stamp = _format_exif_dt(dt)
|
||||
cmd = [
|
||||
"exiftool",
|
||||
"-overwrite_original",
|
||||
"-P",
|
||||
f"-DateTimeOriginal={stamp}",
|
||||
f"-CreateDate={stamp}",
|
||||
f"-ModifyDate={stamp}",
|
||||
filepath,
|
||||
]
|
||||
|
||||
def _run() -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=EXIFTOOL_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(_run)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise ExifWriteError(f"exiftool timed out writing {filepath}") from exc
|
||||
except FileNotFoundError as exc:
|
||||
raise ExifWriteError("exiftool binary not available") from exc
|
||||
|
||||
if result.returncode != 0:
|
||||
msg = (result.stderr or result.stdout or "unknown error").strip()
|
||||
logger.warning("exiftool write failed for %s: %s", filepath, msg)
|
||||
raise ExifWriteError(msg)
|
||||
@@ -1,287 +0,0 @@
|
||||
"""
|
||||
Metadata extraction service using ExifTool
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from celery import shared_task
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo
|
||||
from app.services.date_guess import has_date_warning
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def parse_exif_datetime(date_str: str) -> Optional[datetime]:
|
||||
"""Parse EXIF datetime string to Python datetime"""
|
||||
if not date_str:
|
||||
return None
|
||||
|
||||
# Common EXIF datetime formats
|
||||
formats = [
|
||||
"%Y:%m:%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y:%m:%d %H:%M:%S.%f",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%dT%H:%M:%S.%f",
|
||||
"%Y-%m-%dT%H:%M:%S%z"
|
||||
]
|
||||
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(date_str, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
_DMS_RE = re.compile(
|
||||
r"""\s*
|
||||
(?P<deg>-?\d+(?:\.\d+)?)\s*(?:deg|°|d)?\s*
|
||||
(?:(?P<min>\d+(?:\.\d+)?)\s*[\'’m]?\s*)?
|
||||
(?:(?P<sec>\d+(?:\.\d+)?)\s*[\"”s]?\s*)?
|
||||
(?P<ref>[NSEW])?\s*$""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_coord(value, ref: str | None) -> float | None:
|
||||
"""Coerce a single GPS coordinate from any form ExifTool may emit.
|
||||
|
||||
ExifTool's ``-j`` JSON output applies print conversion by default, so
|
||||
coordinates can come back as:
|
||||
|
||||
* a number (``48.1278``) — happens for some sources / when ``-n`` is set
|
||||
* a plain DMS string (``"48 deg 7' 39.96\\""``) — bare ``EXIF:GPSLatitude``
|
||||
* a DMS-with-ref string (``"48 deg 7' 39.96\\" N"``) — ``Composite:GPSLatitude``
|
||||
|
||||
The optional ``ref`` argument lets the caller pass an explicit
|
||||
``GPSLatitudeRef`` / ``GPSLongitudeRef`` ('N'/'S'/'E'/'W') when the
|
||||
string itself doesn't carry one. Returns signed decimal degrees, or
|
||||
``None`` if the value is unparseable.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
# Numeric path — already decimal degrees, possibly already signed.
|
||||
if isinstance(value, (int, float)):
|
||||
out = float(value)
|
||||
else:
|
||||
m = _DMS_RE.match(str(value))
|
||||
if not m:
|
||||
return None
|
||||
deg = float(m.group('deg'))
|
||||
minutes = float(m.group('min') or 0)
|
||||
seconds = float(m.group('sec') or 0)
|
||||
out = abs(deg) + minutes / 60.0 + seconds / 3600.0
|
||||
if deg < 0:
|
||||
out = -out
|
||||
embedded_ref = m.group('ref')
|
||||
if embedded_ref:
|
||||
ref = embedded_ref
|
||||
if ref:
|
||||
r = ref[0].upper()
|
||||
if r in ('S', 'W'):
|
||||
out = -abs(out)
|
||||
elif r in ('N', 'E'):
|
||||
out = abs(out)
|
||||
return out
|
||||
|
||||
|
||||
def extract_gps(exif_data: Dict) -> tuple:
|
||||
"""Return (lat, lon) in signed decimal degrees, or (None, None).
|
||||
|
||||
With ``exiftool -G -j`` GPS values are keyed under their group.
|
||||
``Composite:GPSLatitude`` / ``Composite:GPSLongitude`` carry the
|
||||
hemisphere reference inline (``"48 deg 7' 39.96\\" N"``) while the bare
|
||||
``EXIF:GPSLatitude`` / ``EXIF:GPSLongitude`` need the separate
|
||||
``EXIF:GPSLatitudeRef`` / ``EXIF:GPSLongitudeRef`` to know the sign.
|
||||
|
||||
Pre-fix this function read the *unprefixed* keys ``GPSLatitude`` /
|
||||
``GPSLongitude`` (which never exist in ``-G`` output) AND assumed
|
||||
they were already floats — so it silently dropped every photo's GPS.
|
||||
"""
|
||||
lat = _parse_coord(exif_data.get('Composite:GPSLatitude'), None)
|
||||
lon = _parse_coord(exif_data.get('Composite:GPSLongitude'), None)
|
||||
if lat is None or lon is None:
|
||||
lat = _parse_coord(
|
||||
exif_data.get('EXIF:GPSLatitude'),
|
||||
exif_data.get('EXIF:GPSLatitudeRef'),
|
||||
)
|
||||
lon = _parse_coord(
|
||||
exif_data.get('EXIF:GPSLongitude'),
|
||||
exif_data.get('EXIF:GPSLongitudeRef'),
|
||||
)
|
||||
if lat is None or lon is None:
|
||||
return None, None
|
||||
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
|
||||
return None, None
|
||||
# Some cameras emit (0, 0) when they have no GPS lock — treat as missing
|
||||
if lat == 0 and lon == 0:
|
||||
return None, None
|
||||
return lat, lon
|
||||
|
||||
|
||||
def extract_key_metadata(exif_data: Dict) -> Dict:
|
||||
"""Extract key metadata fields for FTS indexing"""
|
||||
key_fields = []
|
||||
|
||||
# Camera information
|
||||
if 'EXIF:Make' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:Make'])
|
||||
if 'EXIF:Model' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:Model'])
|
||||
if 'EXIF:LensModel' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:LensModel'])
|
||||
|
||||
# Location information
|
||||
lat, lon = extract_gps(exif_data)
|
||||
if lat is not None and lon is not None:
|
||||
key_fields.append(f"GPS: {lat}, {lon}")
|
||||
|
||||
# IPTC/XMP keywords
|
||||
keywords = exif_data.get('IPTC:Keywords') or exif_data.get('XMP:Subject')
|
||||
if keywords:
|
||||
if isinstance(keywords, list):
|
||||
key_fields.extend(keywords)
|
||||
else:
|
||||
key_fields.append(keywords)
|
||||
|
||||
# Copyright and creator
|
||||
if 'EXIF:Copyright' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:Copyright'])
|
||||
if 'XMP:Creator' in exif_data:
|
||||
key_fields.append(exif_data['XMP:Creator'])
|
||||
if 'EXIF:Artist' in exif_data:
|
||||
key_fields.append(exif_data['EXIF:Artist'])
|
||||
|
||||
return {
|
||||
'exif_text': ' '.join(str(f) for f in key_fields),
|
||||
'camera_make': exif_data.get('EXIF:Make'),
|
||||
'camera_model': exif_data.get('EXIF:Model'),
|
||||
'lens_model': exif_data.get('EXIF:LensModel'),
|
||||
'gps_latitude': lat,
|
||||
'gps_longitude': lon,
|
||||
}
|
||||
|
||||
@shared_task(name='extract_metadata')
|
||||
def extract_metadata(photo_id: str):
|
||||
"""Extract metadata from a photo using ExifTool"""
|
||||
return asyncio.run(_extract_metadata_async(photo_id))
|
||||
|
||||
async def _extract_metadata_async(photo_id: str):
|
||||
"""Async implementation of metadata extraction"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# Get photo from database
|
||||
result = await session.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
|
||||
if not photo:
|
||||
logger.error(f"Photo not found: {photo_id}")
|
||||
return {'status': 'error', 'message': 'Photo not found'}
|
||||
|
||||
# Check if file exists
|
||||
if not Path(photo.filepath).exists():
|
||||
logger.error(f"File not found: {photo.filepath}")
|
||||
return {'status': 'error', 'message': 'File not found'}
|
||||
|
||||
# Run ExifTool to extract metadata
|
||||
cmd = [
|
||||
'exiftool',
|
||||
'-j', # JSON output
|
||||
'-G', # Group names
|
||||
'-s', # Short output format
|
||||
'-All', # All metadata
|
||||
photo.filepath
|
||||
]
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error(f"ExifTool error: {result.stderr}")
|
||||
return {'status': 'error', 'message': result.stderr}
|
||||
|
||||
# Parse JSON output
|
||||
metadata = json.loads(result.stdout)
|
||||
if metadata and len(metadata) > 0:
|
||||
exif_data = metadata[0]
|
||||
|
||||
# Store full metadata as JSON
|
||||
photo.exif_json = json.dumps(exif_data)
|
||||
|
||||
# Extract taken_at date
|
||||
date_fields = [
|
||||
'EXIF:DateTimeOriginal',
|
||||
'EXIF:CreateDate',
|
||||
'QuickTime:MediaCreateDate',
|
||||
'EXIF:ModifyDate'
|
||||
]
|
||||
|
||||
for field in date_fields:
|
||||
if field in exif_data:
|
||||
taken_at = parse_exif_datetime(exif_data[field])
|
||||
if taken_at:
|
||||
photo.taken_at = taken_at
|
||||
photo.taken_at_source = 'exif'
|
||||
break
|
||||
|
||||
# Re-run the path-vs-date heuristic now that we know
|
||||
# whether EXIF provided a real capture date. A true EXIF
|
||||
# date that matches the folder clears the warning the
|
||||
# scanner set during the filesystem-mtime pass.
|
||||
photo.has_date_warning = has_date_warning(
|
||||
photo.filepath, photo.taken_at
|
||||
)
|
||||
|
||||
# Extract dimensions if not already set
|
||||
if not photo.width:
|
||||
photo.width = exif_data.get('EXIF:ImageWidth') or exif_data.get('File:ImageWidth')
|
||||
if not photo.height:
|
||||
photo.height = exif_data.get('EXIF:ImageHeight') or exif_data.get('File:ImageHeight')
|
||||
|
||||
# Extract GPS coordinates into first-class columns so the
|
||||
# Map view can query them without parsing exif_json.
|
||||
lat, lon = extract_gps(exif_data)
|
||||
photo.latitude = lat
|
||||
photo.longitude = lon
|
||||
|
||||
# Extract and store key metadata for search
|
||||
key_metadata = extract_key_metadata(exif_data)
|
||||
|
||||
# Update FTS table (would be done via trigger in production)
|
||||
# For now, we'll store it in a comment
|
||||
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"Metadata extracted for photo {photo_id}")
|
||||
return {
|
||||
'status': 'success',
|
||||
'photo_id': photo_id,
|
||||
'taken_at': photo.taken_at.isoformat() if photo.taken_at else None
|
||||
}
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error(f"ExifTool timeout for {photo.filepath}")
|
||||
return {'status': 'error', 'message': 'ExifTool timeout'}
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse ExifTool output: {e}")
|
||||
return {'status': 'error', 'message': 'Invalid ExifTool output'}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting metadata for {photo_id}: {e}")
|
||||
return {'status': 'error', 'message': str(e)}
|
||||
@@ -1,103 +0,0 @@
|
||||
"""
|
||||
Scanner service for initial library scan and per-user source root bootstrap.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import SourceRoot
|
||||
from app.models.user import User
|
||||
from app.tasks.scan import scan_all_source_roots
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def bootstrap_user_source_root(user: User, session=None) -> None:
|
||||
"""Create the media directory and a source root for a user.
|
||||
|
||||
Called when a new user is created (by the admin or the setup endpoint).
|
||||
If the user already has a source root, this is a no-op.
|
||||
"""
|
||||
own_session = session is None
|
||||
if own_session:
|
||||
session = AsyncSessionLocal()
|
||||
|
||||
try:
|
||||
# Check if user already has a source root
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.user_id == user.id)
|
||||
)
|
||||
if result.scalar_one_or_none() is not None:
|
||||
return
|
||||
|
||||
os.makedirs(user.media_path, exist_ok=True)
|
||||
|
||||
source_root = SourceRoot(
|
||||
name=f"{user.username}'s Library",
|
||||
path=user.media_path,
|
||||
user_id=user.id,
|
||||
)
|
||||
session.add(source_root)
|
||||
if own_session:
|
||||
await session.commit()
|
||||
else:
|
||||
await session.flush()
|
||||
|
||||
logger.info(
|
||||
f"Bootstrapped source root for user '{user.username}': "
|
||||
f"{user.media_path}"
|
||||
)
|
||||
finally:
|
||||
if own_session:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def bootstrap_default_source_root() -> None:
|
||||
"""Legacy bootstrap — for existing installs that have source roots
|
||||
without user_id (pre-auth migration). On fresh installs, source roots
|
||||
are created per-user via bootstrap_user_source_root. If there are
|
||||
already source roots in the DB, this is a no-op.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(select(SourceRoot))
|
||||
if result.scalars().first() is not None:
|
||||
return # Already have source roots.
|
||||
|
||||
# No source roots and no users means fresh install — the setup
|
||||
# endpoint will create the first user + source root.
|
||||
user_count = (await session.execute(
|
||||
select(User)
|
||||
)).scalars().first()
|
||||
if user_count is None:
|
||||
logger.info(
|
||||
"No users or source roots — waiting for first-run setup."
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
async def start_initial_scan():
|
||||
"""Start the initial library scan and optionally the file watcher.
|
||||
|
||||
The file watcher uses a Redis lock to ensure only one instance runs
|
||||
across all workers, so it's safe to dispatch on every startup — only
|
||||
the first one will actually watch, the rest exit immediately.
|
||||
"""
|
||||
try:
|
||||
scan_all_source_roots.delay()
|
||||
logger.info("Initial scan queued successfully")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start initial scan: {e}")
|
||||
|
||||
# Start the file watcher if enabled in config.
|
||||
from app.config import settings
|
||||
if settings.scanner.watch:
|
||||
try:
|
||||
from app.tasks.scan import watch_folders
|
||||
# Countdown gives the initial scan time to register source roots
|
||||
# before the watcher tries to load them.
|
||||
watch_folders.apply_async(countdown=10)
|
||||
logger.info("File watcher queued (Redis-locked, single instance)")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue file watcher: {e}")
|
||||
@@ -1,161 +0,0 @@
|
||||
"""
|
||||
Unified search service — hybrid FTS + semantic (RRF) search.
|
||||
|
||||
Phase 1 (PR4): semantic-only via pgvector cosine similarity.
|
||||
Phase 2 (PR5): adds FTS via tsvector, enables RRF fusion.
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
from sqlalchemy import select, text, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Photo
|
||||
from app.models.embeddings import Embedding
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def hybrid_search(
|
||||
db: AsyncSession,
|
||||
q: Optional[str] = None,
|
||||
tag_ids: Optional[list[str]] = None,
|
||||
date_from: Optional[str] = None,
|
||||
date_to: Optional[str] = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
"""Run hybrid search (FTS + semantic) with RRF fusion.
|
||||
|
||||
Currently semantic-only; FTS leg added in PR5.
|
||||
"""
|
||||
model_name = settings.vision.embedder.name
|
||||
results = {}
|
||||
|
||||
# ── Semantic search (CLIP text → pgvector cosine) ─────────────────
|
||||
if q:
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
query_vec = embedder.embed_text(q)
|
||||
|
||||
# pgvector cosine distance: <=> returns distance (lower = closer).
|
||||
# Join photos so we can filter out discarded / hidden rows
|
||||
# inside the same query — otherwise a hidden-folder photo
|
||||
# can take a top-N rank and starve the visible results.
|
||||
vec_str = "[" + ",".join(str(float(v)) for v in query_vec) + "]"
|
||||
stmt = text("""
|
||||
SELECT e.photo_id,
|
||||
(e.vector <=> :qvec::vector) AS distance
|
||||
FROM embeddings e
|
||||
JOIN photos p ON p.id = e.photo_id
|
||||
WHERE e.model = :model
|
||||
AND p.is_trashed = false
|
||||
AND p.is_hidden = false
|
||||
ORDER BY e.vector <=> :qvec::vector
|
||||
LIMIT 200
|
||||
""")
|
||||
rows = (await db.execute(stmt, {"qvec": vec_str, "model": model_name})).fetchall()
|
||||
|
||||
for rank, (photo_id, distance) in enumerate(rows):
|
||||
if photo_id not in results:
|
||||
results[photo_id] = {"semantic_rank": rank, "fts_rank": None}
|
||||
else:
|
||||
results[photo_id]["semantic_rank"] = rank
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Semantic search failed (models may not be loaded): %s", e)
|
||||
|
||||
# ── FTS search (photos.search_vector + ocr_text) ────────────────
|
||||
if q:
|
||||
try:
|
||||
# Same discarded/hidden filter as the semantic leg.
|
||||
# The OCR branch joins photos (through photo_id) so we can
|
||||
# filter there too; otherwise OCR hits in hidden folders
|
||||
# would leak into results.
|
||||
fts_stmt = text("""
|
||||
SELECT id, ts_rank(search_vector, plainto_tsquery('english', :q)) AS rank
|
||||
FROM photos
|
||||
WHERE search_vector @@ plainto_tsquery('english', :q)
|
||||
AND is_trashed = false
|
||||
AND is_hidden = false
|
||||
UNION
|
||||
SELECT o.photo_id AS id,
|
||||
MAX(o.confidence) AS rank
|
||||
FROM ocr_text o
|
||||
JOIN photos p ON p.id = o.photo_id
|
||||
WHERE to_tsvector('english', o.text) @@ plainto_tsquery('english', :q)
|
||||
AND p.is_trashed = false
|
||||
AND p.is_hidden = false
|
||||
GROUP BY o.photo_id
|
||||
ORDER BY rank DESC
|
||||
LIMIT 200
|
||||
""")
|
||||
fts_rows = (await db.execute(fts_stmt, {"q": q})).fetchall()
|
||||
for rank, (photo_id, score) in enumerate(fts_rows):
|
||||
if photo_id not in results:
|
||||
results[photo_id] = {"semantic_rank": None, "fts_rank": rank}
|
||||
else:
|
||||
results[photo_id]["fts_rank"] = rank
|
||||
except Exception as e:
|
||||
logger.warning("FTS search failed: %s", e)
|
||||
|
||||
# ── RRF fusion ────────────────────────────────────────────────────
|
||||
k = 60
|
||||
scored = []
|
||||
for photo_id, ranks in results.items():
|
||||
score = 0.0
|
||||
if ranks["semantic_rank"] is not None:
|
||||
score += 1.0 / (k + ranks["semantic_rank"])
|
||||
if ranks.get("fts_rank") is not None:
|
||||
score += 1.0 / (k + ranks["fts_rank"])
|
||||
scored.append((photo_id, score))
|
||||
|
||||
scored.sort(key=lambda x: -x[1])
|
||||
|
||||
# If no text query, fall back to recent photos. Always filter out
|
||||
# discarded + hidden here — this path backs the "Tags" and "People"
|
||||
# browse views, which should honor the folder hide flag.
|
||||
if not q:
|
||||
if tag_ids:
|
||||
from app.models.tags import photo_tags
|
||||
# Subquery to get distinct photo_ids matching the tag filter
|
||||
sub = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.tag_id.in_(tag_ids)
|
||||
).distinct().subquery()
|
||||
stmt = select(Photo.id).join(sub, Photo.id == sub.c.photo_id)
|
||||
else:
|
||||
stmt = select(Photo.id)
|
||||
stmt = stmt.where(
|
||||
Photo.is_discarded.is_(False),
|
||||
Photo.is_hidden.is_(False),
|
||||
)
|
||||
stmt = stmt.order_by(Photo.added_at.desc())
|
||||
if date_from:
|
||||
stmt = stmt.where(Photo.taken_at >= date_from)
|
||||
if date_to:
|
||||
stmt = stmt.where(Photo.taken_at <= date_to)
|
||||
stmt = stmt.offset(offset).limit(limit)
|
||||
rows = (await db.execute(stmt)).fetchall()
|
||||
return [{"photo_id": row[0], "score": 0.0} for row in rows]
|
||||
|
||||
# Apply filters to scored results
|
||||
photo_ids = [pid for pid, _ in scored]
|
||||
if not photo_ids:
|
||||
return []
|
||||
|
||||
# Filter by tags if requested
|
||||
if tag_ids:
|
||||
from app.models.tags import photo_tags
|
||||
stmt = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.photo_id.in_(photo_ids),
|
||||
photo_tags.c.tag_id.in_(tag_ids),
|
||||
).distinct()
|
||||
valid_ids = {row[0] for row in (await db.execute(stmt)).fetchall()}
|
||||
scored = [(pid, s) for pid, s in scored if pid in valid_ids]
|
||||
|
||||
# Paginate
|
||||
page = scored[offset : offset + limit]
|
||||
return [{"photo_id": pid, "score": score} for pid, score in page]
|
||||
@@ -1,7 +0,0 @@
|
||||
"""
|
||||
Vision pipeline services — embedding, OCR, object detection, face recognition.
|
||||
|
||||
All inference is done through the ModelRegistry singleton, which lazy-loads
|
||||
ONNX Runtime sessions on first use and caches them for the lifetime of the
|
||||
worker process.
|
||||
"""
|
||||
@@ -1,105 +0,0 @@
|
||||
"""
|
||||
Abstract base classes for vision backends.
|
||||
|
||||
Each ABC defines the contract a backend must satisfy. The default
|
||||
implementation is ONNXBackend (onnx_backend.py). A ROCm backend can be
|
||||
added later by subclassing these ABCs and registering via
|
||||
settings.vision.backend.
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectionBox:
|
||||
"""A single object detection result."""
|
||||
label: str
|
||||
confidence: float
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
|
||||
|
||||
@dataclass
|
||||
class OCRResult:
|
||||
"""A single OCR text region."""
|
||||
text: str
|
||||
confidence: float
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
language: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FaceDetection:
|
||||
"""A detected face with its recognition embedding."""
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
embedding: np.ndarray # float32 vector (128-d for SFace)
|
||||
quality: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassificationResult:
|
||||
"""A content-type classification."""
|
||||
label: str
|
||||
confidence: float
|
||||
|
||||
|
||||
class Embedder(ABC):
|
||||
"""Generates image and text embeddings (e.g. OpenCLIP ViT-B/32)."""
|
||||
|
||||
@abstractmethod
|
||||
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
||||
"""Return a normalized float32 embedding vector for an RGB image."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def embed_text(self, text: str) -> np.ndarray:
|
||||
"""Return a normalized float32 embedding vector for a text query."""
|
||||
...
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def dim(self) -> int:
|
||||
"""Dimensionality of the output embedding."""
|
||||
...
|
||||
|
||||
|
||||
class OCREngine(ABC):
|
||||
"""Extracts text from images (e.g. rapidocr-onnxruntime)."""
|
||||
|
||||
@abstractmethod
|
||||
def run(self, image: np.ndarray) -> list[OCRResult]:
|
||||
"""Return OCR results for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class ObjectDetector(ABC):
|
||||
"""Detects objects in images (e.g. YOLOv8n)."""
|
||||
|
||||
@abstractmethod
|
||||
def detect(self, image: np.ndarray) -> list[DetectionBox]:
|
||||
"""Return detections for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class ContentClassifier(ABC):
|
||||
"""Classifies images into content types (screenshot, document, etc.)."""
|
||||
|
||||
@abstractmethod
|
||||
def classify(self, image: np.ndarray) -> list[ClassificationResult]:
|
||||
"""Return content type classifications for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class FaceProcessor(ABC):
|
||||
"""Detects faces and extracts recognition embeddings (e.g. YuNet + SFace)."""
|
||||
|
||||
@abstractmethod
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
"""Return face detections with embeddings for an RGB image."""
|
||||
...
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def embedding_dim(self) -> int:
|
||||
"""Dimensionality of face embedding vectors."""
|
||||
...
|
||||
@@ -1,109 +0,0 @@
|
||||
"""
|
||||
Download vision model weights on first worker boot.
|
||||
|
||||
Run as: python -m app.services.vision.bootstrap_models
|
||||
|
||||
Or called from the vision worker entrypoint before Celery starts.
|
||||
Downloads are idempotent — existing files are skipped.
|
||||
|
||||
For models that require export (OpenCLIP, YOLOv8n), see export_models.py.
|
||||
Those must be exported once on any machine with pip, then placed in
|
||||
the models volume before the worker starts.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# (relative_path, url, description)
|
||||
# Models with url=None must be pre-exported via export_models.py.
|
||||
# InsightFace (RetinaFace + ArcFace) auto-downloads via the insightface
|
||||
# package on first use — no manual download entries needed.
|
||||
DOWNLOADS = []
|
||||
|
||||
# Models that need manual export via export_models.py
|
||||
EXPORTS = [
|
||||
("embed/visual.onnx", "OpenCLIP ViT-B/32 visual encoder"),
|
||||
("embed/textual.onnx", "OpenCLIP ViT-B/32 textual encoder"),
|
||||
("detect/yolov8n.onnx", "YOLOv8n object detector"),
|
||||
]
|
||||
|
||||
|
||||
def bootstrap(models_dir: str | None = None):
|
||||
"""Ensure all model files are present. Download what we can, warn about
|
||||
files that need manual export."""
|
||||
base = Path(models_dir or settings.vision.models_dir)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Download auto-downloadable models
|
||||
for rel_path, url, desc in DOWNLOADS:
|
||||
dest = base / rel_path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if dest.exists():
|
||||
logger.debug("Already exists: %s (%s)", dest, desc)
|
||||
continue
|
||||
|
||||
logger.info("Downloading %s → %s", desc, dest)
|
||||
try:
|
||||
urlretrieve(url, str(dest))
|
||||
size_kb = dest.stat().st_size / 1024
|
||||
logger.info("Downloaded %s (%.0f KB)", desc, size_kb)
|
||||
except Exception as e:
|
||||
logger.error("Failed to download %s: %s", desc, e)
|
||||
if dest.exists():
|
||||
dest.unlink()
|
||||
|
||||
# Check for manually-exported models
|
||||
missing = []
|
||||
for rel_path, desc in EXPORTS:
|
||||
dest = base / rel_path
|
||||
if not dest.exists():
|
||||
missing.append((rel_path, desc))
|
||||
|
||||
if missing:
|
||||
logger.warning(
|
||||
"Missing %d model file(s); attempting automatic export:",
|
||||
len(missing),
|
||||
)
|
||||
for rel_path, desc in missing:
|
||||
logger.warning(" %s — %s", base / rel_path, desc)
|
||||
|
||||
try:
|
||||
from app.services.vision import export_models
|
||||
|
||||
export_models.export_openclip(base)
|
||||
export_models.export_siglip2(base)
|
||||
export_models.export_yolov8n(base)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Automatic export failed: %s. "
|
||||
"Run `python -m app.services.vision.export_models "
|
||||
"--models-dir %s` manually before starting the worker.",
|
||||
e,
|
||||
base,
|
||||
)
|
||||
return
|
||||
|
||||
# Re-check what's still missing after the export pass.
|
||||
still_missing = [
|
||||
(rel_path, desc)
|
||||
for rel_path, desc in EXPORTS
|
||||
if not (base / rel_path).exists()
|
||||
]
|
||||
if still_missing:
|
||||
for rel_path, desc in still_missing:
|
||||
logger.error(" still missing: %s — %s", base / rel_path, desc)
|
||||
else:
|
||||
logger.info("All model files present in %s", base)
|
||||
else:
|
||||
logger.info("All model files present in %s", base)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
bootstrap()
|
||||
@@ -1,117 +0,0 @@
|
||||
"""
|
||||
CLIP zero-shot content-type classifier.
|
||||
|
||||
Uses the native OpenCLIP PyTorch text encoder for high-quality text
|
||||
embeddings (the ONNX text encoder has degraded quality due to the
|
||||
eot_indices workaround). Image embeddings use the ONNX visual encoder
|
||||
which works well.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import ContentClassifier, ClassificationResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CATEGORY_PROMPTS = {
|
||||
"screenshot": [
|
||||
"a screenshot of a computer screen",
|
||||
"a screenshot of a phone screen",
|
||||
"a screen capture of a user interface",
|
||||
],
|
||||
"document": [
|
||||
"a scanned document",
|
||||
"a photo of a document with printed text",
|
||||
"a photo of a page of text on paper",
|
||||
],
|
||||
"receipt": [
|
||||
"a photo of a receipt",
|
||||
"a photo of a bill or invoice",
|
||||
],
|
||||
"meme": [
|
||||
"an internet meme with text overlay",
|
||||
"a funny image with caption text",
|
||||
],
|
||||
"artwork": [
|
||||
"a painting or drawing",
|
||||
"a sketch or illustration",
|
||||
"digital art or graphic design",
|
||||
],
|
||||
"photograph": [
|
||||
"a photograph taken with a camera",
|
||||
"a real photo of a real scene or person",
|
||||
"a candid photograph",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class CLIPContentClassifier(ContentClassifier):
|
||||
"""Zero-shot content classifier using CLIP text-image similarity.
|
||||
Uses native PyTorch for text encoding, ONNX for image encoding."""
|
||||
|
||||
def __init__(self, settings: VisionSettings):
|
||||
import open_clip
|
||||
|
||||
self._min_confidence = settings.classifier.min_confidence
|
||||
|
||||
# Load native model for text encoding only.
|
||||
# Use whichever model family the embedder is configured for so
|
||||
# the classification text vectors live in the same space as the
|
||||
# image embeddings.
|
||||
embedder_name = settings.embedder.name
|
||||
if embedder_name.startswith("siglip2"):
|
||||
model_arch = "ViT-B-16-SigLIP-384"
|
||||
pretrained = "webli"
|
||||
else:
|
||||
model_arch = "ViT-B-32"
|
||||
pretrained = "laion2b_s34b_b79k"
|
||||
|
||||
logger.info("Loading %s text encoder for content classification", model_arch)
|
||||
model, _, _ = open_clip.create_model_and_transforms(
|
||||
model_arch, pretrained=pretrained
|
||||
)
|
||||
model.eval()
|
||||
self._model = model
|
||||
self._tokenizer = open_clip.get_tokenizer(model_arch)
|
||||
|
||||
# Get the ONNX image embedder from the registry
|
||||
from app.services.vision.registry import registry
|
||||
self._embedder = registry.get_embedder()
|
||||
|
||||
# Pre-compute text embeddings for each category
|
||||
self._category_embeddings: dict[str, np.ndarray] = {}
|
||||
for category, prompts in CATEGORY_PROMPTS.items():
|
||||
tokens = self._tokenizer(prompts)
|
||||
with torch.no_grad():
|
||||
text_features = model.encode_text(tokens)
|
||||
text_features /= text_features.norm(dim=-1, keepdim=True)
|
||||
avg = text_features.mean(dim=0)
|
||||
avg /= avg.norm()
|
||||
self._category_embeddings[category] = avg.numpy().astype(np.float32)
|
||||
|
||||
logger.info("Content classifier ready with %d categories", len(self._category_embeddings))
|
||||
|
||||
def classify(self, image: np.ndarray) -> list[ClassificationResult]:
|
||||
img_vec = self._embedder.embed_image(image)
|
||||
|
||||
# Cosine similarity against each category
|
||||
scores = {}
|
||||
for category, cat_vec in self._category_embeddings.items():
|
||||
scores[category] = float(np.dot(img_vec, cat_vec))
|
||||
|
||||
# Sort by score descending
|
||||
ranked = sorted(scores.items(), key=lambda x: -x[1])
|
||||
best_cat, best_score = ranked[0]
|
||||
second_score = ranked[1][1]
|
||||
|
||||
margin = best_score - second_score
|
||||
# Normalize: 0.01 margin → ~0.5 confidence, 0.03+ → ~1.0
|
||||
confidence = min(1.0, margin * 30)
|
||||
|
||||
if confidence >= self._min_confidence:
|
||||
return [ClassificationResult(label=best_cat, confidence=confidence)]
|
||||
|
||||
return []
|
||||
@@ -1,42 +0,0 @@
|
||||
"""
|
||||
Face embedding clustering using DBSCAN with cosine distance.
|
||||
|
||||
Called by the periodic `recluster_faces` Celery task (PR7).
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
from sklearn.cluster import DBSCAN
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cluster_faces(
|
||||
embeddings: np.ndarray,
|
||||
eps: float = 0.35,
|
||||
min_samples: int = 2,
|
||||
) -> np.ndarray:
|
||||
"""Cluster face embeddings using DBSCAN with cosine metric.
|
||||
|
||||
Args:
|
||||
embeddings: (N, D) float32 array of L2-normalized face embeddings.
|
||||
eps: Maximum cosine distance between two samples to be in the
|
||||
same neighborhood. Lower = tighter clusters.
|
||||
min_samples: Minimum cluster size.
|
||||
|
||||
Returns:
|
||||
(N,) int array of cluster labels. -1 = noise / unclustered.
|
||||
"""
|
||||
if len(embeddings) < min_samples:
|
||||
return np.full(len(embeddings), -1, dtype=int)
|
||||
|
||||
db = DBSCAN(eps=eps, min_samples=min_samples, metric="cosine")
|
||||
labels = db.fit_predict(embeddings)
|
||||
|
||||
n_clusters = len(set(labels) - {-1})
|
||||
n_noise = (labels == -1).sum()
|
||||
logger.info(
|
||||
"Face clustering: %d embeddings → %d clusters, %d noise",
|
||||
len(embeddings), n_clusters, n_noise,
|
||||
)
|
||||
return labels
|
||||
@@ -1,138 +0,0 @@
|
||||
"""
|
||||
YOLOv8n object detector using raw ONNX Runtime.
|
||||
|
||||
Expects {models_dir}/detect/yolov8n.onnx, exported from ultralytics
|
||||
via bootstrap_models.py. We do NOT ship ultralytics at runtime to
|
||||
avoid dragging in torch.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import ObjectDetector, DetectionBox
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_INPUT_SIZE = 640
|
||||
|
||||
# COCO class names (80 classes)
|
||||
COCO_LABELS = [
|
||||
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
|
||||
"truck", "boat", "traffic light", "fire hydrant", "stop sign",
|
||||
"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep",
|
||||
"cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella",
|
||||
"handbag", "tie", "suitcase", "frisbee", "skis", "snowboard",
|
||||
"sports ball", "kite", "baseball bat", "baseball glove", "skateboard",
|
||||
"surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork",
|
||||
"knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
|
||||
"broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
|
||||
"couch", "potted plant", "bed", "dining table", "toilet", "tv",
|
||||
"laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
|
||||
"oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
|
||||
"scissors", "teddy bear", "hair drier", "toothbrush",
|
||||
]
|
||||
|
||||
|
||||
def _preprocess(image: np.ndarray) -> tuple[np.ndarray, float, float]:
|
||||
"""Letterbox-resize + normalize to NCHW float32. Returns input tensor
|
||||
and scale factors for mapping boxes back to original coords."""
|
||||
from PIL import Image
|
||||
|
||||
img = Image.fromarray(image).convert("RGB")
|
||||
orig_w, orig_h = img.size
|
||||
|
||||
scale = min(_INPUT_SIZE / orig_w, _INPUT_SIZE / orig_h)
|
||||
new_w = int(orig_w * scale)
|
||||
new_h = int(orig_h * scale)
|
||||
img = img.resize((new_w, new_h), Image.BICUBIC)
|
||||
|
||||
# Paste onto gray canvas
|
||||
canvas = np.full((_INPUT_SIZE, _INPUT_SIZE, 3), 114, dtype=np.uint8)
|
||||
pad_x = (_INPUT_SIZE - new_w) // 2
|
||||
pad_y = (_INPUT_SIZE - new_h) // 2
|
||||
canvas[pad_y : pad_y + new_h, pad_x : pad_x + new_w] = np.array(img)
|
||||
|
||||
blob = canvas.astype(np.float32) / 255.0
|
||||
blob = blob.transpose(2, 0, 1)[np.newaxis] # NCHW
|
||||
return blob, scale, pad_x, pad_y
|
||||
|
||||
|
||||
def _postprocess(
|
||||
outputs: np.ndarray,
|
||||
scale: float,
|
||||
pad_x: int,
|
||||
pad_y: int,
|
||||
orig_w: int,
|
||||
orig_h: int,
|
||||
conf_threshold: float,
|
||||
max_detections: int,
|
||||
) -> list[DetectionBox]:
|
||||
"""Parse YOLOv8 output (1, 84, N) → list of DetectionBox."""
|
||||
# outputs shape: (1, 84, N) where 84 = 4 box coords + 80 class scores
|
||||
preds = outputs[0] # (84, N)
|
||||
preds = preds.T # (N, 84)
|
||||
|
||||
boxes_xywh = preds[:, :4]
|
||||
scores = preds[:, 4:]
|
||||
|
||||
class_ids = np.argmax(scores, axis=1)
|
||||
confidences = scores[np.arange(len(scores)), class_ids]
|
||||
|
||||
mask = confidences >= conf_threshold
|
||||
boxes_xywh = boxes_xywh[mask]
|
||||
class_ids = class_ids[mask]
|
||||
confidences = confidences[mask]
|
||||
|
||||
if len(confidences) == 0:
|
||||
return []
|
||||
|
||||
# Sort by confidence, take top N
|
||||
order = np.argsort(-confidences)[:max_detections]
|
||||
boxes_xywh = boxes_xywh[order]
|
||||
class_ids = class_ids[order]
|
||||
confidences = confidences[order]
|
||||
|
||||
results = []
|
||||
for i in range(len(confidences)):
|
||||
cx, cy, w, h = boxes_xywh[i]
|
||||
# Remove letterbox padding and rescale to original image
|
||||
x1 = (cx - w / 2 - pad_x) / scale
|
||||
y1 = (cy - h / 2 - pad_y) / scale
|
||||
x2 = (cx + w / 2 - pad_x) / scale
|
||||
y2 = (cy + h / 2 - pad_y) / scale
|
||||
# Normalize to 0-1
|
||||
bbox = [
|
||||
max(0, x1 / orig_w),
|
||||
max(0, y1 / orig_h),
|
||||
min(1, x2 / orig_w),
|
||||
min(1, y2 / orig_h),
|
||||
]
|
||||
label = COCO_LABELS[class_ids[i]] if class_ids[i] < len(COCO_LABELS) else f"class_{class_ids[i]}"
|
||||
results.append(DetectionBox(label=label, confidence=float(confidences[i]), bbox=bbox))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class YOLOv8Detector(ObjectDetector):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
model_path = Path(settings.models_dir) / "detect" / "yolov8n.onnx"
|
||||
|
||||
from app.services.vision.providers import create_session
|
||||
|
||||
logger.info("Loading YOLOv8n from %s", model_path)
|
||||
self._session = create_session(str(model_path), configured_providers=settings.execution_providers)
|
||||
self._conf_threshold = settings.detector.min_confidence
|
||||
self._max_detections = settings.detector.max_detections
|
||||
|
||||
def detect(self, image: np.ndarray) -> list[DetectionBox]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
blob, scale, pad_x, pad_y = _preprocess(image)
|
||||
input_name = self._session.get_inputs()[0].name
|
||||
outputs = self._session.run(None, {input_name: blob})[0]
|
||||
return _postprocess(
|
||||
outputs, scale, pad_x, pad_y, orig_w, orig_h,
|
||||
self._conf_threshold, self._max_detections,
|
||||
)
|
||||
@@ -1,146 +0,0 @@
|
||||
"""
|
||||
CLIP / SigLIP2 embedder using ONNX Runtime.
|
||||
|
||||
Supports two model families:
|
||||
- OpenCLIP ViT-B/32 (512-d) — legacy, config name "openclip_vitb32"
|
||||
- SigLIP2 ViT-B/16 (768-d) — default, config name "siglip2_vitb16"
|
||||
|
||||
Expects two ONNX files under {models_dir}/embed/:
|
||||
- visual.onnx (image encoder)
|
||||
- textual.onnx (text encoder)
|
||||
|
||||
These are exported from open_clip via export_models.py / bootstrap_models.py.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Model-specific constants ──────────────────────────────────────────
|
||||
|
||||
# OpenCLIP ViT-B/32 (ImageNet norm, 224px)
|
||||
_OPENCLIP_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
|
||||
_OPENCLIP_STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32)
|
||||
_OPENCLIP_SIZE = 224
|
||||
|
||||
# SigLIP2 ViT-B/16 (SigLIP norm, 384px)
|
||||
_SIGLIP2_MEAN = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
||||
_SIGLIP2_STD = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
||||
_SIGLIP2_SIZE = 384
|
||||
|
||||
|
||||
def _preprocess_image(
|
||||
image: np.ndarray,
|
||||
input_size: int,
|
||||
mean: np.ndarray,
|
||||
std: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""Resize, center-crop, normalize an RGB uint8 image to NCHW float32."""
|
||||
from PIL import Image
|
||||
|
||||
img = Image.fromarray(image).convert("RGB")
|
||||
w, h = img.size
|
||||
scale = input_size / min(w, h)
|
||||
img = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC)
|
||||
w, h = img.size
|
||||
left = (w - input_size) // 2
|
||||
top = (h - input_size) // 2
|
||||
img = img.crop((left, top, left + input_size, top + input_size))
|
||||
|
||||
arr = np.array(img, dtype=np.float32) / 255.0
|
||||
arr = (arr - mean) / std
|
||||
arr = arr.transpose(2, 0, 1) # HWC → CHW
|
||||
return arr[np.newaxis] # NCHW
|
||||
|
||||
|
||||
class OpenCLIPEmbedder(Embedder):
|
||||
"""Legacy OpenCLIP ViT-B/32 embedder (512-d)."""
|
||||
|
||||
def __init__(self, settings: VisionSettings):
|
||||
model_dir = Path(settings.models_dir) / "embed"
|
||||
visual_path = model_dir / "visual.onnx"
|
||||
textual_path = model_dir / "textual.onnx"
|
||||
|
||||
from app.services.vision.providers import create_session
|
||||
from app.config import settings as app_settings
|
||||
providers = app_settings.vision.execution_providers
|
||||
|
||||
logger.info("Loading OpenCLIP visual encoder from %s", visual_path)
|
||||
self._visual = create_session(str(visual_path), configured_providers=providers)
|
||||
|
||||
logger.info("Loading OpenCLIP textual encoder from %s", textual_path)
|
||||
self._textual = create_session(str(textual_path), configured_providers=providers)
|
||||
|
||||
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
||||
inp = _preprocess_image(image, _OPENCLIP_SIZE, _OPENCLIP_MEAN, _OPENCLIP_STD)
|
||||
input_name = self._visual.get_inputs()[0].name
|
||||
out = self._visual.run(None, {input_name: inp})[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
def embed_text(self, text: str) -> np.ndarray:
|
||||
import open_clip
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
||||
tokens = tokenizer([text]).numpy().astype(np.int64)
|
||||
eot_indices = tokens.argmax(axis=-1).astype(np.int64)
|
||||
inputs = self._textual.get_inputs()
|
||||
out = self._textual.run(None, {
|
||||
inputs[0].name: tokens,
|
||||
inputs[1].name: eot_indices,
|
||||
})[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
@property
|
||||
def dim(self) -> int:
|
||||
return 512
|
||||
|
||||
|
||||
class SigLIP2Embedder(Embedder):
|
||||
"""SigLIP2 ViT-B/16 embedder (768-d) — higher recall than OpenCLIP."""
|
||||
|
||||
def __init__(self, settings: VisionSettings):
|
||||
model_dir = Path(settings.models_dir) / "embed_siglip2"
|
||||
visual_path = model_dir / "visual.onnx"
|
||||
textual_path = model_dir / "textual.onnx"
|
||||
|
||||
from app.services.vision.providers import create_session
|
||||
from app.config import settings as app_settings
|
||||
providers = app_settings.vision.execution_providers
|
||||
|
||||
logger.info("Loading SigLIP2 visual encoder from %s", visual_path)
|
||||
self._visual = create_session(str(visual_path), configured_providers=providers)
|
||||
|
||||
logger.info("Loading SigLIP2 textual encoder from %s", textual_path)
|
||||
self._textual = create_session(str(textual_path), configured_providers=providers)
|
||||
|
||||
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
||||
inp = _preprocess_image(image, _SIGLIP2_SIZE, _SIGLIP2_MEAN, _SIGLIP2_STD)
|
||||
input_name = self._visual.get_inputs()[0].name
|
||||
out = self._visual.run(None, {input_name: inp})[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
def embed_text(self, text: str) -> np.ndarray:
|
||||
import open_clip
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-16-SigLIP-384")
|
||||
tokens = tokenizer([text]).numpy().astype(np.int64)
|
||||
inputs = self._textual.get_inputs()
|
||||
feed = {inputs[0].name: tokens}
|
||||
# SigLIP2 text encoder may need attention mask
|
||||
if len(inputs) > 1:
|
||||
attention_mask = (tokens != 0).astype(np.int64)
|
||||
feed[inputs[1].name] = attention_mask
|
||||
out = self._textual.run(None, feed)[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
@property
|
||||
def dim(self) -> int:
|
||||
return 768
|
||||
@@ -1,254 +0,0 @@
|
||||
"""
|
||||
Export / download all vision model weights to ONNX format.
|
||||
|
||||
Run ONCE on any machine with Python + pip (doesn't need GPU):
|
||||
|
||||
pip install open-clip-torch ultralytics onnx
|
||||
python -m app.services.vision.export_models [--models-dir /data/models]
|
||||
|
||||
This produces:
|
||||
embed/visual.onnx (~350 MB)
|
||||
embed/textual.onnx (~250 MB)
|
||||
detect/yolov8n.onnx (~12 MB)
|
||||
|
||||
YuNet and SFace are downloaded by bootstrap_models.py at worker boot
|
||||
(Apache 2.0, lightweight, no export step needed).
|
||||
|
||||
After export, copy the /data/models directory into your Docker volume:
|
||||
docker cp /data/models mulita-worker:/data/models
|
||||
Or mount a host path in docker-compose.yml.
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def export_openclip(models_dir: Path):
|
||||
"""Export OpenCLIP ViT-B/32 to two ONNX files (visual + textual)."""
|
||||
import torch
|
||||
import open_clip
|
||||
|
||||
out_dir = models_dir / "embed"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
visual_path = out_dir / "visual.onnx"
|
||||
textual_path = out_dir / "textual.onnx"
|
||||
|
||||
if visual_path.exists() and textual_path.exists():
|
||||
logger.info("OpenCLIP ONNX files already exist, skipping export")
|
||||
return
|
||||
|
||||
logger.info("Loading OpenCLIP ViT-B-32 laion2b_s34b_b79k...")
|
||||
model, _, preprocess = open_clip.create_model_and_transforms(
|
||||
"ViT-B-32", pretrained="laion2b_s34b_b79k"
|
||||
)
|
||||
model.eval()
|
||||
|
||||
# Use dynamo=False to get the legacy TorchScript exporter which
|
||||
# produces IR version 9 (compatible with onnxruntime 1.17.x).
|
||||
# The new torch.onnx.export default (dynamo=True) emits IR 10.
|
||||
export_kwargs = dict(opset_version=14, dynamo=False)
|
||||
|
||||
# ── Visual encoder ────────────────────────────────────────────────
|
||||
if not visual_path.exists():
|
||||
logger.info("Exporting visual encoder → %s", visual_path)
|
||||
dummy_image = torch.randn(1, 3, 224, 224)
|
||||
torch.onnx.export(
|
||||
model.visual,
|
||||
dummy_image,
|
||||
str(visual_path),
|
||||
input_names=["image"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"image": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = visual_path.stat().st_size / 1e6
|
||||
logger.info("Visual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
# ── Textual encoder ───────────────────────────────────────────────
|
||||
if not textual_path.exists():
|
||||
logger.info("Exporting textual encoder → %s", textual_path)
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
||||
dummy_text = tokenizer(["a photo"]).to(torch.int64)
|
||||
|
||||
class TextEncoder(torch.nn.Module):
|
||||
"""Wrap the CLIP text encoder to avoid argmax in the ONNX graph.
|
||||
OpenCLIP uses argmax to find the EOT token position, but ORT
|
||||
ARM64 doesn't support ArgMax(13). We pre-compute the EOT index
|
||||
from the token sequence and pass it directly."""
|
||||
def __init__(self, clip_model):
|
||||
super().__init__()
|
||||
self.transformer = clip_model.transformer
|
||||
self.token_embedding = clip_model.token_embedding
|
||||
self.positional_embedding = clip_model.positional_embedding
|
||||
self.ln_final = clip_model.ln_final
|
||||
self.text_projection = clip_model.text_projection
|
||||
|
||||
def forward(self, text, eot_indices):
|
||||
x = self.token_embedding(text)
|
||||
x = x + self.positional_embedding
|
||||
x = x.permute(1, 0, 2) # NLD -> LND
|
||||
x = self.transformer(x)
|
||||
x = x.permute(1, 0, 2) # LND -> NLD
|
||||
x = self.ln_final(x)
|
||||
# Take the feature at the EOT token. The EOT index is
|
||||
# passed in as a separate input (computed outside ONNX)
|
||||
# to avoid ArgMax(13) which ORT ARM64 doesn't support.
|
||||
x = x[torch.arange(x.shape[0]), eot_indices]
|
||||
x = x @ self.text_projection
|
||||
return x
|
||||
|
||||
text_enc = TextEncoder(model)
|
||||
text_enc.eval()
|
||||
|
||||
# Compute EOT indices from dummy tokens (argmax of token ids)
|
||||
dummy_eot = dummy_text.argmax(dim=-1)
|
||||
|
||||
torch.onnx.export(
|
||||
text_enc,
|
||||
(dummy_text, dummy_eot),
|
||||
str(textual_path),
|
||||
input_names=["text", "eot_indices"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"text": {0: "batch"}, "eot_indices": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = textual_path.stat().st_size / 1e6
|
||||
logger.info("Textual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
|
||||
def export_siglip2(models_dir: Path):
|
||||
"""Export SigLIP2 ViT-B/16 to two ONNX files (visual + textual)."""
|
||||
import torch
|
||||
import open_clip
|
||||
|
||||
out_dir = models_dir / "embed_siglip2"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
visual_path = out_dir / "visual.onnx"
|
||||
textual_path = out_dir / "textual.onnx"
|
||||
|
||||
if visual_path.exists() and textual_path.exists():
|
||||
logger.info("SigLIP2 ONNX files already exist, skipping export")
|
||||
return
|
||||
|
||||
logger.info("Loading SigLIP2 ViT-B-16-SigLIP-384 webli...")
|
||||
model, _, preprocess = open_clip.create_model_and_transforms(
|
||||
"ViT-B-16-SigLIP-384", pretrained="webli"
|
||||
)
|
||||
model.eval()
|
||||
|
||||
export_kwargs = dict(opset_version=14, dynamo=False)
|
||||
|
||||
# ── Visual encoder ────────────────────────────────────────────────
|
||||
if not visual_path.exists():
|
||||
logger.info("Exporting SigLIP2 visual encoder → %s", visual_path)
|
||||
dummy_image = torch.randn(1, 3, 384, 384)
|
||||
torch.onnx.export(
|
||||
model.visual,
|
||||
dummy_image,
|
||||
str(visual_path),
|
||||
input_names=["image"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"image": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = visual_path.stat().st_size / 1e6
|
||||
logger.info("SigLIP2 visual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
# ── Textual encoder ───────────────────────────────────────────────
|
||||
if not textual_path.exists():
|
||||
logger.info("Exporting SigLIP2 textual encoder → %s", textual_path)
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-16-SigLIP-384")
|
||||
dummy_text = tokenizer(["a photo"]).to(torch.int64)
|
||||
|
||||
class SigLIP2TextEncoder(torch.nn.Module):
|
||||
"""Wrap the SigLIP2 text transformer for ONNX export."""
|
||||
def __init__(self, clip_model):
|
||||
super().__init__()
|
||||
self.text = clip_model.text
|
||||
|
||||
def forward(self, text):
|
||||
return self.text(text)
|
||||
|
||||
text_enc = SigLIP2TextEncoder(model)
|
||||
text_enc.eval()
|
||||
|
||||
torch.onnx.export(
|
||||
text_enc,
|
||||
dummy_text,
|
||||
str(textual_path),
|
||||
input_names=["text"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"text": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = textual_path.stat().st_size / 1e6
|
||||
logger.info("SigLIP2 textual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
|
||||
def export_yolov8n(models_dir: Path):
|
||||
"""Export YOLOv8n to ONNX."""
|
||||
out_dir = models_dir / "detect"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
onnx_path = out_dir / "yolov8n.onnx"
|
||||
|
||||
if onnx_path.exists():
|
||||
logger.info("YOLOv8n ONNX already exists, skipping export")
|
||||
return
|
||||
|
||||
logger.info("Exporting YOLOv8n → %s", onnx_path)
|
||||
|
||||
from ultralytics import YOLO
|
||||
|
||||
model = YOLO("yolov8n.pt")
|
||||
model.export(format="onnx", imgsz=640, simplify=True)
|
||||
|
||||
# ultralytics exports to cwd as yolov8n.onnx — move to target. Use
|
||||
# shutil.move rather than Path.rename so it works across filesystems
|
||||
# (the cwd is typically /app inside the container, while the target
|
||||
# /data/models is a separately-mounted volume — Path.rename raises
|
||||
# "Invalid cross-device link" in that case).
|
||||
import shutil
|
||||
|
||||
exported = Path("yolov8n.onnx")
|
||||
if exported.exists():
|
||||
shutil.move(str(exported), str(onnx_path))
|
||||
|
||||
size_mb = onnx_path.stat().st_size / 1e6
|
||||
logger.info("YOLOv8n exported (%.1f MB)", size_mb)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Export vision model weights to ONNX")
|
||||
parser.add_argument(
|
||||
"--models-dir",
|
||||
type=Path,
|
||||
default=Path("/data/models"),
|
||||
help="Directory to write model files (default: /data/models)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
|
||||
models_dir = args.models_dir
|
||||
models_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info("Exporting models to %s", models_dir)
|
||||
|
||||
export_openclip(models_dir)
|
||||
export_siglip2(models_dir)
|
||||
export_yolov8n(models_dir)
|
||||
|
||||
logger.info("Done. Run bootstrap_models.py next to download YuNet + SFace.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,146 +0,0 @@
|
||||
"""
|
||||
Face detection (YuNet) + recognition (SFace) using OpenCV DNN.
|
||||
|
||||
YuNet is loaded via cv2.FaceDetectorYN which handles the multi-scale
|
||||
anchor decoding and NMS internally. SFace recognition uses raw ONNX
|
||||
Runtime for the 128-d embedding.
|
||||
|
||||
Both models are from opencv_zoo (Apache 2.0 license).
|
||||
Expects {models_dir}/face/:
|
||||
- yunet.onnx (~233 KB)
|
||||
- sface.onnx (~37 MB, 128-d embeddings)
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import cv2
|
||||
import onnxruntime as ort
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import FaceProcessor, FaceDetection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _align_face(image: np.ndarray, landmarks: np.ndarray) -> np.ndarray:
|
||||
"""Align and crop a 112x112 face patch using 5-point landmarks."""
|
||||
left_eye = landmarks[0]
|
||||
right_eye = landmarks[1]
|
||||
|
||||
dx = right_eye[0] - left_eye[0]
|
||||
dy = right_eye[1] - left_eye[1]
|
||||
angle = np.degrees(np.arctan2(dy, dx))
|
||||
eye_center = ((left_eye[0] + right_eye[0]) / 2, (left_eye[1] + right_eye[1]) / 2)
|
||||
eye_dist = np.sqrt(dx * dx + dy * dy)
|
||||
|
||||
M = cv2.getRotationMatrix2D(eye_center, angle, 1.0)
|
||||
rotated = cv2.warpAffine(image, M, (image.shape[1], image.shape[0]))
|
||||
|
||||
# Crop around face center
|
||||
scale = 64.0 / max(eye_dist, 1e-6)
|
||||
cx, cy = eye_center
|
||||
half = 56.0 / scale
|
||||
x1 = max(0, int(cx - half))
|
||||
y1 = max(0, int(cy - half * 0.8))
|
||||
x2 = min(rotated.shape[1], int(cx + half))
|
||||
y2 = min(rotated.shape[0], int(cy + half * 1.2))
|
||||
crop = rotated[y1:y2, x1:x2]
|
||||
|
||||
if crop.size == 0:
|
||||
return np.zeros((112, 112, 3), dtype=np.float32)
|
||||
|
||||
return cv2.resize(crop, (112, 112)).astype(np.float32)
|
||||
|
||||
|
||||
class YuNetSFaceProcessor(FaceProcessor):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
face_dir = Path(settings.models_dir) / "face"
|
||||
yunet_path = str(face_dir / "yunet.onnx")
|
||||
sface_path = str(face_dir / "sface.onnx")
|
||||
|
||||
# YuNet via OpenCV's FaceDetectorYN — handles anchor decoding + NMS
|
||||
self._detector = cv2.FaceDetectorYN.create(
|
||||
yunet_path,
|
||||
"",
|
||||
(640, 640),
|
||||
settings.faces.recognition_threshold,
|
||||
0.3, # NMS threshold
|
||||
5000, # top_k
|
||||
)
|
||||
logger.info("YuNet face detector loaded via OpenCV")
|
||||
|
||||
# SFace via ONNX Runtime
|
||||
from app.services.vision.providers import create_session
|
||||
ort.set_default_logger_severity(3)
|
||||
self._recognizer = create_session(sface_path, configured_providers=settings.execution_providers)
|
||||
logger.info("SFace recognizer loaded via ONNX Runtime")
|
||||
|
||||
self._min_face_size = settings.faces.min_face_size
|
||||
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
|
||||
# Convert RGB → BGR for OpenCV
|
||||
bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
|
||||
# Set input size to actual image dimensions
|
||||
self._detector.setInputSize((orig_w, orig_h))
|
||||
|
||||
# Detect faces
|
||||
_, faces_raw = self._detector.detect(bgr)
|
||||
|
||||
if faces_raw is None or len(faces_raw) == 0:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for face in faces_raw:
|
||||
# face: [x, y, w, h, right_eye_x, right_eye_y, left_eye_x, left_eye_y,
|
||||
# nose_x, nose_y, right_mouth_x, right_mouth_y, left_mouth_x, left_mouth_y, score]
|
||||
x, y, w, h = int(face[0]), int(face[1]), int(face[2]), int(face[3])
|
||||
score = float(face[14])
|
||||
|
||||
# Filter small faces
|
||||
face_size = max(w, h)
|
||||
if face_size < self._min_face_size:
|
||||
continue
|
||||
|
||||
# Normalized bbox
|
||||
bbox = [
|
||||
max(0, x / orig_w),
|
||||
max(0, y / orig_h),
|
||||
min(1, (x + w) / orig_w),
|
||||
min(1, (y + h) / orig_h),
|
||||
]
|
||||
|
||||
# Extract 5-point landmarks for alignment
|
||||
landmarks = np.array([
|
||||
[face[4], face[5]], # right eye
|
||||
[face[6], face[7]], # left eye
|
||||
[face[8], face[9]], # nose
|
||||
[face[10], face[11]], # right mouth
|
||||
[face[12], face[13]], # left mouth
|
||||
], dtype=np.float32)
|
||||
|
||||
# Align face for recognition
|
||||
face_crop = _align_face(image, landmarks)
|
||||
|
||||
# SFace expects (1, 3, 112, 112) float32, BGR
|
||||
face_bgr = cv2.cvtColor(face_crop.astype(np.uint8), cv2.COLOR_RGB2BGR)
|
||||
face_blob = (face_bgr.astype(np.float32) / 255.0).transpose(2, 0, 1)[np.newaxis]
|
||||
|
||||
rec_input = self._recognizer.get_inputs()[0].name
|
||||
embedding = self._recognizer.run(None, {rec_input: face_blob})[0][0]
|
||||
embedding = embedding / np.linalg.norm(embedding)
|
||||
|
||||
results.append(FaceDetection(
|
||||
bbox=bbox,
|
||||
embedding=embedding.astype(np.float32),
|
||||
quality=score,
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return 128
|
||||
@@ -1,73 +0,0 @@
|
||||
"""
|
||||
Face detection + recognition using InsightFace (RetinaFace + ArcFace).
|
||||
|
||||
Uses the buffalo_l model pack which auto-downloads on first use (~300MB).
|
||||
Produces 512-d ArcFace embeddings. Non-commercial research license —
|
||||
fine for homelab self-hosting.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import FaceProcessor, FaceDetection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InsightFaceProcessor(FaceProcessor):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
from insightface.app import FaceAnalysis
|
||||
|
||||
model_root = str(Path(settings.models_dir) / "face" / "insightface")
|
||||
logger.info("Loading InsightFace buffalo_l from %s", model_root)
|
||||
|
||||
from app.services.vision.providers import get_providers
|
||||
providers = get_providers(settings.execution_providers)
|
||||
|
||||
self._app = FaceAnalysis(
|
||||
name="buffalo_l",
|
||||
root=model_root,
|
||||
providers=providers,
|
||||
)
|
||||
self._app.prepare(ctx_id=-1, det_size=(640, 640))
|
||||
self._min_det_score = settings.faces.recognition_threshold
|
||||
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
|
||||
# InsightFace expects BGR
|
||||
bgr = image[:, :, ::-1].copy()
|
||||
|
||||
faces = self._app.get(bgr)
|
||||
|
||||
if not faces:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for face in faces:
|
||||
if face.det_score < self._min_det_score:
|
||||
continue
|
||||
|
||||
# face.bbox is [x1, y1, x2, y2] in pixel coords
|
||||
x1, y1, x2, y2 = face.bbox
|
||||
bbox = [
|
||||
max(0, float(x1) / orig_w),
|
||||
max(0, float(y1) / orig_h),
|
||||
min(1, float(x2) / orig_w),
|
||||
min(1, float(y2) / orig_h),
|
||||
]
|
||||
|
||||
embedding = face.normed_embedding # already L2-normalized, 512-d
|
||||
results.append(FaceDetection(
|
||||
bbox=bbox,
|
||||
embedding=embedding.astype(np.float32),
|
||||
quality=float(face.det_score),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return 512
|
||||
@@ -1,45 +0,0 @@
|
||||
"""
|
||||
OCR engine using rapidocr-onnxruntime (PP-OCRv4 weights).
|
||||
|
||||
No PaddlePaddle dependency — pure ONNX Runtime. Language packs are
|
||||
downloaded automatically by rapidocr on first use.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import OCREngine, OCRResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RapidOCREngine(OCREngine):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
self._min_confidence = settings.ocr.min_confidence
|
||||
self._engine = RapidOCR()
|
||||
logger.info("RapidOCR engine initialized")
|
||||
|
||||
def run(self, image: np.ndarray) -> list[OCRResult]:
|
||||
result, _ = self._engine(image)
|
||||
if not result:
|
||||
return []
|
||||
|
||||
out = []
|
||||
for box, text, score in result:
|
||||
if score < self._min_confidence:
|
||||
continue
|
||||
# box is [[x1,y1],[x2,y2],[x3,y3],[x4,y4]] — take bounding rect
|
||||
xs = [p[0] for p in box]
|
||||
ys = [p[1] for p in box]
|
||||
h, w = image.shape[:2]
|
||||
bbox = [
|
||||
min(xs) / w,
|
||||
min(ys) / h,
|
||||
max(xs) / w,
|
||||
max(ys) / h,
|
||||
]
|
||||
out.append(OCRResult(text=text, confidence=float(score), bbox=bbox))
|
||||
return out
|
||||
@@ -1,46 +0,0 @@
|
||||
"""
|
||||
ONNX Runtime backend — default CPU inference for all vision models.
|
||||
|
||||
Each create_* method returns a concrete implementation of the
|
||||
corresponding ABC from base.py. Models are loaded from ONNX files
|
||||
under settings.vision.models_dir, downloaded on first boot by
|
||||
bootstrap_models.py.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor, ContentClassifier
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ONNXBackend:
|
||||
"""Factory for ONNX Runtime-based vision model instances."""
|
||||
|
||||
def __init__(self, vision_settings: VisionSettings):
|
||||
self._settings = vision_settings
|
||||
|
||||
def create_embedder(self) -> Embedder:
|
||||
model_name = self._settings.embedder.name
|
||||
if model_name.startswith("siglip2"):
|
||||
from app.services.vision.embed import SigLIP2Embedder
|
||||
return SigLIP2Embedder(self._settings)
|
||||
else:
|
||||
from app.services.vision.embed import OpenCLIPEmbedder
|
||||
return OpenCLIPEmbedder(self._settings)
|
||||
|
||||
def create_ocr(self) -> OCREngine:
|
||||
from app.services.vision.ocr import RapidOCREngine
|
||||
return RapidOCREngine(self._settings)
|
||||
|
||||
def create_detector(self) -> ObjectDetector:
|
||||
from app.services.vision.detect import YOLOv8Detector
|
||||
return YOLOv8Detector(self._settings)
|
||||
|
||||
def create_face_processor(self) -> FaceProcessor:
|
||||
from app.services.vision.insightface_processor import InsightFaceProcessor
|
||||
return InsightFaceProcessor(self._settings)
|
||||
|
||||
def create_classifier(self) -> ContentClassifier:
|
||||
from app.services.vision.classify import CLIPContentClassifier
|
||||
return CLIPContentClassifier(self._settings)
|
||||
@@ -1,86 +0,0 @@
|
||||
"""
|
||||
ONNX Runtime execution provider resolution with GPU auto-detection.
|
||||
|
||||
Resolves configured execution providers against what's actually available
|
||||
in the current ONNX Runtime build. Falls back to CPU if no GPU provider
|
||||
is available. Logs the selected provider so users can confirm GPU is active.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import onnxruntime as ort
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_resolved: list[str] | None = None
|
||||
|
||||
|
||||
def get_providers(configured: list[str] | None = None) -> list[str]:
|
||||
"""Return the best available execution providers.
|
||||
|
||||
1. If `configured` is provided, filter to only those that are
|
||||
actually available in the current ORT build.
|
||||
2. If none of the configured providers are available, fall back
|
||||
to CPUExecutionProvider.
|
||||
3. Auto-detect: if configured is ["auto"], probe for GPU providers.
|
||||
|
||||
Results are cached after first call.
|
||||
"""
|
||||
global _resolved
|
||||
if _resolved is not None:
|
||||
return _resolved
|
||||
|
||||
available = set(ort.get_available_providers())
|
||||
logger.info("ONNX Runtime available providers: %s", sorted(available))
|
||||
|
||||
if configured is None or configured == ["CPUExecutionProvider"]:
|
||||
_resolved = ["CPUExecutionProvider"]
|
||||
return _resolved
|
||||
|
||||
if configured == ["auto"]:
|
||||
# Auto-detect: prefer CUDA > ROCm > OpenVINO > CPU
|
||||
priority = [
|
||||
"CUDAExecutionProvider",
|
||||
"ROCMExecutionProvider",
|
||||
"OpenVINOExecutionProvider",
|
||||
]
|
||||
for p in priority:
|
||||
if p in available:
|
||||
_resolved = [p, "CPUExecutionProvider"]
|
||||
logger.info("Auto-detected GPU provider: %s", p)
|
||||
return _resolved
|
||||
_resolved = ["CPUExecutionProvider"]
|
||||
logger.info("No GPU provider detected, using CPU")
|
||||
return _resolved
|
||||
|
||||
# Filter configured list to available providers.
|
||||
resolved = [p for p in configured if p in available]
|
||||
if not resolved:
|
||||
logger.warning(
|
||||
"None of the configured providers %s are available. "
|
||||
"Falling back to CPU. Available: %s",
|
||||
configured,
|
||||
sorted(available),
|
||||
)
|
||||
resolved = ["CPUExecutionProvider"]
|
||||
else:
|
||||
# Always include CPU as fallback.
|
||||
if "CPUExecutionProvider" not in resolved:
|
||||
resolved.append("CPUExecutionProvider")
|
||||
|
||||
_resolved = resolved
|
||||
logger.info("Using ONNX Runtime providers: %s", _resolved)
|
||||
return _resolved
|
||||
|
||||
|
||||
def create_session(
|
||||
model_path: str,
|
||||
opts: ort.SessionOptions | None = None,
|
||||
configured_providers: list[str] | None = None,
|
||||
) -> ort.InferenceSession:
|
||||
"""Create an ONNX InferenceSession with the best available providers."""
|
||||
providers = get_providers(configured_providers)
|
||||
if opts is None:
|
||||
opts = ort.SessionOptions()
|
||||
opts.inter_op_num_threads = 2
|
||||
opts.intra_op_num_threads = 2
|
||||
return ort.InferenceSession(model_path, opts, providers=providers)
|
||||
@@ -1,84 +0,0 @@
|
||||
"""
|
||||
ModelRegistry — singleton that lazy-loads vision models per worker process.
|
||||
|
||||
Usage from Celery tasks:
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
vec = embedder.embed_image(img)
|
||||
|
||||
Models are created on first access and cached for the worker's lifetime.
|
||||
The registry reads settings.vision to decide which backend to use and
|
||||
where model weights live.
|
||||
"""
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
|
||||
from app.config import settings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor, ContentClassifier
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ModelRegistry:
|
||||
"""Central access point for all vision models."""
|
||||
|
||||
def __init__(self):
|
||||
self._vision = settings.vision
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_embedder(self) -> Embedder:
|
||||
logger.info("Loading embedder: %s (backend=%s)", self._vision.embedder.name, self._vision.backend)
|
||||
return self._load_backend().create_embedder()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_ocr(self) -> OCREngine:
|
||||
logger.info("Loading OCR engine (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_ocr()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_detector(self) -> ObjectDetector:
|
||||
logger.info("Loading object detector (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_detector()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_face_processor(self) -> FaceProcessor:
|
||||
logger.info("Loading face processor (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_face_processor()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_classifier(self) -> ContentClassifier:
|
||||
logger.info("Loading content classifier (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_classifier()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_backend(self):
|
||||
"""Import and instantiate the configured backend."""
|
||||
backend_name = self._vision.backend
|
||||
if backend_name == "onnx":
|
||||
from app.services.vision.onnx_backend import ONNXBackend
|
||||
return ONNXBackend(self._vision)
|
||||
elif backend_name == "rocm":
|
||||
from app.services.vision.rocm_backend import ROCmBackend
|
||||
return ROCmBackend(self._vision)
|
||||
else:
|
||||
raise ValueError(f"Unknown vision backend: {backend_name}")
|
||||
|
||||
def warmup(self):
|
||||
"""Pre-load all enabled models. Called from Celery worker_process_init
|
||||
on the vision queue to avoid cold-start latency on the first task."""
|
||||
logger.info("Warming up vision models...")
|
||||
self.get_embedder()
|
||||
if self._vision.ocr.enabled:
|
||||
self.get_ocr()
|
||||
if self._vision.detector.enabled:
|
||||
self.get_detector()
|
||||
if self._vision.faces.enabled:
|
||||
self.get_face_processor()
|
||||
if self._vision.classifier.enabled:
|
||||
self.get_classifier()
|
||||
logger.info("Vision model warmup complete")
|
||||
|
||||
|
||||
# Module-level singleton. Import this from tasks.
|
||||
registry = ModelRegistry()
|
||||
@@ -1,25 +0,0 @@
|
||||
"""
|
||||
ROCm backend — GPU-accelerated inference for Radeon 760M-class hardware.
|
||||
|
||||
Stub: raises NotImplementedError on all factory methods. To enable,
|
||||
set `vision.backend: rocm` in mulita.yml once ROCm support is implemented.
|
||||
"""
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor
|
||||
|
||||
|
||||
class ROCmBackend:
|
||||
def __init__(self, vision_settings: VisionSettings):
|
||||
self._settings = vision_settings
|
||||
|
||||
def create_embedder(self) -> Embedder:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_ocr(self) -> OCREngine:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_detector(self) -> ObjectDetector:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_face_processor(self) -> FaceProcessor:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
@@ -1,15 +0,0 @@
|
||||
"""
|
||||
Celery tasks module
|
||||
"""
|
||||
from app.tasks.celery import celery_app
|
||||
from app.tasks.scan import scan_folder, scan_all_source_roots, watch_folders
|
||||
from app.tasks.thumbs import generate_thumbnails, regenerate_all_thumbnails
|
||||
|
||||
__all__ = [
|
||||
'celery_app',
|
||||
'scan_folder',
|
||||
'scan_all_source_roots',
|
||||
'watch_folders',
|
||||
'generate_thumbnails',
|
||||
'regenerate_all_thumbnails'
|
||||
]
|
||||
@@ -1,39 +0,0 @@
|
||||
"""
|
||||
Celery configuration and app initialization
|
||||
"""
|
||||
from celery import Celery
|
||||
from app.config import settings
|
||||
|
||||
# Create Celery app
|
||||
celery_app = Celery(
|
||||
'mulita',
|
||||
broker=settings.celery_broker_url,
|
||||
backend=settings.celery_result_backend,
|
||||
include=['app.tasks.scan', 'app.tasks.thumbs', 'app.tasks.vision']
|
||||
)
|
||||
|
||||
# Configure Celery
|
||||
celery_app.conf.update(
|
||||
task_serializer='json',
|
||||
accept_content=['json'],
|
||||
result_serializer='json',
|
||||
timezone='UTC',
|
||||
enable_utc=True,
|
||||
task_routes={
|
||||
'app.tasks.thumbs.*': {'queue': 'high'},
|
||||
'app.tasks.scan.*': {'queue': 'low'},
|
||||
'app.tasks.vision.*': {'queue': 'vision'},
|
||||
'embed_photo': {'queue': 'vision'},
|
||||
'ocr_photo': {'queue': 'vision'},
|
||||
'detect_objects': {'queue': 'vision'},
|
||||
'extract_faces': {'queue': 'vision'},
|
||||
'classify_content': {'queue': 'vision'},
|
||||
'vision_fanout': {'queue': 'vision'},
|
||||
'watch_folders': {'queue': 'watcher'},
|
||||
},
|
||||
task_default_queue='default',
|
||||
task_default_exchange='default',
|
||||
task_default_exchange_type='direct',
|
||||
task_default_routing_key='default',
|
||||
broker_connection_retry_on_startup=True,
|
||||
)
|
||||
@@ -1,612 +0,0 @@
|
||||
"""
|
||||
Celery tasks for scanning folders and indexing photos
|
||||
"""
|
||||
import os
|
||||
import hashlib
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
import logging
|
||||
import json
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
from celery import shared_task
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
import aiofiles
|
||||
import redis
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo, Folder, SourceRoot
|
||||
from app.config import settings
|
||||
from app.tasks.thumbs import generate_thumbnails
|
||||
from app.services.metadata import extract_metadata
|
||||
from app.services.date_guess import has_date_warning
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Redis keys read by GET /api/v1/library/scan/status. The frontend
|
||||
# ScanProgress widget polls that endpoint, so anything we want to surface
|
||||
# in the UI lives here.
|
||||
REDIS_KEY_ACTIVE = 'scan:active'
|
||||
REDIS_KEY_CURRENT_FOLDER = 'scan:current_folder'
|
||||
REDIS_KEY_PROCESSED = 'scan:processed_files'
|
||||
REDIS_KEY_TOTAL = 'scan:total_files'
|
||||
REDIS_KEY_ERRORS = 'scan:errors'
|
||||
MAX_ERROR_ENTRIES = 50 # cap the errors list so a noisy scan doesn't blow Redis
|
||||
|
||||
|
||||
def _get_redis():
|
||||
"""Connect to the broker for progress writes. Returns None on failure
|
||||
so a Redis outage doesn't prevent the scan itself from running."""
|
||||
try:
|
||||
return redis.Redis.from_url(settings.celery_broker_url)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not reach Redis for scan progress: {e}")
|
||||
return None
|
||||
|
||||
# Supported file extensions
|
||||
PHOTO_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.tiff', '.tif', '.webp', '.bmp'}
|
||||
RAW_EXTENSIONS = {'.cr2', '.cr3', '.nef', '.arw', '.raf', '.dng', '.orf', '.rw2', '.pef', '.srw'}
|
||||
HEIC_EXTENSIONS = {'.heic', '.heif'}
|
||||
VIDEO_EXTENSIONS = {'.mp4', '.mov', '.avi', '.mkv', '.mts', '.m2ts', '.3gp', '.wmv', '.flv'}
|
||||
|
||||
SUPPORTED_EXTENSIONS = PHOTO_EXTENSIONS | RAW_EXTENSIONS | HEIC_EXTENSIONS | VIDEO_EXTENSIONS
|
||||
|
||||
def get_media_type(filepath: str) -> str:
|
||||
"""Determine media type from file extension"""
|
||||
ext = Path(filepath).suffix.lower()
|
||||
if ext in PHOTO_EXTENSIONS:
|
||||
return 'photo'
|
||||
elif ext in RAW_EXTENSIONS:
|
||||
return 'raw'
|
||||
elif ext in HEIC_EXTENSIONS:
|
||||
return 'heic'
|
||||
elif ext in VIDEO_EXTENSIONS:
|
||||
return 'video'
|
||||
return 'unknown'
|
||||
|
||||
async def calculate_file_hash(filepath: str) -> str:
|
||||
"""Calculate SHA-256 hash of a file"""
|
||||
hash_sha256 = hashlib.sha256()
|
||||
try:
|
||||
async with aiofiles.open(filepath, 'rb') as f:
|
||||
while chunk := await f.read(8192):
|
||||
hash_sha256.update(chunk)
|
||||
return hash_sha256.hexdigest()
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating hash for {filepath}: {e}")
|
||||
return ""
|
||||
|
||||
@shared_task(bind=True, name='scan_folder')
|
||||
def scan_folder(self, folder_path: str, source_root_id: Optional[str] = None):
|
||||
"""
|
||||
Scan a folder and index all photos/videos
|
||||
"""
|
||||
# Run async function in sync context
|
||||
return asyncio.run(_scan_folder_async(folder_path, source_root_id, self))
|
||||
|
||||
async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], task):
|
||||
"""Async implementation of folder scanning. Writes progress to Redis so
|
||||
GET /api/v1/library/scan/status can surface it to the frontend
|
||||
ScanProgress widget."""
|
||||
logger.info(f"Starting scan of folder: {folder_path}")
|
||||
|
||||
r = _get_redis()
|
||||
|
||||
def progress_set(key: str, value) -> None:
|
||||
if r is None:
|
||||
return
|
||||
try:
|
||||
r.set(key, str(value))
|
||||
except Exception as e:
|
||||
logger.debug(f"scan progress set failed: {e}")
|
||||
|
||||
def progress_push_error(message: str) -> None:
|
||||
if r is None:
|
||||
return
|
||||
try:
|
||||
r.lpush(REDIS_KEY_ERRORS, message)
|
||||
r.ltrim(REDIS_KEY_ERRORS, 0, MAX_ERROR_ENTRIES - 1)
|
||||
except Exception as e:
|
||||
logger.debug(f"scan progress push_error failed: {e}")
|
||||
|
||||
# Mark scan active immediately so the UI starts polling fast.
|
||||
progress_set(REDIS_KEY_ACTIVE, 'true')
|
||||
progress_set(REDIS_KEY_CURRENT_FOLDER, folder_path)
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# Get or create source root
|
||||
if not source_root_id:
|
||||
source_root = await get_or_create_source_root(session, folder_path)
|
||||
source_root_id = source_root.id
|
||||
else:
|
||||
source_root = (await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.id == source_root_id)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
# Inherit user_id from the source root's owner
|
||||
owner_user_id = source_root.user_id if source_root else None
|
||||
|
||||
# Per-scan memoization cache for "is this folder's effective
|
||||
# is_hidden true?" Populated on first lookup by walking the
|
||||
# parent_id chain up to the source root. Keyed by folder_id
|
||||
# so repeated photos in the same folder pay only one lookup.
|
||||
hidden_folder_cache: dict[str, bool] = {}
|
||||
|
||||
async def is_folder_effectively_hidden(folder_row: Folder) -> bool:
|
||||
if folder_row.id in hidden_folder_cache:
|
||||
return hidden_folder_cache[folder_row.id]
|
||||
# Walk parents. If the current folder is hidden, short-
|
||||
# circuit. Otherwise climb until we hit a root (no
|
||||
# parent_id) or a cached ancestor.
|
||||
if folder_row.is_hidden:
|
||||
hidden_folder_cache[folder_row.id] = True
|
||||
return True
|
||||
parent_id = folder_row.parent_id
|
||||
while parent_id is not None:
|
||||
if parent_id in hidden_folder_cache:
|
||||
hidden_folder_cache[folder_row.id] = hidden_folder_cache[parent_id]
|
||||
return hidden_folder_cache[folder_row.id]
|
||||
parent = (
|
||||
await session.execute(
|
||||
select(Folder).where(Folder.id == parent_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if parent is None:
|
||||
break
|
||||
if parent.is_hidden:
|
||||
hidden_folder_cache[folder_row.id] = True
|
||||
return True
|
||||
parent_id = parent.parent_id
|
||||
hidden_folder_cache[folder_row.id] = False
|
||||
return False
|
||||
|
||||
# Pre-walk to compute the total file count upfront. Without this
|
||||
# the progress bar would jump every time a new subfolder is
|
||||
# encountered because the running total kept growing.
|
||||
total_files = 0
|
||||
for _root, _dirs, files in os.walk(folder_path):
|
||||
total_files += sum(
|
||||
1 for f in files if Path(f).suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
)
|
||||
progress_set(REDIS_KEY_TOTAL, total_files)
|
||||
progress_set(REDIS_KEY_PROCESSED, 0)
|
||||
|
||||
processed_files = 0
|
||||
errors = []
|
||||
|
||||
for root, dirs, files in os.walk(folder_path):
|
||||
# Get or create folder entry
|
||||
folder = await get_or_create_folder(session, root, source_root_id, owner_user_id)
|
||||
progress_set(REDIS_KEY_CURRENT_FOLDER, root)
|
||||
|
||||
# Filter supported files
|
||||
supported_files = [f for f in files if Path(f).suffix.lower() in SUPPORTED_EXTENSIONS]
|
||||
|
||||
# Process files in batches
|
||||
batch_size = settings.scanner.batch_size
|
||||
for i in range(0, len(supported_files), batch_size):
|
||||
batch = supported_files[i:i + batch_size]
|
||||
# Defer task dispatch until AFTER commit so workers don't
|
||||
# query for rows that aren't visible to other sessions yet.
|
||||
pending_dispatch: list[str] = []
|
||||
|
||||
for filename in batch:
|
||||
filepath = os.path.join(root, filename)
|
||||
|
||||
try:
|
||||
# Check if file already exists in database
|
||||
existing = await session.execute(
|
||||
select(Photo).where(Photo.filepath == filepath)
|
||||
)
|
||||
if existing.scalar_one_or_none():
|
||||
logger.debug(f"File already indexed: {filepath}")
|
||||
processed_files += 1
|
||||
progress_set(REDIS_KEY_PROCESSED, processed_files)
|
||||
continue
|
||||
|
||||
# Get file stats
|
||||
stat = os.stat(filepath)
|
||||
|
||||
# Calculate file hash for duplicate detection
|
||||
file_hash = await calculate_file_hash(filepath)
|
||||
|
||||
# Check for duplicate by hash. We only care
|
||||
# whether *any* other photo shares this hash, so
|
||||
# use a count rather than scalar_one_or_none()
|
||||
# which raises "Multiple rows were found" the
|
||||
# moment the library has 2+ copies of the same
|
||||
# file (i.e. exactly the case we're trying to
|
||||
# flag).
|
||||
is_dup = False
|
||||
if file_hash:
|
||||
dup_count = (await session.execute(
|
||||
select(func.count(Photo.id)).where(
|
||||
Photo.file_hash == file_hash
|
||||
)
|
||||
)).scalar() or 0
|
||||
is_dup = dup_count > 0
|
||||
|
||||
# Inherit the effective-hidden flag from the
|
||||
# folder's ancestry. If any ancestor folder
|
||||
# has is_hidden=true, the new photo is
|
||||
# immediately marked hidden so it never
|
||||
# briefly appears in cross-cutting views
|
||||
# between scan and the next manual recompute.
|
||||
effective_hidden = await is_folder_effectively_hidden(folder)
|
||||
|
||||
# Create photo entry
|
||||
mtime_dt = datetime.fromtimestamp(stat.st_mtime)
|
||||
photo = Photo(
|
||||
filepath=filepath,
|
||||
filename=filename,
|
||||
folder_id=folder.id,
|
||||
user_id=owner_user_id,
|
||||
file_hash=file_hash,
|
||||
media_type=get_media_type(filepath),
|
||||
original_format=Path(filepath).suffix.upper()[1:],
|
||||
file_size=stat.st_size,
|
||||
taken_at=mtime_dt,
|
||||
taken_at_source='filesystem',
|
||||
# First-pass flag based on the filesystem mtime;
|
||||
# metadata.extract_metadata re-runs this once
|
||||
# EXIF has been parsed so a real DateTimeOriginal
|
||||
# can clear the warning.
|
||||
has_date_warning=has_date_warning(filepath, mtime_dt),
|
||||
is_duplicate=is_dup,
|
||||
is_hidden=effective_hidden,
|
||||
processing_status='pending'
|
||||
)
|
||||
|
||||
session.add(photo)
|
||||
await session.flush() # Assign defaults / FK ids
|
||||
|
||||
# Queue dispatch happens after the batch commit
|
||||
# below; otherwise the worker can race the writer
|
||||
# and see "Photo not found".
|
||||
pending_dispatch.append(photo.id)
|
||||
|
||||
processed_files += 1
|
||||
progress_set(REDIS_KEY_PROCESSED, processed_files)
|
||||
|
||||
# Celery internal progress (used by celery tooling)
|
||||
if processed_files % 10 == 0:
|
||||
task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={
|
||||
'current': processed_files,
|
||||
'total': total_files,
|
||||
'folder': root,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing file {filepath}: {e}")
|
||||
errors.append({'file': filepath, 'error': str(e)})
|
||||
progress_push_error(f"{filepath}: {e}")
|
||||
continue
|
||||
|
||||
# Commit batch, then queue worker tasks. Dispatch order
|
||||
# matters: commit first so workers can find the rows.
|
||||
await session.commit()
|
||||
|
||||
for photo_id in pending_dispatch:
|
||||
generate_thumbnails.delay(photo_id)
|
||||
extract_metadata.delay(photo_id)
|
||||
|
||||
# Update folder scan timestamp
|
||||
folder.last_scanned = datetime.utcnow()
|
||||
folder.photo_count = processed_files
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"Scan complete. Processed {processed_files}/{total_files} files. Errors: {len(errors)}")
|
||||
|
||||
return {
|
||||
'status': 'completed',
|
||||
'processed': processed_files,
|
||||
'total': total_files,
|
||||
'errors': errors,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Scan failed: {e}")
|
||||
progress_push_error(f"scan failed: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
# Always mark inactive on the way out so a crashed scan doesn't
|
||||
# leave the UI thinking we're still scanning.
|
||||
progress_set(REDIS_KEY_ACTIVE, 'false')
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
"""Canonicalise a filesystem path so we don't get duplicate DB rows for
|
||||
the same physical directory due to trailing slashes, redundant separators,
|
||||
or `.` segments. Symlinks are NOT resolved (we want to keep mount paths
|
||||
intact for cross-machine portability)."""
|
||||
return os.path.normpath(path)
|
||||
|
||||
|
||||
async def get_or_create_source_root(session: AsyncSession, path: str) -> SourceRoot:
|
||||
"""Get or create a source root entry, matching by normalized path."""
|
||||
from sqlalchemy import select
|
||||
|
||||
norm = _normalize_path(path)
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.path == norm)
|
||||
)
|
||||
source_root = result.scalar_one_or_none()
|
||||
|
||||
if not source_root:
|
||||
source_root = SourceRoot(
|
||||
name=Path(norm).name,
|
||||
path=norm,
|
||||
)
|
||||
session.add(source_root)
|
||||
await session.flush()
|
||||
|
||||
return source_root
|
||||
|
||||
|
||||
async def get_or_create_folder(
|
||||
session: AsyncSession, path: str, source_root_id: str, user_id: str = None
|
||||
) -> Folder:
|
||||
"""Get or create a folder entry, matching by normalized path."""
|
||||
from sqlalchemy import select
|
||||
|
||||
norm = _normalize_path(path)
|
||||
result = await session.execute(
|
||||
select(Folder).where(Folder.path == norm)
|
||||
)
|
||||
folder = result.scalar_one_or_none()
|
||||
|
||||
if not folder:
|
||||
parent_path = _normalize_path(str(Path(norm).parent))
|
||||
|
||||
if parent_path != norm: # Not the filesystem root
|
||||
parent_result = await session.execute(
|
||||
select(Folder).where(Folder.path == parent_path)
|
||||
)
|
||||
parent = parent_result.scalar_one_or_none()
|
||||
if parent:
|
||||
parent_id = parent.id
|
||||
else:
|
||||
# Recursively create parent
|
||||
parent = await get_or_create_folder(session, parent_path, source_root_id, user_id)
|
||||
parent_id = parent.id
|
||||
else:
|
||||
parent_id = None
|
||||
|
||||
folder = Folder(
|
||||
name=Path(norm).name,
|
||||
path=norm,
|
||||
parent_id=parent_id,
|
||||
source_root_id=source_root_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
session.add(folder)
|
||||
await session.flush()
|
||||
|
||||
return folder
|
||||
|
||||
@shared_task(name='scan_all_source_roots')
|
||||
def scan_all_source_roots():
|
||||
"""Scan every active source root currently registered in the DB."""
|
||||
# Clear stale per-scan progress before queuing new work so the UI sees
|
||||
# a clean slate even if a previous run crashed mid-flight.
|
||||
r = _get_redis()
|
||||
if r is not None:
|
||||
try:
|
||||
r.delete(REDIS_KEY_ERRORS)
|
||||
r.set(REDIS_KEY_PROCESSED, 0)
|
||||
r.set(REDIS_KEY_TOTAL, 0)
|
||||
except Exception as e:
|
||||
logger.debug(f"scan_all_source_roots redis reset failed: {e}")
|
||||
|
||||
return asyncio.run(_scan_all_source_roots_async())
|
||||
|
||||
|
||||
async def _scan_all_source_roots_async():
|
||||
"""Read every active SourceRoot from the DB and queue a scan_folder task
|
||||
for each. Source roots whose path no longer exists on disk are skipped
|
||||
with a warning (the cleanup service surfaces those at startup too).
|
||||
|
||||
After dispatching the scans, queue a delayed `regroup_duplicates`
|
||||
pass so duplicate clusters are recomputed once the new photos have
|
||||
finished thumbnailing (and therefore picked up phashes). The
|
||||
countdown is a best-effort hint — on a big library the user can
|
||||
still hit Settings → Re-detect duplicates to force a fresh pass.
|
||||
"""
|
||||
from app.tasks.thumbs import incremental_regroup_duplicates_task
|
||||
from app.tasks.vision import backfill_vision, recluster_faces
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active == True) # noqa: E712
|
||||
)
|
||||
source_roots = result.scalars().all()
|
||||
dispatched = 0
|
||||
for sr in source_roots:
|
||||
if os.path.exists(sr.path):
|
||||
scan_folder.delay(sr.path, sr.id)
|
||||
dispatched += 1
|
||||
else:
|
||||
logger.warning(f"Source root path does not exist: {sr.path}")
|
||||
|
||||
if dispatched > 0:
|
||||
# 60s gives the thumbs worker a window to compute phashes for
|
||||
# the new photos before regrouping. The task is idempotent, so
|
||||
# firing too early just means the next manual run picks up the
|
||||
# late arrivals — no corrupted state.
|
||||
try:
|
||||
# Use incremental mode: only compare newly added photos
|
||||
# against the full library via CLIP HNSW + pHash.
|
||||
# O(new × log N) instead of O(N²).
|
||||
scan_start = datetime.now(timezone.utc).isoformat()
|
||||
incremental_regroup_duplicates_task.apply_async(
|
||||
kwargs={'since_iso': scan_start},
|
||||
countdown=60,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue post-scan regroup: {e}")
|
||||
|
||||
# 90s lets thumbnails finish so photos reach processing_status
|
||||
# 'completed', which backfill_vision uses as its filter.
|
||||
try:
|
||||
backfill_vision.apply_async(countdown=90)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue post-scan vision backfill: {e}")
|
||||
|
||||
# 300s gives face extraction time to run before reclustering.
|
||||
# Fires even if some faces are still in-flight — the task is
|
||||
# idempotent and the user can re-trigger from Settings.
|
||||
try:
|
||||
recluster_faces.apply_async(countdown=300)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not queue post-scan face recluster: {e}")
|
||||
|
||||
|
||||
WATCHER_LOCK_KEY = "mulita:watch_folders:lock"
|
||||
WATCHER_LOCK_TTL = 300 # 5 min — renewed every 60s
|
||||
|
||||
|
||||
@shared_task(name='watch_folders', bind=True)
|
||||
def watch_folders(self):
|
||||
"""
|
||||
Watch folders for changes using watchfiles. Long-running task that
|
||||
monitors filesystem events under every active source root.
|
||||
|
||||
Uses a Redis lock to ensure only one instance runs across all
|
||||
workers. The lock is renewed periodically so it survives restarts
|
||||
without leaving orphan watchers.
|
||||
"""
|
||||
import redis as redis_lib
|
||||
from watchfiles import watch
|
||||
|
||||
r = redis_lib.from_url(settings.redis_url)
|
||||
|
||||
# Acquire exclusive lock — if another watcher is already running,
|
||||
# this instance exits immediately instead of stacking up.
|
||||
lock = r.lock(WATCHER_LOCK_KEY, timeout=WATCHER_LOCK_TTL)
|
||||
if not lock.acquire(blocking=False):
|
||||
logger.info("watch_folders: another instance is already running, exiting")
|
||||
return {'status': 'skipped', 'reason': 'another instance is running'}
|
||||
|
||||
try:
|
||||
roots: list[tuple[str, str]] = []
|
||||
try:
|
||||
async def _load_roots():
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(SourceRoot).where(SourceRoot.is_active == True) # noqa: E712
|
||||
)
|
||||
return [
|
||||
(os.path.normpath(sr.path), sr.id)
|
||||
for sr in result.scalars().all()
|
||||
if os.path.exists(sr.path)
|
||||
]
|
||||
roots = asyncio.run(_load_roots())
|
||||
except Exception as e:
|
||||
logger.error(f"watch_folders could not load source roots: {e}")
|
||||
return
|
||||
|
||||
if not roots:
|
||||
logger.warning("No valid source roots to watch")
|
||||
return
|
||||
|
||||
paths = [p for p, _ in roots]
|
||||
logger.info(f"Starting folder watcher for: {paths}")
|
||||
|
||||
def find_source_root_for(path: str) -> Optional[str]:
|
||||
"""Return the source_root id whose path contains `path`, or None."""
|
||||
normalized = os.path.normpath(path)
|
||||
for root_path, root_id in roots:
|
||||
if normalized == root_path or normalized.startswith(root_path + os.sep):
|
||||
return root_id
|
||||
return None
|
||||
|
||||
renew_counter = 0
|
||||
for changes in watch(*paths):
|
||||
# Renew the Redis lock periodically so it doesn't expire
|
||||
# while the watcher is idle between events.
|
||||
renew_counter += 1
|
||||
if renew_counter % 10 == 0:
|
||||
try:
|
||||
lock.extend(WATCHER_LOCK_TTL)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for change_type, filepath in changes:
|
||||
filepath = str(filepath)
|
||||
|
||||
if Path(filepath).suffix.lower() not in SUPPORTED_EXTENSIONS:
|
||||
continue
|
||||
|
||||
if change_type == 'added' or change_type == 'modified':
|
||||
parent_dir = str(Path(filepath).parent)
|
||||
source_root_id = find_source_root_for(parent_dir)
|
||||
if source_root_id is None:
|
||||
continue
|
||||
scan_folder.delay(parent_dir, source_root_id)
|
||||
logger.info(f"File {change_type}: {filepath}, queued scan for {parent_dir}")
|
||||
elif change_type == 'deleted':
|
||||
asyncio.run(handle_file_deletion(filepath))
|
||||
finally:
|
||||
try:
|
||||
lock.release()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def handle_file_deletion(filepath: str):
|
||||
"""Handle deletion of a file from the filesystem"""
|
||||
from sqlalchemy import select
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(Photo).where(Photo.filepath == filepath)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
|
||||
if photo:
|
||||
# Mark as missing or delete from database
|
||||
photo.is_discarded = True
|
||||
photo.discarded_at = datetime.utcnow()
|
||||
await session.commit()
|
||||
logger.info(f"Marked photo as discarded: {filepath}")
|
||||
|
||||
|
||||
@shared_task(name='backfill_gps')
|
||||
def backfill_gps():
|
||||
"""Re-run metadata extraction on every non-discarded photo that is
|
||||
missing latitude/longitude. Used both as a one-shot kick-off after the
|
||||
GPS columns are added on an existing install (see app/database.py) and
|
||||
as a manual trigger from POST /api/v1/library/backfill-gps. Each
|
||||
extract_metadata call is itself a Celery task, so this just enqueues —
|
||||
it does not block on extraction completing."""
|
||||
return asyncio.run(_backfill_gps_async())
|
||||
|
||||
|
||||
async def _backfill_gps_async():
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Newest-first so the most recent photos get their GPS + EXIF
|
||||
# written before the worker climbs back through the archive.
|
||||
result = await session.execute(
|
||||
select(Photo.id)
|
||||
.where(
|
||||
Photo.latitude.is_(None),
|
||||
Photo.is_discarded.is_(False),
|
||||
)
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
)
|
||||
photo_ids = [row[0] for row in result.all()]
|
||||
|
||||
for pid in photo_ids:
|
||||
extract_metadata.delay(pid)
|
||||
|
||||
logger.info(f"backfill_gps: queued extract_metadata for {len(photo_ids)} photos")
|
||||
return {'queued': len(photo_ids)}
|
||||
@@ -1,489 +0,0 @@
|
||||
"""
|
||||
Celery tasks for thumbnail generation
|
||||
"""
|
||||
import os
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import logging
|
||||
from typing import Tuple, Optional
|
||||
import json
|
||||
|
||||
from celery import shared_task
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from PIL import Image
|
||||
import imageio
|
||||
from pillow_heif import register_heif_opener
|
||||
import ffmpeg
|
||||
|
||||
# Try to import optional libraries
|
||||
try:
|
||||
import pyvips
|
||||
PYVIPS_AVAILABLE = True
|
||||
except ImportError:
|
||||
PYVIPS_AVAILABLE = False
|
||||
print("pyvips not available, using Pillow for image processing")
|
||||
|
||||
try:
|
||||
import rawpy
|
||||
RAWPY_AVAILABLE = True
|
||||
except ImportError:
|
||||
RAWPY_AVAILABLE = False
|
||||
print("rawpy not available, using exiftool for RAW preview extraction")
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.models import Photo
|
||||
from app.config import settings
|
||||
|
||||
# Register HEIF opener with Pillow
|
||||
register_heif_opener()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thumbnail sizes configuration
|
||||
THUMB_SIZES = {
|
||||
'small': settings.thumbnails.small,
|
||||
'medium': settings.thumbnails.medium,
|
||||
'large': settings.thumbnails.large
|
||||
}
|
||||
|
||||
def get_thumb_path(photo_id: str, size: str, user_id: str = None) -> str:
|
||||
"""Get the path for a thumbnail file.
|
||||
|
||||
When user_id is provided, thumbnails are stored under a user-specific
|
||||
subdirectory to enforce isolation between users.
|
||||
"""
|
||||
if user_id:
|
||||
thumb_dir = f"/data/thumbs/{user_id}/{photo_id}"
|
||||
else:
|
||||
thumb_dir = f"/data/thumbs/{photo_id}"
|
||||
os.makedirs(thumb_dir, exist_ok=True)
|
||||
return f"{thumb_dir}/{size}.{settings.thumbnails.format}"
|
||||
|
||||
def process_standard_image(filepath: str) -> Image.Image:
|
||||
"""Process standard image formats (JPEG, PNG, etc.)"""
|
||||
return Image.open(filepath)
|
||||
|
||||
def process_raw_image(filepath: str) -> Image.Image:
|
||||
"""Process RAW image formats"""
|
||||
if RAWPY_AVAILABLE:
|
||||
try:
|
||||
with rawpy.imread(filepath) as raw:
|
||||
# Use half_size for faster processing
|
||||
rgb = raw.postprocess(use_camera_wb=True, half_size=True)
|
||||
# Convert numpy array to PIL Image
|
||||
return Image.fromarray(rgb, 'RGB')
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing RAW file {filepath}: {e}")
|
||||
# Try to extract embedded JPEG preview
|
||||
return extract_raw_preview(filepath)
|
||||
else:
|
||||
# Use exiftool to extract embedded preview
|
||||
return extract_raw_preview(filepath)
|
||||
|
||||
def extract_raw_preview(filepath: str) -> Optional[Image.Image]:
|
||||
"""Extract embedded JPEG preview from RAW file"""
|
||||
try:
|
||||
# Use exiftool to extract preview
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp:
|
||||
cmd = ['exiftool', '-b', '-PreviewImage', filepath]
|
||||
result = subprocess.run(cmd, capture_output=True)
|
||||
|
||||
if result.returncode == 0 and result.stdout:
|
||||
tmp.write(result.stdout)
|
||||
tmp.flush()
|
||||
return Image.open(tmp.name)
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting RAW preview from {filepath}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def process_heic_image(filepath: str) -> Image.Image:
|
||||
"""Process HEIC/HEIF image formats"""
|
||||
try:
|
||||
# Use pillow-heif to open the image
|
||||
img = Image.open(filepath)
|
||||
# Convert to RGB if needed
|
||||
if img.mode != 'RGB':
|
||||
img = img.convert('RGB')
|
||||
return img
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing HEIC file {filepath}: {e}")
|
||||
raise
|
||||
|
||||
def process_video_thumbnail(filepath: str) -> Image.Image:
|
||||
"""Extract a still frame from a video file as a PIL Image."""
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
|
||||
tmp_path: Optional[str] = None
|
||||
try:
|
||||
# Find a usable seek timestamp. Some camera MOVs only expose
|
||||
# duration at the format level, and stream 0 isn't always the
|
||||
# video stream — search explicitly and fall back to the format
|
||||
# duration, then to t=0 if neither is available.
|
||||
probe = ffmpeg.probe(filepath)
|
||||
duration: Optional[float] = None
|
||||
for stream_info in probe.get('streams', []):
|
||||
if stream_info.get('codec_type') != 'video':
|
||||
continue
|
||||
raw_duration = stream_info.get('duration')
|
||||
if raw_duration is not None:
|
||||
try:
|
||||
duration = float(raw_duration)
|
||||
break
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if duration is None:
|
||||
raw_duration = probe.get('format', {}).get('duration')
|
||||
if raw_duration is not None:
|
||||
try:
|
||||
duration = float(raw_duration)
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
|
||||
# Seek to 10% in for a representative frame; clamp very short
|
||||
# clips to t=0 so we don't seek past the end.
|
||||
timestamp = max(0.0, (duration or 0.0) * 0.1)
|
||||
|
||||
# NamedTemporaryFile creates the file on disk, so we MUST tell
|
||||
# ffmpeg to overwrite it (otherwise it prompts on stdin and the
|
||||
# call hangs/fails — which is why videos were getting the gray
|
||||
# placeholder). We close the handle immediately and clean up
|
||||
# in `finally` ourselves.
|
||||
with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
stream = ffmpeg.input(filepath, ss=timestamp)
|
||||
stream = ffmpeg.output(
|
||||
stream,
|
||||
tmp_path,
|
||||
vframes=1,
|
||||
format='image2',
|
||||
vcodec='mjpeg',
|
||||
)
|
||||
ffmpeg.run(
|
||||
stream,
|
||||
capture_stdout=True,
|
||||
capture_stderr=True,
|
||||
overwrite_output=True,
|
||||
)
|
||||
|
||||
# Load the frame fully into memory so we can delete the temp
|
||||
# file immediately. Pillow's `Image.open` is lazy, which would
|
||||
# otherwise leave the file dangling.
|
||||
with open(tmp_path, 'rb') as fh:
|
||||
data = fh.read()
|
||||
if not data:
|
||||
raise RuntimeError("ffmpeg produced an empty frame")
|
||||
return Image.open(BytesIO(data)).copy()
|
||||
except ffmpeg.Error as e:
|
||||
stderr = (e.stderr or b'').decode('utf-8', errors='replace')
|
||||
logger.error(
|
||||
f"ffmpeg failed extracting video thumbnail from {filepath}: {stderr}"
|
||||
)
|
||||
return create_placeholder_thumbnail('video')
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting video thumbnail from {filepath}: {e}")
|
||||
return create_placeholder_thumbnail('video')
|
||||
finally:
|
||||
if tmp_path and os.path.exists(tmp_path):
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def create_placeholder_thumbnail(media_type: str) -> Image.Image:
|
||||
"""Create a placeholder thumbnail for failed processing"""
|
||||
# Create a simple gray placeholder
|
||||
img = Image.new('RGB', (640, 480), color=(128, 128, 128))
|
||||
return img
|
||||
|
||||
def auto_rotate_image(image: Image.Image) -> Image.Image:
|
||||
"""Auto-rotate image based on EXIF orientation"""
|
||||
try:
|
||||
# Get EXIF data
|
||||
exif = image._getexif()
|
||||
if exif:
|
||||
orientation = exif.get(274) # Orientation tag
|
||||
|
||||
rotation_map = {
|
||||
3: 180,
|
||||
6: 270, # Note: PIL uses different rotation values than vips
|
||||
8: 90
|
||||
}
|
||||
|
||||
if orientation in rotation_map:
|
||||
image = image.rotate(rotation_map[orientation], expand=True)
|
||||
except:
|
||||
pass # No orientation data available
|
||||
|
||||
return image
|
||||
|
||||
def generate_thumbnail(image: Image.Image, size: int, output_path: str):
|
||||
"""Generate a thumbnail of the specified size"""
|
||||
# Maintain aspect ratio
|
||||
image.thumbnail((size, size), Image.Resampling.LANCZOS)
|
||||
|
||||
# Save as WebP with specified quality
|
||||
image.save(
|
||||
output_path,
|
||||
'WEBP',
|
||||
quality=settings.thumbnails.quality,
|
||||
method=4 # Balance between speed and compression
|
||||
)
|
||||
|
||||
@shared_task(bind=True, name='generate_thumbnails')
|
||||
def generate_thumbnails(self, photo_id: str):
|
||||
"""Generate thumbnails for a photo"""
|
||||
return asyncio.run(_generate_thumbnails_async(photo_id, self))
|
||||
|
||||
async def _generate_thumbnails_async(photo_id: str, task):
|
||||
"""Async implementation of thumbnail generation"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Declared up front so the except block below can safely check it
|
||||
# even if the initial SELECT raises (e.g. asyncpg transport error).
|
||||
photo: Optional[Photo] = None
|
||||
try:
|
||||
# Get photo from database
|
||||
result = await session.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
)
|
||||
photo = result.scalar_one_or_none()
|
||||
|
||||
if not photo:
|
||||
logger.error(f"Photo not found: {photo_id}")
|
||||
return {'status': 'error', 'message': 'Photo not found'}
|
||||
|
||||
# Check if file exists
|
||||
if not os.path.exists(photo.filepath):
|
||||
logger.error(f"File not found: {photo.filepath}")
|
||||
photo.processing_status = 'failed'
|
||||
photo.processing_error = 'File not found'
|
||||
await session.commit()
|
||||
return {'status': 'error', 'message': 'File not found'}
|
||||
|
||||
# Update processing status
|
||||
photo.processing_status = 'processing'
|
||||
await session.commit()
|
||||
|
||||
# Load and process the image based on type
|
||||
image = None
|
||||
|
||||
if photo.media_type == 'photo':
|
||||
image = process_standard_image(photo.filepath)
|
||||
elif photo.media_type == 'raw':
|
||||
image = process_raw_image(photo.filepath)
|
||||
elif photo.media_type == 'heic':
|
||||
image = process_heic_image(photo.filepath)
|
||||
elif photo.media_type == 'video':
|
||||
image = process_video_thumbnail(photo.filepath)
|
||||
else:
|
||||
logger.error(f"Unsupported media type: {photo.media_type}")
|
||||
image = create_placeholder_thumbnail(photo.media_type)
|
||||
|
||||
if not image:
|
||||
raise Exception("Failed to process image")
|
||||
|
||||
# Auto-rotate based on EXIF
|
||||
image = auto_rotate_image(image)
|
||||
|
||||
# Store original dimensions
|
||||
photo.width = image.width
|
||||
photo.height = image.height
|
||||
|
||||
# Perceptual hash from the original-resolution decoded frame.
|
||||
# pHash is robust to resize/recompression but the thumbnail
|
||||
# loop below mutates `image` in place, so this MUST run before
|
||||
# the loop sees it. Failures are non-fatal — phash is a
|
||||
# nice-to-have, not a blocker for thumbnail generation.
|
||||
try:
|
||||
import imagehash
|
||||
photo.phash = str(imagehash.phash(image)) # 16-char hex
|
||||
except Exception as e:
|
||||
logger.warning(f"phash failed for {photo_id}: {e}")
|
||||
photo.phash = None
|
||||
|
||||
# Generate thumbnails for each size
|
||||
for size_name, size_value in THUMB_SIZES.items():
|
||||
thumb_path = get_thumb_path(photo_id, size_name, photo.user_id)
|
||||
generate_thumbnail(image, size_value, thumb_path)
|
||||
|
||||
# Update database with thumbnail path
|
||||
setattr(photo, f'thumb_{size_name}', thumb_path)
|
||||
|
||||
# Update progress
|
||||
task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current_size': size_name, 'photo_id': photo_id}
|
||||
)
|
||||
|
||||
# Update processing status
|
||||
photo.processing_status = 'completed'
|
||||
photo.processing_error = None
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"Thumbnails generated for photo {photo_id}")
|
||||
|
||||
# Dispatch vision pipeline (embedding, OCR, detection, faces)
|
||||
# after thumbs are ready so vision tasks have images to read.
|
||||
try:
|
||||
from app.tasks.vision import vision_fanout
|
||||
vision_fanout.delay(photo_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not dispatch vision_fanout for {photo_id}: {e}")
|
||||
|
||||
return {'status': 'success', 'photo_id': photo_id}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating thumbnails for {photo_id}: {e}")
|
||||
|
||||
# Update error status. If the session is in a bad state (e.g.
|
||||
# the original failure was a transport error) rollback first so
|
||||
# the status write has a clean transaction to commit into.
|
||||
try:
|
||||
await session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if photo is not None:
|
||||
try:
|
||||
photo.processing_status = 'failed'
|
||||
photo.processing_error = str(e)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
f"Could not mark photo {photo_id} as failed"
|
||||
)
|
||||
|
||||
return {'status': 'error', 'message': str(e)}
|
||||
|
||||
@shared_task(name='regenerate_all_thumbnails')
|
||||
def regenerate_all_thumbnails():
|
||||
"""Regenerate thumbnails for all photos"""
|
||||
return asyncio.run(_regenerate_all_thumbnails_async())
|
||||
|
||||
async def _regenerate_all_thumbnails_async():
|
||||
"""Async implementation of regenerating all thumbnails.
|
||||
|
||||
Queue order matters on first-boot and recovery runs: we dispatch
|
||||
newest-first (by EXIF taken_at, fallback added_at) so the user's
|
||||
most recent photos become fully-indexed before the 2012 archive even
|
||||
starts. Picking up the library in pipeline order means the grid,
|
||||
timeline and All Photos view populate top-down instead of the worker
|
||||
chewing through random insertion-order rows while the UI still
|
||||
shows grey placeholders.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Get all photos that need thumbnails, newest first.
|
||||
result = await session.execute(
|
||||
select(Photo)
|
||||
.where(Photo.processing_status.in_(['pending', 'failed']))
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
)
|
||||
photos = result.scalars().all()
|
||||
|
||||
logger.info(f"Regenerating thumbnails for {len(photos)} photos")
|
||||
|
||||
for photo in photos:
|
||||
generate_thumbnails.delay(photo.id)
|
||||
|
||||
return {'status': 'queued', 'count': len(photos)}
|
||||
|
||||
|
||||
# ── Perceptual hash backfill ────────────────────────────────────────────
|
||||
#
|
||||
# When phash was added post-launch, every existing photo has phash=NULL.
|
||||
# This task fills them in by reading the existing thumb_large (the cheap
|
||||
# option — pHash is robust to scale, and the thumb is already on local
|
||||
# disk so we avoid re-decoding the original RAW/HEIC). Falls back to the
|
||||
# original filepath if the thumb isn't available for some reason. Runs
|
||||
# in batches to keep memory bounded and to give the user incremental
|
||||
# progress visible in the worker logs.
|
||||
|
||||
@shared_task(name='backfill_phashes')
|
||||
def backfill_phashes():
|
||||
"""Compute and persist phash for every photo currently missing one."""
|
||||
return asyncio.run(_backfill_phashes_async())
|
||||
|
||||
|
||||
async def _backfill_phashes_async():
|
||||
import imagehash
|
||||
from PIL import Image as _PILImage
|
||||
|
||||
BATCH = 100
|
||||
total_done = 0
|
||||
total_failed = 0
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
while True:
|
||||
# Newest-first so the recent end of the library gets phashes
|
||||
# (and therefore duplicate detection) ahead of the archive.
|
||||
result = await session.execute(
|
||||
select(Photo)
|
||||
.where(Photo.phash.is_(None))
|
||||
.where(Photo.processing_status == 'completed')
|
||||
.order_by(
|
||||
Photo.taken_at.desc().nullslast(),
|
||||
Photo.added_at.desc().nullslast(),
|
||||
)
|
||||
.limit(BATCH)
|
||||
)
|
||||
batch = result.scalars().all()
|
||||
if not batch:
|
||||
break
|
||||
|
||||
for photo in batch:
|
||||
source = photo.thumb_large or photo.filepath
|
||||
try:
|
||||
if not source or not os.path.exists(source):
|
||||
photo.phash = None
|
||||
total_failed += 1
|
||||
continue
|
||||
with _PILImage.open(source) as im:
|
||||
photo.phash = str(imagehash.phash(im))
|
||||
total_done += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"phash backfill failed for {photo.id}: {e}")
|
||||
total_failed += 1
|
||||
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"Backfilled phashes: {total_done} done, {total_failed} failed"
|
||||
)
|
||||
|
||||
return {
|
||||
'status': 'success',
|
||||
'computed': total_done,
|
||||
'failed': total_failed,
|
||||
}
|
||||
|
||||
|
||||
@shared_task(name='regroup_duplicates')
|
||||
def regroup_duplicates_task():
|
||||
"""Full recompute of duplicate groups (pHash + CLIP similarity).
|
||||
|
||||
Used by the Settings → Re-detect duplicates button."""
|
||||
from app.services.duplicates import regroup_duplicates
|
||||
return asyncio.run(regroup_duplicates())
|
||||
|
||||
|
||||
@shared_task(name='incremental_regroup_duplicates')
|
||||
def incremental_regroup_duplicates_task(since_iso: str | None = None):
|
||||
"""Incremental duplicate detection for newly added photos.
|
||||
|
||||
Compares only photos added after `since_iso` against the full library
|
||||
using CLIP vector similarity (O(new × log N) via HNSW) plus pHash.
|
||||
Default post-scan path — much faster than a full regroup."""
|
||||
from app.services.duplicates import incremental_regroup
|
||||
from datetime import datetime, timezone
|
||||
since = None
|
||||
if since_iso:
|
||||
since = datetime.fromisoformat(since_iso)
|
||||
return asyncio.run(incremental_regroup(since=since))
|
||||
@@ -1,529 +0,0 @@
|
||||
"""
|
||||
Celery tasks for the vision pipeline — embedding, OCR, object detection,
|
||||
face recognition.
|
||||
|
||||
All tasks run on the dedicated `vision` queue with limited concurrency
|
||||
(memory-bound CPU inference). They read thumbnails generated by
|
||||
generate_thumbnails, so they MUST run after thumbs complete.
|
||||
|
||||
DB access uses sync psycopg2 sessions (not asyncpg) because Celery
|
||||
forks workers and asyncpg connections can't be shared across forks.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from celery import shared_task
|
||||
from sqlalchemy import create_engine, text as sa_text, select, delete
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from PIL import Image
|
||||
|
||||
from app.models.embeddings import Embedding
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_sync_session() -> Session:
|
||||
"""Create a sync DB session for use in Celery workers."""
|
||||
sync_url = settings.database_url.replace("+asyncpg", "+psycopg2").replace("+aiosqlite", "")
|
||||
engine = create_engine(sync_url, pool_pre_ping=True)
|
||||
return sessionmaker(bind=engine)()
|
||||
|
||||
|
||||
def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None:
|
||||
"""Load a thumbnail as an RGB numpy array."""
|
||||
thumb_path = Path(f"/data/thumbs/{photo_id}/{size}.webp")
|
||||
if not thumb_path.exists():
|
||||
logger.warning("Thumbnail not found: %s", thumb_path)
|
||||
return None
|
||||
img = Image.open(thumb_path).convert("RGB")
|
||||
return np.array(img)
|
||||
|
||||
|
||||
@shared_task(name='embed_photo', queue='vision')
|
||||
def embed_photo(photo_id: str):
|
||||
"""Generate CLIP embedding for a photo and store in pgvector."""
|
||||
if not settings.vision.enabled:
|
||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium") # 640px
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
vector = embedder.embed_image(image)
|
||||
|
||||
model_name = settings.vision.embedder.name
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(
|
||||
delete(Embedding).where(
|
||||
Embedding.photo_id == photo_id,
|
||||
Embedding.model == model_name,
|
||||
)
|
||||
)
|
||||
emb = Embedding(
|
||||
photo_id=photo_id,
|
||||
model=model_name,
|
||||
vector=vector.tolist(),
|
||||
)
|
||||
session.add(emb)
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("Embedded photo %s with %s", photo_id, model_name)
|
||||
return {'status': 'success', 'photo_id': photo_id}
|
||||
|
||||
|
||||
@shared_task(name='vision_fanout', queue='vision')
|
||||
def vision_fanout(photo_id: str):
|
||||
"""Dispatch all enabled vision tasks for a photo."""
|
||||
if not settings.vision.enabled:
|
||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||
|
||||
embed_photo.delay(photo_id)
|
||||
|
||||
if settings.vision.ocr.enabled:
|
||||
ocr_photo.delay(photo_id)
|
||||
if settings.vision.detector.enabled:
|
||||
detect_objects.delay(photo_id)
|
||||
if settings.vision.faces.enabled:
|
||||
extract_faces.delay(photo_id)
|
||||
if settings.vision.classifier.enabled:
|
||||
classify_content.delay(photo_id)
|
||||
|
||||
return {'status': 'dispatched', 'photo_id': photo_id}
|
||||
|
||||
|
||||
@shared_task(name='ocr_photo', queue='vision')
|
||||
def ocr_photo(photo_id: str):
|
||||
"""Run OCR on a photo and store text regions."""
|
||||
if not settings.vision.enabled or not settings.vision.ocr.enabled:
|
||||
return {'status': 'skipped', 'reason': 'OCR disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "large") # 1280px for better OCR accuracy
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
ocr_engine = registry.get_ocr()
|
||||
results = ocr_engine.run(image)
|
||||
|
||||
if not results:
|
||||
logger.info("No OCR text found for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'regions': 0}
|
||||
|
||||
from app.models.ocr_text import OCRText
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(delete(OCRText).where(OCRText.photo_id == photo_id))
|
||||
for r in results:
|
||||
session.add(OCRText(
|
||||
photo_id=photo_id,
|
||||
text=r.text,
|
||||
language=r.language,
|
||||
confidence=r.confidence,
|
||||
bbox=r.bbox,
|
||||
))
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("OCR: %d text regions for photo %s", len(results), photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'regions': len(results)}
|
||||
|
||||
|
||||
@shared_task(name='detect_objects', queue='vision')
|
||||
def detect_objects(photo_id: str):
|
||||
"""Detect objects in a photo, create Tag(kind=object) rows, and
|
||||
link via photo_tags with confidence/bbox/source."""
|
||||
if not settings.vision.enabled or not settings.vision.detector.enabled:
|
||||
return {'status': 'skipped', 'reason': 'detection disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium") # 640px
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
detector = registry.get_detector()
|
||||
detections = detector.detect(image)
|
||||
|
||||
if not detections:
|
||||
logger.info("No objects detected for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'objects': 0}
|
||||
|
||||
from app.models.tags import Tag, photo_tags
|
||||
|
||||
source_name = "vision:yolov8n"
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
# Get the photo's user_id so tags inherit ownership.
|
||||
photo = session.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
).scalar_one_or_none()
|
||||
owner_id = photo.user_id if photo else None
|
||||
|
||||
# Wipe previous detection results for this photo from this model
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.photo_id == photo_id,
|
||||
photo_tags.c.source == source_name,
|
||||
)
|
||||
)
|
||||
|
||||
# Group detections by label, keep highest confidence per label
|
||||
best_per_label: dict[str, tuple[float, list]] = {}
|
||||
for det in detections:
|
||||
if det.label not in best_per_label or det.confidence > best_per_label[det.label][0]:
|
||||
best_per_label[det.label] = (det.confidence, det.bbox)
|
||||
|
||||
for label, (confidence, bbox) in best_per_label.items():
|
||||
# Find or create the object tag (scoped to user)
|
||||
tag = session.execute(
|
||||
select(Tag).where(Tag.name == label, Tag.kind == 'object', Tag.user_id == owner_id)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if not tag:
|
||||
tag = Tag(name=label, kind='object', source=source_name, user_id=owner_id)
|
||||
session.add(tag)
|
||||
session.flush() # get tag.id
|
||||
|
||||
# Insert photo_tags association with ML metadata
|
||||
session.execute(
|
||||
photo_tags.insert().values(
|
||||
photo_id=photo_id,
|
||||
tag_id=tag.id,
|
||||
confidence=confidence,
|
||||
bbox=bbox,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
labels = [d.label for d in detections]
|
||||
logger.info("Detected %d objects in photo %s: %s", len(detections), photo_id, labels)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'objects': len(detections)}
|
||||
|
||||
|
||||
@shared_task(name='classify_content', queue='vision')
|
||||
def classify_content(photo_id: str):
|
||||
"""Classify image content type (screenshot, document, artwork, etc.)
|
||||
using CLIP zero-shot classification. Writes Tag(kind=content_type)."""
|
||||
if not settings.vision.enabled or not settings.vision.classifier.enabled:
|
||||
return {'status': 'skipped', 'reason': 'classifier disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium")
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
classifier = registry.get_classifier()
|
||||
results = classifier.classify(image)
|
||||
|
||||
if not results:
|
||||
logger.info("No confident classification for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'content_type': None}
|
||||
|
||||
from app.models.tags import Tag, photo_tags
|
||||
|
||||
source_name = "vision:clip_classifier"
|
||||
best = results[0]
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
# Get the photo's user_id so tags inherit ownership.
|
||||
photo = session.execute(
|
||||
select(Photo).where(Photo.id == photo_id)
|
||||
).scalar_one_or_none()
|
||||
owner_id = photo.user_id if photo else None
|
||||
|
||||
# Wipe previous classification for this photo
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.photo_id == photo_id,
|
||||
photo_tags.c.source == source_name,
|
||||
)
|
||||
)
|
||||
|
||||
# Find or create content_type tag (scoped to user)
|
||||
tag = session.execute(
|
||||
select(Tag).where(Tag.name == best.label, Tag.kind == 'content_type', Tag.user_id == owner_id)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if not tag:
|
||||
tag = Tag(name=best.label, kind='content_type', source=source_name, user_id=owner_id)
|
||||
session.add(tag)
|
||||
session.flush()
|
||||
|
||||
session.execute(
|
||||
photo_tags.insert().values(
|
||||
photo_id=photo_id,
|
||||
tag_id=tag.id,
|
||||
confidence=best.confidence,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("Classified photo %s as '%s' (%.2f)", photo_id, best.label, best.confidence)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'content_type': best.label}
|
||||
|
||||
|
||||
def _load_original(photo_id: str) -> np.ndarray | None:
|
||||
"""Load the original photo file as an RGB numpy array, resized to
|
||||
max 1280px on the longest edge for face detection."""
|
||||
from sqlalchemy import create_engine, select as sa_select, text as sa_text
|
||||
from app.models import Photo
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
photo = session.execute(
|
||||
sa_select(Photo).where(Photo.id == photo_id)
|
||||
).scalar_one_or_none()
|
||||
if not photo or not photo.filepath:
|
||||
return None
|
||||
filepath = photo.filepath
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if not Path(filepath).exists():
|
||||
logger.warning("Original file not found: %s", filepath)
|
||||
return None
|
||||
|
||||
try:
|
||||
img = Image.open(filepath).convert("RGB")
|
||||
# Cap at 4000px on longest edge to avoid OOM, but keep as large
|
||||
# as possible for face detection accuracy
|
||||
max_dim = 4000
|
||||
w, h = img.size
|
||||
if max(w, h) > max_dim:
|
||||
scale = max_dim / max(w, h)
|
||||
img = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC)
|
||||
return np.array(img)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load original %s: %s", filepath, e)
|
||||
return None
|
||||
|
||||
|
||||
@shared_task(name='extract_faces', queue='vision')
|
||||
def extract_faces(photo_id: str):
|
||||
"""Detect faces and store recognition embeddings using InsightFace
|
||||
(RetinaFace + ArcFace). No YOLO workaround needed — RetinaFace has
|
||||
strong human-vs-non-human precision on its own."""
|
||||
if not settings.vision.enabled or not settings.vision.faces.enabled:
|
||||
return {'status': 'skipped', 'reason': 'faces disabled'}
|
||||
|
||||
image = _load_original(photo_id)
|
||||
if image is None:
|
||||
image = _load_thumb(photo_id, "large")
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'no image available'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
face_proc = registry.get_face_processor()
|
||||
faces = face_proc.process(image)
|
||||
|
||||
if not faces:
|
||||
logger.info("No faces detected for photo %s", photo_id)
|
||||
|
||||
return _save_faces(photo_id, faces)
|
||||
|
||||
|
||||
def _save_faces(photo_id: str, faces) -> dict:
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(delete(FaceEmbedding).where(FaceEmbedding.photo_id == photo_id))
|
||||
for face in faces:
|
||||
session.add(FaceEmbedding(
|
||||
photo_id=photo_id,
|
||||
bbox=face.bbox,
|
||||
vector=face.embedding.tolist(),
|
||||
quality=face.quality,
|
||||
cluster_id=None,
|
||||
))
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if faces:
|
||||
logger.info("Extracted %d verified face(s) from photo %s", len(faces), photo_id)
|
||||
_schedule_recluster_debounced()
|
||||
return {'status': 'success', 'photo_id': photo_id, 'faces': len(faces)}
|
||||
|
||||
|
||||
RECLUSTER_DEBOUNCE_KEY = "mule:recluster_faces:pending"
|
||||
RECLUSTER_DELAY = 120 # seconds after last face extraction
|
||||
|
||||
|
||||
def _schedule_recluster_debounced():
|
||||
"""Schedule a recluster_faces run, debounced so rapid-fire face
|
||||
extractions don't spawn hundreds of redundant cluster jobs."""
|
||||
try:
|
||||
import redis as _redis
|
||||
r = _redis.from_url(settings.redis_url)
|
||||
already_pending = r.set(RECLUSTER_DEBOUNCE_KEY, "1",
|
||||
ex=RECLUSTER_DELAY, nx=True)
|
||||
if already_pending:
|
||||
recluster_faces.apply_async(countdown=RECLUSTER_DELAY)
|
||||
logger.info("Scheduled debounced recluster_faces in %ds", RECLUSTER_DELAY)
|
||||
except Exception as e:
|
||||
logger.debug("recluster debounce check failed: %s", e)
|
||||
|
||||
|
||||
@shared_task(name='recluster_faces', queue='vision')
|
||||
def recluster_faces():
|
||||
"""Run DBSCAN clustering over all face embeddings and assign/create
|
||||
Tag(kind=face_cluster) entries."""
|
||||
# Clear debounce key so new face extractions can schedule another round.
|
||||
try:
|
||||
import redis as _redis
|
||||
_redis.from_url(settings.redis_url).delete(RECLUSTER_DEBOUNCE_KEY)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not settings.vision.enabled or not settings.vision.faces.enabled:
|
||||
return {'status': 'skipped', 'reason': 'faces disabled'}
|
||||
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
from app.models.tags import Tag, photo_tags
|
||||
from app.services.vision.clustering import cluster_faces
|
||||
|
||||
source_name = "vision:sface"
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
face_rows = session.execute(
|
||||
select(FaceEmbedding).order_by(FaceEmbedding.created_at)
|
||||
).scalars().all()
|
||||
|
||||
if len(face_rows) < 2:
|
||||
logger.info("Not enough faces for clustering (%d)", len(face_rows))
|
||||
return {'status': 'success', 'clusters': 0}
|
||||
|
||||
embeddings = np.array([f.vector for f in face_rows], dtype=np.float32)
|
||||
labels = cluster_faces(embeddings, eps=settings.vision.faces.cluster_eps)
|
||||
|
||||
# Clean up old face_cluster tags and their photo_tags
|
||||
old_cluster_tags = session.execute(
|
||||
select(Tag).where(Tag.kind == 'face_cluster', Tag.source == source_name)
|
||||
).scalars().all()
|
||||
for old_tag in old_cluster_tags:
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.tag_id == old_tag.id,
|
||||
photo_tags.c.source == source_name,
|
||||
)
|
||||
)
|
||||
session.delete(old_tag)
|
||||
session.flush()
|
||||
|
||||
# Build new clusters
|
||||
cluster_tag_map: dict[int, str] = {}
|
||||
# Track which photos belong to which cluster
|
||||
cluster_photos: dict[int, set[str]] = {}
|
||||
|
||||
for i, label in enumerate(labels):
|
||||
if label == -1:
|
||||
face_rows[i].cluster_id = None
|
||||
continue
|
||||
|
||||
if label not in cluster_photos:
|
||||
cluster_photos[label] = set()
|
||||
cluster_photos[label].add(face_rows[i].photo_id)
|
||||
|
||||
if label not in cluster_tag_map:
|
||||
cluster_name = f"Person {label + 1}"
|
||||
# Inherit user_id from the representative photo.
|
||||
rep_photo = session.execute(
|
||||
select(Photo.user_id).where(Photo.id == face_rows[i].photo_id)
|
||||
).scalar_one_or_none()
|
||||
tag = Tag(
|
||||
name=cluster_name,
|
||||
kind='face_cluster',
|
||||
source=source_name,
|
||||
representative_photo_id=face_rows[i].photo_id,
|
||||
user_id=rep_photo,
|
||||
)
|
||||
session.add(tag)
|
||||
session.flush()
|
||||
cluster_tag_map[label] = tag.id
|
||||
|
||||
face_rows[i].cluster_id = cluster_tag_map[label]
|
||||
|
||||
# Write photo_tags associations so the tag count and tag_ids
|
||||
# filter work for face clusters
|
||||
for label, photo_ids in cluster_photos.items():
|
||||
tag_id = cluster_tag_map[label]
|
||||
for pid in photo_ids:
|
||||
session.execute(
|
||||
photo_tags.insert().values(
|
||||
photo_id=pid,
|
||||
tag_id=tag_id,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
n_clusters = len(cluster_tag_map)
|
||||
logger.info("Face clustering: %d clusters from %d faces", n_clusters, len(face_rows))
|
||||
return {'status': 'success', 'clusters': n_clusters, 'faces': len(face_rows)}
|
||||
|
||||
|
||||
@shared_task(name='backfill_vision')
|
||||
def backfill_vision(task: str | None = None, limit: int | None = None):
|
||||
"""Queue vision tasks for photos that haven't been processed yet.
|
||||
Uses a sync DB connection to avoid asyncpg conflicts in Celery."""
|
||||
model_name = settings.vision.embedder.name
|
||||
# Newest-first ordering — matches regenerate_all_thumbnails so the
|
||||
# whole ingestion pipeline sweeps the library top-down and the user
|
||||
# sees recent photos fully-indexed long before the backlog drains.
|
||||
# `taken_at` is the canonical capture timestamp (from EXIF, falls
|
||||
# back to filesystem mtime in scan); `added_at` is the tie-breaker
|
||||
# when taken_at is null.
|
||||
sql = """
|
||||
SELECT p.id FROM photos p
|
||||
LEFT JOIN embeddings e ON e.photo_id = p.id AND e.model = :model
|
||||
WHERE e.photo_id IS NULL
|
||||
AND p.processing_status = 'completed'
|
||||
ORDER BY p.taken_at DESC NULLS LAST, p.added_at DESC NULLS LAST
|
||||
"""
|
||||
if limit:
|
||||
sql += f" LIMIT {limit}"
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
result = session.execute(sa_text(sql), {"model": model_name})
|
||||
photo_ids = [row[0] for row in result.fetchall()]
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
count = 0
|
||||
for pid in photo_ids:
|
||||
if task == 'embed' or task is None:
|
||||
embed_photo.delay(pid)
|
||||
if task == 'ocr' or task is None:
|
||||
ocr_photo.delay(pid)
|
||||
if task == 'detect' or task is None:
|
||||
detect_objects.delay(pid)
|
||||
if task == 'faces' or task is None:
|
||||
extract_faces.delay(pid)
|
||||
count += 1
|
||||
|
||||
logger.info("Backfill queued %d photos for vision processing", count)
|
||||
return {'status': 'queued', 'count': count}
|
||||
@@ -1,54 +0,0 @@
|
||||
"""Post-init_db bootstrap: run or stamp Alembic migrations.
|
||||
|
||||
On a FRESH Postgres install, init_db's create_all has already built the
|
||||
full schema from the current models. Running `alembic upgrade head` would
|
||||
fail because the older migrations try ADD COLUMN on columns that already
|
||||
exist. So we detect the fresh-install case (alembic_version table is
|
||||
missing or empty) and `stamp head` instead.
|
||||
|
||||
On an EXISTING install, the alembic_version table has a revision and
|
||||
`upgrade head` applies only the new deltas.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from sqlalchemy import create_engine, text, inspect
|
||||
from app.config import settings
|
||||
|
||||
|
||||
def run():
|
||||
# Use a sync engine for this one-shot script.
|
||||
sync_url = settings.database_url.replace("+asyncpg", "").replace("+aiosqlite", "")
|
||||
engine = create_engine(sync_url)
|
||||
|
||||
with engine.connect() as conn:
|
||||
inspector = inspect(engine)
|
||||
tables = inspector.get_table_names()
|
||||
|
||||
if "alembic_version" not in tables:
|
||||
# Fresh install — create_all built everything. Stamp head.
|
||||
print("Fresh install detected — stamping alembic head")
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "stamp", "head"],
|
||||
check=True,
|
||||
)
|
||||
else:
|
||||
row = conn.execute(text("SELECT version_num FROM alembic_version")).first()
|
||||
if row is None:
|
||||
print("Empty alembic_version — stamping head")
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "stamp", "head"],
|
||||
check=True,
|
||||
)
|
||||
else:
|
||||
print(f"Existing install at revision {row[0]} — running alembic upgrade head")
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "upgrade", "head"],
|
||||
check=True,
|
||||
)
|
||||
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@@ -1,64 +0,0 @@
|
||||
# Core dependencies
|
||||
fastapi==0.109.0
|
||||
uvicorn[standard]==0.27.0
|
||||
python-multipart==0.0.6
|
||||
|
||||
# Database
|
||||
sqlalchemy[asyncio]==2.0.25
|
||||
aiosqlite==0.19.0 # SQLite escape hatch (docker-compose.sqlite.yml override)
|
||||
asyncpg==0.29.0 # async Postgres driver (default)
|
||||
psycopg2-binary==2.9.9 # sync Postgres driver, used by Alembic CLI
|
||||
pgvector==0.2.5 # pgvector SQLAlchemy types
|
||||
alembic==1.13.1
|
||||
|
||||
# Redis and Celery
|
||||
redis==5.0.1
|
||||
celery==5.3.6
|
||||
flower==2.0.1
|
||||
|
||||
# Image processing
|
||||
# pyvips==2.2.1 # Optional - having compatibility issues, using Pillow as fallback
|
||||
# rawpy==0.19.0 # Optional - numpy compatibility issues, using Pillow as fallback
|
||||
pillow==10.2.0
|
||||
pillow-heif==0.15.0
|
||||
imagehash==4.3.1 # perceptual hash for duplicate detection
|
||||
imageio==2.33.1
|
||||
imageio-ffmpeg==0.4.9
|
||||
|
||||
# Video processing
|
||||
ffmpeg-python==0.2.0
|
||||
|
||||
# Metadata extraction
|
||||
pyexiftool==0.5.6
|
||||
|
||||
# File watching
|
||||
watchfiles==0.21.0
|
||||
|
||||
# Vision pipeline (ONNX Runtime CPU inference)
|
||||
onnxruntime==1.18.1
|
||||
open-clip-torch==2.24.0 # tokenizer + export helper; inference via ONNX
|
||||
transformers>=4.37.0 # HuggingFace tokenizer for SigLIP models
|
||||
ultralytics==8.4.37 # YOLOv8n export helper; inference via ONNX
|
||||
rapidocr-onnxruntime==1.3.22
|
||||
scikit-learn==1.4.0 # DBSCAN for face clustering
|
||||
insightface>=0.7.3 # RetinaFace + ArcFace face detection/recognition
|
||||
numpy>=1.26.0,<2.0
|
||||
|
||||
# Utilities
|
||||
pyyaml==6.0.1
|
||||
pydantic==2.5.3
|
||||
pydantic-settings==2.1.0
|
||||
python-dotenv==1.0.0
|
||||
httpx==0.26.0
|
||||
aiofiles==23.2.1
|
||||
|
||||
# Security and authentication
|
||||
python-jose[cryptography]==3.3.0
|
||||
passlib[bcrypt]==1.7.4
|
||||
bcrypt==4.0.1
|
||||
|
||||
# Development
|
||||
pytest==7.4.4
|
||||
pytest-asyncio==0.23.3
|
||||
black==23.12.1
|
||||
ruff==0.1.11
|
||||
24
docker-compose.gpu.yml
Normal file
24
docker-compose.gpu.yml
Normal file
@@ -0,0 +1,24 @@
|
||||
# Overlay for hosts with a VA-API-capable GPU passed through (Intel
|
||||
# QSV, AMD VCN/VCE, any VA-API driver). PhotoPrism's :latest image
|
||||
# ships VA-API-enabled ffmpeg; this file just wires the device + group
|
||||
# membership + encoder selection. Layered in by the deploy script on
|
||||
# hosts where /dev/dri/renderD128 exists.
|
||||
#
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
|
||||
|
||||
services:
|
||||
photoprism:
|
||||
devices:
|
||||
- /dev/dri/renderD128:/dev/dri/renderD128
|
||||
- /dev/dri/card0:/dev/dri/card0
|
||||
# Match host GIDs (render=992, video=44 on Debian). PhotoPrism's
|
||||
# container user (PP_UID:PP_GID, typically 33:10000) is not in
|
||||
# these groups by default; group_add grants access to the device
|
||||
# nodes without changing the primary user.
|
||||
group_add:
|
||||
- "992"
|
||||
- "44"
|
||||
environment:
|
||||
PHOTOPRISM_FFMPEG_ENCODER: ${PP_FFMPEG_ENCODER:-vaapi}
|
||||
PHOTOPRISM_FFMPEG_BITRATE: ${PP_FFMPEG_BITRATE:-32}
|
||||
34
docker-compose.podman.yml
Normal file
34
docker-compose.podman.yml
Normal file
@@ -0,0 +1,34 @@
|
||||
# Podman-rootless overlay for the PhotoPrism stack.
|
||||
#
|
||||
# Apply alongside the base compose file:
|
||||
# podman-compose --env-file .env \
|
||||
# -f docker-compose.yml \
|
||||
# -f docker-compose.podman.yml \
|
||||
# up -d
|
||||
#
|
||||
# Adds the podman-specific bits that would break a vanilla docker compose run:
|
||||
# - userns_mode: keep-id maps container UID to the invoking host UID, so
|
||||
# PhotoPrism (running as PP_UID:PP_GID inside) can actually read the
|
||||
# bind-mounted originals volume on the host (which is owned by the host
|
||||
# user, not by uid 1000-in-the-container-namespace).
|
||||
# - the explicit security_opt entries on the base file work in podman as-is.
|
||||
|
||||
services:
|
||||
# MariaDB writes to a named volume managed by podman; its in-container
|
||||
# `mysql` user expects to own that volume. keep-id breaks this by mapping
|
||||
# in-container UID 999 to a podman-subuid that doesn't own the volume,
|
||||
# so let mariadb use the default userns mapping (root-in-namespace).
|
||||
mariadb:
|
||||
# No userns_mode override — use podman defaults.
|
||||
init: true
|
||||
|
||||
# PhotoPrism does need keep-id, so its container UID maps back to the
|
||||
# host UID that owns the bind-mounted originals/.
|
||||
photoprism:
|
||||
userns_mode: keep-id
|
||||
|
||||
# Sidecar mutates the originals tree (rename / folder ops / heap
|
||||
# convert / .duplicates archive) — same keep-id mapping so its writes
|
||||
# land as the host user, not as a podman-subuid the host doesn't own.
|
||||
sidecar:
|
||||
userns_mode: keep-id
|
||||
@@ -1,50 +0,0 @@
|
||||
# SQLite escape hatch override.
|
||||
#
|
||||
# Usage (omit the `db` service from the up command):
|
||||
#
|
||||
# docker compose -f docker-compose.yml -f docker-compose.sqlite.yml \
|
||||
# up frontend backend worker redis
|
||||
#
|
||||
# This pins the backend and worker to the legacy SQLite database file at
|
||||
# /data/db/mulita.db (in the existing db_data volume), drops the dependency
|
||||
# on Postgres, and skips Alembic — the SQLite schema is still managed by
|
||||
# the inline ALTERs in app/database.py:init_db.
|
||||
#
|
||||
# Vision features that depend on pgvector (PR4 onward) will refuse to enable
|
||||
# in this mode; the search/embedding endpoints will return 503 with a clear
|
||||
# error pointing back at the default Postgres setup.
|
||||
|
||||
services:
|
||||
backend:
|
||||
command: sh -c "uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload"
|
||||
environment:
|
||||
- DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-*}
|
||||
- SECRET_KEY=${SECRET_KEY:-mulita-dev-secret-change-me}
|
||||
- ACCESS_TOKEN_EXPIRE_MINUTES=${ACCESS_TOKEN_EXPIRE_MINUTES:-60}
|
||||
- REFRESH_TOKEN_EXPIRE_DAYS=${REFRESH_TOKEN_EXPIRE_DAYS:-30}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
|
||||
worker:
|
||||
environment:
|
||||
- DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- CELERYD_CONCURRENCY=${CELERYD_CONCURRENCY:-4}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
backend:
|
||||
condition: service_started
|
||||
@@ -1,253 +1,209 @@
|
||||
# Compose stack for the PhotoPrism-backed photo app: mariadb + photoprism +
|
||||
# Go sidecar. The SvelteKit web/ frontend runs separately (Vite in dev,
|
||||
# static build in prod) and proxies /api/v1/* to photoprism and
|
||||
# /api/sidecar/* to the sidecar.
|
||||
#
|
||||
# podman-compose --env-file .env \
|
||||
# -f docker-compose.yml -f docker-compose.podman.yml up -d
|
||||
|
||||
services:
|
||||
frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
dockerfile: Dockerfile
|
||||
container_name: mulita-frontend
|
||||
mariadb:
|
||||
# Fully-qualified for podman (which refuses short names by default).
|
||||
# Docker resolves the same digest.
|
||||
image: docker.io/library/mariadb:11
|
||||
container_name: pp-mariadb
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --innodb-buffer-pool-size=512M
|
||||
- --transaction-isolation=READ-COMMITTED
|
||||
- --character-set-server=utf8mb4
|
||||
- --collation-server=utf8mb4_unicode_ci
|
||||
- --max-connections=512
|
||||
- --innodb-rollback-on-timeout=OFF
|
||||
- --innodb-lock-wait-timeout=120
|
||||
environment:
|
||||
MARIADB_AUTO_UPGRADE: "1"
|
||||
MARIADB_INITDB_SKIP_TZINFO: "1"
|
||||
MARIADB_DATABASE: ${PP_DB_NAME:-photoprism}
|
||||
MARIADB_USER: ${PP_DB_USER:-photoprism}
|
||||
MARIADB_PASSWORD: ${PP_DB_PASSWORD:?set PP_DB_PASSWORD in .env}
|
||||
MARIADB_ROOT_PASSWORD: ${PP_DB_ROOT_PASSWORD:?set PP_DB_ROOT_PASSWORD in .env}
|
||||
# Loopback-only host port so the mule-sidecar (running as a host process
|
||||
# in M4) can reach `mule_sidecar.*` over TCP. Not exposed beyond
|
||||
# 127.0.0.1; the photoprism container still resolves mariadb by service
|
||||
# name on the photoprism-network bridge.
|
||||
ports:
|
||||
# Host port is configurable via FRONTEND_PORT in .env so multiple
|
||||
# instances / other services on the same host don't collide.
|
||||
- "${FRONTEND_PORT:-3000}:80"
|
||||
depends_on:
|
||||
- backend
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
|
||||
backend:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
container_name: mulita-backend
|
||||
ports:
|
||||
# Direct backend access on the host is rarely needed (the frontend
|
||||
# talks to it through the nginx /api proxy on the same network),
|
||||
# but it's exposed for debugging / curl. Override with BACKEND_PORT.
|
||||
- "${BACKEND_PORT:-8001}:8000"
|
||||
- "127.0.0.1:${PP_DB_PORT:-3306}:3306"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
# The single host → container mount for your photo library. Set
|
||||
# PHOTO_DIRS in .env to your library root. Mounted :rw because file
|
||||
# operations (rename, move, empty discard pile) need to mutate the
|
||||
# filesystem; flip to :ro for a strict read-only library and the
|
||||
# write endpoints will return EROFS.
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
- thumbs_data:/data/thumbs
|
||||
- proxies_data:/data/proxies
|
||||
- db_data:/data/db # retained so the docker-compose.sqlite.yml override has somewhere to put mulita.db
|
||||
# Run Alembic migrations before starting uvicorn. On a fresh Postgres
|
||||
# the empty 0001 baseline is a no-op stamp; create_all in init_db then
|
||||
# builds the schema.
|
||||
# init_db creates all tables from models (idempotent create_all),
|
||||
# then Alembic runs migrations for existing installs. On fresh DBs
|
||||
# create_all already built the full schema, so bootstrap.py stamps
|
||||
# alembic head to skip redundant ALTER statements.
|
||||
command: sh -c "python -c 'import asyncio; from app.database import init_db; asyncio.run(init_db())' && python bootstrap.py && uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload"
|
||||
environment:
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-*}
|
||||
- SECRET_KEY=${SECRET_KEY:-mulita-dev-secret-change-me}
|
||||
- ACCESS_TOKEN_EXPIRE_MINUTES=${ACCESS_TOKEN_EXPIRE_MINUTES:-60}
|
||||
- REFRESH_TOKEN_EXPIRE_DAYS=${REFRESH_TOKEN_EXPIRE_DAYS:-30}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
|
||||
# ── Celery workers ─────────────────────────────────────────────────────
|
||||
#
|
||||
# The ingestion pipeline is split across two worker services so CPU-heavy
|
||||
# vision tasks (embed / detect / OCR / faces / classify) cannot starve
|
||||
# the fast IO-bound tasks (scan / thumbnails / EXIF / phash / duplicates).
|
||||
#
|
||||
# worker-light listens on default,high,low — IO-bound, cheap
|
||||
# worker-vision listens on vision — CPU-bound, loads ONNX
|
||||
#
|
||||
# Both share the same image, photo volume, and model cache, so there's
|
||||
# no disk duplication and model weights are loaded lazily only by
|
||||
# worker-vision. Each service has its own concurrency knob; both
|
||||
# workers ship their heartbeat to the same Redis broker so the
|
||||
# Settings > Workers panel lists them side-by-side.
|
||||
#
|
||||
# Sizing defaults target a 6-core / 16 GB host:
|
||||
# CELERY_LIGHT_CONCURRENCY=2 (enough for parallel thumbnail + EXIF)
|
||||
# CELERY_VISION_CONCURRENCY=5 (5 × ~2GB ONNX = ~10GB RAM, 5/6 cores)
|
||||
# Raise these in .env and run `docker compose up -d worker-light worker-vision`
|
||||
# to scale. Keep light under ~4 and vision under your physical core
|
||||
# count; more just thrashes.
|
||||
worker-light:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
image: mule-image-worker
|
||||
container_name: mulita-worker-light
|
||||
command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_LIGHT_CONCURRENCY:-2} -Q default,high,low -n light@%h"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
- thumbs_data:/data/thumbs
|
||||
- proxies_data:/data/proxies
|
||||
- db_data:/data/db
|
||||
- models_data:/data/models
|
||||
environment:
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
# NullPool — see app/database.py for rationale.
|
||||
- MULITA_CELERY_WORKER=1
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
backend:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
|
||||
# Dedicated watcher worker — runs the long-lived watch_folders task
|
||||
# on its own queue so it never blocks scan/thumbnail workers.
|
||||
worker-watcher:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
image: mule-image-worker
|
||||
container_name: mulita-worker-watcher
|
||||
command: sh -c "celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=1 -Q watcher -n watcher@%h"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
- db_data:/data/db
|
||||
environment:
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
- MULITA_CELERY_WORKER=1
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
|
||||
worker-vision:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
image: mule-image-worker
|
||||
container_name: mulita-worker-vision
|
||||
command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_VISION_CONCURRENCY:-5} -Q vision -n vision@%h"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
- thumbs_data:/data/thumbs
|
||||
- proxies_data:/data/proxies
|
||||
- db_data:/data/db
|
||||
- models_data:/data/models
|
||||
environment:
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
- MULITA_CELERY_WORKER=1
|
||||
# ONNX Runtime execution providers. Set to "auto" to auto-detect
|
||||
# GPU (CUDA > ROCm > OpenVINO > CPU), or explicitly:
|
||||
# "CUDAExecutionProvider,CPUExecutionProvider"
|
||||
# "ROCMExecutionProvider,CPUExecutionProvider"
|
||||
# Default: CPU only. To enable GPU, also uncomment the deploy
|
||||
# section below and install nvidia-container-toolkit on the host.
|
||||
- VISION_EXECUTION_PROVIDERS=${VISION_EXECUTION_PROVIDERS:-CPUExecutionProvider}
|
||||
# Pin each ONNX session to one intra-op thread so N prefork children
|
||||
# × default-all-cores doesn't oversubscribe the box. With
|
||||
# concurrency=5 and OMP=1, vision peaks at 5 busy cores, leaving
|
||||
# one for worker-light + system. These env vars cover the three
|
||||
# threading runtimes ONNX Runtime might pick up on first use.
|
||||
- OMP_NUM_THREADS=1
|
||||
- OPENBLAS_NUM_THREADS=1
|
||||
- MKL_NUM_THREADS=1
|
||||
# Uncomment for NVIDIA GPU passthrough:
|
||||
# deploy:
|
||||
# resources:
|
||||
# reservations:
|
||||
# devices:
|
||||
# - driver: nvidia
|
||||
# count: all
|
||||
# capabilities: [gpu]
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
backend:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
|
||||
db:
|
||||
image: pgvector/pgvector:pg16
|
||||
container_name: mulita-db
|
||||
environment:
|
||||
POSTGRES_USER: mulita
|
||||
POSTGRES_PASSWORD: mulita
|
||||
POSTGRES_DB: mulita
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
- pp_mariadb_data:/var/lib/mysql
|
||||
# The init script creates the mule_sidecar database + user that the Go
|
||||
# sidecar service will use in M4. Idempotent; no-op on subsequent boots.
|
||||
# ":Z" is the SELinux private-relabel flag — needed on Fedora/RHEL hosts,
|
||||
# silently no-op on Debian/Ubuntu and macOS Docker Desktop.
|
||||
- ./mariadb/init:/docker-entrypoint-initdb.d:ro,Z
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U mulita -d mulita"]
|
||||
interval: 5s
|
||||
test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
retries: 12
|
||||
start_period: 60s
|
||||
networks: [photoprism-network]
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: mulita-redis
|
||||
# Host port exposed only for local debugging; the backend / worker
|
||||
# reach Redis via the internal mulita-network on its container name.
|
||||
ports:
|
||||
- "${REDIS_PORT:-6379}:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
networks:
|
||||
- mulita-network
|
||||
photoprism:
|
||||
image: docker.io/photoprism/photoprism:latest
|
||||
container_name: pp-app
|
||||
restart: unless-stopped
|
||||
command: redis-server --appendonly yes
|
||||
depends_on:
|
||||
mariadb:
|
||||
condition: service_healthy
|
||||
# PhotoPrism's container drops to a non-root user via PHOTOPRISM_UID /
|
||||
# PHOTOPRISM_GID. Match the host user that owns ${PHOTO_DIRS} so the
|
||||
# process can read originals (and later write sidecars).
|
||||
user: "${PP_UID:-1000}:${PP_GID:-1000}"
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
- apparmor:unconfined
|
||||
ports:
|
||||
# Loopback only — the SvelteKit web/ app (Vite dev or built bundle)
|
||||
# is the user-facing surface; PhotoPrism's own UI stays off the
|
||||
# public interface. Vite proxies /api/v1/* here, and the host-mode
|
||||
# sidecar reaches PHOTOPRISM_BASE_URL=http://localhost:2342. Admin
|
||||
# access to PP's UI is via SSH tunnel only.
|
||||
- "127.0.0.1:${PP_PORT:-2342}:2342"
|
||||
environment:
|
||||
PHOTOPRISM_ADMIN_USER: ${PP_ADMIN_USER:-admin}
|
||||
PHOTOPRISM_ADMIN_PASSWORD: ${PP_ADMIN_PASSWORD:?set PP_ADMIN_PASSWORD in .env}
|
||||
PHOTOPRISM_AUTH_MODE: ${PP_AUTH_MODE:-password}
|
||||
PHOTOPRISM_SITE_URL: ${PP_SITE_URL:-http://localhost:2342/}
|
||||
PHOTOPRISM_ORIGINALS_LIMIT: ${PP_ORIGINALS_LIMIT:-50000}
|
||||
PHOTOPRISM_HTTP_COMPRESSION: gzip
|
||||
PHOTOPRISM_LOG_LEVEL: ${PP_LOG_LEVEL:-info}
|
||||
# Indexer concurrency. Defaults to NumCPU/2 (= 3 on a 6-core LXC),
|
||||
# but each worker forks TF + ffmpeg + libvips so effective load is
|
||||
# much higher — a fresh index of 1.2k photos on M0 pushed the LXC
|
||||
# load to 50+ and starved sibling containers. Pin to a low value
|
||||
# for shared hosts; raise on dedicated machines.
|
||||
PHOTOPRISM_WORKERS: ${PP_WORKERS:-2}
|
||||
# podman-compose doesn't expand nested ${A:-${B:-…}}, so keep this
|
||||
# one-level. Override both PP_WORKERS and PP_INDEX_WORKERS if you
|
||||
# want them to differ.
|
||||
PHOTOPRISM_INDEX_WORKERS: ${PP_INDEX_WORKERS:-2}
|
||||
# M0 safety: keep originals read-only. Flip to "false" in M2 when the
|
||||
# right-sidebar enables metadata edits and we want EXIF backwrite.
|
||||
PHOTOPRISM_READONLY: ${PP_READONLY:-true}
|
||||
PHOTOPRISM_EXPERIMENTAL: "false"
|
||||
PHOTOPRISM_DISABLE_CHOWN: "true"
|
||||
PHOTOPRISM_DISABLE_WEBDAV: ${PP_DISABLE_WEBDAV:-false}
|
||||
PHOTOPRISM_DISABLE_SETTINGS: "false"
|
||||
PHOTOPRISM_DISABLE_TLS: "true"
|
||||
PHOTOPRISM_DEFAULT_TLS: "false"
|
||||
# AI/vision pipeline back on — per plan we re-introduce TF labels + faces.
|
||||
PHOTOPRISM_TENSORFLOW_OFF: "false"
|
||||
PHOTOPRISM_DETECT_NSFW: "true"
|
||||
PHOTOPRISM_UPLOAD_NSFW: "true"
|
||||
# Database
|
||||
PHOTOPRISM_DATABASE_DRIVER: mysql
|
||||
PHOTOPRISM_DATABASE_SERVER: mariadb:3306
|
||||
PHOTOPRISM_DATABASE_NAME: ${PP_DB_NAME:-photoprism}
|
||||
PHOTOPRISM_DATABASE_USER: ${PP_DB_USER:-photoprism}
|
||||
PHOTOPRISM_DATABASE_PASSWORD: ${PP_DB_PASSWORD}
|
||||
# Sidecars next to originals — read by the migrator at M5.
|
||||
PHOTOPRISM_SIDECAR_PATH: ""
|
||||
PHOTOPRISM_SIDECAR_YAML: "true"
|
||||
# EXIF backwrite — disabled in M0 (READONLY blocks writes anyway).
|
||||
# Override in .env: PP_BACKUP_DATABASE=true.
|
||||
PHOTOPRISM_DISABLE_BACKUPS: "false"
|
||||
PHOTOPRISM_BACKUP_DATABASE: ${PP_BACKUP_DATABASE:-true}
|
||||
PHOTOPRISM_DISABLE_EXIFTOOL: "false"
|
||||
# OIDC — set in .env when the IdP (Authentik) is wired up.
|
||||
# Empty values keep OIDC dormant; the username/password login still works.
|
||||
# PhotoPrism's CLI flags are --oidc-uri / --oidc-client / --oidc-secret
|
||||
# / --oidc-provider, so the env-var names it actually reads are
|
||||
# PHOTOPRISM_OIDC_URI / _CLIENT / _SECRET / _PROVIDER (NOT _ISSUER_URL
|
||||
# / _CLIENT_ID / _CLIENT_SECRET / _PROVIDER_NAME — those are silently
|
||||
# ignored, OIDC stays dormant, and `photoprism show config` reports
|
||||
# blank oidc-uri / oidc-client). PHOTOPRISM_OIDC_REDIRECT is a bool
|
||||
# (auto-redirect-from-/library/login), not a URL — PhotoPrism builds
|
||||
# the callback from PHOTOPRISM_SITE_URL.
|
||||
PHOTOPRISM_OIDC_PROVIDER: ${OIDC_PROVIDER_NAME:-${OIDC_PROVIDER:-}}
|
||||
PHOTOPRISM_OIDC_URI: ${OIDC_ISSUER_URL:-${OIDC_URI:-}}
|
||||
PHOTOPRISM_OIDC_CLIENT: ${OIDC_CLIENT_ID:-${OIDC_CLIENT:-}}
|
||||
PHOTOPRISM_OIDC_SECRET: ${OIDC_CLIENT_SECRET:-${OIDC_SECRET:-}}
|
||||
PHOTOPRISM_OIDC_SCOPES: ${OIDC_SCOPES:-openid profile email}
|
||||
PHOTOPRISM_OIDC_REGISTER: ${OIDC_REGISTER:-true}
|
||||
PHOTOPRISM_OIDC_ROLE: ${OIDC_ROLE:-user}
|
||||
PHOTOPRISM_OIDC_REDIRECT: ${OIDC_REDIRECT:-false}
|
||||
working_dir: /photoprism
|
||||
volumes:
|
||||
# Existing photo library — mounted read-only in M0; flip to :rw in M2
|
||||
# when the right-sidebar starts saving edits. ",Z" relabels for SELinux
|
||||
# on Fedora/RHEL; silent no-op elsewhere.
|
||||
- "${PHOTO_DIRS:?set PHOTO_DIRS in .env}:/photoprism/originals:${PP_ORIGINALS_MODE:-ro},Z"
|
||||
- "./pp/storage:/photoprism/storage:Z"
|
||||
- "./pp/import:/photoprism/import:Z"
|
||||
networks: [photoprism-network]
|
||||
|
||||
# mule-sidecar — Go + Gin + GORM service for endpoints PhotoPrism's API
|
||||
# does not expose (file rename, folder mutations, heap convert, duplicate
|
||||
# scan, per-photo marks). Same wire contract as the M3 Node prototype;
|
||||
# the SvelteKit dev server proxies /api/sidecar/* here.
|
||||
sidecar:
|
||||
build:
|
||||
context: ./sidecar
|
||||
container_name: pp-sidecar
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
mariadb:
|
||||
condition: service_healthy
|
||||
photoprism:
|
||||
condition: service_started
|
||||
# Match PhotoPrism's UID/GID so renames/folder mutations preserve the
|
||||
# ownership the indexer expects on the bind-mounted originals.
|
||||
user: "${PP_UID:-1000}:${PP_GID:-1000}"
|
||||
ports:
|
||||
# Loopback only — Vite (host) proxies /api/sidecar/* to this port.
|
||||
# Behind a reverse proxy in production; never published beyond the
|
||||
# host.
|
||||
- "127.0.0.1:${SIDECAR_PORT:-8000}:8000"
|
||||
environment:
|
||||
ORIGINALS_ROOT: /photoprism/originals
|
||||
PHOTOPRISM_BASE_URL: http://photoprism:2342
|
||||
# Bind on all interfaces inside the container so the host-side
|
||||
# 127.0.0.1:8000 port mapping can reach the listener. The Go
|
||||
# binary defaults to 127.0.0.1 for the host-mode dev loop.
|
||||
SIDECAR_LISTEN_ADDR: 0.0.0.0
|
||||
SIDECAR_PORT: "8000"
|
||||
SIDECAR_DB_HOST: mariadb
|
||||
SIDECAR_DB_PORT: "3306"
|
||||
SIDECAR_DB_USER: sidecar
|
||||
# Rotate before any non-local deployment. Provisioned by
|
||||
# mariadb/init/01-sidecar.sql on first boot of the mariadb volume.
|
||||
SIDECAR_DB_PASSWORD: ${SIDECAR_DB_PASSWORD:-replace-at-m4-bringup}
|
||||
SIDECAR_DB_NAME: mule_sidecar
|
||||
# Second DB connection for poking PhotoPrism's own schema (only
|
||||
# used by the user-basepath reconciler today). Stays inert if
|
||||
# PP_DB_PASSWORD is empty — the reconciler then silently no-ops.
|
||||
PP_DB_HOST: mariadb
|
||||
PP_DB_PORT: "3306"
|
||||
PP_DB_USER: ${PP_DB_USER:-photoprism}
|
||||
PP_DB_PASSWORD: ${PP_DB_PASSWORD:-}
|
||||
PP_DB_NAME: ${PP_DB_NAME:-photoprism}
|
||||
# Declarative username → originals-relative BasePath mapping.
|
||||
# Format: comma-separated `user:path` pairs. Sidecar applies it
|
||||
# to auth_users on boot and every 60s, and `mkdir -p`s each
|
||||
# target subdirectory so PhotoPrism's ACL filter has somewhere to
|
||||
# point. Leave empty to disable.
|
||||
# USER_BASEPATHS="test:test, alice:family/alice"
|
||||
USER_BASEPATHS: ${USER_BASEPATHS:-}
|
||||
volumes:
|
||||
# Sidecar mutates originals (rename, folder mutations, heap
|
||||
# convert) — always rw regardless of PhotoPrism's mount mode.
|
||||
- "${PHOTO_DIRS:?set PHOTO_DIRS in .env}:/photoprism/originals:rw,Z"
|
||||
networks: [photoprism-network]
|
||||
|
||||
networks:
|
||||
mulita-network:
|
||||
photoprism-network:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
thumbs_data:
|
||||
proxies_data:
|
||||
db_data:
|
||||
redis_data:
|
||||
pg_data:
|
||||
models_data:
|
||||
pp_mariadb_data:
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
# Build stage
|
||||
FROM node:18-alpine as build
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files
|
||||
COPY package*.json ./
|
||||
|
||||
# Install dependencies
|
||||
RUN npm ci
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the application
|
||||
RUN npm run build
|
||||
|
||||
# Production stage
|
||||
FROM nginx:alpine
|
||||
|
||||
# Copy built assets from build stage
|
||||
COPY --from=build /app/dist /usr/share/nginx/html
|
||||
|
||||
# Copy nginx configuration
|
||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
|
||||
# Expose port
|
||||
EXPOSE 80
|
||||
|
||||
# Start nginx
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
@@ -1,15 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en" class="dark">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/png" href="/favicon.png" />
|
||||
<link rel="apple-touch-icon" href="/favicon.png" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="theme-color" content="#0f0f0f" />
|
||||
<title>Mulimago</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,54 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Enable gzip
|
||||
gzip on;
|
||||
gzip_vary on;
|
||||
gzip_min_length 1024;
|
||||
gzip_types text/plain text/css text/xml text/javascript application/javascript application/xml+rss application/json;
|
||||
|
||||
# API proxy
|
||||
location /api/ {
|
||||
proxy_pass http://backend:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket support for real-time updates
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
|
||||
# Thumbnail serving with X-Accel-Redirect
|
||||
location /internal_thumbs/ {
|
||||
internal;
|
||||
alias /data/thumbs/;
|
||||
}
|
||||
|
||||
# SPA routing - serve index.html for all routes
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# Never cache index.html (or any HTML). The asset filenames are
|
||||
# content-hashed by Vite, so a fresh index.html is the only thing
|
||||
# that tells the browser to fetch the new bundle. Without this the
|
||||
# browser happily serves a stale index.html → stale bundle hash →
|
||||
# users see the old build until they hard-reload.
|
||||
location = /index.html {
|
||||
add_header Cache-Control "no-cache, no-store, must-revalidate";
|
||||
add_header Pragma "no-cache";
|
||||
expires 0;
|
||||
}
|
||||
|
||||
# Cache static assets (filenames are content-hashed, so 1y is safe)
|
||||
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
}
|
||||
6094
frontend/package-lock.json
generated
6094
frontend/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,63 +0,0 @@
|
||||
{
|
||||
"name": "mulita-frontend",
|
||||
"private": true,
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"preview": "vite preview",
|
||||
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@radix-ui/react-accordion": "^1.1.2",
|
||||
"@radix-ui/react-alert-dialog": "^1.0.5",
|
||||
"@radix-ui/react-checkbox": "^1.0.4",
|
||||
"@radix-ui/react-context-menu": "^2.1.5",
|
||||
"@radix-ui/react-dialog": "^1.0.5",
|
||||
"@radix-ui/react-dropdown-menu": "^2.0.6",
|
||||
"@radix-ui/react-label": "^2.0.2",
|
||||
"@radix-ui/react-popover": "^1.0.7",
|
||||
"@radix-ui/react-scroll-area": "^1.0.5",
|
||||
"@radix-ui/react-select": "^2.0.0",
|
||||
"@radix-ui/react-separator": "^1.0.3",
|
||||
"@radix-ui/react-slider": "^1.1.2",
|
||||
"@radix-ui/react-switch": "^1.0.3",
|
||||
"@radix-ui/react-tabs": "^1.0.4",
|
||||
"@radix-ui/react-toast": "^1.1.5",
|
||||
"@radix-ui/react-tooltip": "^1.0.7",
|
||||
"@tanstack/react-query": "^5.17.0",
|
||||
"@tanstack/react-virtual": "^3.0.1",
|
||||
"axios": "^1.6.5",
|
||||
"clsx": "^2.1.0",
|
||||
"date-fns": "^3.2.0",
|
||||
"framer-motion": "^10.18.0",
|
||||
"leaflet": "^1.9.4",
|
||||
"lucide-react": "^0.303.0",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-hotkeys-hook": "^4.4.3",
|
||||
"react-intersection-observer": "^9.5.3",
|
||||
"react-leaflet": "^4.2.1",
|
||||
"react-leaflet-cluster": "^2.1.0",
|
||||
"tailwind-merge": "^2.2.0",
|
||||
"zustand": "^4.4.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tanstack/react-query-devtools": "^5.96.2",
|
||||
"@types/leaflet": "^1.9.8",
|
||||
"@types/react": "^18.2.46",
|
||||
"@types/react-dom": "^18.2.18",
|
||||
"@typescript-eslint/eslint-plugin": "^6.17.0",
|
||||
"@typescript-eslint/parser": "^6.17.0",
|
||||
"@vitejs/plugin-react": "^4.2.1",
|
||||
"autoprefixer": "^10.4.16",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-plugin-react-hooks": "^4.6.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.5",
|
||||
"postcss": "^8.4.33",
|
||||
"tailwindcss": "^3.4.0",
|
||||
"typescript": "^5.3.3",
|
||||
"vite": "^5.0.10"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
import { useState } from 'react'
|
||||
import { Timeline } from './components/timeline/Timeline'
|
||||
import { DuplicatesView } from './components/duplicates/DuplicatesView'
|
||||
import { MapView } from './components/map/MapView'
|
||||
import { MemoriesView } from './components/memories/MemoriesView'
|
||||
import { PeopleView } from './components/people/PeopleView'
|
||||
import { TagsView } from './components/tags/TagsView'
|
||||
import { ColorsView } from './components/colors/ColorsView'
|
||||
import { RatedView } from './components/rated/RatedView'
|
||||
import { LeftSidebar } from './components/layout/LeftSidebar'
|
||||
import { RightSidebar } from './components/layout/RightSidebar'
|
||||
import { TopBar } from './components/layout/TopBar'
|
||||
import { ScanProgress } from './components/ScanProgress'
|
||||
import { ToastContainer } from './components/ToastContainer'
|
||||
import { KeyboardHints } from './components/KeyboardHints'
|
||||
import { PreviewView } from './components/preview/PreviewView'
|
||||
import { FilterBar } from './components/filter/FilterBar'
|
||||
import { DiscardActionBar } from './components/discard/DiscardActionBar'
|
||||
import { SettingsPage } from './components/dialogs/SettingsDialog'
|
||||
import { usePhotoStore } from './store/photoStore'
|
||||
import { useFilterStore } from './store/filterStore'
|
||||
import { useKeyboardShortcuts } from './hooks/useKeyboardShortcuts'
|
||||
import { useFilterUrlSync } from './hooks/useFilterUrlSync'
|
||||
import { usePhotosQuery } from './hooks/usePhotosQuery'
|
||||
import { AuthProvider, useAuth } from './contexts/AuthContext'
|
||||
import { LoginPage } from './components/auth/LoginPage'
|
||||
import { SetupPage } from './components/auth/SetupPage'
|
||||
|
||||
function MainApp() {
|
||||
const [leftSidebarOpen, setLeftSidebarOpen] = useState(true)
|
||||
const [rightSidebarOpen, setRightSidebarOpen] = useState(true)
|
||||
const viewMode = usePhotoStore((state) => state.viewMode)
|
||||
const currentSection = useFilterStore((s) => s.currentSection)
|
||||
|
||||
// Bidirectional sync of filter store with URL query params.
|
||||
useFilterUrlSync()
|
||||
|
||||
// Subscribe to the same photos query the Timeline uses, so the keyboard
|
||||
// "open preview on first photo" path can read from the live cache regardless
|
||||
// of what filter key it's stored under.
|
||||
const { data: allPhotos } = usePhotosQuery()
|
||||
|
||||
// Set up global keyboard shortcuts
|
||||
useKeyboardShortcuts({
|
||||
onToggleLeftSidebar: () => setLeftSidebarOpen(!leftSidebarOpen),
|
||||
onToggleRightSidebar: () => setRightSidebarOpen(!rightSidebarOpen),
|
||||
getFirstPhotoId: () => allPhotos?.[0]?.id ?? null,
|
||||
})
|
||||
|
||||
// Settings page is a full-page section — hide filter bar, right sidebar,
|
||||
// and keyboard hints when it's active.
|
||||
const isSettings = currentSection === 'settings'
|
||||
|
||||
// Right sidebar stays open by default and shows whatever's selected
|
||||
// (or an empty state if nothing is). User can still toggle it manually.
|
||||
const showRightSidebar = rightSidebarOpen && !isSettings
|
||||
|
||||
return (
|
||||
<div className="flex flex-col h-screen bg-bg text-text">
|
||||
<TopBar
|
||||
leftSidebarOpen={leftSidebarOpen}
|
||||
rightSidebarOpen={showRightSidebar}
|
||||
onExpandLeft={() => setLeftSidebarOpen(true)}
|
||||
onExpandRight={() => setRightSidebarOpen(true)}
|
||||
/>
|
||||
|
||||
<div className="flex flex-1 overflow-hidden">
|
||||
{/* Left Sidebar */}
|
||||
<div
|
||||
className={`transition-all duration-200 ${
|
||||
leftSidebarOpen ? 'w-60' : 'w-0'
|
||||
} overflow-hidden border-r border-border bg-surface`}
|
||||
>
|
||||
<LeftSidebar
|
||||
onCollapse={() => setLeftSidebarOpen(false)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Main column — filter bar, discard bar, timeline. Lives to the
|
||||
* right of the left sidebar so the filter row doesn't bleed
|
||||
* across the sidebar. relative so the KeyboardHints overlay
|
||||
* centers against this column, not the viewport. */}
|
||||
<div className="relative flex min-w-0 flex-1 flex-col">
|
||||
{!isSettings && <FilterBar />}
|
||||
{!isSettings && <DiscardActionBar />}
|
||||
<div className="flex-1 overflow-auto">
|
||||
{currentSection === 'settings' ? (
|
||||
<SettingsPage />
|
||||
) : currentSection === 'map' ? (
|
||||
<MapView />
|
||||
) : currentSection === 'memories' ? (
|
||||
<MemoriesView />
|
||||
) : currentSection === 'duplicates' ? (
|
||||
<DuplicatesView />
|
||||
) : currentSection === 'people' ? (
|
||||
<PeopleView />
|
||||
) : currentSection === 'tags' ? (
|
||||
<TagsView />
|
||||
) : currentSection === 'colors' ? (
|
||||
<ColorsView />
|
||||
) : currentSection === 'rated' ? (
|
||||
<RatedView />
|
||||
) : (
|
||||
<Timeline />
|
||||
)}
|
||||
</div>
|
||||
{!isSettings && <KeyboardHints />}
|
||||
</div>
|
||||
|
||||
{/* Right Sidebar */}
|
||||
<div
|
||||
className={`transition-all duration-200 ${
|
||||
showRightSidebar ? 'w-72' : 'w-0'
|
||||
} overflow-hidden border-l border-border bg-surface`}
|
||||
>
|
||||
<RightSidebar onCollapse={() => setRightSidebarOpen(false)} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Scan Progress Indicator */}
|
||||
<ScanProgress />
|
||||
|
||||
{/* Toast Notifications */}
|
||||
<ToastContainer />
|
||||
|
||||
{/* Preview overlay — covers TopBar when active */}
|
||||
{viewMode === 'preview' && <PreviewView />}
|
||||
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** Auth-gated shell: shows setup, login, or the main app. */
|
||||
function App() {
|
||||
return (
|
||||
<AuthProvider>
|
||||
<AuthGate />
|
||||
</AuthProvider>
|
||||
)
|
||||
}
|
||||
|
||||
function AuthGate() {
|
||||
const { user, isLoading, needsSetup } = useAuth()
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex min-h-screen items-center justify-center bg-bg">
|
||||
<div className="text-text-muted">Loading…</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (needsSetup) return <SetupPage />
|
||||
if (!user) return <LoginPage />
|
||||
return <MainApp />
|
||||
}
|
||||
|
||||
export default App
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 821 KiB |
@@ -1,62 +0,0 @@
|
||||
import { usePhotoStore } from '../store/photoStore'
|
||||
|
||||
export function KeyboardHints() {
|
||||
const selectedCount = usePhotoStore((state) => state.selectedPhotos.length)
|
||||
const viewMode = usePhotoStore((state) => state.viewMode)
|
||||
|
||||
// In preview mode the viewer has its own context, so the grid hints
|
||||
// would just be confusing. Hide them.
|
||||
if (viewMode === 'preview') return null
|
||||
|
||||
const hints = selectedCount > 0
|
||||
? [
|
||||
{ key: '1-5', action: 'Rate' },
|
||||
{ key: 'P', action: 'Pick → heap' },
|
||||
{ key: 'X', action: 'Discard' },
|
||||
{ key: 'Space', action: 'Preview' },
|
||||
{ key: 'I', action: 'Info panel' },
|
||||
{ key: 'Esc', action: 'Deselect' },
|
||||
]
|
||||
: [
|
||||
{ key: '↑↓←→', action: 'Navigate' },
|
||||
{ key: 'Space', action: 'Preview' },
|
||||
{ key: 'Tab', action: 'Library panel' },
|
||||
{ key: 'I', action: 'Info panel' },
|
||||
{ key: '/', action: 'Search' },
|
||||
]
|
||||
|
||||
return (
|
||||
// Absolute (not fixed) so the parent's flex/position context can
|
||||
// center it relative to the timeline area, not the viewport. Mount
|
||||
// inside the main column in App.tsx so it isn't offset by the
|
||||
// sidebar widths.
|
||||
<div className="pointer-events-none absolute bottom-4 left-1/2 z-30 -translate-x-1/2">
|
||||
{/* Near-opaque dark pill so the hints stay legible against busy
|
||||
* thumbnails. The previous bg-surface/40 + 5% white ring left
|
||||
* text washed out when a bright photo sat directly behind it. */}
|
||||
<div className="pointer-events-auto flex items-center gap-3 whitespace-nowrap rounded-full border border-white/15 bg-black/80 px-4 py-1.5 shadow-xl ring-1 ring-black/40 backdrop-blur-md">
|
||||
{hints.map((hint, i) => (
|
||||
<div key={i} className="flex items-center gap-1.5">
|
||||
<kbd className="rounded bg-white/15 px-1.5 py-0.5 text-[11px] font-medium text-white shadow-sm">
|
||||
{hint.key}
|
||||
</kbd>
|
||||
<span className="whitespace-nowrap text-xs text-white/85">
|
||||
{hint.action}
|
||||
</span>
|
||||
{i < hints.length - 1 && (
|
||||
<span className="ml-1 text-white/30">•</span>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
{selectedCount > 0 && (
|
||||
<>
|
||||
<span className="text-white/30">•</span>
|
||||
<span className="whitespace-nowrap text-xs font-semibold text-primary">
|
||||
{selectedCount} selected
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,264 +0,0 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { FolderOpen, Loader2, Check, AlertCircle, X, Brain, Sparkles } from 'lucide-react'
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { library, WorkerStatus } from '../services/api'
|
||||
import clsx from 'clsx'
|
||||
|
||||
interface ScanStatus {
|
||||
is_scanning: boolean
|
||||
current_folder?: string
|
||||
processed_files: number
|
||||
total_files: number
|
||||
errors: string[]
|
||||
}
|
||||
|
||||
type Phase = 'idle' | 'scanning' | 'processing' | 'done'
|
||||
|
||||
export function ScanProgress() {
|
||||
const [isVisible, setIsVisible] = useState(false)
|
||||
const [isMinimized, setIsMinimized] = useState(false)
|
||||
const queryClient = useQueryClient()
|
||||
const wasScanningRef = useRef(false)
|
||||
const wasProcessingRef = useRef(false)
|
||||
|
||||
const { data: scanStatus } = useQuery<ScanStatus>({
|
||||
queryKey: ['scan-status'],
|
||||
queryFn: () => library.scanStatus(),
|
||||
refetchInterval: (query) =>
|
||||
query.state.data?.is_scanning ? 2000 : 10000,
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
const isScanning = scanStatus?.is_scanning ?? false
|
||||
|
||||
// Poll worker status to track vision queue activity.
|
||||
// Fast polling (3s) while processing, slow (15s) otherwise.
|
||||
const { data: workerStatus } = useQuery<WorkerStatus>({
|
||||
queryKey: ['worker-status-progress'],
|
||||
queryFn: () => library.maintenance.workerStatus(),
|
||||
refetchInterval: (query) => {
|
||||
const q = totalQueued(query.state.data)
|
||||
return q > 0 ? 3000 : 15000
|
||||
},
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
const visionActive = visionQueued(workerStatus)
|
||||
const totalActive = totalQueued(workerStatus)
|
||||
|
||||
const phase: Phase = isScanning
|
||||
? 'scanning'
|
||||
: totalActive > 0
|
||||
? 'processing'
|
||||
: 'idle'
|
||||
|
||||
useEffect(() => {
|
||||
if (phase === 'scanning') {
|
||||
setIsVisible(true)
|
||||
setIsMinimized(false)
|
||||
wasScanningRef.current = true
|
||||
wasProcessingRef.current = false
|
||||
} else if (phase === 'processing') {
|
||||
// Show widget when processing starts (even without a prior scan,
|
||||
// e.g. backfill triggered from Settings).
|
||||
if (!isVisible) setIsVisible(true)
|
||||
wasProcessingRef.current = true
|
||||
|
||||
if (wasScanningRef.current) {
|
||||
// Scan just finished — invalidate data caches.
|
||||
wasScanningRef.current = false
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders', 'tree'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['heaps'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['tags'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['library', 'stats'] })
|
||||
}
|
||||
} else if (phase === 'idle') {
|
||||
if (wasScanningRef.current) {
|
||||
// Scan finished with no queued processing (small import).
|
||||
wasScanningRef.current = false
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders', 'tree'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['heaps'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['tags'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['library', 'stats'] })
|
||||
}
|
||||
if (wasProcessingRef.current) {
|
||||
// Processing just drained — refresh tags (new clusters/objects).
|
||||
wasProcessingRef.current = false
|
||||
queryClient.invalidateQueries({ queryKey: ['tags'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['library', 'stats'] })
|
||||
}
|
||||
if (isVisible) {
|
||||
setTimeout(() => setIsVisible(false), 3000)
|
||||
}
|
||||
}
|
||||
}, [phase, isVisible, queryClient])
|
||||
|
||||
if (!isVisible) return null
|
||||
|
||||
// Scan progress percentage
|
||||
const scanProgress = scanStatus && scanStatus.total_files > 0
|
||||
? (scanStatus.processed_files / scanStatus.total_files) * 100
|
||||
: 0
|
||||
|
||||
const isComplete = phase === 'idle' && (scanStatus?.processed_files ?? 0) > 0
|
||||
const hasErrors = scanStatus?.errors && scanStatus.errors.length > 0
|
||||
|
||||
return (
|
||||
<div
|
||||
className={clsx(
|
||||
'fixed bottom-4 right-4 z-40 overflow-hidden rounded-lg border border-border bg-surface shadow-xl transition-all duration-300',
|
||||
isMinimized ? 'w-12' : 'w-80'
|
||||
)}
|
||||
>
|
||||
{/* Header */}
|
||||
<div
|
||||
className="flex cursor-pointer items-center justify-between bg-surface-2 px-3 py-2"
|
||||
onClick={() => setIsMinimized(!isMinimized)}
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
{phase === 'scanning' ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin text-primary" />
|
||||
) : phase === 'processing' ? (
|
||||
<Sparkles className="h-4 w-4 animate-pulse text-amber-400" />
|
||||
) : isComplete && !hasErrors ? (
|
||||
<Check className="h-4 w-4 text-pick" />
|
||||
) : hasErrors ? (
|
||||
<AlertCircle className="h-4 w-4 text-reject" />
|
||||
) : (
|
||||
<FolderOpen className="h-4 w-4 text-text-muted" />
|
||||
)}
|
||||
{!isMinimized && (
|
||||
<span className="text-sm font-medium text-text">
|
||||
{phase === 'scanning'
|
||||
? 'Scanning Folders'
|
||||
: phase === 'processing'
|
||||
? 'Processing Photos'
|
||||
: isComplete
|
||||
? 'Complete'
|
||||
: 'Status'}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{!isMinimized && (
|
||||
<button
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
setIsVisible(false)
|
||||
}}
|
||||
className="rounded p-0.5 text-text-muted hover:bg-surface-offset hover:text-text"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
{!isMinimized && (
|
||||
<div className="p-3">
|
||||
{/* Scan phase */}
|
||||
{phase === 'scanning' && scanStatus && (
|
||||
<>
|
||||
{scanStatus.current_folder && (
|
||||
<div className="mb-2 text-xs text-text-muted">
|
||||
<span className="font-mono">{scanStatus.current_folder}</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="mb-2">
|
||||
<div className="h-1.5 overflow-hidden rounded-full bg-surface-offset">
|
||||
<div
|
||||
className="h-full bg-primary transition-all duration-300"
|
||||
style={{ width: `${scanProgress}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center justify-between text-xs">
|
||||
<span className="text-text-muted">
|
||||
{scanStatus.processed_files} / {scanStatus.total_files || '?'} files
|
||||
</span>
|
||||
<span className="font-medium text-primary">
|
||||
{Math.round(scanProgress)}%
|
||||
</span>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Processing phase */}
|
||||
{phase === 'processing' && workerStatus && (
|
||||
<>
|
||||
<div className="mb-2 flex items-center gap-2 text-xs text-text-muted">
|
||||
<Brain className="h-3.5 w-3.5 text-amber-400" />
|
||||
<span>Analyzing photos…</span>
|
||||
</div>
|
||||
<div className="space-y-1 text-xs">
|
||||
{visionActive > 0 && (
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-text-muted">Vision pipeline</span>
|
||||
<span className="font-mono text-amber-400">{visionActive} queued</span>
|
||||
</div>
|
||||
)}
|
||||
{(totalActive - visionActive) > 0 && (
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-text-muted">Other tasks</span>
|
||||
<span className="font-mono text-text-muted">{totalActive - visionActive} queued</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-2 text-[11px] text-text-muted/60">
|
||||
Thumbnails, embeddings, faces, tags — runs in background
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Done phase */}
|
||||
{phase === 'idle' && isComplete && (
|
||||
<div className="flex items-center gap-2 text-xs text-pick">
|
||||
<Check className="h-3.5 w-3.5" />
|
||||
<span>All processing complete</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Errors */}
|
||||
{hasErrors && (
|
||||
<div className="mt-2 max-h-20 overflow-y-auto rounded bg-reject/10 p-2">
|
||||
<div className="text-xs text-reject">
|
||||
{scanStatus!.errors.slice(0, 3).map((error, i) => (
|
||||
<div key={i} className="truncate">
|
||||
• {error}
|
||||
</div>
|
||||
))}
|
||||
{scanStatus!.errors.length > 3 && (
|
||||
<div className="mt-1 text-text-muted">
|
||||
+{scanStatus!.errors.length - 3} more errors
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function visionQueued(ws: WorkerStatus | undefined): number {
|
||||
if (!ws) return 0
|
||||
const queued = ws.queues?.['vision'] ?? 0
|
||||
const active = ws.workers
|
||||
?.filter(w => w.queues?.includes('vision'))
|
||||
.reduce((sum, w) => sum + (w.active ?? 0) + (w.reserved ?? 0), 0) ?? 0
|
||||
return queued + active
|
||||
}
|
||||
|
||||
function totalQueued(ws: WorkerStatus | undefined): number {
|
||||
if (!ws) return 0
|
||||
const queued = Object.values(ws.queues ?? {}).reduce((a, b) => a + b, 0)
|
||||
const active = ws.workers?.reduce(
|
||||
(sum, w) => sum + (w.active ?? 0) + (w.reserved ?? 0), 0
|
||||
) ?? 0
|
||||
return queued + active
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { CheckCircle, XCircle, Info, AlertCircle, X } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
|
||||
export interface ToastAction {
|
||||
label: string
|
||||
onClick: () => void
|
||||
}
|
||||
|
||||
export interface Toast {
|
||||
id: string
|
||||
type: 'success' | 'error' | 'info' | 'warning'
|
||||
title: string
|
||||
message?: string
|
||||
duration?: number
|
||||
action?: ToastAction
|
||||
}
|
||||
|
||||
// Global toast state (in production, use Zustand or Context)
|
||||
let toastListeners: ((toasts: Toast[]) => void)[] = []
|
||||
let toastList: Toast[] = []
|
||||
|
||||
export const toast = {
|
||||
success: (title: string, message?: string, action?: ToastAction) =>
|
||||
addToast('success', title, message, 5000, action),
|
||||
error: (title: string, message?: string, action?: ToastAction) =>
|
||||
addToast('error', title, message, 5000, action),
|
||||
info: (title: string, message?: string, action?: ToastAction) =>
|
||||
addToast('info', title, message, 5000, action),
|
||||
warning: (title: string, message?: string, action?: ToastAction) =>
|
||||
addToast('warning', title, message, 5000, action),
|
||||
}
|
||||
|
||||
function addToast(
|
||||
type: Toast['type'],
|
||||
title: string,
|
||||
message?: string,
|
||||
duration = 5000,
|
||||
action?: ToastAction
|
||||
) {
|
||||
const id = Date.now().toString() + Math.random().toString(36).slice(2, 6)
|
||||
const newToast: Toast = { id, type, title, message, duration, action }
|
||||
toastList = [...toastList, newToast]
|
||||
toastListeners.forEach(listener => listener(toastList))
|
||||
|
||||
// Auto-remove after duration. Toasts with an action get a longer window
|
||||
// so the user has time to actually click Undo.
|
||||
const removeAfter = action ? Math.max(duration, 8000) : duration
|
||||
setTimeout(() => {
|
||||
removeToast(id)
|
||||
}, removeAfter)
|
||||
}
|
||||
|
||||
function removeToast(id: string) {
|
||||
toastList = toastList.filter(t => t.id !== id)
|
||||
toastListeners.forEach(listener => listener(toastList))
|
||||
}
|
||||
|
||||
export function ToastContainer() {
|
||||
const [toasts, setToasts] = useState<Toast[]>([])
|
||||
|
||||
useEffect(() => {
|
||||
const listener = (newToasts: Toast[]) => setToasts(newToasts)
|
||||
toastListeners.push(listener)
|
||||
return () => {
|
||||
toastListeners = toastListeners.filter(l => l !== listener)
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Subdued icons — smaller and muted so the toast reads as a
|
||||
// background notification rather than a modal. The colored tint
|
||||
// comes from the border-left accent, not a filled background.
|
||||
const icons = {
|
||||
success: <CheckCircle className="h-3.5 w-3.5 text-pick" />,
|
||||
error: <XCircle className="h-3.5 w-3.5 text-reject" />,
|
||||
info: <Info className="h-3.5 w-3.5 text-primary" />,
|
||||
warning: <AlertCircle className="h-3.5 w-3.5 text-star" />,
|
||||
}
|
||||
|
||||
// Single thin left accent bar per type instead of a full-border +
|
||||
// tinted fill. Keeps the toast visually quiet — the user can still
|
||||
// glance it but it doesn't compete with the rest of the UI.
|
||||
const accents = {
|
||||
success: 'border-l-pick',
|
||||
error: 'border-l-reject',
|
||||
info: 'border-l-primary',
|
||||
warning: 'border-l-star',
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="pointer-events-none fixed bottom-4 left-4 z-50 flex flex-col gap-1.5">
|
||||
{toasts.map((toast) => (
|
||||
<div
|
||||
key={toast.id}
|
||||
className={clsx(
|
||||
'pointer-events-auto flex items-start gap-2 rounded-md border border-border border-l-2 bg-surface/80 px-2.5 py-1.5 text-xs shadow-md backdrop-blur-md transition-all duration-300',
|
||||
'animate-slide-up',
|
||||
accents[toast.type]
|
||||
)}
|
||||
style={{ minWidth: '220px', maxWidth: '320px' }}
|
||||
>
|
||||
<div className="mt-0.5 flex-shrink-0">{icons[toast.type]}</div>
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="truncate font-medium text-text">{toast.title}</div>
|
||||
{toast.message && (
|
||||
<div className="mt-0.5 truncate text-[11px] text-text-muted">
|
||||
{toast.message}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{toast.action && (
|
||||
<button
|
||||
onClick={() => {
|
||||
toast.action!.onClick()
|
||||
removeToast(toast.id)
|
||||
}}
|
||||
className="pointer-events-auto self-center rounded border border-border bg-surface px-1.5 py-0.5 text-[11px] font-medium text-text hover:bg-surface-2"
|
||||
>
|
||||
{toast.action.label}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
onClick={() => removeToast(toast.id)}
|
||||
className="pointer-events-auto rounded p-0.5 text-text-faint hover:bg-surface-offset hover:text-text"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,336 +0,0 @@
|
||||
import { useState, useEffect, useCallback } from 'react'
|
||||
import { Plus, Pencil, UserX, Shield, User as UserIcon } from 'lucide-react'
|
||||
import { admin, type AdminUser } from '../../services/api'
|
||||
|
||||
export function UserManagement() {
|
||||
const [users, setUsers] = useState<AdminUser[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [showCreate, setShowCreate] = useState(false)
|
||||
const [editingUser, setEditingUser] = useState<AdminUser | null>(null)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const fetchUsers = useCallback(async () => {
|
||||
try {
|
||||
const data = await admin.listUsers()
|
||||
setUsers(data.users)
|
||||
} catch {
|
||||
setError('Failed to load users.')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
fetchUsers()
|
||||
}, [fetchUsers])
|
||||
|
||||
if (loading) {
|
||||
return <div className="p-4 text-sm text-text-muted">Loading users…</div>
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<h3 className="text-sm font-semibold text-text">Users</h3>
|
||||
<button
|
||||
onClick={() => setShowCreate(true)}
|
||||
className="flex items-center gap-1 rounded bg-accent px-2 py-1 text-xs text-white hover:bg-accent/80"
|
||||
>
|
||||
<Plus className="h-3 w-3" />
|
||||
Add User
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="rounded bg-red-900/30 px-3 py-2 text-xs text-red-300">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<table className="w-full text-xs">
|
||||
<thead>
|
||||
<tr className="border-b border-border text-left text-text-muted">
|
||||
<th className="pb-1 pr-4">Username</th>
|
||||
<th className="pb-1 pr-4">Role</th>
|
||||
<th className="pb-1 pr-4">Photos</th>
|
||||
<th className="pb-1 pr-4">Status</th>
|
||||
<th className="pb-1">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{users.map((u) => (
|
||||
<tr key={u.id} className="border-b border-border/50">
|
||||
<td className="py-1.5 pr-4">
|
||||
<div className="flex items-center gap-1.5">
|
||||
{u.role === 'admin' ? (
|
||||
<Shield className="h-3 w-3 text-accent" />
|
||||
) : (
|
||||
<UserIcon className="h-3 w-3 text-text-muted" />
|
||||
)}
|
||||
<span className="text-text">{u.username}</span>
|
||||
</div>
|
||||
</td>
|
||||
<td className="py-1.5 pr-4 text-text-muted">{u.role}</td>
|
||||
<td className="py-1.5 pr-4 text-text-muted">
|
||||
{u.photo_count.toLocaleString()}
|
||||
</td>
|
||||
<td className="py-1.5 pr-4">
|
||||
<span
|
||||
className={
|
||||
u.is_active
|
||||
? 'text-green-400'
|
||||
: 'text-red-400'
|
||||
}
|
||||
>
|
||||
{u.is_active ? 'Active' : 'Inactive'}
|
||||
</span>
|
||||
</td>
|
||||
<td className="py-1.5">
|
||||
<div className="flex gap-1">
|
||||
<button
|
||||
onClick={() => setEditingUser(u)}
|
||||
className="rounded p-1 text-text-muted hover:bg-bg hover:text-text"
|
||||
title="Edit user"
|
||||
>
|
||||
<Pencil className="h-3 w-3" />
|
||||
</button>
|
||||
{u.is_active && (
|
||||
<button
|
||||
onClick={async () => {
|
||||
if (!confirm(`Deactivate user "${u.username}"? Their photos will be preserved.`)) return
|
||||
try {
|
||||
await admin.deleteUser(u.id)
|
||||
fetchUsers()
|
||||
} catch (err: any) {
|
||||
setError(err.response?.data?.detail ?? 'Failed to deactivate user.')
|
||||
}
|
||||
}}
|
||||
className="rounded p-1 text-text-muted hover:bg-bg hover:text-red-400"
|
||||
title="Deactivate user"
|
||||
>
|
||||
<UserX className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{showCreate && (
|
||||
<CreateUserModal
|
||||
onClose={() => setShowCreate(false)}
|
||||
onCreated={() => {
|
||||
setShowCreate(false)
|
||||
fetchUsers()
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{editingUser && (
|
||||
<EditUserModal
|
||||
user={editingUser}
|
||||
onClose={() => setEditingUser(null)}
|
||||
onSaved={() => {
|
||||
setEditingUser(null)
|
||||
fetchUsers()
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ── Create User Modal ──────────────────────────────────────────────────
|
||||
|
||||
function CreateUserModal({
|
||||
onClose,
|
||||
onCreated,
|
||||
}: {
|
||||
onClose: () => void
|
||||
onCreated: () => void
|
||||
}) {
|
||||
const [username, setUsername] = useState('')
|
||||
const [password, setPassword] = useState('')
|
||||
const [role, setRole] = useState<'user' | 'admin'>('user')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
|
||||
const handleSubmit = async () => {
|
||||
setError(null)
|
||||
setLoading(true)
|
||||
try {
|
||||
await admin.createUser({ username: username.trim(), password, role })
|
||||
onCreated()
|
||||
} catch (err: any) {
|
||||
setError(err.response?.data?.detail ?? 'Failed to create user.')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<ModalOverlay onClose={onClose} title="Add User">
|
||||
{error && (
|
||||
<div className="rounded bg-red-900/30 px-3 py-2 text-xs text-red-300">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
<div className="space-y-3">
|
||||
<Field label="Username">
|
||||
<input
|
||||
value={username}
|
||||
onChange={(e) => setUsername(e.target.value)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text outline-none focus:border-accent"
|
||||
autoFocus
|
||||
/>
|
||||
</Field>
|
||||
<Field label="Password">
|
||||
<input
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text outline-none focus:border-accent"
|
||||
/>
|
||||
</Field>
|
||||
<Field label="Role">
|
||||
<select
|
||||
value={role}
|
||||
onChange={(e) => setRole(e.target.value as 'user' | 'admin')}
|
||||
className="rounded border border-border bg-bg px-2 py-1 text-xs text-text outline-none focus:border-accent"
|
||||
>
|
||||
<option value="user">User</option>
|
||||
<option value="admin">Admin</option>
|
||||
</select>
|
||||
</Field>
|
||||
</div>
|
||||
<div className="mt-4 flex justify-end gap-2">
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="rounded border border-border px-3 py-1 text-xs text-text hover:bg-bg"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
onClick={handleSubmit}
|
||||
disabled={loading}
|
||||
className="rounded bg-accent px-3 py-1 text-xs text-white hover:bg-accent/80 disabled:opacity-50"
|
||||
>
|
||||
{loading ? 'Creating\u2026' : 'Create'}
|
||||
</button>
|
||||
</div>
|
||||
</ModalOverlay>
|
||||
)
|
||||
}
|
||||
|
||||
// ── Edit User Modal ────────────────────────────────────────────────────
|
||||
|
||||
function EditUserModal({
|
||||
user,
|
||||
onClose,
|
||||
onSaved,
|
||||
}: {
|
||||
user: AdminUser
|
||||
onClose: () => void
|
||||
onSaved: () => void
|
||||
}) {
|
||||
const [role, setRole] = useState(user.role)
|
||||
const [newPassword, setNewPassword] = useState('')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
|
||||
const handleSubmit = async () => {
|
||||
setError(null)
|
||||
setLoading(true)
|
||||
try {
|
||||
const data: { role?: string; new_password?: string } = {}
|
||||
if (role !== user.role) data.role = role
|
||||
if (newPassword) data.new_password = newPassword
|
||||
if (Object.keys(data).length > 0) {
|
||||
await admin.updateUser(user.id, data)
|
||||
}
|
||||
onSaved()
|
||||
} catch (err: any) {
|
||||
setError(err.response?.data?.detail ?? 'Failed to update user.')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<ModalOverlay onClose={onClose} title={`Edit: ${user.username}`}>
|
||||
{error && (
|
||||
<div className="rounded bg-red-900/30 px-3 py-2 text-xs text-red-300">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
<div className="space-y-3">
|
||||
<Field label="Role">
|
||||
<select
|
||||
value={role}
|
||||
onChange={(e) => setRole(e.target.value as 'user' | 'admin')}
|
||||
className="rounded border border-border bg-bg px-2 py-1 text-xs text-text outline-none focus:border-accent"
|
||||
>
|
||||
<option value="user">User</option>
|
||||
<option value="admin">Admin</option>
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="New Password (leave blank to keep current)">
|
||||
<input
|
||||
type="password"
|
||||
value={newPassword}
|
||||
onChange={(e) => setNewPassword(e.target.value)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text outline-none focus:border-accent"
|
||||
placeholder="Unchanged"
|
||||
/>
|
||||
</Field>
|
||||
</div>
|
||||
<div className="mt-4 flex justify-end gap-2">
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="rounded border border-border px-3 py-1 text-xs text-text hover:bg-bg"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
onClick={handleSubmit}
|
||||
disabled={loading}
|
||||
className="rounded bg-accent px-3 py-1 text-xs text-white hover:bg-accent/80 disabled:opacity-50"
|
||||
>
|
||||
{loading ? 'Saving\u2026' : 'Save'}
|
||||
</button>
|
||||
</div>
|
||||
</ModalOverlay>
|
||||
)
|
||||
}
|
||||
|
||||
// ── Shared helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function ModalOverlay({
|
||||
onClose: _onClose,
|
||||
title,
|
||||
children,
|
||||
}: {
|
||||
onClose: () => void
|
||||
title: string
|
||||
children: React.ReactNode
|
||||
}) {
|
||||
return (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/50" onClick={_onClose}>
|
||||
<div className="w-full max-w-sm rounded-lg border border-border bg-surface p-5 shadow-xl" onClick={(e) => e.stopPropagation()}>
|
||||
<h4 className="mb-3 text-sm font-semibold text-text">{title}</h4>
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Field({ label, children }: { label: string; children: React.ReactNode }) {
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<label className="block text-[11px] text-text-muted">{label}</label>
|
||||
{children}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
import { useState, type FormEvent } from 'react'
|
||||
import { useAuth } from '../../contexts/AuthContext'
|
||||
|
||||
export function LoginPage() {
|
||||
const { login } = useAuth()
|
||||
const [username, setUsername] = useState('')
|
||||
const [password, setPassword] = useState('')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
|
||||
const handleSubmit = async (e: FormEvent) => {
|
||||
e.preventDefault()
|
||||
setError(null)
|
||||
setLoading(true)
|
||||
try {
|
||||
await login(username, password)
|
||||
} catch (err: any) {
|
||||
setError(
|
||||
err.response?.data?.detail ?? 'Unable to sign in. Check your credentials.',
|
||||
)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex min-h-screen items-center justify-center bg-bg px-4">
|
||||
<form
|
||||
onSubmit={handleSubmit}
|
||||
className="w-full max-w-sm space-y-5 rounded-lg border border-border bg-surface p-8 shadow-xl"
|
||||
>
|
||||
<h1 className="text-center text-xl font-semibold text-text">
|
||||
Sign in to Mulita
|
||||
</h1>
|
||||
|
||||
{error && (
|
||||
<div className="rounded bg-red-900/30 px-3 py-2 text-sm text-red-300">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-1">
|
||||
<label htmlFor="login-user" className="block text-sm text-text-muted">
|
||||
Username
|
||||
</label>
|
||||
<input
|
||||
id="login-user"
|
||||
type="text"
|
||||
value={username}
|
||||
onChange={(e) => setUsername(e.target.value)}
|
||||
required
|
||||
autoFocus
|
||||
className="w-full rounded border border-border bg-bg px-3 py-2 text-sm text-text outline-none focus:border-accent"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<label htmlFor="login-pass" className="block text-sm text-text-muted">
|
||||
Password
|
||||
</label>
|
||||
<input
|
||||
id="login-pass"
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
required
|
||||
className="w-full rounded border border-border bg-bg px-3 py-2 text-sm text-text outline-none focus:border-accent"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="submit"
|
||||
disabled={loading}
|
||||
className="w-full rounded bg-accent px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-accent/80 disabled:opacity-50"
|
||||
>
|
||||
{loading ? 'Signing in\u2026' : 'Sign In'}
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
import { useState, type FormEvent } from 'react'
|
||||
import { useAuth } from '../../contexts/AuthContext'
|
||||
import api from '../../services/api'
|
||||
|
||||
export function SetupPage() {
|
||||
const { onSetupComplete } = useAuth()
|
||||
const [username, setUsername] = useState('')
|
||||
const [password, setPassword] = useState('')
|
||||
const [confirmPassword, setConfirmPassword] = useState('')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
|
||||
const handleSubmit = async (e: FormEvent) => {
|
||||
e.preventDefault()
|
||||
setError(null)
|
||||
|
||||
if (password !== confirmPassword) {
|
||||
setError('Passwords do not match.')
|
||||
return
|
||||
}
|
||||
if (password.length < 6) {
|
||||
setError('Password must be at least 6 characters.')
|
||||
return
|
||||
}
|
||||
if (username.trim().length < 2) {
|
||||
setError('Username must be at least 2 characters.')
|
||||
return
|
||||
}
|
||||
|
||||
setLoading(true)
|
||||
try {
|
||||
const res = await api.post('/auth/setup', {
|
||||
username: username.trim(),
|
||||
password,
|
||||
})
|
||||
const { access_token, refresh_token } = res.data
|
||||
await onSetupComplete(access_token, refresh_token)
|
||||
} catch (err: any) {
|
||||
setError(
|
||||
err.response?.data?.detail ?? 'Setup failed. Please try again.',
|
||||
)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex min-h-screen items-center justify-center bg-bg px-4">
|
||||
<form
|
||||
onSubmit={handleSubmit}
|
||||
className="w-full max-w-sm space-y-5 rounded-lg border border-border bg-surface p-8 shadow-xl"
|
||||
>
|
||||
<div className="space-y-1 text-center">
|
||||
<h1 className="text-xl font-semibold text-text">Welcome to Mulita</h1>
|
||||
<p className="text-sm text-text-muted">
|
||||
Create your admin account to get started.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="rounded bg-red-900/30 px-3 py-2 text-sm text-red-300">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-1">
|
||||
<label htmlFor="setup-user" className="block text-sm text-text-muted">
|
||||
Username
|
||||
</label>
|
||||
<input
|
||||
id="setup-user"
|
||||
type="text"
|
||||
value={username}
|
||||
onChange={(e) => setUsername(e.target.value)}
|
||||
required
|
||||
autoFocus
|
||||
className="w-full rounded border border-border bg-bg px-3 py-2 text-sm text-text outline-none focus:border-accent"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<label htmlFor="setup-pass" className="block text-sm text-text-muted">
|
||||
Password
|
||||
</label>
|
||||
<input
|
||||
id="setup-pass"
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
required
|
||||
className="w-full rounded border border-border bg-bg px-3 py-2 text-sm text-text outline-none focus:border-accent"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<label htmlFor="setup-confirm" className="block text-sm text-text-muted">
|
||||
Confirm Password
|
||||
</label>
|
||||
<input
|
||||
id="setup-confirm"
|
||||
type="password"
|
||||
value={confirmPassword}
|
||||
onChange={(e) => setConfirmPassword(e.target.value)}
|
||||
required
|
||||
className="w-full rounded border border-border bg-bg px-3 py-2 text-sm text-text outline-none focus:border-accent"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="submit"
|
||||
disabled={loading}
|
||||
className="w-full rounded bg-accent px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-accent/80 disabled:opacity-50"
|
||||
>
|
||||
{loading ? 'Creating account\u2026' : 'Create Admin Account'}
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
import { useState, useMemo, useCallback } from 'react'
|
||||
import { Palette, ArrowLeft, Loader2 } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { photos as photosApi } from '../../services/api'
|
||||
import { useFilterStore } from '../../store/filterStore'
|
||||
import { usePhotosQuery } from '../../hooks/usePhotosQuery'
|
||||
import { COLOR_LABEL_OPTIONS, type ColorLabel } from '../../constants/colorLabels'
|
||||
import { useCardGridNav } from '../../hooks/useCardGridNav'
|
||||
import { Timeline } from '../timeline/Timeline'
|
||||
import type { Photo } from '../../types/photo'
|
||||
|
||||
interface ColorGroup {
|
||||
label: string
|
||||
value: ColorLabel | null
|
||||
className: string
|
||||
count: number
|
||||
representative: Photo | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Colors view — two states:
|
||||
* 1. Grid of color label cards (default) — arrow keys + Enter to browse
|
||||
* 2. Detail view showing a color's photos in the full Timeline — Esc to go back
|
||||
*/
|
||||
export function ColorsView() {
|
||||
const { data: allPhotos = [], isLoading } = usePhotosQuery()
|
||||
const setColorLabel = useFilterStore((s) => s.setColorLabel)
|
||||
const [selectedGroup, setSelectedGroup] = useState<ColorGroup | null>(null)
|
||||
|
||||
const groups = useMemo(() => {
|
||||
const buckets = new Map<string, Photo[]>()
|
||||
const uncolored: Photo[] = []
|
||||
|
||||
for (const photo of allPhotos) {
|
||||
if (photo.color_label) {
|
||||
const arr = buckets.get(photo.color_label) ?? []
|
||||
arr.push(photo)
|
||||
buckets.set(photo.color_label, arr)
|
||||
} else {
|
||||
uncolored.push(photo)
|
||||
}
|
||||
}
|
||||
|
||||
const result: ColorGroup[] = []
|
||||
for (const { value, className } of COLOR_LABEL_OPTIONS) {
|
||||
const photos = buckets.get(value) ?? []
|
||||
if (photos.length === 0) continue
|
||||
result.push({
|
||||
label: value.charAt(0).toUpperCase() + value.slice(1),
|
||||
value,
|
||||
className,
|
||||
count: photos.length,
|
||||
representative: photos[0],
|
||||
})
|
||||
}
|
||||
if (uncolored.length > 0) {
|
||||
result.push({
|
||||
label: 'Uncolored',
|
||||
value: null,
|
||||
className: 'bg-neutral-400',
|
||||
count: uncolored.length,
|
||||
representative: uncolored[0],
|
||||
})
|
||||
}
|
||||
result.sort((a, b) => b.count - a.count)
|
||||
return result
|
||||
}, [allPhotos])
|
||||
|
||||
const enterDetail = useCallback(
|
||||
(group: ColorGroup) => {
|
||||
setColorLabel((group.value ?? 'none') as ColorLabel)
|
||||
setSelectedGroup(group)
|
||||
},
|
||||
[setColorLabel]
|
||||
)
|
||||
|
||||
const exitDetail = useCallback(() => {
|
||||
setColorLabel(null)
|
||||
setSelectedGroup(null)
|
||||
}, [setColorLabel])
|
||||
|
||||
const { activeIndex, gridRef } = useCardGridNav({
|
||||
items: groups,
|
||||
inDetail: selectedGroup !== null,
|
||||
onEnter: enterDetail,
|
||||
onExit: exitDetail,
|
||||
})
|
||||
|
||||
if (selectedGroup) {
|
||||
return (
|
||||
<div className="flex h-full flex-col overflow-hidden">
|
||||
<div className="flex items-center gap-3 border-b border-border px-4 py-3">
|
||||
<button
|
||||
onClick={exitDetail}
|
||||
className="rounded p-1 text-text-muted transition-colors hover:bg-surface-2 hover:text-text"
|
||||
title="Back to colors"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</button>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className={`inline-block h-3 w-3 rounded-full ${selectedGroup.className}`} />
|
||||
<h2 className="text-sm font-semibold text-text">{selectedGroup.label}</h2>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex-1 overflow-hidden">
|
||||
<Timeline />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center text-text-muted">
|
||||
<Loader2 className="mr-2 h-5 w-5 animate-spin" />
|
||||
Loading colors...
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (groups.length === 0) {
|
||||
return (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-3 text-text-muted">
|
||||
<Palette className="h-12 w-12 opacity-40" />
|
||||
<p className="text-sm">No color labels assigned yet</p>
|
||||
<p className="max-w-xs text-center text-xs opacity-70">
|
||||
Color labels will appear here once you assign them to photos.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto p-4 pb-20">
|
||||
<div className="mb-4 flex items-center gap-2 text-text-muted">
|
||||
<Palette className="h-4 w-4" />
|
||||
<span className="text-sm font-medium">
|
||||
{groups.length} {groups.length === 1 ? 'color' : 'colors'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
ref={gridRef}
|
||||
className="grid grid-cols-[repeat(auto-fill,minmax(140px,1fr))] gap-3"
|
||||
>
|
||||
{groups.map((group, i) => (
|
||||
<div
|
||||
key={group.label}
|
||||
className={clsx(
|
||||
'group cursor-pointer overflow-hidden rounded-lg border bg-surface transition-all hover:border-primary/50 hover:shadow-md',
|
||||
i === activeIndex
|
||||
? 'border-primary ring-1 ring-primary/30'
|
||||
: 'border-border'
|
||||
)}
|
||||
onClick={() => enterDetail(group)}
|
||||
>
|
||||
<div className="relative aspect-square overflow-hidden bg-surface-2">
|
||||
{group.representative ? (
|
||||
<img
|
||||
src={photosApi.getThumbnailUrl(group.representative.id, 'small')}
|
||||
alt={group.label}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center">
|
||||
<Palette className="h-10 w-10 text-text-muted/30" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span className="absolute bottom-1.5 right-1.5 rounded-full bg-black/60 px-2 py-0.5 text-[11px] font-medium text-white backdrop-blur-sm">
|
||||
{group.count}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-1.5 px-2 py-1.5">
|
||||
<span className={`inline-block h-2.5 w-2.5 rounded-full ${group.className}`} />
|
||||
<p className="truncate text-xs font-medium text-text">{group.label}</p>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import { useEffect } from 'react'
|
||||
import clsx from 'clsx'
|
||||
|
||||
interface ConfirmDialogProps {
|
||||
isOpen: boolean
|
||||
title: string
|
||||
message: React.ReactNode
|
||||
confirmLabel?: string
|
||||
cancelLabel?: string
|
||||
/** When true, the confirm button uses the destructive accent. */
|
||||
destructive?: boolean
|
||||
onConfirm: () => void
|
||||
onClose: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Tiny modal-confirmation dialog. Mirrors the AddSourceFolderDialog overlay
|
||||
* pattern (custom fixed inset-0 backdrop, no shadcn Dialog dep). Esc closes.
|
||||
*/
|
||||
export function ConfirmDialog({
|
||||
isOpen,
|
||||
title,
|
||||
message,
|
||||
confirmLabel = 'Confirm',
|
||||
cancelLabel = 'Cancel',
|
||||
destructive = false,
|
||||
onConfirm,
|
||||
onClose,
|
||||
}: ConfirmDialogProps) {
|
||||
// Esc to close.
|
||||
useEffect(() => {
|
||||
if (!isOpen) return
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') onClose()
|
||||
}
|
||||
window.addEventListener('keydown', handler)
|
||||
return () => window.removeEventListener('keydown', handler)
|
||||
}, [isOpen, onClose])
|
||||
|
||||
if (!isOpen) return null
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50">
|
||||
<div
|
||||
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
|
||||
onClick={onClose}
|
||||
/>
|
||||
<div className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2">
|
||||
<div className="relative z-10 w-96 rounded-lg border border-border bg-surface p-5 shadow-2xl">
|
||||
<h2 className="mb-2 text-base font-semibold text-text">{title}</h2>
|
||||
<div className="mb-4 text-sm text-text-muted">{message}</div>
|
||||
<div className="flex justify-end gap-2">
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="rounded border border-border px-3 py-1.5 text-sm text-text hover:bg-surface-2"
|
||||
>
|
||||
{cancelLabel}
|
||||
</button>
|
||||
<button
|
||||
onClick={onConfirm}
|
||||
className={clsx(
|
||||
'rounded px-3 py-1.5 text-sm font-medium text-white',
|
||||
destructive
|
||||
? 'bg-reject hover:bg-reject/80'
|
||||
: 'bg-primary hover:bg-primary/80'
|
||||
)}
|
||||
>
|
||||
{confirmLabel}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import clsx from 'clsx'
|
||||
import { Trash2, Archive } from 'lucide-react'
|
||||
|
||||
interface DeleteFolderDialogProps {
|
||||
isOpen: boolean
|
||||
folderName: string
|
||||
/** Number of photos under this folder, including descendants. Surfaced
|
||||
* in the dialog copy so the user understands the blast radius. */
|
||||
photoCount?: number
|
||||
onClose: () => void
|
||||
/** Called with the chosen mode when the user confirms. */
|
||||
onConfirm: (mode: 'discard' | 'permanent') => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-mode folder delete dialog:
|
||||
*
|
||||
* - Move to discard pile (default, soft, recoverable)
|
||||
* - Permanently delete (destructive, irreversible)
|
||||
*
|
||||
* The user picks a mode via the radio cards then clicks Delete. Esc /
|
||||
* backdrop click cancels.
|
||||
*/
|
||||
export function DeleteFolderDialog({
|
||||
isOpen,
|
||||
folderName,
|
||||
photoCount,
|
||||
onClose,
|
||||
onConfirm,
|
||||
}: DeleteFolderDialogProps) {
|
||||
const [mode, setMode] = useState<'discard' | 'permanent'>('discard')
|
||||
|
||||
// Reset mode when re-opening so the safe option is always the default.
|
||||
useEffect(() => {
|
||||
if (isOpen) setMode('discard')
|
||||
}, [isOpen])
|
||||
|
||||
// Esc to close.
|
||||
useEffect(() => {
|
||||
if (!isOpen) return
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') onClose()
|
||||
}
|
||||
window.addEventListener('keydown', handler)
|
||||
return () => window.removeEventListener('keydown', handler)
|
||||
}, [isOpen, onClose])
|
||||
|
||||
if (!isOpen) return null
|
||||
|
||||
const photoBlurb =
|
||||
photoCount === undefined
|
||||
? 'photos in this folder'
|
||||
: photoCount === 0
|
||||
? 'this empty folder'
|
||||
: `${photoCount} photo${photoCount === 1 ? '' : 's'} in this folder`
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50">
|
||||
<div
|
||||
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
|
||||
onClick={onClose}
|
||||
/>
|
||||
<div className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2">
|
||||
<div className="relative z-10 w-[420px] rounded-lg border border-border bg-surface p-5 shadow-2xl">
|
||||
<h2 className="mb-1 text-base font-semibold text-text">
|
||||
Delete folder "{folderName}"?
|
||||
</h2>
|
||||
<p className="mb-4 text-sm text-text-muted">
|
||||
What should happen to {photoBlurb}?
|
||||
</p>
|
||||
|
||||
<div className="space-y-2">
|
||||
<ModeCard
|
||||
icon={<Archive className="h-4 w-4" />}
|
||||
title="Move photos to discard pile"
|
||||
description="Photos can be restored later from Discarded. The folder and files stay on disk."
|
||||
selected={mode === 'discard'}
|
||||
onClick={() => setMode('discard')}
|
||||
/>
|
||||
<ModeCard
|
||||
icon={<Trash2 className="h-4 w-4" />}
|
||||
title="Permanently delete folder and photos"
|
||||
description="Removes the folder, every photo inside it, and the directory from disk. This cannot be undone."
|
||||
selected={mode === 'permanent'}
|
||||
destructive
|
||||
onClick={() => setMode('permanent')}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="mt-5 flex justify-end gap-2">
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="rounded border border-border px-3 py-1.5 text-sm text-text hover:bg-surface-2"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
onClick={() => onConfirm(mode)}
|
||||
className={clsx(
|
||||
'rounded px-3 py-1.5 text-sm font-medium text-white',
|
||||
mode === 'permanent'
|
||||
? 'bg-reject hover:bg-reject/80'
|
||||
: 'bg-primary hover:bg-primary/80'
|
||||
)}
|
||||
>
|
||||
{mode === 'permanent' ? 'Delete forever' : 'Move to discard pile'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ModeCard({
|
||||
icon,
|
||||
title,
|
||||
description,
|
||||
selected,
|
||||
destructive = false,
|
||||
onClick,
|
||||
}: {
|
||||
icon: React.ReactNode
|
||||
title: string
|
||||
description: string
|
||||
selected: boolean
|
||||
destructive?: boolean
|
||||
onClick: () => void
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
onClick={onClick}
|
||||
className={clsx(
|
||||
'flex w-full gap-3 rounded-lg border p-3 text-left transition-colors',
|
||||
selected
|
||||
? destructive
|
||||
? 'border-reject/60 bg-reject/10'
|
||||
: 'border-primary/60 bg-primary/10'
|
||||
: 'border-border bg-surface-2 hover:bg-surface-offset'
|
||||
)}
|
||||
>
|
||||
<div
|
||||
className={clsx(
|
||||
'mt-0.5 flex-shrink-0',
|
||||
selected ? (destructive ? 'text-reject' : 'text-primary') : 'text-text-muted'
|
||||
)}
|
||||
>
|
||||
{icon}
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<div
|
||||
className={clsx(
|
||||
'text-sm font-medium',
|
||||
selected ? (destructive ? 'text-reject' : 'text-primary') : 'text-text'
|
||||
)}
|
||||
>
|
||||
{title}
|
||||
</div>
|
||||
<div className="mt-0.5 text-xs text-text-muted">{description}</div>
|
||||
</div>
|
||||
</button>
|
||||
)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,188 +0,0 @@
|
||||
import { useState } from 'react'
|
||||
import { RotateCcw, Trash2 } from 'lucide-react'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { usePhotoStore } from '../../store/photoStore'
|
||||
import { useFilterStore } from '../../store/filterStore'
|
||||
import { usePhotosQuery, stripPhotosFromCache } from '../../hooks/usePhotosQuery'
|
||||
import { discard as discardApi, photos as photosApi } from '../../services/api'
|
||||
import { toast } from '../ToastContainer'
|
||||
import { ConfirmDialog } from '../dialogs/ConfirmDialog'
|
||||
import { registerUndoable } from '../../store/undoStore'
|
||||
import { LIBRARY_STATS_QUERY_KEY } from '../../hooks/useLibraryStatsQuery'
|
||||
|
||||
/**
|
||||
* Top-of-timeline bar visible only when the discarded filter is active.
|
||||
* Shows a count, lets the user restore the current selection, and lets them
|
||||
* permanently empty the discard pile (with confirmation).
|
||||
*/
|
||||
export function DiscardActionBar() {
|
||||
const flag = useFilterStore((s) => s.flag)
|
||||
const selectedPhotos = usePhotoStore((s) => s.selectedPhotos)
|
||||
const clearSelection = usePhotoStore((s) => s.clearSelection)
|
||||
const queryClient = useQueryClient()
|
||||
const { data: photos = [] } = usePhotosQuery()
|
||||
|
||||
const [confirmOpen, setConfirmOpen] = useState(false)
|
||||
const [deleteSelectedOpen, setDeleteSelectedOpen] = useState(false)
|
||||
|
||||
const restoreMutation = useMutation({
|
||||
mutationFn: (ids: string[]) => discardApi.restore(ids),
|
||||
// Pull the restored ids out of the discard view immediately. The
|
||||
// user is sitting on flag=discarded so they should disappear from
|
||||
// sight the moment the click lands; the onSuccess invalidate still
|
||||
// reconciles with server truth shortly after.
|
||||
onMutate: (ids) => {
|
||||
usePhotoStore.getState().removePhotosFromTimeline(ids)
|
||||
stripPhotosFromCache(queryClient, ids)
|
||||
},
|
||||
onSuccess: (_, ids) => {
|
||||
registerUndoable(
|
||||
`Restored ${ids.length} photo${ids.length === 1 ? '' : 's'}`,
|
||||
async () => {
|
||||
await photosApi.bulkDiscard(ids)
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||||
}
|
||||
)
|
||||
clearSelection()
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||||
},
|
||||
onError: (e: any) => toast.error('Restore failed', e.message || 'Unknown error'),
|
||||
})
|
||||
|
||||
const deleteSelectedMutation = useMutation({
|
||||
mutationFn: (ids: string[]) => discardApi.deletePermanent(ids),
|
||||
onMutate: (ids) => {
|
||||
usePhotoStore.getState().removePhotosFromTimeline(ids)
|
||||
stripPhotosFromCache(queryClient, ids)
|
||||
},
|
||||
onSuccess: (data: any) => {
|
||||
const count = data?.deleted ?? 0
|
||||
const errors = data?.file_errors ?? 0
|
||||
if (errors > 0) {
|
||||
toast.error(
|
||||
`Deleted with ${errors} error${errors > 1 ? 's' : ''}`,
|
||||
`${count} record${count === 1 ? '' : 's'} deleted; some files could not be removed`
|
||||
)
|
||||
} else {
|
||||
toast.success(
|
||||
'Permanently deleted',
|
||||
`${count} photo${count === 1 ? '' : 's'} removed from disk`
|
||||
)
|
||||
}
|
||||
clearSelection()
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||||
setDeleteSelectedOpen(false)
|
||||
},
|
||||
onError: (e: any) =>
|
||||
toast.error('Delete failed', e.message || 'Unknown error'),
|
||||
})
|
||||
|
||||
const emptyMutation = useMutation({
|
||||
mutationFn: () => discardApi.empty(),
|
||||
onSuccess: (data: any) => {
|
||||
const count = data?.deleted ?? 0
|
||||
const errors = data?.file_errors ?? 0
|
||||
if (errors > 0) {
|
||||
toast.error(
|
||||
`Emptied with ${errors} error${errors > 1 ? 's' : ''}`,
|
||||
`${count} record${count > 1 ? 's' : ''} deleted; some files could not be removed`
|
||||
)
|
||||
} else {
|
||||
toast.success('Discard pile emptied', `${count} photo${count > 1 ? 's' : ''} permanently deleted`)
|
||||
}
|
||||
clearSelection()
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||||
setConfirmOpen(false)
|
||||
},
|
||||
onError: (e: any) => toast.error('Empty failed', e.message || 'Unknown error'),
|
||||
})
|
||||
|
||||
if (flag !== 'discarded') return null
|
||||
|
||||
const total = photos.length
|
||||
const selected = selectedPhotos.length
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex items-center justify-between gap-3 border-b border-border bg-reject/10 px-4 py-2 text-sm">
|
||||
<div className="flex items-center gap-2 text-text">
|
||||
<Trash2 className="h-4 w-4 text-reject" />
|
||||
<span className="font-medium">Discarded</span>
|
||||
<span className="text-text-muted">
|
||||
{total} photo{total === 1 ? '' : 's'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
{selected > 0 && (
|
||||
<>
|
||||
<button
|
||||
onClick={() => restoreMutation.mutate(selectedPhotos)}
|
||||
disabled={restoreMutation.isPending}
|
||||
className="flex items-center gap-1.5 rounded bg-surface-2 px-3 py-1 text-text hover:bg-surface-offset disabled:opacity-50"
|
||||
title="Restore selected (U)"
|
||||
>
|
||||
<RotateCcw className="h-3.5 w-3.5" />
|
||||
Restore {selected}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setDeleteSelectedOpen(true)}
|
||||
disabled={deleteSelectedMutation.isPending}
|
||||
className="flex items-center gap-1.5 rounded bg-reject/20 px-3 py-1 text-reject hover:bg-reject/30 disabled:opacity-50"
|
||||
title="Permanently delete selected"
|
||||
>
|
||||
<Trash2 className="h-3.5 w-3.5" />
|
||||
Delete {selected}
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
<button
|
||||
onClick={() => setConfirmOpen(true)}
|
||||
disabled={total === 0 || emptyMutation.isPending}
|
||||
className="flex items-center gap-1.5 rounded bg-reject/20 px-3 py-1 text-reject hover:bg-reject/30 disabled:opacity-50"
|
||||
title="Permanently delete all discarded photos and files"
|
||||
>
|
||||
<Trash2 className="h-3.5 w-3.5" />
|
||||
Empty discard pile
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ConfirmDialog
|
||||
isOpen={deleteSelectedOpen}
|
||||
title={`Delete ${selected} photo${selected === 1 ? '' : 's'}?`}
|
||||
message={
|
||||
<>
|
||||
This will <strong className="text-text">permanently delete</strong>{' '}
|
||||
{selected} photo{selected === 1 ? '' : 's'} and remove the file
|
||||
{selected === 1 ? '' : 's'} from disk. This cannot be undone.
|
||||
</>
|
||||
}
|
||||
confirmLabel="Delete"
|
||||
destructive
|
||||
onConfirm={() => deleteSelectedMutation.mutate(selectedPhotos)}
|
||||
onClose={() => setDeleteSelectedOpen(false)}
|
||||
/>
|
||||
|
||||
<ConfirmDialog
|
||||
isOpen={confirmOpen}
|
||||
title="Empty discard pile?"
|
||||
message={
|
||||
<>
|
||||
This will <strong className="text-text">permanently delete</strong>{' '}
|
||||
{total} photo{total === 1 ? '' : 's'} and remove the file
|
||||
{total === 1 ? '' : 's'} from disk. This cannot be undone.
|
||||
</>
|
||||
}
|
||||
confirmLabel="Empty pile"
|
||||
destructive
|
||||
onConfirm={() => emptyMutation.mutate()}
|
||||
onClose={() => setConfirmOpen(false)}
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
@@ -1,433 +0,0 @@
|
||||
import { useMemo, useState, useEffect, useCallback } from 'react'
|
||||
import { Copy, Layers, Sparkles, Trash2, Loader2, Info, Crown } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import {
|
||||
useDuplicateGroupsQuery,
|
||||
DUPLICATE_GROUPS_QUERY_KEY,
|
||||
} from '../../hooks/useDuplicateGroupsQuery'
|
||||
import {
|
||||
photos as photosApi,
|
||||
type DuplicateGroup,
|
||||
type DuplicateGroupMember,
|
||||
} from '../../services/api'
|
||||
import {
|
||||
PhotoThumbnail,
|
||||
THUMB_BADGE_BASE,
|
||||
THUMB_BADGE_ICON,
|
||||
THUMB_BADGE_NEUTRAL,
|
||||
THUMB_BADGE_PICK,
|
||||
} from '../timeline/PhotoThumbnail'
|
||||
import { usePhotoStore } from '../../store/photoStore'
|
||||
import { registerUndoable } from '../../store/undoStore'
|
||||
import { LIBRARY_STATS_QUERY_KEY } from '../../hooks/useLibraryStatsQuery'
|
||||
import { toast } from '../ToastContainer'
|
||||
import type { Photo } from '../../types/photo'
|
||||
|
||||
/**
|
||||
* Sectioned grid view of duplicate clusters. Replaces the old flat
|
||||
* "is_duplicate=true" timeline. Each section is one cluster the
|
||||
* regroup_duplicates task identified — header on top with a count and a
|
||||
* "keep best, discard rest" button, members rendered as PhotoThumbnail
|
||||
* cards below.
|
||||
*
|
||||
* Mounted from App.tsx in place of <Timeline /> when the user is in the
|
||||
* duplicates section. Touches no filter store state.
|
||||
*/
|
||||
export function DuplicatesView() {
|
||||
const { data, isLoading, isError, error } = useDuplicateGroupsQuery()
|
||||
const queryClient = useQueryClient()
|
||||
const openPreview = usePhotoStore((s) => s.openPreview)
|
||||
const selectPhoto = usePhotoStore((s) => s.selectPhoto)
|
||||
const selectedPhotos = usePhotoStore((s) => s.selectedPhotos)
|
||||
const activePhotoId = usePhotoStore((s) => s.activePhotoId)
|
||||
|
||||
// Bulk discard with the same undoable wrapper the timeline uses, so
|
||||
// Cmd+Z restores the discarded copies. invalidate ['library', 'duplicates']
|
||||
// so the group disappears from the view immediately.
|
||||
const discardMutation = useMutation({
|
||||
mutationFn: (ids: string[]) => photosApi.bulkDiscard(ids),
|
||||
onSuccess: (_, ids) => {
|
||||
registerUndoable(
|
||||
`Discarded ${ids.length} duplicate${ids.length === 1 ? '' : 's'}`,
|
||||
async () => {
|
||||
await photosApi.bulkRestore(ids)
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: DUPLICATE_GROUPS_QUERY_KEY })
|
||||
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||||
}
|
||||
)
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: DUPLICATE_GROUPS_QUERY_KEY })
|
||||
queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
|
||||
},
|
||||
onError: (e: any) =>
|
||||
toast.error('Discard failed', e?.message || 'Unknown error'),
|
||||
})
|
||||
|
||||
// Hooks below this point must run on every render — rules of hooks
|
||||
// forbid early returns above any useState/useEffect/useMemo. The early
|
||||
// loading/error/empty branches sit AFTER the hook block.
|
||||
const groups = data?.groups ?? []
|
||||
|
||||
// Flat sequence of member ids in visual order. Drives both preview
|
||||
// navigation and the in-grid keyboard walker. useMemo so the keyboard
|
||||
// effect doesn't tear down on every render.
|
||||
const allMemberIds = useMemo(
|
||||
() => groups.flatMap((g) => g.members.map((m) => m.id)),
|
||||
[groups]
|
||||
)
|
||||
|
||||
// Track the rendered column count of the duplicates grid so ↑/↓ can
|
||||
// skip a row instead of jumping a single cell. The grid uses
|
||||
// `repeat(auto-fill, minmax(180px, 1fr))` so columns = floor(width/180).
|
||||
// We measure the FIRST section's grid container — every section uses
|
||||
// the same auto-fill rule so any one is representative.
|
||||
const [columns, setColumns] = useState(4)
|
||||
const sampleGridRef = useCallback((el: HTMLDivElement | null) => {
|
||||
if (!el) return
|
||||
const measure = () => {
|
||||
const cols = Math.max(1, Math.floor(el.clientWidth / 180))
|
||||
setColumns(cols)
|
||||
}
|
||||
measure()
|
||||
const ro = new ResizeObserver(measure)
|
||||
ro.observe(el)
|
||||
// Caller doesn't get the cleanup hook but ResizeObserver disconnects
|
||||
// when the element unmounts, which is fine for our lifecycle.
|
||||
}, [])
|
||||
|
||||
// Window-level keyboard nav. Mirrors Timeline's handler but walks
|
||||
// `allMemberIds` directly — duplicate groups don't have a uniform row
|
||||
// grid so we approximate ↑/↓ via the measured `columns` count and
|
||||
// wrap ←/→ across group boundaries.
|
||||
useEffect(() => {
|
||||
if (allMemberIds.length === 0) return
|
||||
|
||||
const onKeyDown = (e: KeyboardEvent) => {
|
||||
const target = e.target as HTMLElement | null
|
||||
if (target && (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA')) {
|
||||
return
|
||||
}
|
||||
const key = e.key
|
||||
if (
|
||||
key !== 'ArrowLeft' &&
|
||||
key !== 'ArrowRight' &&
|
||||
key !== 'ArrowUp' &&
|
||||
key !== 'ArrowDown'
|
||||
) {
|
||||
return
|
||||
}
|
||||
e.preventDefault()
|
||||
const currentIdx = activePhotoId ? allMemberIds.indexOf(activePhotoId) : -1
|
||||
const startIdx = currentIdx >= 0 ? currentIdx : 0
|
||||
let nextIdx = startIdx
|
||||
if (key === 'ArrowLeft') nextIdx = startIdx - 1
|
||||
else if (key === 'ArrowRight') nextIdx = startIdx + 1
|
||||
else if (key === 'ArrowUp') nextIdx = startIdx - columns
|
||||
else if (key === 'ArrowDown') nextIdx = startIdx + columns
|
||||
// Clamp to bounds — we don't wrap on out-of-range vertical moves
|
||||
// since the grid is partitioned into groups and a "wrap" would
|
||||
// skip across visually unrelated content.
|
||||
nextIdx = Math.max(0, Math.min(allMemberIds.length - 1, nextIdx))
|
||||
const nextId = allMemberIds[nextIdx]
|
||||
if (!nextId) return
|
||||
selectPhoto(nextId)
|
||||
// Scroll the now-active cell into view if it's off-screen. The
|
||||
// PhotoThumbnail wrapper carries data-dup-id so we can find it
|
||||
// without threading refs through every cell.
|
||||
const el = document.querySelector<HTMLElement>(
|
||||
`[data-dup-id="${nextId}"]`
|
||||
)
|
||||
el?.scrollIntoView({ block: 'nearest', inline: 'nearest' })
|
||||
}
|
||||
|
||||
window.addEventListener('keydown', onKeyDown)
|
||||
return () => window.removeEventListener('keydown', onKeyDown)
|
||||
}, [allMemberIds, activePhotoId, columns, selectPhoto])
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center text-text-muted">
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
Loading duplicate groups…
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (isError) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center text-text-muted">
|
||||
Could not load duplicate groups: {(error as any)?.message ?? 'unknown error'}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (groups.length === 0) {
|
||||
return (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-2 px-6 text-center text-text-muted">
|
||||
<Sparkles className="h-8 w-8" />
|
||||
<div className="text-sm font-medium text-text">No duplicates found</div>
|
||||
<p className="max-w-sm text-xs">
|
||||
Nothing in your library matches another photo at the perceptual-hash
|
||||
level. If you've just added new photos, give the worker a minute and
|
||||
re-run "Re-detect duplicates" from Settings.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto bg-bg p-4 pb-20">
|
||||
<div className="mb-4 flex items-center gap-2 text-xs text-text-muted">
|
||||
<Info className="h-3.5 w-3.5" />
|
||||
<span>
|
||||
{data?.total_groups} group{data?.total_groups === 1 ? '' : 's'} ·{' '}
|
||||
{data?.total_members} photo{data?.total_members === 1 ? '' : 's'}.
|
||||
Click "Keep best" to auto-discard all but the highest-resolution
|
||||
copy of each group. Cmd+Z to undo.
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="space-y-6">
|
||||
{groups.map((group, idx) => (
|
||||
<DuplicateGroupSection
|
||||
key={group.group_id}
|
||||
group={group}
|
||||
onKeepBest={(discardIds) => discardMutation.mutate(discardIds)}
|
||||
onPreviewMember={(memberId) => openPreview(memberId, allMemberIds)}
|
||||
onSelectMember={(memberId) => selectPhoto(memberId)}
|
||||
selectedPhotos={selectedPhotos}
|
||||
isPending={discardMutation.isPending}
|
||||
// Hand the column-measurement ref to the first section only
|
||||
// — every section's grid uses the same auto-fill rule so any
|
||||
// one is representative of the rendered column count.
|
||||
gridRef={idx === 0 ? sampleGridRef : undefined}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
interface DuplicateGroupSectionProps {
|
||||
group: DuplicateGroup
|
||||
onKeepBest: (discardIds: string[]) => void
|
||||
onPreviewMember: (memberId: string) => void
|
||||
onSelectMember: (memberId: string) => void
|
||||
selectedPhotos: string[]
|
||||
isPending: boolean
|
||||
/** Optional callback ref attached to this section's grid container.
|
||||
* Used by DuplicatesView to measure the rendered column count for
|
||||
* ↑/↓ keyboard navigation. Only the first section gets one. */
|
||||
gridRef?: (el: HTMLDivElement | null) => void
|
||||
}
|
||||
|
||||
function DuplicateGroupSection({
|
||||
group,
|
||||
onKeepBest,
|
||||
onPreviewMember,
|
||||
onSelectMember,
|
||||
selectedPhotos,
|
||||
isPending,
|
||||
gridRef,
|
||||
}: DuplicateGroupSectionProps) {
|
||||
// Auto-pick "best" copy: highest pixel count, ties broken by file_size,
|
||||
// then earliest taken_at, then id for determinism. This is just the
|
||||
// default — the user can override it by clicking the crown button on
|
||||
// any other thumbnail (see `manualBestId`).
|
||||
const autoBest = useMemo(() => pickBestMember(group.members), [group.members])
|
||||
// When the user clicks "make this the best" on a non-default thumb,
|
||||
// we override the auto-pick. Local to the section so different groups
|
||||
// remember independent overrides; resets if the group itself changes.
|
||||
const [manualBestId, setManualBestId] = useState<string | null>(null)
|
||||
const bestId =
|
||||
manualBestId && group.members.some((m) => m.id === manualBestId)
|
||||
? manualBestId
|
||||
: autoBest.id
|
||||
const best = group.members.find((m) => m.id === bestId) ?? autoBest
|
||||
const discardCount = group.member_count - 1
|
||||
const isExact = group.reason === 'exact'
|
||||
|
||||
return (
|
||||
<section className="rounded-lg border border-border bg-surface">
|
||||
<header className="flex items-center justify-between gap-3 border-b border-border px-3 py-2">
|
||||
<div className="flex items-center gap-2 text-sm">
|
||||
{isExact ? (
|
||||
<Copy className="h-4 w-4 text-text-muted" />
|
||||
) : (
|
||||
<Layers className="h-4 w-4 text-text-muted" />
|
||||
)}
|
||||
<span className="font-medium text-text">
|
||||
{group.member_count} {isExact ? 'exact' : 'similar'} photos
|
||||
</span>
|
||||
<span className="text-xs text-text-faint">
|
||||
keeping: {formatDimensions(best)}
|
||||
{best.file_size != null && ` · ${formatBytes(best.file_size)}`}
|
||||
{manualBestId && manualBestId !== autoBest.id && (
|
||||
<span className="ml-1 text-text-muted">(manual)</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => {
|
||||
const discardIds = group.members
|
||||
.filter((m) => m.id !== best.id)
|
||||
.map((m) => m.id)
|
||||
if (discardIds.length === 0) return
|
||||
onKeepBest(discardIds)
|
||||
}}
|
||||
disabled={isPending}
|
||||
className={clsx(
|
||||
'flex items-center gap-1.5 rounded border border-border px-2 py-1 text-xs font-medium transition-colors',
|
||||
'hover:border-reject/50 hover:bg-reject/10 hover:text-reject',
|
||||
isPending && 'cursor-not-allowed opacity-50'
|
||||
)}
|
||||
title="Keep the highest-resolution copy and discard the rest"
|
||||
>
|
||||
<Trash2 className="h-3 w-3" />
|
||||
Keep best, discard {discardCount}
|
||||
</button>
|
||||
</header>
|
||||
|
||||
<div
|
||||
ref={gridRef}
|
||||
className="grid gap-1 p-2"
|
||||
style={{
|
||||
gridTemplateColumns:
|
||||
'repeat(auto-fill, minmax(180px, 1fr))',
|
||||
}}
|
||||
>
|
||||
{group.members.map((member) => {
|
||||
const isBest = member.id === best.id
|
||||
return (
|
||||
<div
|
||||
key={member.id}
|
||||
data-dup-id={member.id}
|
||||
className="group/dup relative"
|
||||
>
|
||||
<PhotoThumbnail
|
||||
photo={memberToPhoto(member)}
|
||||
size={180}
|
||||
fill
|
||||
isSelected={selectedPhotos.includes(member.id)}
|
||||
onClick={() => onSelectMember(member.id)}
|
||||
onDoubleClick={() => onPreviewMember(member.id)}
|
||||
/>
|
||||
{/* BEST pill — top-right, pick-coloured. Composes the same
|
||||
* THUMB_BADGE_* family used by PhotoThumbnail so the full
|
||||
* set of ornaments (selection, rating, flags, BEST) reads
|
||||
* as one consistent chip system. Inset 1.5 (6px) rather
|
||||
* than 1 (4px) because these are SIBLINGS of the thumbnail,
|
||||
* not inside its overflow-hidden box, so they need
|
||||
* clearance from PhotoThumbnail's outer selection ring. */}
|
||||
{isBest && (
|
||||
<span
|
||||
className={clsx(
|
||||
'pointer-events-none absolute right-1.5 top-1.5 z-10 uppercase',
|
||||
THUMB_BADGE_BASE,
|
||||
THUMB_BADGE_PICK
|
||||
)}
|
||||
>
|
||||
<Crown className={THUMB_BADGE_ICON} />
|
||||
Best
|
||||
</span>
|
||||
)}
|
||||
{/* "Keep this" — shown on hover for non-best thumbnails.
|
||||
* Mirrors BEST's placement so the eye doesn't retarget
|
||||
* while scanning. Starts neutral and shifts to pick on
|
||||
* hover as a preview of the state it'll set. */}
|
||||
{!isBest && (
|
||||
<button
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
setManualBestId(member.id)
|
||||
}}
|
||||
className={clsx(
|
||||
'absolute right-1.5 top-1.5 z-10 hidden uppercase transition hover:bg-pick group-hover/dup:inline-flex',
|
||||
THUMB_BADGE_BASE,
|
||||
THUMB_BADGE_NEUTRAL
|
||||
)}
|
||||
title="Keep this one instead"
|
||||
>
|
||||
<Crown className={THUMB_BADGE_ICON} />
|
||||
Keep this
|
||||
</button>
|
||||
)}
|
||||
{/* Dimensions chip — bottom-LEFT. Neutral metadata variant
|
||||
* matches the family. Rare collision with a manual rating
|
||||
* (also bottom-left) is tolerated: rated duplicates are
|
||||
* uncommon in practice. */}
|
||||
<div
|
||||
className={clsx(
|
||||
'pointer-events-none absolute bottom-1 left-1 z-10 font-mono',
|
||||
THUMB_BADGE_BASE,
|
||||
THUMB_BADGE_NEUTRAL
|
||||
)}
|
||||
>
|
||||
{formatDimensions(member)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** Score a member by (pixels, file_size, -taken_at) and return the winner.
|
||||
* Larger pixel count wins; ties broken by file_size; final tie by earliest
|
||||
* taken_at (more likely the original capture). */
|
||||
function pickBestMember(members: DuplicateGroupMember[]): DuplicateGroupMember {
|
||||
return members.reduce((best, m) => {
|
||||
const bestPixels = (best.width ?? 0) * (best.height ?? 0)
|
||||
const mPixels = (m.width ?? 0) * (m.height ?? 0)
|
||||
if (mPixels !== bestPixels) return mPixels > bestPixels ? m : best
|
||||
const bestSize = best.file_size ?? 0
|
||||
const mSize = m.file_size ?? 0
|
||||
if (mSize !== bestSize) return mSize > bestSize ? m : best
|
||||
// Earliest taken_at wins (treat null as far-future).
|
||||
const bestTaken = best.taken_at ?? '9999'
|
||||
const mTaken = m.taken_at ?? '9999'
|
||||
if (mTaken !== bestTaken) return mTaken < bestTaken ? m : best
|
||||
return best
|
||||
})
|
||||
}
|
||||
|
||||
function formatDimensions(m: DuplicateGroupMember): string {
|
||||
if (!m.width || !m.height) return '?'
|
||||
const mp = (m.width * m.height) / 1_000_000
|
||||
if (mp >= 1) return `${mp.toFixed(1)}MP`
|
||||
return `${m.width}×${m.height}`
|
||||
}
|
||||
|
||||
function formatBytes(n: number): string {
|
||||
if (n >= 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(1)}MB`
|
||||
if (n >= 1024) return `${(n / 1024).toFixed(0)}KB`
|
||||
return `${n}B`
|
||||
}
|
||||
|
||||
/** Adapt a DuplicateGroupMember (the slim API shape) to a Photo, which
|
||||
* is what PhotoThumbnail expects. We deliberately set is_duplicate=false
|
||||
* on the synthetic Photo so the duplicate badge isn't drawn on every
|
||||
* cell — the entire view is duplicates, the badge would be redundant. */
|
||||
function memberToPhoto(m: DuplicateGroupMember): Photo {
|
||||
return {
|
||||
id: m.id,
|
||||
filepath: m.filename, // good enough for the RAW/video extension regex
|
||||
filename: m.filename,
|
||||
media_type: m.media_type,
|
||||
width: m.width,
|
||||
height: m.height,
|
||||
taken_at: m.taken_at,
|
||||
rating: 0,
|
||||
is_discarded: false,
|
||||
is_duplicate: false,
|
||||
file_hash: m.file_hash ?? '',
|
||||
folder_id: m.folder_id,
|
||||
added_at: null,
|
||||
thumb_small: m.thumb_small ?? undefined,
|
||||
}
|
||||
}
|
||||
@@ -1,556 +0,0 @@
|
||||
import { useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { Star, X, ArrowDown, ArrowUp, Search, AlertTriangle, Check } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import {
|
||||
useFilterStore,
|
||||
hasActiveFilters,
|
||||
type MediaType,
|
||||
type SortField,
|
||||
} from '../../store/filterStore'
|
||||
import { useTagsQuery } from '../../hooks/useTagsQuery'
|
||||
import type { Tag } from '../../services/api'
|
||||
import { FilterPill } from './FilterPill'
|
||||
import { COLOR_LABEL_OPTIONS } from '../../constants/colorLabels'
|
||||
|
||||
const SEARCH_DEBOUNCE_MS = 300
|
||||
|
||||
const MEDIA_TYPES: { value: MediaType; label: string }[] = [
|
||||
{ value: 'photo', label: 'Photo' },
|
||||
{ value: 'video', label: 'Video' },
|
||||
{ value: 'raw', label: 'RAW' },
|
||||
{ value: 'heic', label: 'HEIC' },
|
||||
]
|
||||
|
||||
const SORT_OPTIONS: { value: SortField; label: string }[] = [
|
||||
{ value: 'taken_at', label: 'Date taken' },
|
||||
{ value: 'added_at', label: 'Date added' },
|
||||
{ value: 'filename', label: 'Filename' },
|
||||
{ value: 'file_size', label: 'File size' },
|
||||
{ value: 'rating', label: 'Rating' },
|
||||
]
|
||||
|
||||
/**
|
||||
* Compact, always-visible filter toolbar built out of FilterPill primitives.
|
||||
* Each pill represents a filter category, opens a popover with the
|
||||
* underlying control, and shows a short value summary inline when active.
|
||||
* Replaces the old expandable FilterBar + ActiveFilterChips combo.
|
||||
*/
|
||||
export function FilterBar() {
|
||||
const filterState = useFilterStore()
|
||||
const dateFrom = useFilterStore((s) => s.dateFrom)
|
||||
const dateTo = useFilterStore((s) => s.dateTo)
|
||||
const mediaTypes = useFilterStore((s) => s.mediaTypes)
|
||||
const ratingMin = useFilterStore((s) => s.ratingMin)
|
||||
const colorLabel = useFilterStore((s) => s.colorLabel)
|
||||
const flag = useFilterStore((s) => s.flag)
|
||||
const sortBy = useFilterStore((s) => s.sortBy)
|
||||
const sortOrder = useFilterStore((s) => s.sortOrder)
|
||||
const tagIds = useFilterStore((s) => s.tagIds)
|
||||
const currentSection = useFilterStore((s) => s.currentSection)
|
||||
|
||||
// Only the Flag pill is hidden inside the Discarded section. Flag has
|
||||
// exactly two values and the section locks one of them, so the pill
|
||||
// would only ever toggle the section off — useless. Rating + Tags
|
||||
// pills stay visible in their sections because the user can refine
|
||||
// them further (ratingMin >= 3, restrict to specific tag ids).
|
||||
const hideFlagPill = currentSection === 'discarded'
|
||||
|
||||
const setDateFrom = useFilterStore((s) => s.setDateFrom)
|
||||
const setDateTo = useFilterStore((s) => s.setDateTo)
|
||||
const toggleMediaType = useFilterStore((s) => s.toggleMediaType)
|
||||
const setRatingMin = useFilterStore((s) => s.setRatingMin)
|
||||
const setColorLabel = useFilterStore((s) => s.setColorLabel)
|
||||
const setFlag = useFilterStore((s) => s.setFlag)
|
||||
const setTagIds = useFilterStore((s) => s.setTagIds)
|
||||
const toggleTagId = useFilterStore((s) => s.toggleTagId)
|
||||
const setSortBy = useFilterStore((s) => s.setSortBy)
|
||||
const toggleSortOrder = useFilterStore((s) => s.toggleSortOrder)
|
||||
const clearAll = useFilterStore((s) => s.clearAll)
|
||||
|
||||
const { data: allTags = [] } = useTagsQuery()
|
||||
|
||||
// Search box. Local state mirrors the store so typing stays responsive
|
||||
// while we debounce store writes (each store write triggers a re-fetch).
|
||||
const storeQ = useFilterStore((s) => s.q)
|
||||
const setStoreQ = useFilterStore((s) => s.setQ)
|
||||
const [searchQuery, setSearchQuery] = useState(storeQ)
|
||||
useEffect(() => {
|
||||
setSearchQuery(storeQ)
|
||||
}, [storeQ])
|
||||
const debounceRef = useRef<number | null>(null)
|
||||
useEffect(() => {
|
||||
if (searchQuery === storeQ) return
|
||||
if (debounceRef.current) window.clearTimeout(debounceRef.current)
|
||||
debounceRef.current = window.setTimeout(() => {
|
||||
setStoreQ(searchQuery)
|
||||
}, SEARCH_DEBOUNCE_MS)
|
||||
return () => {
|
||||
if (debounceRef.current) window.clearTimeout(debounceRef.current)
|
||||
}
|
||||
}, [searchQuery, storeQ, setStoreQ])
|
||||
|
||||
// Pre-compute pill values + active flags so the JSX stays terse.
|
||||
const dateActive = dateFrom !== null || dateTo !== null
|
||||
const dateValue = dateActive
|
||||
? `${dateFrom ?? '…'} → ${dateTo ?? '…'}`
|
||||
: null
|
||||
|
||||
const typeActive = mediaTypes.length > 0
|
||||
const typeValue = typeActive
|
||||
? mediaTypes.map((t) => t.toUpperCase()).join(', ')
|
||||
: null
|
||||
|
||||
const ratingActive = ratingMin > 0
|
||||
const ratingValue = ratingActive ? `≥ ${ratingMin}★` : null
|
||||
|
||||
const colorActive = colorLabel !== null
|
||||
const colorValue = colorActive ? colorLabel : null
|
||||
|
||||
const flagActive = flag !== 'any'
|
||||
const flagValue = flagActive
|
||||
? flag === 'date_warning'
|
||||
? 'date issues'
|
||||
: flag
|
||||
: null
|
||||
|
||||
const tagActive = tagIds.length > 0
|
||||
const activeTagNames = allTags
|
||||
.filter((t) => tagIds.includes(t.id))
|
||||
.map((t) => t.name)
|
||||
const tagValue = tagActive
|
||||
? activeTagNames.length <= 2
|
||||
? activeTagNames.join(', ')
|
||||
: `${activeTagNames.slice(0, 2).join(', ')} +${activeTagNames.length - 2}`
|
||||
: null
|
||||
|
||||
const sortLabel = SORT_OPTIONS.find((o) => o.value === sortBy)?.label ?? sortBy
|
||||
const sortValue = `${sortLabel} ${sortOrder === 'desc' ? '↓' : '↑'}`
|
||||
|
||||
const anyActive = hasActiveFilters(filterState)
|
||||
|
||||
return (
|
||||
// Fixed bar height + py-0 so neither the active filter pills nor the
|
||||
// clear-all button can stretch the bar vertically. The fixed h-11
|
||||
// matches the h-7 pills + 8px symmetric vertical padding.
|
||||
<div className="flex h-11 items-center gap-3 border-b border-border bg-surface px-3 py-0">
|
||||
{/* Pills — left side, scroll horizontally if they overflow. */}
|
||||
<div className="flex min-w-0 flex-1 items-center gap-1.5 overflow-x-auto">
|
||||
{/* Date */}
|
||||
<FilterPill
|
||||
label="Date"
|
||||
value={dateValue}
|
||||
isActive={dateActive}
|
||||
onClear={() => {
|
||||
setDateFrom(null)
|
||||
setDateTo(null)
|
||||
}}
|
||||
>
|
||||
<div className="space-y-2">
|
||||
<div>
|
||||
<label className="mb-1 block text-[11px] text-text-muted">From</label>
|
||||
<input
|
||||
type="date"
|
||||
value={dateFrom ?? ''}
|
||||
onChange={(e) => setDateFrom(e.target.value || null)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label className="mb-1 block text-[11px] text-text-muted">To</label>
|
||||
<input
|
||||
type="date"
|
||||
value={dateTo ?? ''}
|
||||
onChange={(e) => setDateTo(e.target.value || null)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Type */}
|
||||
<FilterPill
|
||||
label="Type"
|
||||
value={typeValue}
|
||||
isActive={typeActive}
|
||||
onClear={() => mediaTypes.forEach((t) => toggleMediaType(t))}
|
||||
>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{MEDIA_TYPES.map(({ value, label }) => {
|
||||
const active = mediaTypes.includes(value)
|
||||
return (
|
||||
<button
|
||||
key={value}
|
||||
onClick={() => toggleMediaType(value)}
|
||||
className={clsx(
|
||||
'rounded px-2 py-1 text-xs transition-colors',
|
||||
active
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Rating */}
|
||||
<FilterPill
|
||||
label="Rating"
|
||||
value={ratingValue}
|
||||
isActive={ratingActive}
|
||||
onClear={() => setRatingMin(0)}
|
||||
>
|
||||
<div>
|
||||
<p className="mb-1 text-[11px] text-text-muted">Minimum</p>
|
||||
<div className="flex gap-1">
|
||||
{[1, 2, 3, 4, 5].map((n) => (
|
||||
<button
|
||||
key={n}
|
||||
onClick={() => setRatingMin(ratingMin === n ? 0 : n)}
|
||||
className="p-0.5"
|
||||
title={`At least ${n} star${n > 1 ? 's' : ''}`}
|
||||
>
|
||||
<Star
|
||||
className={clsx(
|
||||
'h-5 w-5 transition-colors',
|
||||
n <= ratingMin
|
||||
? 'fill-star text-star'
|
||||
: 'text-text-muted hover:text-star'
|
||||
)}
|
||||
/>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Color */}
|
||||
<FilterPill
|
||||
label="Color"
|
||||
value={colorValue}
|
||||
isActive={colorActive}
|
||||
onClear={() => setColorLabel(null)}
|
||||
>
|
||||
<div className="flex items-center gap-1.5">
|
||||
{COLOR_LABEL_OPTIONS.map(({ value, className }) => {
|
||||
const active = colorLabel === value
|
||||
return (
|
||||
<button
|
||||
key={value}
|
||||
onClick={() => setColorLabel(active ? null : value)}
|
||||
className={clsx(
|
||||
'h-5 w-5 rounded-full ring-offset-2 ring-offset-surface transition-all',
|
||||
className,
|
||||
active ? 'ring-2 ring-primary' : 'opacity-60 hover:opacity-100'
|
||||
)}
|
||||
title={value}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
{colorLabel && (
|
||||
<button
|
||||
onClick={() => setColorLabel(null)}
|
||||
className="ml-1 rounded p-0.5 text-text-muted hover:bg-surface-2 hover:text-text"
|
||||
title="Clear color"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Flag — hidden in the Discarded section, where the flag is
|
||||
* pinned to "discarded" by the section preset. */}
|
||||
{!hideFlagPill && (
|
||||
<FilterPill
|
||||
label="Flag"
|
||||
value={flagValue}
|
||||
isActive={flagActive}
|
||||
onClear={() => setFlag('any')}
|
||||
>
|
||||
<div className="flex flex-col gap-1">
|
||||
<button
|
||||
onClick={() => setFlag('any')}
|
||||
className={clsx(
|
||||
'rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'any'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
>
|
||||
Any
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setFlag('discarded')}
|
||||
className={clsx(
|
||||
'rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'discarded'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
>
|
||||
Discarded
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setFlag('date_warning')}
|
||||
className={clsx(
|
||||
'flex items-center gap-1.5 rounded px-2 py-1 text-left text-xs transition-colors',
|
||||
flag === 'date_warning'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
title="Photos whose folder/filename suggests a different date than the stored taken_at"
|
||||
>
|
||||
<AlertTriangle className="h-3 w-3" />
|
||||
Date issues
|
||||
</button>
|
||||
</div>
|
||||
</FilterPill>
|
||||
)}
|
||||
|
||||
{/* Tags */}
|
||||
{allTags.length > 0 && (
|
||||
<FilterPill
|
||||
label="Tags"
|
||||
value={tagValue}
|
||||
isActive={tagActive}
|
||||
onClear={() => setTagIds([])}
|
||||
>
|
||||
<TagFilterPopover
|
||||
allTags={allTags}
|
||||
selectedIds={tagIds}
|
||||
onToggle={toggleTagId}
|
||||
onClear={() => setTagIds([])}
|
||||
/>
|
||||
</FilterPill>
|
||||
)}
|
||||
|
||||
{/* Sort — always present, never "active/inactive" since there's
|
||||
always a value. */}
|
||||
<FilterPill label="Sort" value={sortValue} isActive>
|
||||
<div className="space-y-2">
|
||||
<select
|
||||
value={sortBy}
|
||||
onChange={(e) => setSortBy(e.target.value as SortField)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1 text-xs text-text focus:border-primary focus:outline-none"
|
||||
>
|
||||
{SORT_OPTIONS.map((opt) => (
|
||||
<option key={opt.value} value={opt.value}>
|
||||
{opt.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<button
|
||||
onClick={toggleSortOrder}
|
||||
className="flex w-full items-center justify-center gap-1 rounded bg-surface-2 px-2 py-1 text-xs text-text-muted hover:bg-surface-offset hover:text-text"
|
||||
>
|
||||
{sortOrder === 'desc' ? (
|
||||
<>
|
||||
<ArrowDown className="h-3.5 w-3.5" />
|
||||
Descending
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<ArrowUp className="h-3.5 w-3.5" />
|
||||
Ascending
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
</FilterPill>
|
||||
|
||||
{/* Clear-all — borderless text affordance pinned next to the pill
|
||||
* cluster on the right. Lives inside the pills container so it
|
||||
* shares the same flex group and gap and reads as "another
|
||||
* pill". Only renders when any filter is active. */}
|
||||
{anyActive && (
|
||||
<button
|
||||
onClick={clearAll}
|
||||
className="ml-1 flex h-7 flex-shrink-0 items-center whitespace-nowrap px-1 text-xs text-text-muted underline-offset-2 hover:text-text hover:underline"
|
||||
title="Clear all filters in this section"
|
||||
>
|
||||
Clear all
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Search — pinned to the right edge of the bar. Same id as before
|
||||
* so the global "/" focus shortcut still finds it. */}
|
||||
<div className="relative w-56 flex-shrink-0">
|
||||
<Search className="pointer-events-none absolute left-2.5 top-1/2 h-3.5 w-3.5 -translate-y-1/2 text-text-muted" />
|
||||
<input
|
||||
id="topbar-search"
|
||||
type="text"
|
||||
value={searchQuery}
|
||||
onChange={(e) => setSearchQuery(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Escape') {
|
||||
setSearchQuery('')
|
||||
setStoreQ('')
|
||||
e.currentTarget.blur()
|
||||
}
|
||||
}}
|
||||
placeholder="Search photos…"
|
||||
className="h-7 w-full rounded-full border border-border bg-surface-2 pl-8 pr-7 text-xs text-text placeholder-text-muted focus:border-primary focus:outline-none"
|
||||
/>
|
||||
{searchQuery && (
|
||||
<button
|
||||
onClick={() => {
|
||||
setSearchQuery('')
|
||||
setStoreQ('')
|
||||
}}
|
||||
className="absolute right-1.5 top-1/2 -translate-y-1/2 rounded-full p-0.5 text-text-muted hover:bg-surface-offset hover:text-text"
|
||||
title="Clear search (Esc)"
|
||||
aria-label="Clear search"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
interface TagFilterPopoverProps {
|
||||
allTags: Tag[]
|
||||
selectedIds: string[]
|
||||
onToggle: (id: string) => void
|
||||
onClear: () => void
|
||||
}
|
||||
|
||||
function TagFilterPopover({
|
||||
allTags,
|
||||
selectedIds,
|
||||
onToggle,
|
||||
onClear,
|
||||
}: TagFilterPopoverProps) {
|
||||
const [query, setQuery] = useState('')
|
||||
const inputRef = useRef<HTMLInputElement>(null)
|
||||
|
||||
useEffect(() => {
|
||||
inputRef.current?.focus()
|
||||
}, [])
|
||||
|
||||
const selectedSet = useMemo(() => new Set(selectedIds), [selectedIds])
|
||||
|
||||
// Selected tags pinned at top, remaining sorted by photo_count desc
|
||||
// then name. Filtered by query (case-insensitive substring).
|
||||
const orderedTags = useMemo(() => {
|
||||
const q = query.trim().toLowerCase()
|
||||
const match = (t: Tag) => !q || t.name.toLowerCase().includes(q)
|
||||
const selected = allTags.filter((t) => selectedSet.has(t.id) && match(t))
|
||||
const unselected = allTags
|
||||
.filter((t) => !selectedSet.has(t.id) && match(t))
|
||||
.sort((a, b) => {
|
||||
if (b.photo_count !== a.photo_count) return b.photo_count - a.photo_count
|
||||
return a.name.localeCompare(b.name)
|
||||
})
|
||||
return { selected, unselected }
|
||||
}, [allTags, selectedSet, query])
|
||||
|
||||
const totalVisible = orderedTags.selected.length + orderedTags.unselected.length
|
||||
|
||||
return (
|
||||
<div className="w-64">
|
||||
<div className="relative mb-2">
|
||||
<Search className="pointer-events-none absolute left-2 top-1/2 h-3.5 w-3.5 -translate-y-1/2 text-text-muted" />
|
||||
<input
|
||||
ref={inputRef}
|
||||
type="text"
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Escape' && query) {
|
||||
e.stopPropagation()
|
||||
setQuery('')
|
||||
}
|
||||
}}
|
||||
placeholder="Search tags…"
|
||||
className="h-7 w-full rounded border border-border bg-bg pl-7 pr-6 text-xs text-text placeholder-text-muted focus:border-primary focus:outline-none"
|
||||
/>
|
||||
{query && (
|
||||
<button
|
||||
onClick={() => setQuery('')}
|
||||
className="absolute right-1 top-1/2 -translate-y-1/2 rounded p-0.5 text-text-muted hover:bg-surface-2 hover:text-text"
|
||||
aria-label="Clear search"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="mb-1 flex items-center justify-between px-0.5 text-[11px] text-text-muted">
|
||||
<span>
|
||||
{selectedIds.length > 0
|
||||
? `${selectedIds.length} selected`
|
||||
: `${totalVisible} tag${totalVisible === 1 ? '' : 's'}`}
|
||||
</span>
|
||||
{selectedIds.length > 0 && (
|
||||
<button
|
||||
onClick={onClear}
|
||||
className="underline-offset-2 hover:text-text hover:underline"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="max-h-64 overflow-y-auto rounded border border-border bg-bg">
|
||||
{totalVisible === 0 ? (
|
||||
<div className="px-2 py-3 text-center text-xs text-text-muted">
|
||||
No tags match
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
{orderedTags.selected.map((tag) => (
|
||||
<TagRow key={tag.id} tag={tag} selected onToggle={onToggle} />
|
||||
))}
|
||||
{orderedTags.selected.length > 0 && orderedTags.unselected.length > 0 && (
|
||||
<div className="my-0.5 border-t border-border" />
|
||||
)}
|
||||
{orderedTags.unselected.map((tag) => (
|
||||
<TagRow key={tag.id} tag={tag} selected={false} onToggle={onToggle} />
|
||||
))}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function TagRow({
|
||||
tag,
|
||||
selected,
|
||||
onToggle,
|
||||
}: {
|
||||
tag: Tag
|
||||
selected: boolean
|
||||
onToggle: (id: string) => void
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
onClick={() => onToggle(tag.id)}
|
||||
className={clsx(
|
||||
'flex w-full items-center gap-2 px-2 py-1.5 text-left text-xs transition-colors',
|
||||
selected
|
||||
? 'bg-primary/15 text-text hover:bg-primary/25'
|
||||
: 'text-text-muted hover:bg-surface-2 hover:text-text'
|
||||
)}
|
||||
>
|
||||
<span
|
||||
className={clsx(
|
||||
'flex h-3.5 w-3.5 flex-shrink-0 items-center justify-center rounded border',
|
||||
selected ? 'border-primary bg-primary text-white' : 'border-border'
|
||||
)}
|
||||
>
|
||||
{selected && <Check className="h-2.5 w-2.5" strokeWidth={3} />}
|
||||
</span>
|
||||
<span className="min-w-0 flex-1 truncate">{tag.name}</span>
|
||||
<span className="flex-shrink-0 tabular-nums text-[10px] text-text-muted">
|
||||
{tag.photo_count}
|
||||
</span>
|
||||
</button>
|
||||
)
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
import { useEffect, useLayoutEffect, useRef, useState } from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import { ChevronDown, X } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
|
||||
interface FilterPillProps {
|
||||
/** Category label, always shown ("Date", "Type", etc.). */
|
||||
label: string
|
||||
/** Currently unused in the rendered output — the inline value display
|
||||
* was making active pills wider than inactive ones. Kept on the
|
||||
* interface so callers don't have to change. The value is still
|
||||
* surfaced via the title attribute for hover discovery. */
|
||||
value?: string | null
|
||||
isActive?: boolean
|
||||
/** When provided + isActive, an X appears inside the pill that clears
|
||||
* this filter without opening the popover. */
|
||||
onClear?: () => void
|
||||
/** Popover contents — usually the existing control for this filter. */
|
||||
children: React.ReactNode
|
||||
/** Force the popover open programmatically (rare). */
|
||||
defaultOpen?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* A toolbar pill that hosts a filter category. Click the pill to open a
|
||||
* small popover with the actual control; the popover closes on outside
|
||||
* click or Escape. Active filters tint the pill primary and show their
|
||||
* current value inline.
|
||||
*/
|
||||
export function FilterPill({
|
||||
label,
|
||||
value,
|
||||
isActive = false,
|
||||
onClear,
|
||||
children,
|
||||
defaultOpen = false,
|
||||
}: FilterPillProps) {
|
||||
const [open, setOpen] = useState(defaultOpen)
|
||||
const buttonRef = useRef<HTMLButtonElement>(null)
|
||||
const popoverRef = useRef<HTMLDivElement>(null)
|
||||
const [popoverPos, setPopoverPos] = useState<{ top: number; left: number } | null>(null)
|
||||
|
||||
// Compute the popover's screen position from the trigger button. Done
|
||||
// imperatively (not via CSS absolute) so the popover can live in a portal
|
||||
// and escape the FilterBar's overflow-x-auto clipping. Re-computed on
|
||||
// open, scroll, and resize.
|
||||
useLayoutEffect(() => {
|
||||
if (!open) return
|
||||
const update = () => {
|
||||
const btn = buttonRef.current
|
||||
if (!btn) return
|
||||
const rect = btn.getBoundingClientRect()
|
||||
// Default left-align under the trigger; clamp to viewport so the
|
||||
// last pill on the right doesn't overflow.
|
||||
const popWidth = popoverRef.current?.offsetWidth ?? 240
|
||||
const margin = 8
|
||||
let left = rect.left
|
||||
if (left + popWidth + margin > window.innerWidth) {
|
||||
left = Math.max(margin, window.innerWidth - popWidth - margin)
|
||||
}
|
||||
setPopoverPos({ top: rect.bottom + 4, left })
|
||||
}
|
||||
update()
|
||||
window.addEventListener('resize', update)
|
||||
window.addEventListener('scroll', update, true)
|
||||
return () => {
|
||||
window.removeEventListener('resize', update)
|
||||
window.removeEventListener('scroll', update, true)
|
||||
}
|
||||
}, [open])
|
||||
|
||||
// Close on outside click + Escape. Outside means neither the trigger
|
||||
// button nor the (portaled) popover.
|
||||
useEffect(() => {
|
||||
if (!open) return
|
||||
const onDocMouseDown = (e: MouseEvent) => {
|
||||
const target = e.target as Node
|
||||
if (buttonRef.current?.contains(target)) return
|
||||
if (popoverRef.current?.contains(target)) return
|
||||
setOpen(false)
|
||||
}
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') setOpen(false)
|
||||
}
|
||||
document.addEventListener('mousedown', onDocMouseDown)
|
||||
document.addEventListener('keydown', onKey)
|
||||
return () => {
|
||||
document.removeEventListener('mousedown', onDocMouseDown)
|
||||
document.removeEventListener('keydown', onKey)
|
||||
}
|
||||
}, [open])
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
ref={buttonRef}
|
||||
onClick={() => setOpen((v) => !v)}
|
||||
// Hover to see the active value as a tooltip — keeps the pill at
|
||||
// a constant width regardless of state. The popover is the
|
||||
// canonical place to read/edit the filter value.
|
||||
title={isActive && value ? `${label}: ${value}` : label}
|
||||
className={clsx(
|
||||
// Fixed height + py-0 so neither the X clear icon nor the
|
||||
// chevron can stretch the pill vertically when the active
|
||||
// state swaps them in.
|
||||
'flex h-7 items-center gap-1 rounded-full border px-2.5 py-0 text-xs transition-colors',
|
||||
isActive
|
||||
? 'border-primary/40 bg-primary/15 text-primary'
|
||||
: 'border-border bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
>
|
||||
<span className={clsx(isActive && 'font-medium')}>{label}</span>
|
||||
{isActive && onClear ? (
|
||||
<span
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
onClear()
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter' || e.key === ' ') {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
onClear()
|
||||
}
|
||||
}}
|
||||
// Same h-4 w-4 as the chevron slot below so swapping the
|
||||
// two doesn't change the pill's footprint.
|
||||
className="ml-0.5 inline-flex h-4 w-4 cursor-pointer items-center justify-center rounded-full hover:bg-primary/30"
|
||||
title={`Clear ${label}`}
|
||||
aria-label={`Clear ${label}`}
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</span>
|
||||
) : (
|
||||
<span className="inline-flex h-4 w-4 items-center justify-center">
|
||||
<ChevronDown className="h-3 w-3 opacity-60" />
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
|
||||
{open &&
|
||||
createPortal(
|
||||
<div
|
||||
ref={popoverRef}
|
||||
style={{
|
||||
position: 'fixed',
|
||||
top: popoverPos?.top ?? -9999,
|
||||
left: popoverPos?.left ?? -9999,
|
||||
visibility: popoverPos ? 'visible' : 'hidden',
|
||||
}}
|
||||
className="z-50 min-w-[220px] rounded-lg border border-border bg-surface p-3 shadow-xl"
|
||||
>
|
||||
{children}
|
||||
</div>,
|
||||
document.body
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
@@ -1,169 +0,0 @@
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { motion, AnimatePresence } from 'framer-motion'
|
||||
import { ShoppingBasket } from 'lucide-react'
|
||||
import { useActiveHeapMembers } from '../../hooks/useActiveHeapMembersQuery'
|
||||
import { useFilterStore } from '../../store/filterStore'
|
||||
import { photos as photosApi, heaps as heapsApi } from '../../services/api'
|
||||
import cardBg from '../../assets/card.png'
|
||||
|
||||
/** How many thumbnails fan out across the stack at once. The newest is
|
||||
* drawn last (top), older ones fan back-left and back-right. */
|
||||
const STACK_SIZE = 5
|
||||
|
||||
/**
|
||||
* Pinned card at the bottom of the LeftSidebar showing the currently
|
||||
* active heap. Renders nothing when no heap is active — the parent
|
||||
* layout collapses around it cleanly.
|
||||
*
|
||||
* The card is a fast nav shortcut + a satisfying landing spot for the P
|
||||
* pick action: every new pick optimistically updates the
|
||||
* ['heap-photo-ids', heapId] cache that the existing pick mutation
|
||||
* already maintains, so we just subscribe to the same query and let
|
||||
* framer-motion's AnimatePresence handle the entrance/exit animation
|
||||
* when ids appear or disappear.
|
||||
*/
|
||||
export function ActiveHeapCard() {
|
||||
const { activeHeap } = useActiveHeapMembers()
|
||||
const navigateToSection = useFilterStore((s) => s.navigateToSection)
|
||||
|
||||
// Subscribes to the same cache key the pick mutation optimistically
|
||||
// updates. The data is already fetched (and kept fresh) by
|
||||
// useActiveHeapMembers above; this useQuery just gives us a render-
|
||||
// dependency on the array contents and stable insertion order.
|
||||
const { data: orderedIds = [] } = useQuery<string[]>({
|
||||
queryKey: ['heap-photo-ids', activeHeap?.id],
|
||||
queryFn: () => heapsApi.photoIds(activeHeap!.id),
|
||||
enabled: !!activeHeap,
|
||||
staleTime: 30_000,
|
||||
})
|
||||
|
||||
if (!activeHeap) return null
|
||||
|
||||
// Show the latest STACK_SIZE photos. The backend returns ids in
|
||||
// insertion order so the last one in the array is the most recently
|
||||
// picked — that's the one we want at the front of the stack.
|
||||
const visible = orderedIds.slice(-STACK_SIZE)
|
||||
// We render newest LAST so it draws on top via z-index. Reverse so
|
||||
// index 0 is the back card and index N-1 is the front.
|
||||
const stack = visible.map((id, i) => ({
|
||||
id,
|
||||
// Symmetric fan: front card has rotate=0, x=0; cards behind it
|
||||
// alternate left/right as you walk back through the stack.
|
||||
rotate: stackRotate(i, visible.length),
|
||||
x: stackOffsetX(i, visible.length),
|
||||
y: stackOffsetY(i, visible.length),
|
||||
z: i,
|
||||
}))
|
||||
|
||||
return (
|
||||
<div className="m-1.5 rounded-md border border-border bg-surface-2 shadow-sm">
|
||||
{/* Header — clickable, navigates to the heap section. */}
|
||||
<button
|
||||
onClick={() =>
|
||||
navigateToSection(`heap-${activeHeap.id}`, { heapId: activeHeap.id })
|
||||
}
|
||||
className="flex w-full items-center gap-1.5 rounded-t-md px-2 py-1.5 text-left hover:bg-surface-offset"
|
||||
title={`Open "${activeHeap.name}"`}
|
||||
>
|
||||
<ShoppingBasket className="h-3.5 w-3.5 flex-shrink-0 text-pick" />
|
||||
<span className="min-w-0 flex-1 truncate text-[12px] font-semibold text-text">
|
||||
{activeHeap.name}
|
||||
</span>
|
||||
<span className="flex h-4 min-w-[20px] items-center justify-center rounded bg-surface px-1 text-[10px] font-medium text-text-muted">
|
||||
{orderedIds.length}
|
||||
</span>
|
||||
</button>
|
||||
|
||||
{/* Stack row. Re-keyed on activeHeap.id so switching heaps tears
|
||||
* the animation context down cleanly instead of trying to
|
||||
* crossfade unrelated photos. The desert scene sits behind the
|
||||
* fanned thumbnails — `cover` + `bottom` keeps the dunes anchored
|
||||
* so the cacti frame the photos rather than the (transparent) sky. */}
|
||||
<div
|
||||
key={activeHeap.id}
|
||||
className="relative h-20 overflow-hidden rounded-b-md px-2 pb-2"
|
||||
style={{
|
||||
backgroundImage: `url(${cardBg})`,
|
||||
backgroundSize: 'cover',
|
||||
backgroundPosition: 'center bottom',
|
||||
backgroundRepeat: 'no-repeat',
|
||||
imageRendering: 'pixelated',
|
||||
opacity: 0.7,
|
||||
}}
|
||||
>
|
||||
{visible.length === 0 ? (
|
||||
<div className="flex h-full items-center justify-center px-2 text-center text-[11px] text-text-faint">
|
||||
Pick photos with P to fill the heap
|
||||
</div>
|
||||
) : (
|
||||
<div className="relative h-full">
|
||||
<AnimatePresence initial={false}>
|
||||
{stack.map((item) => (
|
||||
<motion.img
|
||||
key={item.id}
|
||||
src={photosApi.getThumbnailUrl(item.id, 'small')}
|
||||
alt=""
|
||||
initial={{ x: 80, y: 0, scale: 0.6, rotate: 0, opacity: 0 }}
|
||||
animate={{
|
||||
x: item.x,
|
||||
y: item.y,
|
||||
scale: 1,
|
||||
rotate: item.rotate,
|
||||
opacity: 1,
|
||||
}}
|
||||
exit={{ x: -60, scale: 0.6, opacity: 0 }}
|
||||
transition={{ type: 'spring', stiffness: 360, damping: 28 }}
|
||||
style={{
|
||||
zIndex: item.z,
|
||||
position: 'absolute',
|
||||
left: '50%',
|
||||
top: '50%',
|
||||
marginLeft: -26, // half of w-13
|
||||
marginTop: -26, // half of h-13
|
||||
}}
|
||||
className="h-[52px] w-[52px] rounded object-cover shadow-[0_4px_10px_rgba(0,0,0,0.65),0_2px_4px_rgba(0,0,0,0.5)] ring-1 ring-black/60"
|
||||
/>
|
||||
))}
|
||||
</AnimatePresence>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ── Stack geometry ───────────────────────────────────────────────────────
|
||||
//
|
||||
// `i` is the position in the visible array (0 = oldest, last = newest).
|
||||
// We want the newest card at center (rotate 0, x 0) and earlier cards
|
||||
// fanning symmetrically outward — so we score each card by how far it
|
||||
// is from the front, alternating sign.
|
||||
|
||||
const X_STEP = 18 // pixels per fan step
|
||||
const Y_STEP = 2 // tiny vertical lift so the back cards peek above
|
||||
const ROTATE_STEP = 6 // degrees per fan step
|
||||
|
||||
function stackRotate(i: number, len: number): number {
|
||||
// Distance from the front (newest). Front card → 0, then alternating
|
||||
// -1, +1, -2, +2 ... to spread cards outward.
|
||||
const fromFront = len - 1 - i
|
||||
if (fromFront === 0) return 0
|
||||
const sign = fromFront % 2 === 1 ? -1 : 1
|
||||
const magnitude = Math.ceil(fromFront / 2)
|
||||
return sign * magnitude * ROTATE_STEP
|
||||
}
|
||||
|
||||
function stackOffsetX(i: number, len: number): number {
|
||||
const fromFront = len - 1 - i
|
||||
if (fromFront === 0) return 0
|
||||
const sign = fromFront % 2 === 1 ? -1 : 1
|
||||
const magnitude = Math.ceil(fromFront / 2)
|
||||
return sign * magnitude * X_STEP
|
||||
}
|
||||
|
||||
function stackOffsetY(i: number, len: number): number {
|
||||
// Back cards lift up a couple pixels so they're visible above the
|
||||
// front card's top edge — gives the stack its sense of depth.
|
||||
const fromFront = len - 1 - i
|
||||
return fromFront * -Y_STEP
|
||||
}
|
||||
@@ -1,247 +0,0 @@
|
||||
import { useState, useEffect, useMemo } from 'react'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { X, Folder, AlertCircle } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import {
|
||||
heaps as heapsApi,
|
||||
type Heap,
|
||||
type FolderTreeNode,
|
||||
} from '../../services/api'
|
||||
import { HEAPS_QUERY_KEY } from '../../hooks/useHeapsQuery'
|
||||
import { useFolderTreeQuery } from '../../hooks/useFolderTreeQuery'
|
||||
import { toast } from '../ToastContainer'
|
||||
|
||||
interface FlatFolder {
|
||||
id: string
|
||||
name: string
|
||||
path: string
|
||||
depth: number
|
||||
}
|
||||
|
||||
/** Walk the folder tree depth-first into a flat list with depth info so
|
||||
* the picker can render every node — including nested subfolders — as
|
||||
* one indented option. */
|
||||
function flattenTree(nodes: FolderTreeNode[], depth = 0): FlatFolder[] {
|
||||
const out: FlatFolder[] = []
|
||||
for (const n of nodes) {
|
||||
out.push({ id: n.id, name: n.name, path: n.path, depth })
|
||||
if (n.children.length > 0) {
|
||||
out.push(...flattenTree(n.children, depth + 1))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
interface HeapConvertDialogProps {
|
||||
heap: Heap | null
|
||||
onClose: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Modal that converts a heap into a folder. The user picks a target folder
|
||||
* (any source root, today — sub-folder picking is a follow-up), chooses
|
||||
* move vs copy semantics, and optionally has the heap deleted on success.
|
||||
*/
|
||||
export function HeapConvertDialog({ heap, onClose }: HeapConvertDialogProps) {
|
||||
const queryClient = useQueryClient()
|
||||
const [targetId, setTargetId] = useState('')
|
||||
const [mode, setMode] = useState<'move' | 'copy'>('move')
|
||||
const [deleteHeap, setDeleteHeap] = useState(false)
|
||||
const [subfolderName, setSubfolderName] = useState('')
|
||||
|
||||
// Use the recursive folder tree, not the flat source-root list, so the
|
||||
// user can pick a sub-folder at any depth as the target.
|
||||
const { data: tree = [] } = useFolderTreeQuery()
|
||||
const folders = useMemo<FlatFolder[]>(() => flattenTree(tree), [tree])
|
||||
|
||||
// Default to the first folder when the dialog opens or folders load.
|
||||
useEffect(() => {
|
||||
if (!targetId && folders.length > 0) {
|
||||
setTargetId(folders[0].id)
|
||||
}
|
||||
}, [folders, targetId])
|
||||
|
||||
// Reset state on close, prefill subfolder name when opened.
|
||||
useEffect(() => {
|
||||
if (heap) {
|
||||
setSubfolderName(heap.name)
|
||||
} else {
|
||||
setTargetId('')
|
||||
setMode('move')
|
||||
setDeleteHeap(false)
|
||||
setSubfolderName('')
|
||||
}
|
||||
}, [heap])
|
||||
|
||||
const convertMutation = useMutation({
|
||||
mutationFn: () =>
|
||||
heapsApi.convert(heap!.id, {
|
||||
target_id: targetId,
|
||||
mode,
|
||||
delete_heap: deleteHeap,
|
||||
// Empty subfolder = drop directly into the parent. Trim and only
|
||||
// send if the user kept it populated.
|
||||
subfolder_name: subfolderName.trim() || null,
|
||||
}),
|
||||
onSuccess: (data) => {
|
||||
const total = (data.moved ?? 0) + (data.copied ?? 0)
|
||||
const verb = data.mode === 'move' ? 'Moved' : 'Copied'
|
||||
toast.success(
|
||||
`${verb} ${total} photo${total === 1 ? '' : 's'}`,
|
||||
data.heap_deleted ? `Heap "${heap?.name}" deleted` : undefined
|
||||
)
|
||||
queryClient.invalidateQueries({ queryKey: HEAPS_QUERY_KEY })
|
||||
queryClient.invalidateQueries({ queryKey: ['photos'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['folders'] })
|
||||
onClose()
|
||||
},
|
||||
onError: (e: any) =>
|
||||
toast.error('Convert failed', e?.response?.data?.detail || e.message),
|
||||
})
|
||||
|
||||
if (!heap) return null
|
||||
|
||||
const targetFolder = folders.find((f) => f.id === targetId)
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center">
|
||||
<div className="absolute inset-0 bg-black/60 backdrop-blur-sm" onClick={onClose} />
|
||||
|
||||
<div className="relative z-10 w-full max-w-md rounded-lg border border-border bg-surface p-6 shadow-xl">
|
||||
<div className="mb-4 flex items-center justify-between">
|
||||
<h2 className="text-lg font-semibold text-text">
|
||||
Convert "{heap.name}" to folder
|
||||
</h2>
|
||||
<button
|
||||
onClick={onClose}
|
||||
disabled={convertMutation.isPending}
|
||||
className="rounded p-1 text-text-muted hover:bg-surface-2 hover:text-text"
|
||||
>
|
||||
<X className="h-5 w-5" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Target picker */}
|
||||
<div className="mb-4">
|
||||
<label className="mb-1 block text-xs text-text-muted">Target folder</label>
|
||||
{folders.length === 0 ? (
|
||||
<div className="rounded border border-border bg-bg px-3 py-2 text-xs text-text-muted">
|
||||
No folders available
|
||||
</div>
|
||||
) : (
|
||||
<select
|
||||
value={targetId}
|
||||
onChange={(e) => setTargetId(e.target.value)}
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1.5 text-sm text-text focus:border-primary focus:outline-none"
|
||||
>
|
||||
{folders.map((f) => (
|
||||
<option key={f.id} value={f.id}>
|
||||
{/* Two non-breaking spaces per depth so nested
|
||||
* subfolders read as a tree in the native dropdown. */}
|
||||
{'\u00A0\u00A0'.repeat(f.depth) + f.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
{targetFolder && (
|
||||
<p className="mt-1 flex items-center gap-1 text-xs text-text-faint">
|
||||
<Folder className="h-3 w-3" />
|
||||
{targetFolder.path}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Subfolder name */}
|
||||
<div className="mb-4">
|
||||
<label className="mb-1 block text-xs text-text-muted">
|
||||
Subfolder name
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
value={subfolderName}
|
||||
onChange={(e) => setSubfolderName(e.target.value)}
|
||||
placeholder="(none — use parent directly)"
|
||||
className="w-full rounded border border-border bg-bg px-2 py-1.5 text-sm text-text placeholder-text-faint focus:border-primary focus:outline-none"
|
||||
/>
|
||||
<p className="mt-1 text-xs text-text-faint">
|
||||
{subfolderName.trim() && targetFolder
|
||||
? `Will create ${targetFolder.path}/${subfolderName.trim()} if missing.`
|
||||
: 'Photos go directly into the parent folder.'}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Mode toggle */}
|
||||
<div className="mb-4">
|
||||
<label className="mb-1 block text-xs text-text-muted">Mode</label>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={() => setMode('move')}
|
||||
className={clsx(
|
||||
'flex-1 rounded px-3 py-1.5 text-sm transition-colors',
|
||||
mode === 'move'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
>
|
||||
Move
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setMode('copy')}
|
||||
className={clsx(
|
||||
'flex-1 rounded px-3 py-1.5 text-sm transition-colors',
|
||||
mode === 'copy'
|
||||
? 'bg-primary text-white'
|
||||
: 'bg-surface-2 text-text-muted hover:bg-surface-offset hover:text-text'
|
||||
)}
|
||||
>
|
||||
Copy
|
||||
</button>
|
||||
</div>
|
||||
<p className="mt-1 text-xs text-text-faint">
|
||||
{mode === 'move'
|
||||
? 'Files are moved on disk; original photos update their folder.'
|
||||
: 'Files are copied on disk; new photo records are created.'}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Delete heap toggle */}
|
||||
<div className="mb-4 flex items-center gap-2">
|
||||
<input
|
||||
id="delete-heap"
|
||||
type="checkbox"
|
||||
checked={deleteHeap}
|
||||
onChange={(e) => setDeleteHeap(e.target.checked)}
|
||||
className="h-4 w-4 rounded border-border bg-bg text-primary focus:ring-2 focus:ring-primary focus:ring-offset-0"
|
||||
/>
|
||||
<label htmlFor="delete-heap" className="text-sm text-text">
|
||||
Delete heap after conversion
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{convertMutation.isError && (
|
||||
<div className="mb-3 flex items-center gap-2 rounded bg-reject/10 p-3 text-sm text-reject">
|
||||
<AlertCircle className="h-4 w-4 flex-shrink-0" />
|
||||
<span>{(convertMutation.error as any)?.message || 'Conversion failed'}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex justify-end gap-2">
|
||||
<button
|
||||
onClick={onClose}
|
||||
disabled={convertMutation.isPending}
|
||||
className="rounded bg-surface-2 px-4 py-2 text-sm text-text hover:bg-surface-offset disabled:opacity-50"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
onClick={() => convertMutation.mutate()}
|
||||
disabled={!targetId || convertMutation.isPending}
|
||||
className="rounded bg-primary px-4 py-2 text-sm font-medium text-white hover:bg-primary/90 disabled:opacity-50"
|
||||
>
|
||||
{convertMutation.isPending ? 'Converting…' : 'Convert'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user