diff --git a/.env b/.env deleted file mode 100644 index c11e4e0..0000000 --- a/.env +++ /dev/null @@ -1,22 +0,0 @@ -# Mulita / PhotoVault local environment. -# See .env.example for the full list of knobs and their docs. - -# REQUIRED — host path to your photo library. -PHOTO_DIRS=/mnt/library/homecloud/admin/files/ - -# Ports — change if 3000 / 8001 collide with other services on the host. -FRONTEND_PORT=3000 -BACKEND_PORT=8001 -REDIS_PORT=6379 - -# CORS — wildcard for local dev. Lock down for real deployments. -ALLOWED_ORIGINS=* - -# Logging + timezone. -LOG_LEVEL=INFO -TZ=UTC - -# Celery worker pools — split worker-light (IO) and worker-vision (CPU). -# Defaults target a 6-core / 16 GB host. -CELERY_LIGHT_CONCURRENCY=2 -CELERY_VISION_CONCURRENCY=5 diff --git a/.env.example b/.env.example index 9974db3..d139cbe 100644 --- a/.env.example +++ b/.env.example @@ -1,113 +1,70 @@ -# ───────────────────────────────────────────────────────────────────────────── -# Mulita / PhotoVault — example environment file +# Example environment file. Copy to `.env` and adjust. # -# Copy this file to `.env` and adjust the values for your setup. Every key -# below has a sensible default in docker-compose.yml, so you only need to -# uncomment the ones you actually want to change. -# ───────────────────────────────────────────────────────────────────────────── +# podman-compose --env-file .env \ +# -f docker-compose.yml -f docker-compose.podman.yml up -d # ── REQUIRED ───────────────────────────────────────────────────────────────── -# Host path to your photo library. The compose file mounts this at /photos -# inside the backend + worker containers. The backend creates a default -# source root pointing at /photos on first boot, so once this is set the -# library is scanned with zero further configuration. +# Host path to your photo library. PhotoPrism reads this in place and +# writes EXIF backwrites next to originals (when PP_ORIGINALS_MODE=rw). +PHOTO_DIRS=/mnt/library/homecloud/admin/files/ + +# Bootstrap admin password. The first PhotoPrism boot creates an `admin` +# account with this password. Rotate after first login from the UI. +PP_ADMIN_PASSWORD=please-change-me + +# MariaDB passwords. Generate with `openssl rand -hex 24`. +PP_DB_PASSWORD=please-change-me +PP_DB_ROOT_PASSWORD=please-change-me + + +# ── OPTIONAL ───────────────────────────────────────────────────────────────── + +# Loopback host port for PhotoPrism's API (and UI, if you tunnel to it). +# Vite proxies /api/v1/* here and the host-mode sidecar reaches it on +# localhost. Not published on the public interface. +PP_PORT=2342 + +# Site URL — used for share links, OIDC redirect URI, and reverse-proxy aware +# URL generation. Set to the public hostname once the proxy is in front. +PP_SITE_URL=http://localhost:2342/ + +# Auth mode — "password" for username/password (default), "public" for an +# unauthenticated kiosk mode (don't use this on a multi-user library). +PP_AUTH_MODE=password + +# Library mount mode. "rw" allows rename / folder mutations / EXIF backwrite; +# "ro" is safe-for-archives but disables those sidecar endpoints. Set in +# lockstep with PP_READONLY below. +PP_ORIGINALS_MODE=rw +PP_READONLY=false + +# UID/GID inside the PhotoPrism container. Set these to the host UID/GID that +# owns ${PHOTO_DIRS}. `id -u` and `id -g`. +PP_UID=1000 +PP_GID=1000 + + +# ── OIDC SSO (Authentik or equivalent) ─────────────────────────────────────── +# Leave blank to keep OIDC dormant. Fill in to enable the "Sign in with OIDC" +# button on the login page; OIDC_REGISTER=true auto-creates accounts at role +# `user` (override to `admin` to grant full access on first SSO login). # -# Examples: -# macOS / Linux: PHOTO_DIRS=/Users/you/Pictures -# Network share: PHOTO_DIRS=/mnt/nas/photos -# Windows (WSL): PHOTO_DIRS=/mnt/c/Users/you/Pictures -PHOTO_DIRS=./photos +# The compose file reads these and maps them to PhotoPrism's actual env-var +# names (PHOTOPRISM_OIDC_URI / _CLIENT / _SECRET / _PROVIDER) — see the +# comment in docker-compose.yml. The PhotoPrism callback URI is auto-derived +# from PP_SITE_URL; do not set it manually. + +# OIDC_PROVIDER_NAME=Authentik +# OIDC_ISSUER_URL=https://auth.example.com/application/o/photoprism/ +# OIDC_CLIENT_ID=... +# OIDC_CLIENT_SECRET=... +# OIDC_SCOPES=openid profile email +# OIDC_REGISTER=true +# OIDC_ROLE=user -# ── PORTS ──────────────────────────────────────────────────────────────────── +# ── LOGGING ────────────────────────────────────────────────────────────────── -# Host port the SPA is served on. Browse to http://:/. -FRONTEND_PORT=3000 - -# Host port for the backend API. Almost never needed directly — the frontend -# nginx proxies /api/ to the backend over the internal compose network. Kept -# exposed for debugging / curl. -BACKEND_PORT=8001 - -# Redis host port. Internal services reach Redis on its container name; this -# is just for local debugging. -REDIS_PORT=6379 - - -# ── AUTH ───────────────────────────────────────────────────────────────────── - -# Secret key used to sign JWT tokens. Generate a strong random value for -# production (e.g. `openssl rand -base64 32`). The default is a deterministic -# placeholder acceptable only for local/homelab use. -# SECRET_KEY=change-me-to-a-random-string - -# How long access and refresh tokens stay valid. Access tokens are short-lived -# and silently refreshed by the frontend; refresh tokens let a session survive -# across browser restarts. -# ACCESS_TOKEN_EXPIRE_MINUTES=60 -# REFRESH_TOKEN_EXPIRE_DAYS=30 - - -# ── CORS ───────────────────────────────────────────────────────────────────── - -# Comma-separated list of allowed origins for direct browser access to the -# backend. Same-origin requests through the nginx / vite proxy never trip -# CORS, so this only matters when something hits the backend port directly -# from a different origin (e.g. another machine, dev tools, a reverse proxy -# under a different hostname). -# -# Default "*" is permissive, fine for a single-user homelab. Lock it down in -# real deployments: -# ALLOWED_ORIGINS=https://photos.example.com -# ALLOWED_ORIGINS=https://photos.example.com,http://192.168.1.10:3000 -ALLOWED_ORIGINS=* - - -# ── LOGGING / TIMEZONE ─────────────────────────────────────────────────────── - -# Python log level for the backend and Celery worker. Bump to DEBUG when -# chasing scan / thumbnail issues. -LOG_LEVEL=INFO - -# Container timezone. Affects the timestamps in logs and the "added at" -# field on newly imported photos. Defaults to UTC. -# TZ=Europe/Berlin -# TZ=America/New_York -TZ=UTC - - -# ── WORKER CONCURRENCY ─────────────────────────────────────────────────────── -# -# The ingestion pipeline runs on two Celery worker services with separate -# concurrency knobs so heavy vision tasks can't starve cheap IO tasks: -# -# worker-light (default / high / low queues) -# Runs: scan, thumbnails, EXIF, pHash, duplicate regrouping. -# Mostly IO-bound — 2 prefork children keep a library streaming in. -# -# worker-vision (vision queue) -# Runs: embeddings, object detection, OCR, face extraction, content -# classification. Each prefork child loads ~2 GB of ONNX model weights, -# so set this to roughly (physical_cores − 1) and watch RAM. -# -# Defaults target a ~6 core / 16 GB host. Raise these, then -# docker compose up -d worker-light worker-vision -# to pick them up. Lower for a Pi; go higher on a workstation. -# -# The old `CELERYD_CONCURRENCY=N` single-worker variable is no longer -# read — delete it from your .env if it's set. -CELERY_LIGHT_CONCURRENCY=2 -CELERY_VISION_CONCURRENCY=5 - - -# ── INTERNAL (rarely overridden) ───────────────────────────────────────────── - -# These point at the in-compose Redis and the bind-mounted SQLite db. Override -# only if you're running Mulita without docker-compose or against an external -# Redis. -# REDIS_URL=redis://redis:6379 -# CELERY_BROKER_URL=redis://redis:6379 -# CELERY_RESULT_BACKEND=redis://redis:6379 -# DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db +PP_LOG_LEVEL=info diff --git a/.gitignore b/.gitignore index 2fe2140..5a0b223 100644 --- a/.gitignore +++ b/.gitignore @@ -61,9 +61,18 @@ build/ # Docker docker-compose.override.yml +# PhotoPrism state (sidecars, cache, thumbs, db backups) — regenerable. +/pp/storage/ +/pp/import/ + +# Sidecar runtime state (per-user marks etc.) — generated, not seed data. +/sidecar/data/ + +# Sidecar Go build output. +/sidecar/mule-sidecar + # Photos (for development) /photos/ # Thumbnails /thumbs/ -/trash/backend/yolov8n.pt diff --git a/README.md b/README.md index 866de8d..ccff048 100644 --- a/README.md +++ b/README.md @@ -1,230 +1,125 @@ -# Mulita - Self-Hosted Photo Management Application +# mule-image -A self-hosted, Docker-deployed photo management application inspired by Lightroom's workflow. Mulita provides a fast, keyboard-driven interface to browse, organize, tag, and manage your photo library. - -## Features - -- **Photo Organization**: Browse photos in a timeline view with virtual scrolling for performance -- **Thumbnail Generation**: Automatic thumbnail generation for all photo formats including RAW -- **Metadata Extraction**: Full EXIF/XMP metadata extraction and GPS mapping -- **Keyboard Shortcuts**: Lightroom-style keyboard navigation and actions -- **File Support**: JPEG, PNG, RAW formats (CR2, CR3, NEF, ARW, etc.), HEIC/HEIF, and videos -- **Heaps**: Temporary collections for organizing photos -- **Tags & Ratings**: Organize with tags, star ratings, and color labels — each with a card-grid browse view that drills into a full Timeline detail -- **Dark Mode**: Photography-optimized dark interface -- **Vision Pipeline**: YOLO object detection, OCR text extraction, CLIP embeddings for semantic search, InsightFace face detection and clustering -- **People View**: Browse identified people as cards, click to see all photos of a person -- **Map View**: Browse GPS-tagged photos on an interactive Leaflet map -- **Duplicate Detection**: Perceptual hash-based duplicate grouping with best-pick UI -- **Semantic Search**: Natural-language photo search powered by CLIP embeddings - -## Tech Stack - -### Backend -- Python 3.12 with FastAPI -- PostgreSQL + pgvector with SQLAlchemy (async) and Alembic migrations -- Celery + Redis for background tasks -- pyvips for fast thumbnail generation -- ExifTool for metadata extraction -- ONNX Runtime for vision models (YOLO, CLIP, InsightFace) - -### Frontend -- React 18 with TypeScript -- Vite for fast development -- TanStack Query for data fetching -- TanStack Virtual for virtualized scrolling -- Tailwind CSS for styling -- Zustand for state management - -## Quick Start - -### Prerequisites -- Docker and Docker Compose - -### Setup (one variable) - -1. Clone the repo: -```bash -git clone -cd muleimage -``` - -2. Copy the example env file and set **one** variable — the **host** - directory that contains your photo library. Whatever you point at - will become your library inside Mulita. - - ```bash - cp .env.example .env - # then edit .env and set PHOTO_DIRS: - # macOS / Linux: PHOTO_DIRS=/Users/you/Pictures - # Network share: PHOTO_DIRS=/mnt/nas/photos - # Windows (WSL): PHOTO_DIRS=/mnt/c/Users/you/Pictures - ``` - -3. Start the stack: -```bash -docker compose up -d -``` - -4. Open `http://localhost:3000`. On first boot Mulita will: - - Mount your `PHOTO_DIRS` at `/photos` inside the container - - Auto-create a source root called **Library** pointing at `/photos` - - Queue an initial scan, generate thumbnails, and start serving them - -You don't need to touch `mulita.yml` or the API to get started. - -### Configuration knobs - -Everything is environment-driven. `PHOTO_DIRS` is the only required -value; the rest have sensible defaults documented in `.env.example`: - -| Variable | Default | Notes | -|----------------------|---------|----------------------------------------------------| -| `PHOTO_DIRS` | — | **Required.** Host path mounted at `/photos`. | -| `FRONTEND_PORT` | `3000` | SPA host port. Bump if `3000` is taken. | -| `BACKEND_PORT` | `8001` | Direct backend port (debug only — frontend uses internal nginx proxy). | -| `REDIS_PORT` | `6379` | Redis host port (internal services don't need it). | -| `ALLOWED_ORIGINS` | `*` | Comma-separated CORS origins for direct backend access. Lock down for prod, e.g. `https://photos.example.com`. | -| `LOG_LEVEL` | `INFO` | Backend + worker log level. `DEBUG` for chasing scan issues. | -| `TZ` | `UTC` | Container timezone. Affects log timestamps and "added at". | -| `CELERYD_CONCURRENCY`| `4` | Parallel worker processes (scans, thumbs, metadata). Lower on a Pi, higher on a beefy host. | - -### Accessing from another machine - -The frontend talks to the backend through its bundled nginx, which -proxies `/api/` to the backend on the internal compose network. That -means requests are always **same-origin** as the page, so accessing -Mulita from another host works without any CORS dance: - -``` -http://:3000 -``` - -If you want to put it behind a reverse proxy at e.g. -`https://photos.your.tld`, set `ALLOWED_ORIGINS` to that host so the -backend's direct port (`BACKEND_PORT`) also accepts cross-origin -requests if anything bypasses the proxy. - -### How libraries are managed - -Mulita is **config-driven**: the host directory you mount via -`PHOTO_DIRS` becomes your library, and the backend automatically -registers it as a source root on startup. There is no UI for adding -or removing source roots — to change what Mulita scans, edit `.env` -(or `docker-compose.yml` for multi-mount setups) and restart the -stack. - -This keeps the model simple: **the docker mount IS the library**. -No two layers, no confusion about which view to use. - -### Changing or adding libraries - -To point at a different library: -1. Edit `PHOTO_DIRS` in `.env` -2. `docker compose down` -3. (Optional, for a clean slate) `docker volume rm muleimage_db_data muleimage_thumbs_data muleimage_proxies_data` -4. `docker compose up -d` - -The new library shows up automatically. Without step 3 the old -library's metadata stays in the DB and you'll see a warning at -startup that the old source root's path is missing on disk — -that's a hint to clean up. - -For multiple libraries, edit `docker-compose.yml` and add additional -mount lines: - -```yaml -volumes: - - ${PHOTO_DIRS}:/photos:rw - - /Volumes/Archive:/archive:rw # additional library -``` - -Each mounted directory will need a corresponding source root row in -the DB; today that means `POST /api/v1/folders` via curl, or wait -for the multi-mount auto-registration that's on the roadmap. - -### Read-only libraries - -The default mount is `:rw` because file operations (rename, move, -empty discard pile) need to mutate the filesystem. If you want a -strict read-only library — pointing at a network share, an -authoritative archive, etc. — flip `:rw` to `:ro` in -`docker-compose.yml`. Mulita will keep working for browsing, rating, -color labels, picks, heaps, and the (soft) discard flag, but the -following will return an OS error: - -- `PATCH /photos/{id}` with a new `filename` (rename) -- `POST /photos/move` (bulk move) -- `DELETE /discard/empty` (file unlinks) - -**Heads up**: with `:rw`, Mulita has full write access to whatever -host directory you mount. Treat the same way you would Lightroom's -catalog folder. +Self-hosted photo management built on top of [PhotoPrism][pp]. A SvelteKit +frontend ([`web/`](web/)) plus a small Go service ([`sidecar/`](sidecar/)) +fill in the keyboard-driven UI and the file/folder/mark endpoints +PhotoPrism's REST API does not expose. PhotoPrism itself handles +indexing, originals, thumbnails, and the database; we never re-implement +those. ## Architecture -The application consists of 5 Docker services: - -- **frontend**: React SPA served by Nginx -- **backend**: FastAPI REST API -- **worker**: Celery workers for background tasks (thumbnails, metadata, vision pipeline) -- **redis**: Message broker for Celery -- **db**: PostgreSQL with pgvector extension (for CLIP/face embeddings) - -## Keyboard Shortcuts - -| Key | Action | -|-----|--------| -| `←` `→` `↑` `↓` | Navigate photos | -| `Space` | Quick preview | -| `Enter` | Open loupe view | -| `T` | Add to active heap | -| `1-5` | Set star rating | -| `Tab` | Toggle left sidebar | -| `I` | Toggle metadata panel | -| `G` | Grid view | -| `E` | Loupe view | -| `Delete` | Move to trash | - -## Development - -### Backend Development -```bash -cd backend -pip install -r requirements.txt -uvicorn app.main:app --reload +```text +┌──────────────────┐ /api/v1/* ┌──────────────┐ +│ SvelteKit web/ │ ───────────────▶ │ photoprism │ ──▶ mariadb +│ (Vite : 5173) │ /api/sidecar/* │ :2342 │ +│ │ ─────────┐ └──────────────┘ +└──────────────────┘ ▼ + ┌──────────────┐ + │ sidecar │ ──▶ mariadb (mule_sidecar.*) + │ :8000 │ ──▶ originals FS (rename / folders / dups) + └──────────────┘ ``` -### Frontend Development +Three compose services — `mariadb`, `photoprism`, `sidecar` — plus the +SvelteKit `web/` app served separately. PhotoPrism's port `2342` is +**bound to `127.0.0.1` only**; it isn't a user-facing surface. The +SvelteKit app is. + +What the sidecar adds on top of PhotoPrism (full list in +[`sidecar/README.md`](sidecar/README.md)): + +- Per-photo marks (rating + color) persisted to `mule_sidecar.marks` +- File rename + folder create/rename/delete with PhotoPrism reindex +- Heap (album) → folder conversion +- Perceptual-hash duplicate scan + archive + +## Quick start + ```bash -cd frontend +cp .env.example .env +# edit .env: set PHOTO_DIRS to the host path holding your library +# rotate PP_ADMIN_PASSWORD, PP_DB_PASSWORD, PP_DB_ROOT_PASSWORD +# before any non-local deployment. + +podman-compose --env-file .env \ + -f docker-compose.yml \ + -f docker-compose.podman.yml \ + up -d +``` + +Then serve the frontend. For local use the simplest path is the Vite +dev server: + +```bash +cd web npm install npm run dev +# open http://localhost:5173 ``` +For a static deployment, `npm run build` produces a bundle under +`web/build/` that any static file host (nginx, Caddy, GitHub Pages-style) +can serve. Reverse-proxy `/api/v1/*` to `http://127.0.0.1:2342` and +`/api/sidecar/*` to `http://127.0.0.1:8000`. + +PhotoPrism's own UI is still reachable from the host at +`http://127.0.0.1:2342` if you need admin features (user management, +settings) — set up an SSH tunnel from your laptop if the server is +remote. + ## Configuration -Source roots are managed by the UI / API (the database owns them). Edit -`mulita.yml` to configure operational settings only: +All knobs live in [`.env.example`](.env.example). The required ones: -- Thumbnail sizes, quality, and format -- Scanner behaviour (watch, batch size, initial scan) -- Performance tuning (concurrency, cache TTLs, DB pool) +| Variable | Notes | +|----------------------|-----------------------------------------------------------------------------------------------| +| `PHOTO_DIRS` | Host path mounted at `/photoprism/originals`. The library. | +| `PP_ADMIN_PASSWORD` | First-boot admin password. Rotate. | +| `PP_DB_PASSWORD` | MariaDB password for the `photoprism` user. Rotate. | +| `PP_DB_ROOT_PASSWORD`| MariaDB root password. Rotate. | +| `PP_UID` / `PP_GID` | Host UID/GID that owns `PHOTO_DIRS`. PhotoPrism + sidecar drop to this user inside. | +| `PP_PORT` | Loopback host port for PhotoPrism (default `2342`). | +| `PP_ORIGINALS_MODE` | `rw` (default) or `ro` — see [Read-only libraries](#read-only-libraries). | +| `SIDECAR_PORT` | Loopback host port for the sidecar (default `8000`). | -## Performance +Sidecar-specific env (DB DSN, `USER_BASEPATHS`, etc.) is documented in +[`sidecar/README.md`](sidecar/README.md). -- Handles 100,000+ photos efficiently -- Virtual scrolling for smooth timeline navigation -- Thumbnail generation at 10+ photos/second -- PostgreSQL full-text search with tsvector indexing -- pgvector for fast nearest-neighbor embedding search +## Read-only libraries -## Future Features +The default originals mount is `:rw` because file operations (rename, +folder mutations, duplicate archive, heap convert) need to mutate the +filesystem. To run against a read-only archive, set +`PP_ORIGINALS_MODE=ro` in `.env`. Browsing, marks, ratings, and color +labels still work; the following sidecar endpoints return an OS error: -- Smart albums (auto-populated by saved filters) -- Export presets -- Multi-user support +- `POST /api/sidecar/files/:uid/rename` +- `POST /api/sidecar/folders` / `:rel/rename` / `DELETE /:rel` +- `POST /api/sidecar/albums/:uid/convert` +- `POST /api/sidecar/duplicates/archive` -## License +PhotoPrism's `PHOTOPRISM_READONLY` is controlled separately by +`PP_READONLY` and gates its own backwrite / import paths. -MIT \ No newline at end of file +## Dev iteration loop + +For fast iteration on the sidecar without rebuilding its image on every +change, run it as a host process — bring up just `mariadb` and +`photoprism` from compose, then build and run the Go binary locally. +Full instructions in [`sidecar/README.md`](sidecar/README.md#dev-iteration-loop-host-build). + +## Layout + +```text +. +├── docker-compose.yml base stack: mariadb + photoprism + sidecar +├── docker-compose.podman.yml rootless-podman overlay (keep-id mapping) +├── .env.example required env vars (copy to .env) +├── mariadb/init/ first-boot SQL: creates mule_sidecar DB + user +├── pp/ PhotoPrism bind-mounted state (storage, import) +├── sidecar/ Go service — see sidecar/README.md +└── web/ SvelteKit frontend +``` + +[pp]: https://photoprism.app/ diff --git a/backend/Dockerfile b/backend/Dockerfile deleted file mode 100644 index f9dbe1b..0000000 --- a/backend/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# syntax=docker/dockerfile:1.7 -FROM python:3.12-slim - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - # Build dependencies - gcc \ - g++ \ - make \ - # Image processing libraries - libvips42 \ - libvips-dev \ - # ExifTool for metadata extraction - libimage-exiftool-perl \ - # FFmpeg for video processing - ffmpeg \ - # Git for some Python packages - git \ - # PostgreSQL client (for potential future use) - postgresql-client \ - # Clean up - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -COPY requirements.txt . -# buildkit cache mount keeps pip's download cache on disk across builds -# so even when this layer is invalidated, wheels are reused locally. -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r requirements.txt - -# Copy application code -COPY . . - -# Create necessary directories -RUN mkdir -p /data/thumbs /data/db /data/proxies /data/models /app/config - -# Expose port -EXPOSE 8000 - -# Run the application -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] \ No newline at end of file diff --git a/backend/alembic.ini b/backend/alembic.ini deleted file mode 100644 index 48fdc2a..0000000 --- a/backend/alembic.ini +++ /dev/null @@ -1,48 +0,0 @@ -# Alembic configuration for PhotoVault. -# -# The actual database URL is loaded at runtime by alembic/env.py from the -# DATABASE_URL environment variable (with the async driver suffix stripped). -# The placeholder below is only used for `alembic revision --autogenerate` -# when no env var is set. - -[alembic] -script_location = alembic -prepend_sys_path = . -version_path_separator = os -sqlalchemy.url = postgresql+psycopg2://mulita:mulita@localhost:5432/mulita - -[post_write_hooks] - -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARNING -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARNING -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/backend/alembic/env.py b/backend/alembic/env.py deleted file mode 100644 index 285044b..0000000 --- a/backend/alembic/env.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Alembic environment for PhotoVault. - -Pulls DATABASE_URL from the environment so the same migrations work in -docker compose and locally. Strips the async driver suffix because Alembic -runs synchronously via psycopg2. - -Future-migration note ---------------------- -Fresh installs run `Base.metadata.create_all` in `app.database.init_db` -*before* migrations would normally apply, so any migration that adds a -column / index / table to an object the model already declares will see -that object already present. Write migrations defensively: - - op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS new_col TEXT") - op.execute("CREATE INDEX IF NOT EXISTS ix_foo ON foo(bar)") - -For brand-new tables that the model also declares, the same applies — use -`op.execute("CREATE TABLE IF NOT EXISTS ...")` or check first. -""" -from logging.config import fileConfig -import os -import sys -from pathlib import Path - -from sqlalchemy import engine_from_config, pool -from alembic import context - -# Make `app` importable from this script. -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from app.database import Base # noqa: E402 -# Import all models so they're registered on Base.metadata for autogenerate. -from app.models import ( # noqa: E402, F401 - Photo, - Folder, - SourceRoot, - Tag, - Heap, - HeapPhoto, -) - -config = context.config - -# Resolve DATABASE_URL from env. Strip async driver suffixes — Alembic -# uses sync drivers. -db_url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url") -if db_url: - if "+asyncpg" in db_url: - db_url = db_url.replace("+asyncpg", "+psycopg2") - elif db_url.startswith("postgresql://"): - db_url = db_url.replace("postgresql://", "postgresql+psycopg2://", 1) - elif "+aiosqlite" in db_url: - db_url = db_url.replace("+aiosqlite", "") - config.set_main_option("sqlalchemy.url", db_url) - -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -target_metadata = Base.metadata - - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode (emit SQL only).""" - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - """Run migrations against a live database.""" - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata, - ) - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako deleted file mode 100644 index fbc4b07..0000000 --- a/backend/alembic/script.py.mako +++ /dev/null @@ -1,26 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} - - -def upgrade() -> None: - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - ${downgrades if downgrades else "pass"} diff --git a/backend/alembic/versions/0001_baseline.py b/backend/alembic/versions/0001_baseline.py deleted file mode 100644 index 2ee73a7..0000000 --- a/backend/alembic/versions/0001_baseline.py +++ /dev/null @@ -1,27 +0,0 @@ -"""baseline (empty) - -Revision ID: 0001_baseline -Revises: -Create Date: 2026-04-10 - -The current schema is created by SQLAlchemy `Base.metadata.create_all` in -`app.database.init_db()` on first boot. Alembic only owns deltas from -PR3 onward. This baseline is intentionally empty so `alembic upgrade head` -on a fresh DB simply creates the `alembic_version` table and stamps it. -""" -from typing import Sequence, Union - - -# revision identifiers, used by Alembic. -revision: str = "0001_baseline" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - pass - - -def downgrade() -> None: - pass diff --git a/backend/alembic/versions/0002_extend_tags_for_vision.py b/backend/alembic/versions/0002_extend_tags_for_vision.py deleted file mode 100644 index 238ac78..0000000 --- a/backend/alembic/versions/0002_extend_tags_for_vision.py +++ /dev/null @@ -1,85 +0,0 @@ -"""extend tags for vision pipeline - -Revision ID: 0002_extend_tags -Revises: 0001_baseline -Create Date: 2026-04-10 - -Add kind, source, representative_photo_id to tags table. -Add confidence, bbox, source to photo_tags association. -Switch uniqueness from (name) to (name, kind). -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import JSONB - -revision: str = "0002_extend_tags" -down_revision: Union[str, None] = "0001_baseline" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ── tags table ──────────────────────────────────────────────────── - op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS kind VARCHAR NOT NULL DEFAULT 'user'") - op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS source VARCHAR") - op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS representative_photo_id VARCHAR REFERENCES photos(id) ON DELETE SET NULL") - - # Create index on kind for filtering - op.execute("CREATE INDEX IF NOT EXISTS ix_tags_kind ON tags(kind)") - - # Drop old unique constraint on name (if it exists) and add (name, kind). - # SQLAlchemy create_all may have created either — handle both cases. - op.execute(""" - DO $$ - BEGIN - -- Drop the old single-column unique index/constraint if present. - IF EXISTS ( - SELECT 1 FROM pg_indexes - WHERE tablename = 'tags' AND indexname = 'ix_tags_name' - ) THEN - DROP INDEX ix_tags_name; - END IF; - - -- Some SQLAlchemy versions create a unique constraint directly. - IF EXISTS ( - SELECT 1 FROM information_schema.table_constraints - WHERE table_name = 'tags' AND constraint_name = 'tags_name_key' - ) THEN - ALTER TABLE tags DROP CONSTRAINT tags_name_key; - END IF; - END $$; - """) - - op.execute(""" - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_constraint WHERE conname = 'uq_tags_name_kind' - ) THEN - ALTER TABLE tags ADD CONSTRAINT uq_tags_name_kind UNIQUE (name, kind); - END IF; - END $$; - """) - - # ── photo_tags table ────────────────────────────────────────────── - op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS confidence FLOAT") - op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS bbox JSONB") - op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS source VARCHAR") - - -def downgrade() -> None: - # photo_tags columns - op.drop_column("photo_tags", "source") - op.drop_column("photo_tags", "bbox") - op.drop_column("photo_tags", "confidence") - - # tags: restore old unique constraint - op.execute("ALTER TABLE tags DROP CONSTRAINT IF EXISTS uq_tags_name_kind") - op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tags_name ON tags(name)") - - # tags columns - op.drop_column("tags", "representative_photo_id") - op.drop_column("tags", "source") - op.drop_column("tags", "kind") diff --git a/backend/alembic/versions/0003_pgvector_embeddings.py b/backend/alembic/versions/0003_pgvector_embeddings.py deleted file mode 100644 index e17a311..0000000 --- a/backend/alembic/versions/0003_pgvector_embeddings.py +++ /dev/null @@ -1,52 +0,0 @@ -"""pgvector embeddings - -Revision ID: 0003_pgvector_embeddings -Revises: 0002_extend_tags -Create Date: 2026-04-10 - -Rewrite the embeddings table to use pgvector Vector(512) instead of -LargeBinary. Add composite PK (photo_id, model), created_at, and -HNSW index on vector column. -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - -revision: str = "0003_pgvector_embeddings" -down_revision: Union[str, None] = "0002_extend_tags" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Drop the old placeholder table and recreate with pgvector types. - # No data to preserve — it was never populated. - op.execute("DROP TABLE IF EXISTS embeddings") - op.execute(""" - CREATE TABLE embeddings ( - photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE, - model VARCHAR(64) NOT NULL, - vector vector(512), - created_at TIMESTAMPTZ DEFAULT now(), - PRIMARY KEY (photo_id, model) - ) - """) - # HNSW index for cosine similarity search. - # Defer creation on large backfills — drop and recreate afterward. - op.execute(""" - CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw - ON embeddings USING hnsw (vector vector_cosine_ops) - """) - - -def downgrade() -> None: - op.execute("DROP TABLE IF EXISTS embeddings") - op.execute(""" - CREATE TABLE embeddings ( - photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE, - model VARCHAR, - vector BYTEA, - PRIMARY KEY (photo_id) - ) - """) diff --git a/backend/alembic/versions/0004_ocr_text_and_fts.py b/backend/alembic/versions/0004_ocr_text_and_fts.py deleted file mode 100644 index a8989dd..0000000 --- a/backend/alembic/versions/0004_ocr_text_and_fts.py +++ /dev/null @@ -1,82 +0,0 @@ -"""ocr_text table and Postgres FTS - -Revision ID: 0004_ocr_fts -Revises: 0003_pgvector_embeddings -Create Date: 2026-04-10 - -Create ocr_text table for storing OCR results. Add a tsvector column -to photos for unified full-text search (filename + user_title + -user_notes) with a GIN index. OCR text is rolled up into a materialized -view or joined at query time. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0004_ocr_fts" -down_revision: Union[str, None] = "0003_pgvector_embeddings" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ── ocr_text table ──────────────────────────────────────────────── - op.execute(""" - CREATE TABLE IF NOT EXISTS ocr_text ( - id VARCHAR PRIMARY KEY, - photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE, - text TEXT NOT NULL, - language VARCHAR(8) DEFAULT '', - confidence FLOAT, - bbox JSONB, - created_at TIMESTAMPTZ DEFAULT now() - ) - """) - op.execute("CREATE INDEX IF NOT EXISTS ix_ocr_text_photo_id ON ocr_text(photo_id)") - - # ── tsvector column on photos ───────────────────────────────────── - op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS search_vector tsvector") - op.execute("CREATE INDEX IF NOT EXISTS ix_photos_search_vector ON photos USING GIN (search_vector)") - - # Trigger to auto-update search_vector on INSERT/UPDATE - op.execute(""" - CREATE OR REPLACE FUNCTION photos_search_vector_update() RETURNS trigger AS $$ - BEGIN - NEW.search_vector := - setweight(to_tsvector('english', coalesce(NEW.filename, '')), 'A') || - setweight(to_tsvector('english', coalesce(NEW.user_title, '')), 'A') || - setweight(to_tsvector('english', coalesce(NEW.user_notes, '')), 'B'); - RETURN NEW; - END - $$ LANGUAGE plpgsql; - """) - op.execute(""" - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_trigger WHERE tgname = 'photos_search_vector_trigger' - ) THEN - CREATE TRIGGER photos_search_vector_trigger - BEFORE INSERT OR UPDATE OF filename, user_title, user_notes - ON photos - FOR EACH ROW - EXECUTE FUNCTION photos_search_vector_update(); - END IF; - END $$; - """) - - # Backfill existing rows - op.execute(""" - UPDATE photos SET search_vector = - setweight(to_tsvector('english', coalesce(filename, '')), 'A') || - setweight(to_tsvector('english', coalesce(user_title, '')), 'A') || - setweight(to_tsvector('english', coalesce(user_notes, '')), 'B') - """) - - -def downgrade() -> None: - op.execute("DROP TRIGGER IF EXISTS photos_search_vector_trigger ON photos") - op.execute("DROP FUNCTION IF EXISTS photos_search_vector_update()") - op.execute("DROP INDEX IF EXISTS ix_photos_search_vector") - op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS search_vector") - op.execute("DROP TABLE IF EXISTS ocr_text") diff --git a/backend/alembic/versions/0005_face_embeddings.py b/backend/alembic/versions/0005_face_embeddings.py deleted file mode 100644 index 05a19cc..0000000 --- a/backend/alembic/versions/0005_face_embeddings.py +++ /dev/null @@ -1,41 +0,0 @@ -"""face_embeddings table - -Revision ID: 0005_face_embeddings -Revises: 0004_ocr_fts -Create Date: 2026-04-10 - -Create face_embeddings table with pgvector Vector(128) for SFace -recognition embeddings and HNSW index. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0005_face_embeddings" -down_revision: Union[str, None] = "0004_ocr_fts" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute(""" - CREATE TABLE IF NOT EXISTS face_embeddings ( - id VARCHAR PRIMARY KEY, - photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE, - bbox JSONB, - vector vector(128), - cluster_id VARCHAR REFERENCES tags(id) ON DELETE SET NULL, - quality FLOAT, - created_at TIMESTAMPTZ DEFAULT now() - ) - """) - op.execute("CREATE INDEX IF NOT EXISTS ix_face_embeddings_photo_id ON face_embeddings(photo_id)") - op.execute("CREATE INDEX IF NOT EXISTS ix_face_embeddings_cluster_id ON face_embeddings(cluster_id)") - op.execute(""" - CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw - ON face_embeddings USING hnsw (vector vector_cosine_ops) - """) - - -def downgrade() -> None: - op.execute("DROP TABLE IF EXISTS face_embeddings") diff --git a/backend/alembic/versions/0006_face_embeddings_512d.py b/backend/alembic/versions/0006_face_embeddings_512d.py deleted file mode 100644 index b8eaf93..0000000 --- a/backend/alembic/versions/0006_face_embeddings_512d.py +++ /dev/null @@ -1,39 +0,0 @@ -"""face_embeddings vector 128 -> 512 - -Revision ID: 0006_face_512d -Revises: 0005_face_embeddings -Create Date: 2026-04-10 - -Resize face_embeddings.vector from Vector(128) to Vector(512) for -ArcFace embeddings (InsightFace). Drops existing data and HNSW index, -recreates both. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0006_face_512d" -down_revision: Union[str, None] = "0005_face_embeddings" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Drop index, truncate (old 128-d vectors are incompatible), resize - op.execute("DROP INDEX IF EXISTS ix_face_embeddings_vector_hnsw") - op.execute("DELETE FROM face_embeddings") - op.execute("ALTER TABLE face_embeddings ALTER COLUMN vector TYPE vector(512)") - op.execute(""" - CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw - ON face_embeddings USING hnsw (vector vector_cosine_ops) - """) - - -def downgrade() -> None: - op.execute("DROP INDEX IF EXISTS ix_face_embeddings_vector_hnsw") - op.execute("DELETE FROM face_embeddings") - op.execute("ALTER TABLE face_embeddings ALTER COLUMN vector TYPE vector(128)") - op.execute(""" - CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw - ON face_embeddings USING hnsw (vector vector_cosine_ops) - """) diff --git a/backend/alembic/versions/0007_folder_hidden.py b/backend/alembic/versions/0007_folder_hidden.py deleted file mode 100644 index 78aa9d9..0000000 --- a/backend/alembic/versions/0007_folder_hidden.py +++ /dev/null @@ -1,67 +0,0 @@ -"""folders + photos is_hidden flag - -Revision ID: 0007_folder_hidden -Revises: 0006_face_512d -Create Date: 2026-04-11 - -Adds an "exclude from cross-cutting views" flag: - - folders.is_hidden — user-toggled on a folder or source root. When - true, photos in that subtree are hidden from - library-wide views (All Photos, Map, Tags, - People, Search, Duplicates, sidebar counts) but - remain indexed and visible when the user - navigates into the folder directly. - - photos.is_hidden — denormalized: true iff any ancestor folder in - the photo's folder chain has is_hidden=true. - Kept as a real column (rather than a recursive - query per read) because the filter runs on - essentially every photo query in the app, and - the toggle operation that recomputes it is - rare. Indexed so `WHERE NOT is_hidden` doesn't - fall off the rating/taken_at indexes. - -Both columns default to false so existing rows need no backfill. -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - -revision: str = "0007_folder_hidden" -down_revision: Union[str, None] = "0006_face_512d" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "folders", - sa.Column( - "is_hidden", - sa.Boolean(), - nullable=False, - server_default=sa.false(), - ), - ) - op.add_column( - "photos", - sa.Column( - "is_hidden", - sa.Boolean(), - nullable=False, - server_default=sa.false(), - ), - ) - op.create_index( - "ix_photos_is_hidden", - "photos", - ["is_hidden"], - ) - - -def downgrade() -> None: - op.drop_index("ix_photos_is_hidden", table_name="photos") - op.drop_column("photos", "is_hidden") - op.drop_column("folders", "is_hidden") diff --git a/backend/alembic/versions/0008_photos_date_warning.py b/backend/alembic/versions/0008_photos_date_warning.py deleted file mode 100644 index 8f9e183..0000000 --- a/backend/alembic/versions/0008_photos_date_warning.py +++ /dev/null @@ -1,49 +0,0 @@ -"""photos has_date_warning flag - -Revision ID: 0008_photos_date_warning -Revises: 0007_folder_hidden -Create Date: 2026-04-11 - -Adds `photos.has_date_warning` — a denormalized boolean that's true when -the scanner's folder/filename date guesser disagrees with the stored -taken_at by more than 24h (or taken_at is missing and the path would -provide a date). Surfacing this as a real column means the filter bar -can restrict the timeline to suspicious photos without the client -recomputing the heuristic for every row. - -Indexed because the filter is meant to run on top of the existing -taken_at / folder queries that dominate the timeline, and we want the -partial `WHERE has_date_warning` scan to stay cheap as the library -grows. -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - -revision: str = "0008_photos_date_warning" -down_revision: Union[str, None] = "0007_folder_hidden" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "photos", - sa.Column( - "has_date_warning", - sa.Boolean(), - nullable=False, - server_default=sa.false(), - ), - ) - op.create_index( - "ix_photos_has_date_warning", - "photos", - ["has_date_warning"], - ) - - -def downgrade() -> None: - op.drop_index("ix_photos_has_date_warning", table_name="photos") - op.drop_column("photos", "has_date_warning") diff --git a/backend/alembic/versions/0009_users_and_auth.py b/backend/alembic/versions/0009_users_and_auth.py deleted file mode 100644 index e35a7f3..0000000 --- a/backend/alembic/versions/0009_users_and_auth.py +++ /dev/null @@ -1,144 +0,0 @@ -"""users table and user_id foreign keys - -Revision ID: 0009_users_and_auth -Revises: 0008_photos_date_warning -Create Date: 2026-04-12 - -Introduces multi-user support: - 1. Creates the `users` table. - 2. Adds `user_id` FK columns to photos, folders, source_roots, heaps, tags. - 3. For existing installs: creates a default admin user and assigns all - existing rows to that user. The generated password is printed to the - backend logs — the admin should change it on first login. - 4. Replaces the unique constraint on tags (name, kind) with - (name, kind, user_id) so each user can have their own tags. -""" -from typing import Sequence, Union -import uuid -import secrets - -from alembic import op -import sqlalchemy as sa - -revision: str = "0009_users_and_auth" -down_revision: Union[str, None] = "0008_photos_date_warning" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - conn = op.get_bind() - - # 1. Create users table (IF NOT EXISTS — safe on fresh installs where - # init_db's create_all has already laid down the schema). - conn.execute(sa.text(""" - CREATE TABLE IF NOT EXISTS users ( - id VARCHAR NOT NULL PRIMARY KEY, - username VARCHAR(50) NOT NULL UNIQUE, - email VARCHAR UNIQUE, - hashed_password VARCHAR NOT NULL, - role VARCHAR NOT NULL DEFAULT 'user', - is_active BOOLEAN NOT NULL DEFAULT true, - created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), - media_path VARCHAR NOT NULL - ) - """)) - conn.execute(sa.text( - "CREATE INDEX IF NOT EXISTS ix_users_username ON users (username)" - )) - - # 2. Add user_id columns (nullable initially for the data migration) - for table in ("photos", "folders", "source_roots", "heaps", "tags"): - conn.execute(sa.text( - f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS user_id VARCHAR" - )) - conn.execute(sa.text( - f"CREATE INDEX IF NOT EXISTS ix_{table}_user_id ON {table} (user_id)" - )) - # FK — check if it already exists before adding - fk_name = f"fk_{table}_user_id" - fk_exists = conn.execute(sa.text( - "SELECT 1 FROM information_schema.table_constraints " - "WHERE constraint_name = :name AND table_name = :tbl" - ), {"name": fk_name, "tbl": table}).scalar() - if not fk_exists: - conn.execute(sa.text( - f"ALTER TABLE {table} ADD CONSTRAINT {fk_name} " - f"FOREIGN KEY (user_id) REFERENCES users(id)" - )) - - # 3. Data migration: if rows exist, create a default admin and assign - conn = op.get_bind() - photo_count = conn.execute(sa.text("SELECT COUNT(*) FROM photos")).scalar() - - if photo_count > 0: - admin_id = str(uuid.uuid4()) - generated_password = secrets.token_urlsafe(16) - - # Hash the password using passlib at migration time - from passlib.context import CryptContext - pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") - hashed = pwd_context.hash(generated_password) - - # Every user gets a subfolder — including the migrated admin. - conn.execute( - sa.text( - "INSERT INTO users (id, username, hashed_password, role, media_path) " - "VALUES (:id, :username, :hashed, :role, :media_path)" - ), - { - "id": admin_id, - "username": "admin", - "hashed": hashed, - "role": "admin", - "media_path": "/photos/admin", - }, - ) - - # Assign all existing rows to the default admin - for table in ("photos", "folders", "source_roots", "heaps", "tags"): - conn.execute( - sa.text(f"UPDATE {table} SET user_id = :uid WHERE user_id IS NULL"), - {"uid": admin_id}, - ) - - import logging - logger = logging.getLogger("alembic.migration") - logger.warning( - f"=== MIGRATION 0009 === Default admin created. " - f"Username: admin | Password: {generated_password} | " - f"Change this password on first login!" - ) - - # 4. Replace tag unique constraint to include user_id - # Check whether the old constraint exists before trying to drop it - # (on fresh installs create_all creates the new constraint directly). - old_uq_exists = conn.execute(sa.text( - "SELECT 1 FROM information_schema.table_constraints " - "WHERE constraint_name = 'uq_tags_name_kind' AND table_name = 'tags'" - )).scalar() - if old_uq_exists: - op.drop_constraint("uq_tags_name_kind", "tags", type_="unique") - - new_uq_exists = conn.execute(sa.text( - "SELECT 1 FROM information_schema.table_constraints " - "WHERE constraint_name = 'uq_tags_name_kind_user' AND table_name = 'tags'" - )).scalar() - if not new_uq_exists: - op.create_unique_constraint("uq_tags_name_kind_user", "tags", ["name", "kind", "user_id"]) - - -def downgrade() -> None: - # Reverse the tag constraint - op.drop_constraint("uq_tags_name_kind_user", "tags", type_="unique") - op.create_unique_constraint("uq_tags_name_kind", "tags", ["name", "kind"]) - - # Drop user_id columns and FKs - for table in ("photos", "folders", "source_roots", "heaps", "tags"): - op.drop_constraint(f"fk_{table}_user_id", table, type_="foreignkey") - op.drop_index(f"ix_{table}_user_id", table_name=table) - op.drop_column(table, "user_id") - - # Drop users table - op.drop_index("ix_users_username", table_name="users") - op.drop_table("users") diff --git a/backend/alembic/versions/0010_embeddings_768d_siglip2.py b/backend/alembic/versions/0010_embeddings_768d_siglip2.py deleted file mode 100644 index 0a9152c..0000000 --- a/backend/alembic/versions/0010_embeddings_768d_siglip2.py +++ /dev/null @@ -1,39 +0,0 @@ -"""embeddings vector 512 -> 768 - -Revision ID: 0010_embeddings_768d -Revises: 0009_users_and_auth -Create Date: 2026-04-12 - -Resize embeddings.vector from Vector(512) to Vector(768) for -SigLIP2 ViT-B/16 embeddings. Drops existing data and HNSW index, -recreates with the new dimension. Existing embeddings will be -regenerated by the vision backfill task. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0010_embeddings_768d" -down_revision: Union[str, None] = "0009_users_and_auth" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute("DROP INDEX IF EXISTS ix_embeddings_vector_hnsw") - op.execute("DELETE FROM embeddings") - op.execute("ALTER TABLE embeddings ALTER COLUMN vector TYPE vector(768)") - op.execute(""" - CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw - ON embeddings USING hnsw (vector vector_cosine_ops) - """) - - -def downgrade() -> None: - op.execute("DROP INDEX IF EXISTS ix_embeddings_vector_hnsw") - op.execute("DELETE FROM embeddings") - op.execute("ALTER TABLE embeddings ALTER COLUMN vector TYPE vector(512)") - op.execute(""" - CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw - ON embeddings USING hnsw (vector vector_cosine_ops) - """) diff --git a/backend/alembic/versions/0011_sharing.py b/backend/alembic/versions/0011_sharing.py deleted file mode 100644 index c60ca16..0000000 --- a/backend/alembic/versions/0011_sharing.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Add sharing tables for heaps and folders - -Revision ID: 0011_sharing -Revises: 0010_embeddings_768d -Create Date: 2026-04-13 - -Adds heap_shares and folder_shares tables so users can share -heaps and folders with other users (read or read+write). -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - -revision: str = "0011_sharing" -down_revision: Union[str, None] = "0010_embeddings_768d" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute(""" - CREATE TABLE IF NOT EXISTS heap_shares ( - id VARCHAR NOT NULL PRIMARY KEY, - heap_id VARCHAR NOT NULL REFERENCES heaps(id) ON DELETE CASCADE, - owner_id VARCHAR NOT NULL REFERENCES users(id), - shared_with_id VARCHAR NOT NULL REFERENCES users(id), - permission VARCHAR NOT NULL DEFAULT 'read', - created_at TIMESTAMP DEFAULT now(), - CONSTRAINT uq_heap_share UNIQUE (heap_id, shared_with_id) - ) - """) - op.execute("CREATE INDEX IF NOT EXISTS ix_heap_shares_shared_with ON heap_shares(shared_with_id)") - op.execute("CREATE INDEX IF NOT EXISTS ix_heap_shares_heap_id ON heap_shares(heap_id)") - - op.execute(""" - CREATE TABLE IF NOT EXISTS folder_shares ( - id VARCHAR NOT NULL PRIMARY KEY, - folder_id VARCHAR NOT NULL, - folder_type VARCHAR NOT NULL DEFAULT 'folder', - owner_id VARCHAR NOT NULL REFERENCES users(id), - shared_with_id VARCHAR NOT NULL REFERENCES users(id), - permission VARCHAR NOT NULL DEFAULT 'read', - created_at TIMESTAMP DEFAULT now(), - CONSTRAINT uq_folder_share UNIQUE (folder_id, shared_with_id) - ) - """) - op.execute("CREATE INDEX IF NOT EXISTS ix_folder_shares_shared_with ON folder_shares(shared_with_id)") - op.execute("CREATE INDEX IF NOT EXISTS ix_folder_shares_folder_id ON folder_shares(folder_id)") - - -def downgrade() -> None: - op.execute("DROP TABLE IF EXISTS folder_shares") - op.execute("DROP TABLE IF EXISTS heap_shares") diff --git a/backend/alembic/versions/0012_strip_ai_pipeline.py b/backend/alembic/versions/0012_strip_ai_pipeline.py deleted file mode 100644 index 98a9edd..0000000 --- a/backend/alembic/versions/0012_strip_ai_pipeline.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Strip AI pipeline to binary classifier only - -Revision ID: 0012_strip_ai -Revises: 0011_sharing -Create Date: 2026-04-14 - -Removes face recognition, OCR, object detection, and semantic embeddings. -The remaining AI is a single binary 'photography' vs 'other' classifier -whose output feeds Tag(kind='content_type') and a new Photo.needs_review -flag. - -Drops: embeddings, face_embeddings, ocr_text tables. -Drops: photo_tags rows produced by 'vision:yolov8n' and 'vision:sface'. -Drops: tags with kind IN ('object','scene','face_cluster'). -Drops: tags.representative_photo_id column. -Adds: photos.needs_review (bool, default false) + partial index. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0012_strip_ai" -down_revision: Union[str, None] = "0011_sharing" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Drop dropped-AI tables. CASCADE clears any lingering FKs/indices. - op.execute("DROP TABLE IF EXISTS embeddings CASCADE") - op.execute("DROP TABLE IF EXISTS face_embeddings CASCADE") - op.execute("DROP TABLE IF EXISTS ocr_text CASCADE") - - # Clear ML-produced photo_tags rows and their parent tags. - op.execute( - "DELETE FROM photo_tags WHERE source IN ('vision:yolov8n','vision:sface')" - ) - op.execute( - "DELETE FROM tags WHERE kind IN ('object','scene','face_cluster')" - ) - - # Drop the face-cluster representative column. - op.execute("ALTER TABLE tags DROP COLUMN IF EXISTS representative_photo_id") - - # Add the needs_review flag. - op.execute( - "ALTER TABLE photos ADD COLUMN IF NOT EXISTS needs_review " - "BOOLEAN NOT NULL DEFAULT false" - ) - op.execute( - "CREATE INDEX IF NOT EXISTS ix_photos_needs_review " - "ON photos(needs_review) WHERE needs_review" - ) - - -def downgrade() -> None: - # Data is not recoverable on downgrade — only the schema stubs are - # put back so a future reinstall of the old pipeline can re-populate. - op.execute("DROP INDEX IF EXISTS ix_photos_needs_review") - op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS needs_review") - - op.execute( - "ALTER TABLE tags ADD COLUMN IF NOT EXISTS representative_photo_id " - "VARCHAR REFERENCES photos(id) ON DELETE SET NULL" - ) diff --git a/backend/alembic/versions/0013_drop_old_content_types.py b/backend/alembic/versions/0013_drop_old_content_types.py deleted file mode 100644 index a676501..0000000 --- a/backend/alembic/versions/0013_drop_old_content_types.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Drop legacy content_type tags from the 6-category classifier - -Revision ID: 0013_drop_old_ct -Revises: 0012_strip_ai -Create Date: 2026-04-14 - -The previous classifier wrote Tag(kind='content_type', name IN -('photograph','screenshot','document','receipt','meme','artwork')). -The new binary classifier writes names ('photography','other'). Both -coexisted after the cutover so users saw duplicate groupings like -'photography' alongside 'photograph'. Drop the old names — photo_tags -rows cascade-delete via the FK. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0013_drop_old_ct" -down_revision: Union[str, None] = "0012_strip_ai" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -LEGACY_NAMES = ('photograph', 'screenshot', 'document', 'receipt', 'meme', 'artwork') - - -def upgrade() -> None: - op.execute( - "DELETE FROM tags WHERE kind = 'content_type' " - f"AND name IN {LEGACY_NAMES}" - ) - - -def downgrade() -> None: - pass diff --git a/backend/alembic/versions/0014_share_status.py b/backend/alembic/versions/0014_share_status.py deleted file mode 100644 index 5654064..0000000 --- a/backend/alembic/versions/0014_share_status.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Add status + accepted_at to heap_shares and folder_shares - -Revision ID: 0014_share_status -Revises: 0013_drop_old_ct -Create Date: 2026-04-21 - -Shares used to activate instantly on the owner's side. We now want a -pending/accepted lifecycle so the recipient gets a notification bell and -chooses to accept or decline before the shared item shows up in their -sidebar. - -Backfill note: every pre-existing row is treated as `accepted` with -accepted_at = created_at. This is a pragmatic fiction — it keeps the -sidebar populated after the migration without anyone having to click -accept on shares that were already live. Any future "accepted X ago" UI -inheriting this backfilled timestamp should be aware it's not a real -user-action moment. - -The one-migration trick: we add `status` with `server_default="accepted"` -so the backfill happens in-place, then drop the default so new inserts -fall through to the Python-side model default ("pending"). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0014_share_status" -down_revision: Union[str, None] = "0013_drop_old_ct" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - for table in ("heap_shares", "folder_shares"): - op.add_column( - table, - sa.Column( - "status", - sa.String(), - nullable=False, - server_default="accepted", - ), - ) - op.add_column( - table, - sa.Column("accepted_at", sa.DateTime(), nullable=True), - ) - op.execute( - f"UPDATE {table} SET accepted_at = created_at " - "WHERE accepted_at IS NULL" - ) - # Drop the DB default so new rows inherit the Python-side - # model default ("pending") instead of silently auto-accepting. - op.alter_column(table, "status", server_default=None) - op.create_index( - f"ix_{table}_shared_with_status", - table, - ["shared_with_id", "status"], - ) - - -def downgrade() -> None: - for table in ("heap_shares", "folder_shares"): - op.drop_index(f"ix_{table}_shared_with_status", table_name=table) - op.drop_column(table, "accepted_at") - op.drop_column(table, "status") diff --git a/backend/alembic/versions/0015_oidc_and_avatar.py b/backend/alembic/versions/0015_oidc_and_avatar.py deleted file mode 100644 index abdc6eb..0000000 --- a/backend/alembic/versions/0015_oidc_and_avatar.py +++ /dev/null @@ -1,83 +0,0 @@ -"""OIDC identity + avatar / display_name on users - -Revision ID: 0015_oidc_and_avatar -Revises: 0014_share_status -Create Date: 2026-04-22 - -Lets users sign in via an OIDC provider (Authentik) and carry a profile -image / display name from the provider. Password-only users are -unaffected. - - 1. Add users.oidc_issuer, users.oidc_sub (identity pair from the IdP). - 2. Add users.avatar_url, users.display_name (profile bits from claims - or manually set). - 3. Make users.hashed_password nullable — OIDC-only users have no local - password. Existing rows all have hashes so the NULLability change - is backwards-compatible. - 4. Partial unique index on (oidc_issuer, oidc_sub) WHERE oidc_sub IS - NOT NULL so multiple password-only users (both NULL) don't collide. -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - -revision: str = "0015_oidc_and_avatar" -down_revision: Union[str, None] = "0014_share_status" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - conn = op.get_bind() - - # 1 + 2. Add new columns (idempotent — create_all on fresh installs - # already built them from the model). - for col_def in ( - "oidc_issuer VARCHAR", - "oidc_sub VARCHAR", - "avatar_url VARCHAR", - "display_name VARCHAR", - ): - conn.execute(sa.text(f"ALTER TABLE users ADD COLUMN IF NOT EXISTS {col_def}")) - - # 3. Drop NOT NULL on hashed_password. Postgres only — SQLite can't - # alter column nullability in place, but the SQLite escape hatch is - # used for fresh local dev where create_all already wrote the new - # nullable definition. - if conn.dialect.name == "postgresql": - conn.execute(sa.text( - "ALTER TABLE users ALTER COLUMN hashed_password DROP NOT NULL" - )) - - # 4. Partial unique index — Postgres supports the WHERE clause so - # NULLs don't collide; SQLite treats NULLs as distinct in unique - # indexes already, so a plain unique index is safe there too. - if conn.dialect.name == "postgresql": - conn.execute(sa.text( - "CREATE UNIQUE INDEX IF NOT EXISTS ix_users_oidc_identity " - "ON users (oidc_issuer, oidc_sub) WHERE oidc_sub IS NOT NULL" - )) - else: - conn.execute(sa.text( - "CREATE UNIQUE INDEX IF NOT EXISTS ix_users_oidc_identity " - "ON users (oidc_issuer, oidc_sub)" - )) - - -def downgrade() -> None: - conn = op.get_bind() - conn.execute(sa.text("DROP INDEX IF EXISTS ix_users_oidc_identity")) - - if conn.dialect.name == "postgresql": - # Can't re-apply NOT NULL if any OIDC-only user has NULL — so - # only do it when safe. - conn.execute(sa.text( - "UPDATE users SET hashed_password = '' WHERE hashed_password IS NULL" - )) - conn.execute(sa.text( - "ALTER TABLE users ALTER COLUMN hashed_password SET NOT NULL" - )) - - for col in ("display_name", "avatar_url", "oidc_sub", "oidc_issuer"): - conn.execute(sa.text(f"ALTER TABLE users DROP COLUMN IF EXISTS {col}")) diff --git a/backend/alembic/versions/0016_nextcloud_integration.py b/backend/alembic/versions/0016_nextcloud_integration.py deleted file mode 100644 index 673fe17..0000000 --- a/backend/alembic/versions/0016_nextcloud_integration.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Nextcloud integration: per-user username override + encrypted app password - -Revision ID: 0016_nextcloud_integration -Revises: 0015_oidc_and_avatar -Create Date: 2026-04-26 - -Lets each mule-image user wire their account to a Nextcloud account so -photos can be browsed, indexed, and mutated under their own Nextcloud -file tree (`/mnt/library/homecloud//files/...` mounted into the -backend + workers as `/nextcloud-users`). The OIDC `preferred_username` -claim is the default mapping; the override field handles cases where the -authentik username and the Nextcloud username don't match. - - 1. users.nextcloud_username — default sourced from preferred_username - on OIDC login (only when null), editable via PATCH /api/v1/auth/me. - 2. users.nextcloud_app_password_enc — Fernet-encrypted Nextcloud app - password used for HTTP Basic auth on WebDAV calls. Set from the - Settings UI; the cleartext is never persisted. -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - -revision: str = "0016_nextcloud_integration" -down_revision: Union[str, None] = "0015_oidc_and_avatar" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - conn = op.get_bind() - - for col_def in ( - "nextcloud_username VARCHAR", - "nextcloud_app_password_enc VARCHAR", - ): - conn.execute(sa.text(f"ALTER TABLE users ADD COLUMN IF NOT EXISTS {col_def}")) - - # Index the username for the per-user path-scoping check on /browse - # and /source-roots — keeps lookups fast even on tiny user tables. - conn.execute(sa.text( - "CREATE INDEX IF NOT EXISTS ix_users_nextcloud_username " - "ON users (nextcloud_username)" - )) - - -def downgrade() -> None: - conn = op.get_bind() - conn.execute(sa.text("DROP INDEX IF EXISTS ix_users_nextcloud_username")) - for col in ("nextcloud_app_password_enc", "nextcloud_username"): - conn.execute(sa.text(f"ALTER TABLE users DROP COLUMN IF EXISTS {col}")) diff --git a/backend/alembic/versions/0017_photos_list_index.py b/backend/alembic/versions/0017_photos_list_index.py deleted file mode 100644 index cfb7b65..0000000 --- a/backend/alembic/versions/0017_photos_list_index.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Partial index for the photos list query - -Revision ID: 0017_photos_list_index -Revises: 0016_nextcloud_integration -Create Date: 2026-05-10 - -The default photos-list query (GET /api/v1/photos?per_page=N&sort=taken_at&order=desc) -filters `NOT is_trashed AND NOT is_hidden` and sorts by -`(taken_at DESC NULLS LAST, id DESC)`. EXPLAIN ANALYZE on a 21k-row -table showed a seq-scan + top-N heapsort (~20ms standalone, worse under -concurrency) — Postgres can't use the single-column ix_photos_taken_at -when the leading WHERE clause is two booleans. - -A partial index on the sort key, scoped to the visible subset, lets the -planner index-scan in reverse and stop at LIMIT N. Two booleans select -~95% of rows, so the partial predicate is tighter than the full table -without losing common queries. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0017_photos_list_index" -down_revision: Union[str, None] = "0016_nextcloud_integration" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute( - """ - CREATE INDEX IF NOT EXISTS ix_photos_list_visible - ON photos (taken_at DESC NULLS LAST, id DESC) - WHERE NOT is_trashed AND NOT is_hidden - """ - ) - - -def downgrade() -> None: - op.execute("DROP INDEX IF EXISTS ix_photos_list_visible") diff --git a/backend/alembic/versions/0018_photos_nextcloud_fileid.py b/backend/alembic/versions/0018_photos_nextcloud_fileid.py deleted file mode 100644 index 69388a1..0000000 --- a/backend/alembic/versions/0018_photos_nextcloud_fileid.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Add photos.nextcloud_fileid for NC preview proxying - -Revision ID: 0018_photos_nextcloud_fileid -Revises: 0017_photos_list_index -Create Date: 2026-05-11 - -The thumbnail endpoint will proxy Nextcloud's /index.php/core/preview -instead of generating and serving its own WebP cache under /data/thumbs. -That requires storing each photo's Nextcloud numeric fileid alongside -the row. NULL is allowed because legacy / non-NC photos still exist -and the handler keeps the on-disk fallback for them. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0018_photos_nextcloud_fileid" -down_revision: Union[str, None] = "0017_photos_list_index" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "photos", - sa.Column("nextcloud_fileid", sa.Integer(), nullable=True), - ) - op.create_index( - "ix_photos_nextcloud_fileid", - "photos", - ["nextcloud_fileid"], - ) - - -def downgrade() -> None: - op.drop_index("ix_photos_nextcloud_fileid", table_name="photos") - op.drop_column("photos", "nextcloud_fileid") diff --git a/backend/alembic/versions/0019_drop_ai_remnants.py b/backend/alembic/versions/0019_drop_ai_remnants.py deleted file mode 100644 index 288e2e4..0000000 --- a/backend/alembic/versions/0019_drop_ai_remnants.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Drop AI remnants: photos.needs_review and the pgvector extension - -Revision ID: 0019_drop_ai_remnants -Revises: 0018_photos_nextcloud_fileid -Create Date: 2026-05-14 - -The vision pipeline has been removed entirely (no more classifier, no -worker-vision service, no torch/onnxruntime/open-clip-torch deps). The -`needs_review` boolean and its partial index were populated only by -that classifier and have no remaining writers or readers. - -The pgvector extension was originally added by 0003_pgvector_embeddings -for the embeddings table that 0012_strip_ai_pipeline dropped; the -extension itself is now unused, and the next deploy switches the -Postgres image from pgvector/pgvector:pg16 to plain postgres:16. The -extension must be dropped *before* that image swap or the new -container will fail to load existing CREATE EXTENSION declarations. -""" -from typing import Sequence, Union - -from alembic import op - - -revision: str = "0019_drop_ai_remnants" -down_revision: Union[str, None] = "0018_photos_nextcloud_fileid" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute("DROP INDEX IF EXISTS ix_photos_needs_review") - op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS needs_review") - op.execute("DROP EXTENSION IF EXISTS vector") - - -def downgrade() -> None: - # Re-create the column as a no-op (data is gone). The pgvector - # extension is intentionally NOT re-added — matches the precedent - # set by 0012_strip_ai_pipeline for its dropped tables. - op.execute( - "ALTER TABLE photos ADD COLUMN IF NOT EXISTS needs_review " - "BOOLEAN NOT NULL DEFAULT FALSE" - ) - op.execute( - "CREATE INDEX IF NOT EXISTS ix_photos_needs_review " - "ON photos(needs_review) WHERE needs_review" - ) diff --git a/backend/app/auth.py b/backend/app/auth.py deleted file mode 100644 index 34050d4..0000000 --- a/backend/app/auth.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Authentication utilities — password hashing and JWT token management. -""" -from datetime import datetime, timedelta, timezone - -from jose import jwt, JWTError -from passlib.context import CryptContext - -from app.config import settings - -pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") - -ALGORITHM = "HS256" - - -def hash_password(plain: str) -> str: - return pwd_context.hash(plain) - - -def verify_password(plain: str, hashed: str) -> bool: - return pwd_context.verify(plain, hashed) - - -def create_access_token(user_id: str, role: str) -> str: - expire = datetime.now(timezone.utc) + timedelta(minutes=settings.access_token_expire_minutes) - payload = { - "sub": user_id, - "role": role, - "exp": expire, - "type": "access", - } - return jwt.encode(payload, settings.secret_key, algorithm=ALGORITHM) - - -def create_refresh_token(user_id: str) -> str: - expire = datetime.now(timezone.utc) + timedelta(days=settings.refresh_token_expire_days) - payload = { - "sub": user_id, - "exp": expire, - "type": "refresh", - } - return jwt.encode(payload, settings.secret_key, algorithm=ALGORITHM) - - -def decode_token(token: str) -> dict: - """Decode and validate a JWT. Raises JWTError on any problem.""" - return jwt.decode(token, settings.secret_key, algorithms=[ALGORITHM]) diff --git a/backend/app/auth_oidc.py b/backend/app/auth_oidc.py deleted file mode 100644 index 1f9c72e..0000000 --- a/backend/app/auth_oidc.py +++ /dev/null @@ -1,77 +0,0 @@ -"""OIDC (OpenID Connect) client setup — used for Authentik SSO today, -generic enough to register other providers later. - -Authlib handles the Authorization Code + PKCE flow, including discovery -via the provider's `.well-known/openid-configuration` document. We keep -a single registered client named "authentik" regardless of label, so the -router code always knows where to find it. - -If OIDC isn't fully configured the module stays inert — `is_enabled()` -returns False and the registry has no client. Callers must guard. -""" -from typing import Optional - -from authlib.integrations.starlette_client import OAuth - -from app.config import settings - - -PROVIDER_NAME = "authentik" - - -def is_configured() -> bool: - """True when every required OIDC setting is present.""" - return bool( - settings.oidc_issuer - and settings.oidc_client_id - and settings.oidc_client_secret - and settings.oidc_redirect_uri - ) - - -def is_enabled() -> bool: - """True when OIDC is both configured and the admin flipped it on.""" - return bool(settings.oidc_enabled) and is_configured() - - -def provider_label() -> str: - return settings.oidc_provider_label or "Authentik" - - -def _build_oauth() -> OAuth: - """Build the Authlib OAuth registry. Always safe to call; only - registers the provider when credentials are present so importing - this module never fails on a fresh install. - """ - registry = OAuth() - if not is_configured(): - return registry - - # Authentik exposes discovery at `{issuer}/.well-known/openid-configuration`. - # Trailing slash handling varies by Authentik version, so normalise. - issuer = settings.oidc_issuer.rstrip("/") - discovery_url = f"{issuer}/.well-known/openid-configuration" - - registry.register( - name=PROVIDER_NAME, - client_id=settings.oidc_client_id, - client_secret=settings.oidc_client_secret, - server_metadata_url=discovery_url, - client_kwargs={ - "scope": settings.oidc_scopes, - # Force PKCE — cheap win for public clients, harmless for - # confidential ones. - "code_challenge_method": "S256", - }, - ) - return registry - - -oauth: OAuth = _build_oauth() - - -def get_client() -> Optional[object]: - """Return the registered provider client, or None when not configured.""" - if not is_configured(): - return None - return oauth.create_client(PROVIDER_NAME) diff --git a/backend/app/config.py b/backend/app/config.py deleted file mode 100644 index 262c179..0000000 --- a/backend/app/config.py +++ /dev/null @@ -1,189 +0,0 @@ -""" -Application configuration using Pydantic Settings -""" -from pydantic_settings import BaseSettings -from pydantic import BaseModel, Field -from typing import Optional -import yaml -from pathlib import Path - -class ThumbnailSettings(BaseModel): - """Thumbnail generation settings""" - small: int = 240 - medium: int = 640 - large: int = 1280 - quality: int = 85 - format: str = "webp" - -class ScannerSettings(BaseModel): - """File scanner settings""" - watch: bool = True - initial_scan_on_start: bool = True - batch_size: int = 100 - concurrent_workers: int = 4 - -class PerformanceSettings(BaseModel): - """Performance tuning settings""" - max_concurrent_thumbnails: int = 10 - cache_ttl: int = 3600 - db_pool_size: int = 10 - db_pool_max_overflow: int = 10 - db_pool_recycle: int = 3600 - -class MulitaConfig(BaseModel): - """Main configuration from YAML file. Source roots and the discard - workflow are owned by the database now — only operational settings - live here.""" - thumbnails: ThumbnailSettings = ThumbnailSettings() - scanner: ScannerSettings = ScannerSettings() - performance: PerformanceSettings = PerformanceSettings() - -class Settings(BaseSettings): - """Application settings""" - # Database — plain Postgres. The SQLite escape hatch remains - # supported via the docker-compose.sqlite.yml override and by setting - # DATABASE_URL=sqlite+aiosqlite:///... in .env for local dev. - database_url: str = Field( - default="postgresql+asyncpg://mulita:mulita@db:5432/mulita", - env="DATABASE_URL" - ) - - # Redis - redis_url: str = Field( - default="redis://localhost:6379", - env="REDIS_URL" - ) - - # Celery - celery_broker_url: str = Field( - default="redis://localhost:6379", - env="CELERY_BROKER_URL" - ) - celery_result_backend: str = Field( - default="redis://localhost:6379", - env="CELERY_RESULT_BACKEND" - ) - - # Photo directories - photo_dirs: str = Field( - default="/photos", - env="PHOTO_DIRS" - ) - - # API settings - api_host: str = Field(default="0.0.0.0", env="API_HOST") - api_port: int = Field(default=8000, env="API_PORT") - - # CORS — comma-separated list of allowed origins, or "*" for any. - # Same-origin requests (the normal case behind nginx / vite proxy) - # never trip CORS, so this is only for direct browser access from - # other origins (LAN IP, reverse proxy, dev tools). - allowed_origins: str = Field(default="*", env="ALLOWED_ORIGINS") - - # Logging — accepts standard python levels (DEBUG, INFO, WARNING, - # ERROR, CRITICAL). Bumped from INFO when chasing a problem. - log_level: str = Field(default="INFO", env="LOG_LEVEL") - - # Auth — JWT signing key. Set SECRET_KEY in .env for production. - # If unset, a deterministic fallback is used (acceptable for - # single-machine homelab deploys, but set a real key if the instance - # is network-exposed). - secret_key: str = Field( - default="mulita-dev-secret-change-me", - env="SECRET_KEY", - ) - access_token_expire_minutes: int = Field(default=525600, env="ACCESS_TOKEN_EXPIRE_MINUTES") # 1 year - refresh_token_expire_days: int = Field(default=3650, env="REFRESH_TOKEN_EXPIRE_DAYS") # 10 years - - # ── OIDC / Authentik single sign-on ──────────────────────────────── - # Disabled by default; enable by setting OIDC_ENABLED=true and the - # issuer + client credentials. When enabled the login page shows a - # "Sign in with {label}" button alongside the username/password form. - oidc_enabled: bool = Field(default=False, env="OIDC_ENABLED") - oidc_issuer: Optional[str] = Field(default=None, env="OIDC_ISSUER") - oidc_client_id: Optional[str] = Field(default=None, env="OIDC_CLIENT_ID") - oidc_client_secret: Optional[str] = Field(default=None, env="OIDC_CLIENT_SECRET") - # Absolute URL the IdP redirects back to. Must match the Redirect URI - # configured on the Authentik side exactly. - oidc_redirect_uri: Optional[str] = Field(default=None, env="OIDC_REDIRECT_URI") - oidc_scopes: str = Field(default="openid profile email", env="OIDC_SCOPES") - oidc_provider_label: str = Field(default="Authentik", env="OIDC_PROVIDER_LABEL") - # When true, a successful OIDC login for a subject we've never seen - # auto-creates a local user + their /photos/{username} folder. When - # false, unknown subjects get 403 and must be pre-provisioned. - oidc_allow_signup: bool = Field(default=True, env="OIDC_ALLOW_SIGNUP") - # Comma-separated Authentik group names. Any group-claim match - # promotes the user to role=admin; otherwise role=user. Role is - # refreshed on every sign-in so removals demote automatically. - oidc_admin_groups: str = Field(default="", env="OIDC_ADMIN_GROUPS") - # Last-resort link step: if (issuer, sub) AND email fallback both - # miss, try matching the IdP's `preferred_username` claim against - # `users.username`. Safe in single-tenant setups where the IdP is - # the source of truth for usernames (homelab, family instance). - # Leave off in multi-tenant — a name collision would hand someone - # else's account to a new SSO user. - oidc_link_by_username: bool = Field(default=False, env="OIDC_LINK_BY_USERNAME") - # Starlette session cookie secret — only used to hold PKCE/state - # during the brief OIDC round-trip. Falls back to secret_key when - # unset. - session_secret: Optional[str] = Field(default=None, env="SESSION_SECRET") - - @property - def oidc_admin_group_list(self) -> list[str]: - raw = (self.oidc_admin_groups or "").strip() - return [g.strip() for g in raw.split(",") if g.strip()] - - @property - def effective_session_secret(self) -> str: - return self.session_secret or self.secret_key - - @property - def cors_origins(self) -> list[str]: - """Parse the ALLOWED_ORIGINS env var into a list. Accepts: - - "*" → wildcard (single-element list ["*"]) - - "http://a.com,http://b.com" → split + strip - Empty entries are dropped. - """ - raw = (self.allowed_origins or "").strip() - if not raw or raw == "*": - return ["*"] - return [o.strip() for o in raw.split(",") if o.strip()] - - # App configuration from YAML - _config: Optional[MulitaConfig] = None - - @property - def config(self) -> MulitaConfig: - """Load configuration from YAML file""" - if self._config is None: - config_path = Path("/app/config/mulita.yml") - if not config_path.exists(): - config_path = Path("mulita.yml") - - if config_path.exists(): - with open(config_path, "r") as f: - config_data = yaml.safe_load(f) - self._config = MulitaConfig(**config_data) - else: - self._config = MulitaConfig() - - return self._config - - @property - def thumbnails(self) -> ThumbnailSettings: - return self.config.thumbnails - - @property - def scanner(self) -> ScannerSettings: - return self.config.scanner - - @property - def performance(self) -> PerformanceSettings: - return self.config.performance - - class Config: - env_file = ".env" - case_sensitive = False - -# Global settings instance -settings = Settings() \ No newline at end of file diff --git a/backend/app/database.py b/backend/app/database.py deleted file mode 100644 index e00c26b..0000000 --- a/backend/app/database.py +++ /dev/null @@ -1,204 +0,0 @@ -""" -Database configuration and session management. - -Schema management strategy --------------------------- -Postgres (default): Alembic owns schema deltas. `alembic upgrade head` is -run before the app starts (in the container CMD). `init_db()` calls -`create_all` afterward as the source of truth for fresh installs — it is -idempotent for existing tables and creates any tables defined on -`Base.metadata` that don't yet exist. Future Alembic migrations should be -written defensively (`IF NOT EXISTS` etc.) so they remain safe to run on a -fresh DB where `create_all` has already laid down the same objects. - -SQLite (escape hatch via docker-compose.sqlite.yml): no Alembic. The -historical inline ALTER TABLE block stays in place so existing dev -installs keep upgrading. -""" -import os -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker -from sqlalchemy.orm import declarative_base -from sqlalchemy.pool import NullPool -from sqlalchemy import text -import logging -from pathlib import Path - -from app.config import settings - -logger = logging.getLogger(__name__) - -_is_sqlite = settings.database_url.startswith("sqlite") -_is_postgres = settings.database_url.startswith("postgresql") - -# When running inside a Celery worker we use NullPool rather than the -# default connection pool. The reasons stack up: -# -# 1. Celery's prefork model forks the master *after* imports, so every -# child inherits the same asyncpg Connection objects — they share -# a socket, and two children using one concurrently raises -# "another operation is in progress". -# -# 2. Task bodies run under `asyncio.run()`, which spins up a fresh -# event loop per invocation. A pooled asyncpg Connection created -# on loop A, returned to the pool, and checked out on loop B -# raises "Future attached to a different loop". -# -# NullPool dodges both: every session checkout opens a brand-new -# connection on the *current* loop and the connection is closed at -# session end. Connection setup is cheap compared to task cost, so this -# is the right default for the worker. The FastAPI backend keeps the -# normal pool because it serves many short requests on a single long- -# lived event loop, where pooling is a clear win. -_is_celery_worker = os.environ.get("MULITA_CELERY_WORKER") == "1" - -if _is_sqlite: - db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", "")) - db_path.parent.mkdir(parents=True, exist_ok=True) - engine = create_async_engine( - settings.database_url, - echo=False, - connect_args={ - "check_same_thread": False, - "timeout": 30, - }, - ) -elif _is_celery_worker: - engine = create_async_engine( - settings.database_url, - echo=False, - poolclass=NullPool, - ) -else: - engine = create_async_engine( - settings.database_url, - echo=False, - pool_size=settings.performance.db_pool_size, - max_overflow=settings.performance.db_pool_max_overflow, - pool_recycle=settings.performance.db_pool_recycle, - pool_pre_ping=True, - pool_timeout=10, # fail fast if pool exhausted (default 30) - # Kill connections idle in a transaction for >60s. Prevents leaked - # sessions from thumbnail requests that disconnect mid-flight. - connect_args={"server_settings": {"idle_in_transaction_session_timeout": "60000"}}, - ) - -# Create async session factory -AsyncSessionLocal = async_sessionmaker( - engine, - class_=AsyncSession, - expire_on_commit=False -) - -# Base class for models -Base = declarative_base() - -async def get_db() -> AsyncSession: - """Dependency to get database session. - - Rolls back any uncommitted transaction before closing so a client - disconnect doesn't leave idle-in-transaction connections in the pool. - """ - async with AsyncSessionLocal() as session: - try: - yield session - except Exception: - await session.rollback() - raise - finally: - await session.close() - -async def init_db(): - """Initialize database, create tables if they don't exist""" - async with engine.begin() as conn: - # Import all models to register them with Base - from app.models import User, Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto # noqa: F401 - - # Create all tables. Note: create_all only creates *missing* tables — - # it does NOT add new columns to existing tables when the model gains - # them. On Postgres, Alembic handles deltas; on SQLite, the inline - # ALTER block below is the legacy fallback. - await conn.run_sync(Base.metadata.create_all) - - gps_columns_added = False - - if _is_sqlite: - # WAL mode for better concurrency. - await conn.execute(text("PRAGMA journal_mode=WAL")) - await conn.execute(text("PRAGMA synchronous=NORMAL")) - await conn.execute(text("PRAGMA cache_size=10000")) - await conn.execute(text("PRAGMA temp_store=MEMORY")) - - # ── Idempotent column adds (SQLite only) ───────────────────── - # SQLite supports ADD COLUMN but not "IF NOT EXISTS" for - # columns, so introspect via PRAGMA first. Each entry is - # (column_name, ALTER statement). Add new columns at the - # bottom. On Postgres these live in Alembic migrations. - existing_cols = { - row[1] - for row in ( - await conn.execute(text("PRAGMA table_info(photos)")) - ).fetchall() - } - pending_alters: list[tuple[str, str]] = [ - ("phash", "ALTER TABLE photos ADD COLUMN phash VARCHAR(16)"), - ( - "duplicate_group_id", - "ALTER TABLE photos ADD COLUMN duplicate_group_id VARCHAR", - ), - ("latitude", "ALTER TABLE photos ADD COLUMN latitude REAL"), - ("longitude", "ALTER TABLE photos ADD COLUMN longitude REAL"), - ] - for col_name, alter_sql in pending_alters: - if col_name not in existing_cols: - logger.info(f"Adding photos.{col_name} column") - await conn.execute(text(alter_sql)) - if col_name in ("latitude", "longitude"): - gps_columns_added = True - await conn.execute( - text("CREATE INDEX IF NOT EXISTS ix_photos_phash ON photos(phash)") - ) - await conn.execute( - text( - "CREATE INDEX IF NOT EXISTS ix_photos_duplicate_group_id " - "ON photos(duplicate_group_id)" - ) - ) - await conn.execute( - text( - "CREATE INDEX IF NOT EXISTS ix_photos_lat_lon " - "ON photos(latitude, longitude)" - ) - ) - - logger.info("Database initialized successfully") - - # If we just introduced the GPS columns on an existing SQLite - # install, kick off a one-shot backfill so the Map view is - # populated without a manual full re-scan. Postgres installs are - # always fresh (no SQLite→PG migration path), so this code path - # is SQLite-only. - if _is_sqlite and gps_columns_added: - try: - from app.tasks.scan import backfill_gps - backfill_gps.delay() - logger.info("Queued one-shot backfill_gps task after column add") - except Exception as e: - logger.warning(f"Could not queue backfill_gps task: {e}") - -async def create_fts_table(): - """Create Full-Text Search table for SQLite. On Postgres this is - replaced by a tsvector column on the photos table (added in PR5).""" - if _is_sqlite: - async with engine.begin() as conn: - # Create FTS5 virtual table for full-text search - await conn.execute(text(""" - CREATE VIRTUAL TABLE IF NOT EXISTS photos_fts USING fts5( - photo_id UNINDEXED, - filename, - user_title, - user_notes, - exif_text, - tokenize='unicode61' - ) - """)) - logger.info("FTS5 table created successfully") diff --git a/backend/app/dependencies.py b/backend/app/dependencies.py deleted file mode 100644 index a2fab9c..0000000 --- a/backend/app/dependencies.py +++ /dev/null @@ -1,355 +0,0 @@ -""" -FastAPI dependencies for authentication and user-scoped data access. -""" -from typing import Optional - -from fastapi import Depends, HTTPException, Query, Request, status -from fastapi.security import OAuth2PasswordBearer -from jose import JWTError -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import selectinload - -from app.auth import decode_token -from app.database import get_db -from app.models.user import User -from app.models.photos import Photo -from app.models.folders import Folder, SourceRoot -from app.models.heaps import Heap, heap_photos -from app.models.tags import Tag -from app.models.sharing import HeapShare, FolderShare - -oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/login") - - -async def get_current_user( - token: str = Depends(oauth2_scheme), - db: AsyncSession = Depends(get_db), -) -> User: - """Decode JWT, look up user, raise 401 if invalid or inactive.""" - credentials_exception = HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid or expired token", - headers={"WWW-Authenticate": "Bearer"}, - ) - try: - payload = decode_token(token) - user_id: str = payload.get("sub") - token_type: str = payload.get("type") - if user_id is None or token_type != "access": - raise credentials_exception - except JWTError: - raise credentials_exception - - result = await db.execute(select(User).where(User.id == user_id)) - user = result.scalar_one_or_none() - if user is None or not user.is_active: - raise credentials_exception - return user - - -async def get_current_user_media( - request: Request, - token: Optional[str] = Query(None, alias="token"), - db: AsyncSession = Depends(get_db), -) -> User: - """Authenticate via Authorization header OR ?token= query parameter. - - Used for media endpoints (thumbnails, originals, proxies) where the - URL is set as an or