diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..48fdc2a --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,48 @@ +# Alembic configuration for PhotoVault. +# +# The actual database URL is loaded at runtime by alembic/env.py from the +# DATABASE_URL environment variable (with the async driver suffix stripped). +# The placeholder below is only used for `alembic revision --autogenerate` +# when no env var is set. + +[alembic] +script_location = alembic +prepend_sys_path = . +version_path_separator = os +sqlalchemy.url = postgresql+psycopg2://mulita:mulita@localhost:5432/mulita + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 0000000..efe903f --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,96 @@ +""" +Alembic environment for PhotoVault. + +Pulls DATABASE_URL from the environment so the same migrations work in +docker compose and locally. Strips the async driver suffix because Alembic +runs synchronously via psycopg2. + +Future-migration note +--------------------- +Fresh installs run `Base.metadata.create_all` in `app.database.init_db` +*before* migrations would normally apply, so any migration that adds a +column / index / table to an object the model already declares will see +that object already present. Write migrations defensively: + + op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS new_col TEXT") + op.execute("CREATE INDEX IF NOT EXISTS ix_foo ON foo(bar)") + +For brand-new tables that the model also declares, the same applies — use +`op.execute("CREATE TABLE IF NOT EXISTS ...")` or check first. +""" +from logging.config import fileConfig +import os +import sys +from pathlib import Path + +from sqlalchemy import engine_from_config, pool +from alembic import context + +# Make `app` importable from this script. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app.database import Base # noqa: E402 +# Import all models so they're registered on Base.metadata for autogenerate. +from app.models import ( # noqa: E402, F401 + Photo, + Folder, + SourceRoot, + Tag, + Heap, + HeapPhoto, + Embedding, +) + +config = context.config + +# Resolve DATABASE_URL from env. Strip async driver suffixes — Alembic +# uses sync drivers. +db_url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url") +if db_url: + if "+asyncpg" in db_url: + db_url = db_url.replace("+asyncpg", "+psycopg2") + elif db_url.startswith("postgresql://"): + db_url = db_url.replace("postgresql://", "postgresql+psycopg2://", 1) + elif "+aiosqlite" in db_url: + db_url = db_url.replace("+aiosqlite", "") + config.set_main_option("sqlalchemy.url", db_url) + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode (emit SQL only).""" + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations against a live database.""" + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/backend/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/backend/alembic/versions/0001_baseline.py b/backend/alembic/versions/0001_baseline.py new file mode 100644 index 0000000..2ee73a7 --- /dev/null +++ b/backend/alembic/versions/0001_baseline.py @@ -0,0 +1,27 @@ +"""baseline (empty) + +Revision ID: 0001_baseline +Revises: +Create Date: 2026-04-10 + +The current schema is created by SQLAlchemy `Base.metadata.create_all` in +`app.database.init_db()` on first boot. Alembic only owns deltas from +PR3 onward. This baseline is intentionally empty so `alembic upgrade head` +on a fresh DB simply creates the `alembic_version` table and stamps it. +""" +from typing import Sequence, Union + + +# revision identifiers, used by Alembic. +revision: str = "0001_baseline" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/backend/app/config.py b/backend/app/config.py index 9e5803b..998e6a5 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -39,9 +39,11 @@ class MulitaConfig(BaseModel): class Settings(BaseSettings): """Application settings""" - # Database + # Database — Postgres + pgvector by default. The SQLite escape hatch + # remains supported via the docker-compose.sqlite.yml override and by + # setting DATABASE_URL=sqlite+aiosqlite:///... in .env for local dev. database_url: str = Field( - default="sqlite+aiosqlite:///data/db/mulita.db", + default="postgresql+asyncpg://mulita:mulita@db:5432/mulita", env="DATABASE_URL" ) diff --git a/backend/app/database.py b/backend/app/database.py index 2f4261b..d016fb9 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -1,38 +1,50 @@ """ -Database configuration and session management +Database configuration and session management. + +Schema management strategy +-------------------------- +Postgres (default): Alembic owns schema deltas. `alembic upgrade head` is +run before the app starts (in the container CMD). `init_db()` calls +`create_all` afterward as the source of truth for fresh installs — it is +idempotent for existing tables and creates any tables defined on +`Base.metadata` that don't yet exist. Future Alembic migrations should be +written defensively (`IF NOT EXISTS` etc.) so they remain safe to run on a +fresh DB where `create_all` has already laid down the same objects. + +SQLite (escape hatch via docker-compose.sqlite.yml): no Alembic. The +historical inline ALTER TABLE block stays in place so existing dev +installs keep upgrading. """ from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker from sqlalchemy.orm import declarative_base -from sqlalchemy import event, text +from sqlalchemy import text import logging -import os from pathlib import Path from app.config import settings logger = logging.getLogger(__name__) -# Create database directory if it doesn't exist -db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", "")) -db_path.parent.mkdir(parents=True, exist_ok=True) +_is_sqlite = settings.database_url.startswith("sqlite") +_is_postgres = settings.database_url.startswith("postgresql") -# Create async engine -# SQLite doesn't support pool configuration -if "sqlite" in settings.database_url: +if _is_sqlite: + db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", "")) + db_path.parent.mkdir(parents=True, exist_ok=True) engine = create_async_engine( settings.database_url, - echo=False, # Set to True for SQL debugging + echo=False, connect_args={ - "check_same_thread": False, # SQLite specific - "timeout": 30 - } + "check_same_thread": False, + "timeout": 30, + }, ) else: engine = create_async_engine( settings.database_url, - echo=False, # Set to True for SQL debugging + echo=False, pool_size=settings.performance.db_pool_size, - pool_recycle=settings.performance.db_pool_recycle + pool_recycle=settings.performance.db_pool_recycle, ) # Create async session factory @@ -59,26 +71,32 @@ async def init_db(): # Import all models to register them with Base from app.models import Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto, Embedding + # Postgres: ensure pgvector is available before create_all touches + # any Vector columns (added in later PRs but the extension is cheap + # and idempotent to create now). + if _is_postgres: + await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + # Create all tables. Note: create_all only creates *missing* tables — # it does NOT add new columns to existing tables when the model gains - # them. Anything new on an existing table needs an explicit ALTER - # below. + # them. On Postgres, Alembic handles deltas; on SQLite, the inline + # ALTER block below is the legacy fallback. await conn.run_sync(Base.metadata.create_all) - # Enable WAL mode for SQLite (better concurrency) - if "sqlite" in settings.database_url: + gps_columns_added = False + + if _is_sqlite: + # WAL mode for better concurrency. await conn.execute(text("PRAGMA journal_mode=WAL")) await conn.execute(text("PRAGMA synchronous=NORMAL")) await conn.execute(text("PRAGMA cache_size=10000")) await conn.execute(text("PRAGMA temp_store=MEMORY")) - # ── Idempotent column adds ──────────────────────────────────────── - # The project does not use Alembic; we lean on create_all + a small - # set of inline ALTER TABLE statements for the columns we've added - # post-launch. SQLite supports ADD COLUMN but not "IF NOT EXISTS" - # for columns, so introspect via PRAGMA first. Each entry is - # (column_name, ALTER statement). Add new columns at the bottom. - if "sqlite" in settings.database_url: + # ── Idempotent column adds (SQLite only) ───────────────────── + # SQLite supports ADD COLUMN but not "IF NOT EXISTS" for + # columns, so introspect via PRAGMA first. Each entry is + # (column_name, ALTER statement). Add new columns at the + # bottom. On Postgres these live in Alembic migrations. existing_cols = { row[1] for row in ( @@ -94,18 +112,12 @@ async def init_db(): ("latitude", "ALTER TABLE photos ADD COLUMN latitude REAL"), ("longitude", "ALTER TABLE photos ADD COLUMN longitude REAL"), ] - # Track whether the GPS columns were just added so we can kick - # off a one-shot backfill of existing photos at the end of init. - gps_columns_added = False for col_name, alter_sql in pending_alters: if col_name not in existing_cols: logger.info(f"Adding photos.{col_name} column") await conn.execute(text(alter_sql)) if col_name in ("latitude", "longitude"): gps_columns_added = True - # Indexes for the new duplicate-detection columns. CREATE INDEX - # IF NOT EXISTS is supported on SQLite so this is safe to run - # every startup. await conn.execute( text("CREATE INDEX IF NOT EXISTS ix_photos_phash ON photos(phash)") ) @@ -124,11 +136,12 @@ async def init_db(): logger.info("Database initialized successfully") - # If we just introduced the GPS columns on an existing install, kick - # off a one-shot backfill so the Map view is populated without a - # manual full re-scan. Imported lazily to avoid pulling Celery into - # the import graph for non-worker processes that don't need it. - if "sqlite" in settings.database_url and gps_columns_added: + # If we just introduced the GPS columns on an existing SQLite + # install, kick off a one-shot backfill so the Map view is + # populated without a manual full re-scan. Postgres installs are + # always fresh (no SQLite→PG migration path), so this code path + # is SQLite-only. + if _is_sqlite and gps_columns_added: try: from app.tasks.scan import backfill_gps backfill_gps.delay() @@ -137,8 +150,9 @@ async def init_db(): logger.warning(f"Could not queue backfill_gps task: {e}") async def create_fts_table(): - """Create Full-Text Search table for SQLite""" - if "sqlite" in settings.database_url: + """Create Full-Text Search table for SQLite. On Postgres this is + replaced by a tsvector column on the photos table (added in PR5).""" + if _is_sqlite: async with engine.begin() as conn: # Create FTS5 virtual table for full-text search await conn.execute(text(""" @@ -151,4 +165,4 @@ async def create_fts_table(): tokenize='unicode61' ) """)) - logger.info("FTS5 table created successfully") \ No newline at end of file + logger.info("FTS5 table created successfully") diff --git a/backend/requirements.txt b/backend/requirements.txt index 0aaa005..ceeac45 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,7 +5,10 @@ python-multipart==0.0.6 # Database sqlalchemy[asyncio]==2.0.25 -aiosqlite==0.19.0 +aiosqlite==0.19.0 # SQLite escape hatch (docker-compose.sqlite.yml override) +asyncpg==0.29.0 # async Postgres driver (default) +psycopg2-binary==2.9.9 # sync Postgres driver, used by Alembic CLI +pgvector==0.2.5 # pgvector SQLAlchemy types alembic==1.13.1 # Redis and Celery diff --git a/docker-compose.sqlite.yml b/docker-compose.sqlite.yml new file mode 100644 index 0000000..37adffb --- /dev/null +++ b/docker-compose.sqlite.yml @@ -0,0 +1,47 @@ +# SQLite escape hatch override. +# +# Usage (omit the `db` service from the up command): +# +# docker compose -f docker-compose.yml -f docker-compose.sqlite.yml \ +# up frontend backend worker redis +# +# This pins the backend and worker to the legacy SQLite database file at +# /data/db/mulita.db (in the existing db_data volume), drops the dependency +# on Postgres, and skips Alembic — the SQLite schema is still managed by +# the inline ALTERs in app/database.py:init_db. +# +# Vision features that depend on pgvector (PR4 onward) will refuse to enable +# in this mode; the search/embedding endpoints will return 503 with a clear +# error pointing back at the default Postgres setup. + +services: + backend: + command: sh -c "uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload" + environment: + - DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db + - REDIS_URL=redis://redis:6379 + - CELERY_BROKER_URL=redis://redis:6379 + - CELERY_RESULT_BACKEND=redis://redis:6379 + - PHOTO_DIRS=${PHOTO_DIRS:-/photos} + - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-*} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - TZ=${TZ:-UTC} + depends_on: + redis: + condition: service_started + + worker: + environment: + - DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db + - REDIS_URL=redis://redis:6379 + - CELERY_BROKER_URL=redis://redis:6379 + - CELERY_RESULT_BACKEND=redis://redis:6379 + - PHOTO_DIRS=${PHOTO_DIRS:-/photos} + - CELERYD_CONCURRENCY=${CELERYD_CONCURRENCY:-4} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - TZ=${TZ:-UTC} + depends_on: + redis: + condition: service_started + backend: + condition: service_started diff --git a/docker-compose.yml b/docker-compose.yml index db714f1..b60b5b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: frontend: build: @@ -36,9 +34,13 @@ services: - ${PHOTO_DIRS:-./photos}:/photos:rw - thumbs_data:/data/thumbs - proxies_data:/data/proxies - - db_data:/data/db + - db_data:/data/db # retained so the docker-compose.sqlite.yml override has somewhere to put mulita.db + # Run Alembic migrations before starting uvicorn. On a fresh Postgres + # the empty 0001 baseline is a no-op stamp; create_all in init_db then + # builds the schema. + command: sh -c "alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload" environment: - - DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db + - DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita - REDIS_URL=redis://redis:6379 - CELERY_BROKER_URL=redis://redis:6379 - CELERY_RESULT_BACKEND=redis://redis:6379 @@ -47,7 +49,10 @@ services: - LOG_LEVEL=${LOG_LEVEL:-INFO} - TZ=${TZ:-UTC} depends_on: - - redis + redis: + condition: service_started + db: + condition: service_healthy networks: - mulita-network restart: unless-stopped @@ -65,7 +70,7 @@ services: - proxies_data:/data/proxies - db_data:/data/db environment: - - DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db + - DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita - REDIS_URL=redis://redis:6379 - CELERY_BROKER_URL=redis://redis:6379 - CELERY_RESULT_BACKEND=redis://redis:6379 @@ -74,12 +79,34 @@ services: - LOG_LEVEL=${LOG_LEVEL:-INFO} - TZ=${TZ:-UTC} depends_on: - - redis - - backend + redis: + condition: service_started + backend: + condition: service_started + db: + condition: service_healthy networks: - mulita-network restart: unless-stopped + db: + image: pgvector/pgvector:pg16 + container_name: mulita-db + environment: + POSTGRES_USER: mulita + POSTGRES_PASSWORD: mulita + POSTGRES_DB: mulita + volumes: + - pg_data:/var/lib/postgresql/data + networks: + - mulita-network + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U mulita -d mulita"] + interval: 5s + timeout: 5s + retries: 10 + redis: image: redis:7-alpine container_name: mulita-redis @@ -102,4 +129,5 @@ volumes: thumbs_data: proxies_data: db_data: - redis_data: \ No newline at end of file + redis_data: + pg_data: \ No newline at end of file