feat: migrate to Postgres + pgvector with Alembic scaffolding
Switch the default database from SQLite to Postgres + pgvector (via pgvector/pgvector:pg16 Docker image) to support the upcoming vision pipeline (embeddings, OCR, object detection, face clustering). - Add `db` service to docker-compose.yml with healthcheck - Wire `alembic upgrade head` into backend CMD before uvicorn - Bootstrap empty 0001_baseline revision (schema still owned by create_all) - Guard SQLite-only PRAGMAs and inline ALTERs behind _is_sqlite flag - Run `CREATE EXTENSION IF NOT EXISTS vector` on Postgres init - Add asyncpg, psycopg2-binary, pgvector to requirements - Provide docker-compose.sqlite.yml escape hatch for legacy SQLite mode Fresh DB + rescan assumed — no SQLite→Postgres data migration. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -39,9 +39,11 @@ class MulitaConfig(BaseModel):
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings"""
|
||||
# Database
|
||||
# Database — Postgres + pgvector by default. The SQLite escape hatch
|
||||
# remains supported via the docker-compose.sqlite.yml override and by
|
||||
# setting DATABASE_URL=sqlite+aiosqlite:///... in .env for local dev.
|
||||
database_url: str = Field(
|
||||
default="sqlite+aiosqlite:///data/db/mulita.db",
|
||||
default="postgresql+asyncpg://mulita:mulita@db:5432/mulita",
|
||||
env="DATABASE_URL"
|
||||
)
|
||||
|
||||
|
||||
@@ -1,38 +1,50 @@
|
||||
"""
|
||||
Database configuration and session management
|
||||
Database configuration and session management.
|
||||
|
||||
Schema management strategy
|
||||
--------------------------
|
||||
Postgres (default): Alembic owns schema deltas. `alembic upgrade head` is
|
||||
run before the app starts (in the container CMD). `init_db()` calls
|
||||
`create_all` afterward as the source of truth for fresh installs — it is
|
||||
idempotent for existing tables and creates any tables defined on
|
||||
`Base.metadata` that don't yet exist. Future Alembic migrations should be
|
||||
written defensively (`IF NOT EXISTS` etc.) so they remain safe to run on a
|
||||
fresh DB where `create_all` has already laid down the same objects.
|
||||
|
||||
SQLite (escape hatch via docker-compose.sqlite.yml): no Alembic. The
|
||||
historical inline ALTER TABLE block stays in place so existing dev
|
||||
installs keep upgrading.
|
||||
"""
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy import event, text
|
||||
from sqlalchemy import text
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Create database directory if it doesn't exist
|
||||
db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", ""))
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
_is_sqlite = settings.database_url.startswith("sqlite")
|
||||
_is_postgres = settings.database_url.startswith("postgresql")
|
||||
|
||||
# Create async engine
|
||||
# SQLite doesn't support pool configuration
|
||||
if "sqlite" in settings.database_url:
|
||||
if _is_sqlite:
|
||||
db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", ""))
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False, # Set to True for SQL debugging
|
||||
echo=False,
|
||||
connect_args={
|
||||
"check_same_thread": False, # SQLite specific
|
||||
"timeout": 30
|
||||
}
|
||||
"check_same_thread": False,
|
||||
"timeout": 30,
|
||||
},
|
||||
)
|
||||
else:
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False, # Set to True for SQL debugging
|
||||
echo=False,
|
||||
pool_size=settings.performance.db_pool_size,
|
||||
pool_recycle=settings.performance.db_pool_recycle
|
||||
pool_recycle=settings.performance.db_pool_recycle,
|
||||
)
|
||||
|
||||
# Create async session factory
|
||||
@@ -59,26 +71,32 @@ async def init_db():
|
||||
# Import all models to register them with Base
|
||||
from app.models import Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto, Embedding
|
||||
|
||||
# Postgres: ensure pgvector is available before create_all touches
|
||||
# any Vector columns (added in later PRs but the extension is cheap
|
||||
# and idempotent to create now).
|
||||
if _is_postgres:
|
||||
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
|
||||
|
||||
# Create all tables. Note: create_all only creates *missing* tables —
|
||||
# it does NOT add new columns to existing tables when the model gains
|
||||
# them. Anything new on an existing table needs an explicit ALTER
|
||||
# below.
|
||||
# them. On Postgres, Alembic handles deltas; on SQLite, the inline
|
||||
# ALTER block below is the legacy fallback.
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
# Enable WAL mode for SQLite (better concurrency)
|
||||
if "sqlite" in settings.database_url:
|
||||
gps_columns_added = False
|
||||
|
||||
if _is_sqlite:
|
||||
# WAL mode for better concurrency.
|
||||
await conn.execute(text("PRAGMA journal_mode=WAL"))
|
||||
await conn.execute(text("PRAGMA synchronous=NORMAL"))
|
||||
await conn.execute(text("PRAGMA cache_size=10000"))
|
||||
await conn.execute(text("PRAGMA temp_store=MEMORY"))
|
||||
|
||||
# ── Idempotent column adds ────────────────────────────────────────
|
||||
# The project does not use Alembic; we lean on create_all + a small
|
||||
# set of inline ALTER TABLE statements for the columns we've added
|
||||
# post-launch. SQLite supports ADD COLUMN but not "IF NOT EXISTS"
|
||||
# for columns, so introspect via PRAGMA first. Each entry is
|
||||
# (column_name, ALTER statement). Add new columns at the bottom.
|
||||
if "sqlite" in settings.database_url:
|
||||
# ── Idempotent column adds (SQLite only) ─────────────────────
|
||||
# SQLite supports ADD COLUMN but not "IF NOT EXISTS" for
|
||||
# columns, so introspect via PRAGMA first. Each entry is
|
||||
# (column_name, ALTER statement). Add new columns at the
|
||||
# bottom. On Postgres these live in Alembic migrations.
|
||||
existing_cols = {
|
||||
row[1]
|
||||
for row in (
|
||||
@@ -94,18 +112,12 @@ async def init_db():
|
||||
("latitude", "ALTER TABLE photos ADD COLUMN latitude REAL"),
|
||||
("longitude", "ALTER TABLE photos ADD COLUMN longitude REAL"),
|
||||
]
|
||||
# Track whether the GPS columns were just added so we can kick
|
||||
# off a one-shot backfill of existing photos at the end of init.
|
||||
gps_columns_added = False
|
||||
for col_name, alter_sql in pending_alters:
|
||||
if col_name not in existing_cols:
|
||||
logger.info(f"Adding photos.{col_name} column")
|
||||
await conn.execute(text(alter_sql))
|
||||
if col_name in ("latitude", "longitude"):
|
||||
gps_columns_added = True
|
||||
# Indexes for the new duplicate-detection columns. CREATE INDEX
|
||||
# IF NOT EXISTS is supported on SQLite so this is safe to run
|
||||
# every startup.
|
||||
await conn.execute(
|
||||
text("CREATE INDEX IF NOT EXISTS ix_photos_phash ON photos(phash)")
|
||||
)
|
||||
@@ -124,11 +136,12 @@ async def init_db():
|
||||
|
||||
logger.info("Database initialized successfully")
|
||||
|
||||
# If we just introduced the GPS columns on an existing install, kick
|
||||
# off a one-shot backfill so the Map view is populated without a
|
||||
# manual full re-scan. Imported lazily to avoid pulling Celery into
|
||||
# the import graph for non-worker processes that don't need it.
|
||||
if "sqlite" in settings.database_url and gps_columns_added:
|
||||
# If we just introduced the GPS columns on an existing SQLite
|
||||
# install, kick off a one-shot backfill so the Map view is
|
||||
# populated without a manual full re-scan. Postgres installs are
|
||||
# always fresh (no SQLite→PG migration path), so this code path
|
||||
# is SQLite-only.
|
||||
if _is_sqlite and gps_columns_added:
|
||||
try:
|
||||
from app.tasks.scan import backfill_gps
|
||||
backfill_gps.delay()
|
||||
@@ -137,8 +150,9 @@ async def init_db():
|
||||
logger.warning(f"Could not queue backfill_gps task: {e}")
|
||||
|
||||
async def create_fts_table():
|
||||
"""Create Full-Text Search table for SQLite"""
|
||||
if "sqlite" in settings.database_url:
|
||||
"""Create Full-Text Search table for SQLite. On Postgres this is
|
||||
replaced by a tsvector column on the photos table (added in PR5)."""
|
||||
if _is_sqlite:
|
||||
async with engine.begin() as conn:
|
||||
# Create FTS5 virtual table for full-text search
|
||||
await conn.execute(text("""
|
||||
@@ -151,4 +165,4 @@ async def create_fts_table():
|
||||
tokenize='unicode61'
|
||||
)
|
||||
"""))
|
||||
logger.info("FTS5 table created successfully")
|
||||
logger.info("FTS5 table created successfully")
|
||||
|
||||
Reference in New Issue
Block a user