Compare commits
20 Commits
f01b5ed77e
...
a4f64fad58
| Author | SHA1 | Date | |
|---|---|---|---|
| a4f64fad58 | |||
| 4bc6dc1dc8 | |||
| fa9b21856f | |||
| f48e099bd2 | |||
| 40d570f2c2 | |||
| 5f3fa5240e | |||
| aba061dd43 | |||
| 229611b4c3 | |||
| db20cbb7d8 | |||
| 17a69a271e | |||
| 7558aeb5e6 | |||
| 2a6661f779 | |||
| 29177f0c1a | |||
| ad007e4cd4 | |||
| 1ebc4bfe73 | |||
| 842a4fc864 | |||
| 649437dc85 | |||
| b1c2bdf7f0 | |||
| 9282a5c734 | |||
| dea04ceed9 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -65,4 +65,4 @@ docker-compose.override.yml
|
||||
|
||||
# Thumbnails
|
||||
/thumbs/
|
||||
/trash/
|
||||
/trash/backend/yolov8n.pt
|
||||
|
||||
29
README.md
29
README.md
@@ -6,21 +6,27 @@ A self-hosted, Docker-deployed photo management application inspired by Lightroo
|
||||
|
||||
- **Photo Organization**: Browse photos in a timeline view with virtual scrolling for performance
|
||||
- **Thumbnail Generation**: Automatic thumbnail generation for all photo formats including RAW
|
||||
- **Metadata Extraction**: Full EXIF/XMP metadata extraction and search
|
||||
- **Metadata Extraction**: Full EXIF/XMP metadata extraction and GPS mapping
|
||||
- **Keyboard Shortcuts**: Lightroom-style keyboard navigation and actions
|
||||
- **File Support**: JPEG, PNG, RAW formats (CR2, CR3, NEF, ARW, etc.), HEIC/HEIF, and videos
|
||||
- **Heaps**: Temporary collections for organizing photos
|
||||
- **Tags & Ratings**: Organize with tags, star ratings, and color labels
|
||||
- **Tags & Ratings**: Organize with tags, star ratings, and color labels — each with a card-grid browse view that drills into a full Timeline detail
|
||||
- **Dark Mode**: Photography-optimized dark interface
|
||||
- **Vision Pipeline**: YOLO object detection, OCR text extraction, CLIP embeddings for semantic search, InsightFace face detection and clustering
|
||||
- **People View**: Browse identified people as cards, click to see all photos of a person
|
||||
- **Map View**: Browse GPS-tagged photos on an interactive Leaflet map
|
||||
- **Duplicate Detection**: Perceptual hash-based duplicate grouping with best-pick UI
|
||||
- **Semantic Search**: Natural-language photo search powered by CLIP embeddings
|
||||
|
||||
## Tech Stack
|
||||
|
||||
### Backend
|
||||
- Python 3.12 with FastAPI
|
||||
- SQLite with SQLAlchemy (async)
|
||||
- PostgreSQL + pgvector with SQLAlchemy (async) and Alembic migrations
|
||||
- Celery + Redis for background tasks
|
||||
- pyvips for fast thumbnail generation
|
||||
- ExifTool for metadata extraction
|
||||
- ONNX Runtime for vision models (YOLO, CLIP, InsightFace)
|
||||
|
||||
### Frontend
|
||||
- React 18 with TypeScript
|
||||
@@ -161,9 +167,9 @@ The application consists of 5 Docker services:
|
||||
|
||||
- **frontend**: React SPA served by Nginx
|
||||
- **backend**: FastAPI REST API
|
||||
- **worker**: Celery workers for background tasks
|
||||
- **worker**: Celery workers for background tasks (thumbnails, metadata, vision pipeline)
|
||||
- **redis**: Message broker for Celery
|
||||
- **db**: SQLite database (file-based)
|
||||
- **db**: PostgreSQL with pgvector extension (for CLIP/face embeddings)
|
||||
|
||||
## Keyboard Shortcuts
|
||||
|
||||
@@ -172,8 +178,7 @@ The application consists of 5 Docker services:
|
||||
| `←` `→` `↑` `↓` | Navigate photos |
|
||||
| `Space` | Quick preview |
|
||||
| `Enter` | Open loupe view |
|
||||
| `P` | Pick photo |
|
||||
| `X` | Reject photo |
|
||||
| `T` | Add to active heap |
|
||||
| `1-5` | Set star rating |
|
||||
| `Tab` | Toggle left sidebar |
|
||||
| `I` | Toggle metadata panel |
|
||||
@@ -211,14 +216,12 @@ Source roots are managed by the UI / API (the database owns them). Edit
|
||||
- Handles 100,000+ photos efficiently
|
||||
- Virtual scrolling for smooth timeline navigation
|
||||
- Thumbnail generation at 10+ photos/second
|
||||
- SQLite FTS5 for fast full-text search
|
||||
- PostgreSQL full-text search with tsvector indexing
|
||||
- pgvector for fast nearest-neighbor embedding search
|
||||
|
||||
## Future Features (Phase 2)
|
||||
## Future Features
|
||||
|
||||
- AI-powered scene classification
|
||||
- Face detection and clustering
|
||||
- Smart albums
|
||||
- Duplicate detection
|
||||
- Smart albums (auto-populated by saved filters)
|
||||
- Export presets
|
||||
- Multi-user support
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY . .
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p /data/thumbs /data/db /data/proxies /app/config
|
||||
RUN mkdir -p /data/thumbs /data/db /data/proxies /data/models /app/config
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8000
|
||||
|
||||
48
backend/alembic.ini
Normal file
48
backend/alembic.ini
Normal file
@@ -0,0 +1,48 @@
|
||||
# Alembic configuration for PhotoVault.
|
||||
#
|
||||
# The actual database URL is loaded at runtime by alembic/env.py from the
|
||||
# DATABASE_URL environment variable (with the async driver suffix stripped).
|
||||
# The placeholder below is only used for `alembic revision --autogenerate`
|
||||
# when no env var is set.
|
||||
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
version_path_separator = os
|
||||
sqlalchemy.url = postgresql+psycopg2://mulita:mulita@localhost:5432/mulita
|
||||
|
||||
[post_write_hooks]
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
96
backend/alembic/env.py
Normal file
96
backend/alembic/env.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""
|
||||
Alembic environment for PhotoVault.
|
||||
|
||||
Pulls DATABASE_URL from the environment so the same migrations work in
|
||||
docker compose and locally. Strips the async driver suffix because Alembic
|
||||
runs synchronously via psycopg2.
|
||||
|
||||
Future-migration note
|
||||
---------------------
|
||||
Fresh installs run `Base.metadata.create_all` in `app.database.init_db`
|
||||
*before* migrations would normally apply, so any migration that adds a
|
||||
column / index / table to an object the model already declares will see
|
||||
that object already present. Write migrations defensively:
|
||||
|
||||
op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS new_col TEXT")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_foo ON foo(bar)")
|
||||
|
||||
For brand-new tables that the model also declares, the same applies — use
|
||||
`op.execute("CREATE TABLE IF NOT EXISTS ...")` or check first.
|
||||
"""
|
||||
from logging.config import fileConfig
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
from alembic import context
|
||||
|
||||
# Make `app` importable from this script.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.database import Base # noqa: E402
|
||||
# Import all models so they're registered on Base.metadata for autogenerate.
|
||||
from app.models import ( # noqa: E402, F401
|
||||
Photo,
|
||||
Folder,
|
||||
SourceRoot,
|
||||
Tag,
|
||||
Heap,
|
||||
HeapPhoto,
|
||||
Embedding,
|
||||
)
|
||||
|
||||
config = context.config
|
||||
|
||||
# Resolve DATABASE_URL from env. Strip async driver suffixes — Alembic
|
||||
# uses sync drivers.
|
||||
db_url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url")
|
||||
if db_url:
|
||||
if "+asyncpg" in db_url:
|
||||
db_url = db_url.replace("+asyncpg", "+psycopg2")
|
||||
elif db_url.startswith("postgresql://"):
|
||||
db_url = db_url.replace("postgresql://", "postgresql+psycopg2://", 1)
|
||||
elif "+aiosqlite" in db_url:
|
||||
db_url = db_url.replace("+aiosqlite", "")
|
||||
config.set_main_option("sqlalchemy.url", db_url)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode (emit SQL only)."""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations against a live database."""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
26
backend/alembic/script.py.mako
Normal file
26
backend/alembic/script.py.mako
Normal file
@@ -0,0 +1,26 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
27
backend/alembic/versions/0001_baseline.py
Normal file
27
backend/alembic/versions/0001_baseline.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""baseline (empty)
|
||||
|
||||
Revision ID: 0001_baseline
|
||||
Revises:
|
||||
Create Date: 2026-04-10
|
||||
|
||||
The current schema is created by SQLAlchemy `Base.metadata.create_all` in
|
||||
`app.database.init_db()` on first boot. Alembic only owns deltas from
|
||||
PR3 onward. This baseline is intentionally empty so `alembic upgrade head`
|
||||
on a fresh DB simply creates the `alembic_version` table and stamps it.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "0001_baseline"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
85
backend/alembic/versions/0002_extend_tags_for_vision.py
Normal file
85
backend/alembic/versions/0002_extend_tags_for_vision.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""extend tags for vision pipeline
|
||||
|
||||
Revision ID: 0002_extend_tags
|
||||
Revises: 0001_baseline
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Add kind, source, representative_photo_id to tags table.
|
||||
Add confidence, bbox, source to photo_tags association.
|
||||
Switch uniqueness from (name) to (name, kind).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision: str = "0002_extend_tags"
|
||||
down_revision: Union[str, None] = "0001_baseline"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ── tags table ────────────────────────────────────────────────────
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS kind VARCHAR NOT NULL DEFAULT 'user'")
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS source VARCHAR")
|
||||
op.execute("ALTER TABLE tags ADD COLUMN IF NOT EXISTS representative_photo_id VARCHAR REFERENCES photos(id) ON DELETE SET NULL")
|
||||
|
||||
# Create index on kind for filtering
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_tags_kind ON tags(kind)")
|
||||
|
||||
# Drop old unique constraint on name (if it exists) and add (name, kind).
|
||||
# SQLAlchemy create_all may have created either — handle both cases.
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Drop the old single-column unique index/constraint if present.
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_indexes
|
||||
WHERE tablename = 'tags' AND indexname = 'ix_tags_name'
|
||||
) THEN
|
||||
DROP INDEX ix_tags_name;
|
||||
END IF;
|
||||
|
||||
-- Some SQLAlchemy versions create a unique constraint directly.
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM information_schema.table_constraints
|
||||
WHERE table_name = 'tags' AND constraint_name = 'tags_name_key'
|
||||
) THEN
|
||||
ALTER TABLE tags DROP CONSTRAINT tags_name_key;
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'uq_tags_name_kind'
|
||||
) THEN
|
||||
ALTER TABLE tags ADD CONSTRAINT uq_tags_name_kind UNIQUE (name, kind);
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
# ── photo_tags table ──────────────────────────────────────────────
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS confidence FLOAT")
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS bbox JSONB")
|
||||
op.execute("ALTER TABLE photo_tags ADD COLUMN IF NOT EXISTS source VARCHAR")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# photo_tags columns
|
||||
op.drop_column("photo_tags", "source")
|
||||
op.drop_column("photo_tags", "bbox")
|
||||
op.drop_column("photo_tags", "confidence")
|
||||
|
||||
# tags: restore old unique constraint
|
||||
op.execute("ALTER TABLE tags DROP CONSTRAINT IF EXISTS uq_tags_name_kind")
|
||||
op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tags_name ON tags(name)")
|
||||
|
||||
# tags columns
|
||||
op.drop_column("tags", "representative_photo_id")
|
||||
op.drop_column("tags", "source")
|
||||
op.drop_column("tags", "kind")
|
||||
52
backend/alembic/versions/0003_pgvector_embeddings.py
Normal file
52
backend/alembic/versions/0003_pgvector_embeddings.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""pgvector embeddings
|
||||
|
||||
Revision ID: 0003_pgvector_embeddings
|
||||
Revises: 0002_extend_tags
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Rewrite the embeddings table to use pgvector Vector(512) instead of
|
||||
LargeBinary. Add composite PK (photo_id, model), created_at, and
|
||||
HNSW index on vector column.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0003_pgvector_embeddings"
|
||||
down_revision: Union[str, None] = "0002_extend_tags"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Drop the old placeholder table and recreate with pgvector types.
|
||||
# No data to preserve — it was never populated.
|
||||
op.execute("DROP TABLE IF EXISTS embeddings")
|
||||
op.execute("""
|
||||
CREATE TABLE embeddings (
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
model VARCHAR(64) NOT NULL,
|
||||
vector vector(512),
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
PRIMARY KEY (photo_id, model)
|
||||
)
|
||||
""")
|
||||
# HNSW index for cosine similarity search.
|
||||
# Defer creation on large backfills — drop and recreate afterward.
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
||||
ON embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS embeddings")
|
||||
op.execute("""
|
||||
CREATE TABLE embeddings (
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
model VARCHAR,
|
||||
vector BYTEA,
|
||||
PRIMARY KEY (photo_id)
|
||||
)
|
||||
""")
|
||||
82
backend/alembic/versions/0004_ocr_text_and_fts.py
Normal file
82
backend/alembic/versions/0004_ocr_text_and_fts.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""ocr_text table and Postgres FTS
|
||||
|
||||
Revision ID: 0004_ocr_fts
|
||||
Revises: 0003_pgvector_embeddings
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Create ocr_text table for storing OCR results. Add a tsvector column
|
||||
to photos for unified full-text search (filename + user_title +
|
||||
user_notes) with a GIN index. OCR text is rolled up into a materialized
|
||||
view or joined at query time.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0004_ocr_fts"
|
||||
down_revision: Union[str, None] = "0003_pgvector_embeddings"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ── ocr_text table ────────────────────────────────────────────────
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ocr_text (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
language VARCHAR(8) DEFAULT '',
|
||||
confidence FLOAT,
|
||||
bbox JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT now()
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_ocr_text_photo_id ON ocr_text(photo_id)")
|
||||
|
||||
# ── tsvector column on photos ─────────────────────────────────────
|
||||
op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS search_vector tsvector")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_photos_search_vector ON photos USING GIN (search_vector)")
|
||||
|
||||
# Trigger to auto-update search_vector on INSERT/UPDATE
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION photos_search_vector_update() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_vector :=
|
||||
setweight(to_tsvector('english', coalesce(NEW.filename, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.user_title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.user_notes, '')), 'B');
|
||||
RETURN NEW;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_trigger WHERE tgname = 'photos_search_vector_trigger'
|
||||
) THEN
|
||||
CREATE TRIGGER photos_search_vector_trigger
|
||||
BEFORE INSERT OR UPDATE OF filename, user_title, user_notes
|
||||
ON photos
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION photos_search_vector_update();
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
# Backfill existing rows
|
||||
op.execute("""
|
||||
UPDATE photos SET search_vector =
|
||||
setweight(to_tsvector('english', coalesce(filename, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(user_title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(user_notes, '')), 'B')
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TRIGGER IF EXISTS photos_search_vector_trigger ON photos")
|
||||
op.execute("DROP FUNCTION IF EXISTS photos_search_vector_update()")
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_search_vector")
|
||||
op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS search_vector")
|
||||
op.execute("DROP TABLE IF EXISTS ocr_text")
|
||||
41
backend/alembic/versions/0005_face_embeddings.py
Normal file
41
backend/alembic/versions/0005_face_embeddings.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""face_embeddings table
|
||||
|
||||
Revision ID: 0005_face_embeddings
|
||||
Revises: 0004_ocr_fts
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Create face_embeddings table with pgvector Vector(128) for SFace
|
||||
recognition embeddings and HNSW index.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0005_face_embeddings"
|
||||
down_revision: Union[str, None] = "0004_ocr_fts"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS face_embeddings (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
bbox JSONB,
|
||||
vector vector(128),
|
||||
cluster_id VARCHAR REFERENCES tags(id) ON DELETE SET NULL,
|
||||
quality FLOAT,
|
||||
created_at TIMESTAMPTZ DEFAULT now()
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_face_embeddings_photo_id ON face_embeddings(photo_id)")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_face_embeddings_cluster_id ON face_embeddings(cluster_id)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS face_embeddings")
|
||||
39
backend/alembic/versions/0006_face_embeddings_512d.py
Normal file
39
backend/alembic/versions/0006_face_embeddings_512d.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""face_embeddings vector 128 -> 512
|
||||
|
||||
Revision ID: 0006_face_512d
|
||||
Revises: 0005_face_embeddings
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Resize face_embeddings.vector from Vector(128) to Vector(512) for
|
||||
ArcFace embeddings (InsightFace). Drops existing data and HNSW index,
|
||||
recreates both.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0006_face_512d"
|
||||
down_revision: Union[str, None] = "0005_face_embeddings"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Drop index, truncate (old 128-d vectors are incompatible), resize
|
||||
op.execute("DROP INDEX IF EXISTS ix_face_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM face_embeddings")
|
||||
op.execute("ALTER TABLE face_embeddings ALTER COLUMN vector TYPE vector(512)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_face_embeddings_vector_hnsw")
|
||||
op.execute("DELETE FROM face_embeddings")
|
||||
op.execute("ALTER TABLE face_embeddings ALTER COLUMN vector TYPE vector(128)")
|
||||
op.execute("""
|
||||
CREATE INDEX IF NOT EXISTS ix_face_embeddings_vector_hnsw
|
||||
ON face_embeddings USING hnsw (vector vector_cosine_ops)
|
||||
""")
|
||||
@@ -29,6 +29,48 @@ class PerformanceSettings(BaseModel):
|
||||
db_pool_size: int = 20
|
||||
db_pool_recycle: int = 3600
|
||||
|
||||
class EmbedderSettings(BaseModel):
|
||||
"""CLIP / SigLIP embedding model settings"""
|
||||
name: str = "openclip_vitb32"
|
||||
batch_size: int = 8
|
||||
|
||||
class OCRSettings(BaseModel):
|
||||
"""PaddleOCR / rapidocr settings"""
|
||||
enabled: bool = True
|
||||
languages: list[str] = ["en"]
|
||||
min_confidence: float = 0.5
|
||||
|
||||
class DetectorSettings(BaseModel):
|
||||
"""YOLOv8n object detection settings"""
|
||||
enabled: bool = True
|
||||
min_confidence: float = 0.35
|
||||
max_detections: int = 50
|
||||
|
||||
class FacesSettings(BaseModel):
|
||||
"""YuNet + SFace face detection/recognition settings"""
|
||||
enabled: bool = True
|
||||
min_face_size: int = 40
|
||||
recognition_threshold: float = 0.65
|
||||
cluster_eps: float = 0.5
|
||||
|
||||
class ClassifierSettings(BaseModel):
|
||||
"""CLIP zero-shot content classification settings"""
|
||||
enabled: bool = True
|
||||
min_confidence: float = 0.3
|
||||
|
||||
class VisionSettings(BaseModel):
|
||||
"""AI vision pipeline settings. Disabled when running on SQLite
|
||||
(pgvector is required for embedding storage)."""
|
||||
enabled: bool = True
|
||||
backend: str = "onnx" # "onnx" | "rocm" (future)
|
||||
models_dir: str = "/data/models"
|
||||
embedder: EmbedderSettings = EmbedderSettings()
|
||||
ocr: OCRSettings = OCRSettings()
|
||||
detector: DetectorSettings = DetectorSettings()
|
||||
faces: FacesSettings = FacesSettings()
|
||||
classifier: ClassifierSettings = ClassifierSettings()
|
||||
worker_concurrency: int = 2
|
||||
|
||||
class MulitaConfig(BaseModel):
|
||||
"""Main configuration from YAML file. Source roots and the discard
|
||||
workflow are owned by the database now — only operational settings
|
||||
@@ -36,12 +78,15 @@ class MulitaConfig(BaseModel):
|
||||
thumbnails: ThumbnailSettings = ThumbnailSettings()
|
||||
scanner: ScannerSettings = ScannerSettings()
|
||||
performance: PerformanceSettings = PerformanceSettings()
|
||||
vision: VisionSettings = VisionSettings()
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings"""
|
||||
# Database
|
||||
# Database — Postgres + pgvector by default. The SQLite escape hatch
|
||||
# remains supported via the docker-compose.sqlite.yml override and by
|
||||
# setting DATABASE_URL=sqlite+aiosqlite:///... in .env for local dev.
|
||||
database_url: str = Field(
|
||||
default="sqlite+aiosqlite:///data/db/mulita.db",
|
||||
default="postgresql+asyncpg://mulita:mulita@db:5432/mulita",
|
||||
env="DATABASE_URL"
|
||||
)
|
||||
|
||||
@@ -125,6 +170,10 @@ class Settings(BaseSettings):
|
||||
def performance(self) -> PerformanceSettings:
|
||||
return self.config.performance
|
||||
|
||||
@property
|
||||
def vision(self) -> VisionSettings:
|
||||
return self.config.vision
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
case_sensitive = False
|
||||
|
||||
@@ -1,38 +1,50 @@
|
||||
"""
|
||||
Database configuration and session management
|
||||
Database configuration and session management.
|
||||
|
||||
Schema management strategy
|
||||
--------------------------
|
||||
Postgres (default): Alembic owns schema deltas. `alembic upgrade head` is
|
||||
run before the app starts (in the container CMD). `init_db()` calls
|
||||
`create_all` afterward as the source of truth for fresh installs — it is
|
||||
idempotent for existing tables and creates any tables defined on
|
||||
`Base.metadata` that don't yet exist. Future Alembic migrations should be
|
||||
written defensively (`IF NOT EXISTS` etc.) so they remain safe to run on a
|
||||
fresh DB where `create_all` has already laid down the same objects.
|
||||
|
||||
SQLite (escape hatch via docker-compose.sqlite.yml): no Alembic. The
|
||||
historical inline ALTER TABLE block stays in place so existing dev
|
||||
installs keep upgrading.
|
||||
"""
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy import event, text
|
||||
from sqlalchemy import text
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Create database directory if it doesn't exist
|
||||
db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", ""))
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
_is_sqlite = settings.database_url.startswith("sqlite")
|
||||
_is_postgres = settings.database_url.startswith("postgresql")
|
||||
|
||||
# Create async engine
|
||||
# SQLite doesn't support pool configuration
|
||||
if "sqlite" in settings.database_url:
|
||||
if _is_sqlite:
|
||||
db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", ""))
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False, # Set to True for SQL debugging
|
||||
echo=False,
|
||||
connect_args={
|
||||
"check_same_thread": False, # SQLite specific
|
||||
"timeout": 30
|
||||
}
|
||||
"check_same_thread": False,
|
||||
"timeout": 30,
|
||||
},
|
||||
)
|
||||
else:
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=False, # Set to True for SQL debugging
|
||||
echo=False,
|
||||
pool_size=settings.performance.db_pool_size,
|
||||
pool_recycle=settings.performance.db_pool_recycle
|
||||
pool_recycle=settings.performance.db_pool_recycle,
|
||||
)
|
||||
|
||||
# Create async session factory
|
||||
@@ -59,26 +71,32 @@ async def init_db():
|
||||
# Import all models to register them with Base
|
||||
from app.models import Photo, Folder, SourceRoot, Tag, PhotoTag, Heap, HeapPhoto, Embedding
|
||||
|
||||
# Postgres: ensure pgvector is available before create_all touches
|
||||
# any Vector columns (added in later PRs but the extension is cheap
|
||||
# and idempotent to create now).
|
||||
if _is_postgres:
|
||||
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
|
||||
|
||||
# Create all tables. Note: create_all only creates *missing* tables —
|
||||
# it does NOT add new columns to existing tables when the model gains
|
||||
# them. Anything new on an existing table needs an explicit ALTER
|
||||
# below.
|
||||
# them. On Postgres, Alembic handles deltas; on SQLite, the inline
|
||||
# ALTER block below is the legacy fallback.
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
# Enable WAL mode for SQLite (better concurrency)
|
||||
if "sqlite" in settings.database_url:
|
||||
gps_columns_added = False
|
||||
|
||||
if _is_sqlite:
|
||||
# WAL mode for better concurrency.
|
||||
await conn.execute(text("PRAGMA journal_mode=WAL"))
|
||||
await conn.execute(text("PRAGMA synchronous=NORMAL"))
|
||||
await conn.execute(text("PRAGMA cache_size=10000"))
|
||||
await conn.execute(text("PRAGMA temp_store=MEMORY"))
|
||||
|
||||
# ── Idempotent column adds ────────────────────────────────────────
|
||||
# The project does not use Alembic; we lean on create_all + a small
|
||||
# set of inline ALTER TABLE statements for the columns we've added
|
||||
# post-launch. SQLite supports ADD COLUMN but not "IF NOT EXISTS"
|
||||
# for columns, so introspect via PRAGMA first. Each entry is
|
||||
# (column_name, ALTER statement). Add new columns at the bottom.
|
||||
if "sqlite" in settings.database_url:
|
||||
# ── Idempotent column adds (SQLite only) ─────────────────────
|
||||
# SQLite supports ADD COLUMN but not "IF NOT EXISTS" for
|
||||
# columns, so introspect via PRAGMA first. Each entry is
|
||||
# (column_name, ALTER statement). Add new columns at the
|
||||
# bottom. On Postgres these live in Alembic migrations.
|
||||
existing_cols = {
|
||||
row[1]
|
||||
for row in (
|
||||
@@ -94,18 +112,12 @@ async def init_db():
|
||||
("latitude", "ALTER TABLE photos ADD COLUMN latitude REAL"),
|
||||
("longitude", "ALTER TABLE photos ADD COLUMN longitude REAL"),
|
||||
]
|
||||
# Track whether the GPS columns were just added so we can kick
|
||||
# off a one-shot backfill of existing photos at the end of init.
|
||||
gps_columns_added = False
|
||||
for col_name, alter_sql in pending_alters:
|
||||
if col_name not in existing_cols:
|
||||
logger.info(f"Adding photos.{col_name} column")
|
||||
await conn.execute(text(alter_sql))
|
||||
if col_name in ("latitude", "longitude"):
|
||||
gps_columns_added = True
|
||||
# Indexes for the new duplicate-detection columns. CREATE INDEX
|
||||
# IF NOT EXISTS is supported on SQLite so this is safe to run
|
||||
# every startup.
|
||||
await conn.execute(
|
||||
text("CREATE INDEX IF NOT EXISTS ix_photos_phash ON photos(phash)")
|
||||
)
|
||||
@@ -124,11 +136,12 @@ async def init_db():
|
||||
|
||||
logger.info("Database initialized successfully")
|
||||
|
||||
# If we just introduced the GPS columns on an existing install, kick
|
||||
# off a one-shot backfill so the Map view is populated without a
|
||||
# manual full re-scan. Imported lazily to avoid pulling Celery into
|
||||
# the import graph for non-worker processes that don't need it.
|
||||
if "sqlite" in settings.database_url and gps_columns_added:
|
||||
# If we just introduced the GPS columns on an existing SQLite
|
||||
# install, kick off a one-shot backfill so the Map view is
|
||||
# populated without a manual full re-scan. Postgres installs are
|
||||
# always fresh (no SQLite→PG migration path), so this code path
|
||||
# is SQLite-only.
|
||||
if _is_sqlite and gps_columns_added:
|
||||
try:
|
||||
from app.tasks.scan import backfill_gps
|
||||
backfill_gps.delay()
|
||||
@@ -137,8 +150,9 @@ async def init_db():
|
||||
logger.warning(f"Could not queue backfill_gps task: {e}")
|
||||
|
||||
async def create_fts_table():
|
||||
"""Create Full-Text Search table for SQLite"""
|
||||
if "sqlite" in settings.database_url:
|
||||
"""Create Full-Text Search table for SQLite. On Postgres this is
|
||||
replaced by a tsvector column on the photos table (added in PR5)."""
|
||||
if _is_sqlite:
|
||||
async with engine.begin() as conn:
|
||||
# Create FTS5 virtual table for full-text search
|
||||
await conn.execute(text("""
|
||||
@@ -151,4 +165,4 @@ async def create_fts_table():
|
||||
tokenize='unicode61'
|
||||
)
|
||||
"""))
|
||||
logger.info("FTS5 table created successfully")
|
||||
logger.info("FTS5 table created successfully")
|
||||
|
||||
@@ -11,7 +11,7 @@ import os
|
||||
|
||||
from app.config import settings
|
||||
from app.database import init_db
|
||||
from app.routers import photos, folders, heaps, tags, discard, library
|
||||
from app.routers import photos, folders, heaps, tags, discard, library, search
|
||||
from app.services.scanner import start_initial_scan, bootstrap_default_source_root
|
||||
from app.services.cleanup import cleanup_data_integrity
|
||||
|
||||
@@ -91,6 +91,7 @@ app.include_router(heaps.router, prefix="/api/v1/heaps", tags=["heaps"])
|
||||
app.include_router(tags.router, prefix="/api/v1/tags", tags=["tags"])
|
||||
app.include_router(discard.router, prefix="/api/v1/discard", tags=["discard"])
|
||||
app.include_router(library.router, prefix="/api/v1/library", tags=["library"])
|
||||
app.include_router(search.router, prefix="/api/v1/photos/search", tags=["search"])
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
|
||||
@@ -6,6 +6,8 @@ from app.models.folders import Folder, SourceRoot
|
||||
from app.models.tags import Tag, PhotoTag
|
||||
from app.models.heaps import Heap, HeapPhoto
|
||||
from app.models.embeddings import Embedding
|
||||
from app.models.ocr_text import OCRText
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
|
||||
__all__ = [
|
||||
'Photo',
|
||||
@@ -15,5 +17,7 @@ __all__ = [
|
||||
'PhotoTag',
|
||||
'Heap',
|
||||
'HeapPhoto',
|
||||
'Embedding'
|
||||
'Embedding',
|
||||
'OCRText',
|
||||
'FaceEmbedding',
|
||||
]
|
||||
@@ -1,17 +1,19 @@
|
||||
"""
|
||||
Embedding model definition (placeholder for AI features)
|
||||
Embedding model — stores CLIP/SigLIP image embeddings via pgvector.
|
||||
|
||||
Composite PK (photo_id, model) allows re-embedding with newer models
|
||||
without clobbering old vectors.
|
||||
"""
|
||||
from sqlalchemy import Column, String, ForeignKey, LargeBinary
|
||||
import uuid
|
||||
from sqlalchemy import Column, String, ForeignKey, DateTime, func
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Embedding(Base):
|
||||
"""
|
||||
Placeholder table for future AI embeddings (CLIP, face recognition, etc.)
|
||||
"""
|
||||
__tablename__ = 'embeddings'
|
||||
|
||||
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True)
|
||||
model = Column(String) # e.g., 'clip-vit-b32', 'face-recognition', etc.
|
||||
vector = Column(LargeBinary) # raw float32 bytes for embedding vector
|
||||
model = Column(String(64), primary_key=True) # e.g. 'openclip_vitb32'
|
||||
vector = Column(Vector(512)) # OpenCLIP ViT-B/32 → 512-d
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
24
backend/app/models/face_embedding.py
Normal file
24
backend/app/models/face_embedding.py
Normal file
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
Face embedding model — stores per-face detection + recognition vectors.
|
||||
|
||||
cluster_id FKs to tags.id where kind='face_cluster'. Null means
|
||||
unclustered (will be assigned by recluster_faces).
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, DateTime, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from pgvector.sqlalchemy import Vector
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class FaceEmbedding(Base):
|
||||
__tablename__ = 'face_embeddings'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
bbox = Column(JSONB) # [x1, y1, x2, y2] normalized 0-1
|
||||
vector = Column(Vector(512)) # ArcFace → 512-d
|
||||
cluster_id = Column(String, ForeignKey('tags.id', ondelete='SET NULL'), nullable=True, index=True)
|
||||
quality = Column(Float)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
20
backend/app/models/ocr_text.py
Normal file
20
backend/app/models/ocr_text.py
Normal file
@@ -0,0 +1,20 @@
|
||||
"""
|
||||
OCR text model — stores text regions extracted from photos via rapidocr.
|
||||
"""
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, Text, DateTime, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class OCRText(Base):
|
||||
__tablename__ = 'ocr_text'
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
photo_id = Column(String, ForeignKey('photos.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
text = Column(Text, nullable=False)
|
||||
language = Column(String(8), default='')
|
||||
confidence = Column(Float)
|
||||
bbox = Column(JSONB) # [x1, y1, x2, y2] normalized 0-1
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -1,8 +1,14 @@
|
||||
"""
|
||||
Tag model definitions
|
||||
Tag model definitions.
|
||||
|
||||
Tags are unified across user-created tags, ML-detected objects, scene
|
||||
labels, and face clusters via the `kind` column. The `photo_tags`
|
||||
association carries per-photo ML metadata (confidence, bounding box,
|
||||
source model).
|
||||
"""
|
||||
from sqlalchemy import Column, String, ForeignKey, Table, Index
|
||||
from sqlalchemy import Column, String, Float, ForeignKey, Table, Index, UniqueConstraint
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
import uuid
|
||||
|
||||
from app.database import Base
|
||||
@@ -13,20 +19,41 @@ photo_tags = Table(
|
||||
Base.metadata,
|
||||
Column('photo_id', String, ForeignKey('photos.id', ondelete='CASCADE'), primary_key=True),
|
||||
Column('tag_id', String, ForeignKey('tags.id', ondelete='CASCADE'), primary_key=True),
|
||||
# ML metadata — null for user-applied tags
|
||||
Column('confidence', Float, nullable=True),
|
||||
Column('bbox', JSONB, nullable=True), # [x1, y1, x2, y2] normalized 0-1
|
||||
Column('source', String, nullable=True), # e.g. "vision:yolov8n", "vision:sface"
|
||||
Index('ix_photo_tags_photo_id', 'photo_id'),
|
||||
Index('ix_photo_tags_tag_id', 'tag_id'),
|
||||
)
|
||||
|
||||
class Tag(Base):
|
||||
__tablename__ = 'tags'
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint('name', 'kind', name='uq_tags_name_kind'),
|
||||
)
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, unique=True, nullable=False, index=True)
|
||||
name = Column(String, nullable=False, index=True)
|
||||
color = Column(String) # Hex color code for UI display
|
||||
|
||||
|
||||
# Tag classification
|
||||
kind = Column(String, nullable=False, default='user', index=True)
|
||||
# kind values: 'user' | 'object' | 'scene' | 'face_cluster'
|
||||
|
||||
# Which model produced this tag (null for user-created)
|
||||
source = Column(String, nullable=True)
|
||||
# e.g. "vision:yolov8n", "vision:sface", null
|
||||
|
||||
# For face clusters: the photo used as the cluster representative thumbnail
|
||||
representative_photo_id = Column(
|
||||
String, ForeignKey('photos.id', ondelete='SET NULL'), nullable=True
|
||||
)
|
||||
|
||||
# Relationships
|
||||
photos = relationship("Photo", secondary=photo_tags, backref="tags")
|
||||
|
||||
|
||||
class PhotoTag:
|
||||
"""Helper class for photo-tag associations (not a table model)"""
|
||||
pass
|
||||
pass
|
||||
|
||||
74
backend/app/routers/search.py
Normal file
74
backend/app/routers/search.py
Normal file
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Search API router — unified hybrid search endpoint.
|
||||
"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Photo
|
||||
from app.services.search import hybrid_search
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
q: Optional[str] = None
|
||||
filters: Optional[dict] = None
|
||||
limit: int = 50
|
||||
offset: int = 0
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def search_photos(body: SearchRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""Unified search endpoint. Every query runs hybrid (FTS + semantic)
|
||||
by default — the user never picks a mode.
|
||||
|
||||
Filters:
|
||||
- tag_ids: list of tag IDs (any kind: user, object, face_cluster)
|
||||
- date_from / date_to: ISO date strings
|
||||
"""
|
||||
filters = body.filters or {}
|
||||
|
||||
results = await hybrid_search(
|
||||
db=db,
|
||||
q=body.q,
|
||||
tag_ids=filters.get("tag_ids"),
|
||||
date_from=filters.get("date_from"),
|
||||
date_to=filters.get("date_to"),
|
||||
limit=body.limit,
|
||||
offset=body.offset,
|
||||
)
|
||||
|
||||
if not results:
|
||||
return {"results": [], "total": 0}
|
||||
|
||||
# Hydrate with photo data
|
||||
photo_ids = [r["photo_id"] for r in results]
|
||||
stmt = select(Photo).where(Photo.id.in_(photo_ids))
|
||||
rows = (await db.execute(stmt)).scalars().all()
|
||||
photo_map = {p.id: p for p in rows}
|
||||
|
||||
hydrated = []
|
||||
for r in results:
|
||||
photo = photo_map.get(r["photo_id"])
|
||||
if not photo:
|
||||
continue
|
||||
hydrated.append({
|
||||
"id": photo.id,
|
||||
"filename": photo.filename,
|
||||
"filepath": photo.filepath,
|
||||
"media_type": photo.media_type,
|
||||
"width": photo.width,
|
||||
"height": photo.height,
|
||||
"taken_at": photo.taken_at.isoformat() if photo.taken_at else None,
|
||||
"rating": photo.rating,
|
||||
"color_label": photo.color_label,
|
||||
"thumb_small": photo.thumb_small,
|
||||
"thumb_medium": photo.thumb_medium,
|
||||
"score": r["score"],
|
||||
})
|
||||
|
||||
return {"results": hydrated, "total": len(hydrated)}
|
||||
@@ -1,10 +1,15 @@
|
||||
"""
|
||||
Tags API router
|
||||
Tags API router.
|
||||
|
||||
Unified across user tags, ML-detected objects, and face clusters via
|
||||
the `kind` query parameter. Default behaviour (no kind filter) returns
|
||||
all tags — the frontend's "Hide auto-generated tags" toggle filters
|
||||
client-side or passes `kind=user`.
|
||||
"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func, insert, delete
|
||||
from sqlalchemy import select, func, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
@@ -19,6 +24,7 @@ router = APIRouter()
|
||||
class TagCreate(BaseModel):
|
||||
name: str
|
||||
color: Optional[str] = None
|
||||
kind: str = "user"
|
||||
|
||||
|
||||
class TagUpdate(BaseModel):
|
||||
@@ -26,24 +32,35 @@ class TagUpdate(BaseModel):
|
||||
color: Optional[str] = None
|
||||
|
||||
|
||||
class TagMerge(BaseModel):
|
||||
target_id: str # tag to merge INTO
|
||||
|
||||
|
||||
# ── Endpoints ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("")
|
||||
async def list_tags(db: AsyncSession = Depends(get_db)):
|
||||
"""List all tags with their photo counts."""
|
||||
async def list_tags(
|
||||
kind: Optional[str] = Query(None, description="Filter by kind: user, object, scene, face_cluster"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all tags with their photo counts, optionally filtered by kind."""
|
||||
count_subq = (
|
||||
select(
|
||||
photo_tags.c.tag_id,
|
||||
func.count(photo_tags.c.photo_id).label("photo_count"),
|
||||
func.min(photo_tags.c.photo_id).label("first_photo_id"),
|
||||
)
|
||||
.group_by(photo_tags.c.tag_id)
|
||||
.subquery()
|
||||
)
|
||||
stmt = (
|
||||
select(Tag, count_subq.c.photo_count)
|
||||
select(Tag, count_subq.c.photo_count, count_subq.c.first_photo_id)
|
||||
.outerjoin(count_subq, Tag.id == count_subq.c.tag_id)
|
||||
.order_by(Tag.name.asc())
|
||||
)
|
||||
if kind:
|
||||
stmt = stmt.where(Tag.kind == kind)
|
||||
stmt = stmt.order_by(Tag.name.asc())
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = result.all()
|
||||
|
||||
@@ -52,38 +69,48 @@ async def list_tags(db: AsyncSession = Depends(get_db)):
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"color": tag.color,
|
||||
"kind": tag.kind,
|
||||
"source": tag.source,
|
||||
"representative_photo_id": tag.representative_photo_id or first_photo_id,
|
||||
"photo_count": int(count or 0),
|
||||
}
|
||||
for tag, count in rows
|
||||
for tag, count, first_photo_id in rows
|
||||
]
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_tag(body: TagCreate, db: AsyncSession = Depends(get_db)):
|
||||
"""Create a new tag. Names are unique — re-creating an existing name
|
||||
returns the existing row instead of erroring (idempotent for the
|
||||
autocomplete UI flow)."""
|
||||
"""Create a new tag. The (name, kind) pair is unique — re-creating an
|
||||
existing pair returns the existing row (idempotent for autocomplete)."""
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="Tag name is required")
|
||||
|
||||
existing = await db.execute(select(Tag).where(Tag.name == name))
|
||||
existing = await db.execute(
|
||||
select(Tag).where(Tag.name == name, Tag.kind == body.kind)
|
||||
)
|
||||
found = existing.scalar_one_or_none()
|
||||
if found:
|
||||
return {"id": found.id, "name": found.name, "color": found.color, "photo_count": 0}
|
||||
return {
|
||||
"id": found.id, "name": found.name, "color": found.color,
|
||||
"kind": found.kind, "photo_count": 0,
|
||||
}
|
||||
|
||||
tag = Tag(name=name, color=body.color)
|
||||
tag = Tag(name=name, color=body.color, kind=body.kind)
|
||||
db.add(tag)
|
||||
await db.commit()
|
||||
await db.refresh(tag)
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "photo_count": 0}
|
||||
return {
|
||||
"id": tag.id, "name": tag.name, "color": tag.color,
|
||||
"kind": tag.kind, "photo_count": 0,
|
||||
}
|
||||
|
||||
|
||||
@router.patch("/{tag_id}")
|
||||
async def update_tag(
|
||||
tag_id: str, body: TagUpdate, db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Rename or recolor a tag."""
|
||||
"""Rename or recolor a tag (works for any kind — user, object, face_cluster)."""
|
||||
result = await db.execute(select(Tag).where(Tag.id == tag_id))
|
||||
tag = result.scalar_one_or_none()
|
||||
if not tag:
|
||||
@@ -99,7 +126,52 @@ async def update_tag(
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(tag)
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color}
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "kind": tag.kind}
|
||||
|
||||
|
||||
@router.post("/{tag_id}/merge")
|
||||
async def merge_tag(
|
||||
tag_id: str, body: TagMerge, db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""Merge tag_id INTO target_id. All photo associations from the source
|
||||
tag are moved to the target, then the source tag is deleted.
|
||||
|
||||
Useful for merging auto-detected face clusters (e.g. "Person 3" → "Alice")
|
||||
or merging duplicate object labels."""
|
||||
if tag_id == body.target_id:
|
||||
raise HTTPException(status_code=400, detail="Cannot merge a tag into itself")
|
||||
|
||||
source = (await db.execute(select(Tag).where(Tag.id == tag_id))).scalar_one_or_none()
|
||||
target = (await db.execute(select(Tag).where(Tag.id == body.target_id))).scalar_one_or_none()
|
||||
if not source:
|
||||
raise HTTPException(status_code=404, detail="Source tag not found")
|
||||
if not target:
|
||||
raise HTTPException(status_code=404, detail="Target tag not found")
|
||||
|
||||
# Move photo associations: update tag_id from source → target.
|
||||
# Skip any that would violate the PK (photo already tagged with target).
|
||||
existing_target_photos = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.tag_id == body.target_id
|
||||
)
|
||||
await db.execute(
|
||||
update(photo_tags)
|
||||
.where(
|
||||
photo_tags.c.tag_id == tag_id,
|
||||
photo_tags.c.photo_id.notin_(existing_target_photos),
|
||||
)
|
||||
.values(tag_id=body.target_id)
|
||||
)
|
||||
# Delete remaining source associations (duplicates that couldn't move)
|
||||
from sqlalchemy import delete as sa_delete
|
||||
await db.execute(
|
||||
sa_delete(photo_tags).where(photo_tags.c.tag_id == tag_id)
|
||||
)
|
||||
|
||||
# Delete source tag
|
||||
await db.delete(source)
|
||||
await db.commit()
|
||||
|
||||
return {"merged_into": target.id, "target_name": target.name}
|
||||
|
||||
|
||||
@router.delete("/{tag_id}", status_code=204)
|
||||
|
||||
140
backend/app/services/search.py
Normal file
140
backend/app/services/search.py
Normal file
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
Unified search service — hybrid FTS + semantic (RRF) search.
|
||||
|
||||
Phase 1 (PR4): semantic-only via pgvector cosine similarity.
|
||||
Phase 2 (PR5): adds FTS via tsvector, enables RRF fusion.
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
from sqlalchemy import select, text, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Photo
|
||||
from app.models.embeddings import Embedding
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def hybrid_search(
|
||||
db: AsyncSession,
|
||||
q: Optional[str] = None,
|
||||
tag_ids: Optional[list[str]] = None,
|
||||
date_from: Optional[str] = None,
|
||||
date_to: Optional[str] = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
"""Run hybrid search (FTS + semantic) with RRF fusion.
|
||||
|
||||
Currently semantic-only; FTS leg added in PR5.
|
||||
"""
|
||||
model_name = settings.vision.embedder.name
|
||||
results = {}
|
||||
|
||||
# ── Semantic search (CLIP text → pgvector cosine) ─────────────────
|
||||
if q:
|
||||
try:
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
query_vec = embedder.embed_text(q)
|
||||
|
||||
# pgvector cosine distance: <=> returns distance (lower = closer)
|
||||
vec_str = "[" + ",".join(str(float(v)) for v in query_vec) + "]"
|
||||
stmt = text("""
|
||||
SELECT e.photo_id,
|
||||
(e.vector <=> :qvec::vector) AS distance
|
||||
FROM embeddings e
|
||||
WHERE e.model = :model
|
||||
ORDER BY e.vector <=> :qvec::vector
|
||||
LIMIT 200
|
||||
""")
|
||||
rows = (await db.execute(stmt, {"qvec": vec_str, "model": model_name})).fetchall()
|
||||
|
||||
for rank, (photo_id, distance) in enumerate(rows):
|
||||
if photo_id not in results:
|
||||
results[photo_id] = {"semantic_rank": rank, "fts_rank": None}
|
||||
else:
|
||||
results[photo_id]["semantic_rank"] = rank
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Semantic search failed (models may not be loaded): %s", e)
|
||||
|
||||
# ── FTS search (photos.search_vector + ocr_text) ────────────────
|
||||
if q:
|
||||
try:
|
||||
fts_stmt = text("""
|
||||
SELECT id, ts_rank(search_vector, plainto_tsquery('english', :q)) AS rank
|
||||
FROM photos
|
||||
WHERE search_vector @@ plainto_tsquery('english', :q)
|
||||
UNION
|
||||
SELECT o.photo_id AS id,
|
||||
MAX(o.confidence) AS rank
|
||||
FROM ocr_text o
|
||||
WHERE to_tsvector('english', o.text) @@ plainto_tsquery('english', :q)
|
||||
GROUP BY o.photo_id
|
||||
ORDER BY rank DESC
|
||||
LIMIT 200
|
||||
""")
|
||||
fts_rows = (await db.execute(fts_stmt, {"q": q})).fetchall()
|
||||
for rank, (photo_id, score) in enumerate(fts_rows):
|
||||
if photo_id not in results:
|
||||
results[photo_id] = {"semantic_rank": None, "fts_rank": rank}
|
||||
else:
|
||||
results[photo_id]["fts_rank"] = rank
|
||||
except Exception as e:
|
||||
logger.warning("FTS search failed: %s", e)
|
||||
|
||||
# ── RRF fusion ────────────────────────────────────────────────────
|
||||
k = 60
|
||||
scored = []
|
||||
for photo_id, ranks in results.items():
|
||||
score = 0.0
|
||||
if ranks["semantic_rank"] is not None:
|
||||
score += 1.0 / (k + ranks["semantic_rank"])
|
||||
if ranks.get("fts_rank") is not None:
|
||||
score += 1.0 / (k + ranks["fts_rank"])
|
||||
scored.append((photo_id, score))
|
||||
|
||||
scored.sort(key=lambda x: -x[1])
|
||||
|
||||
# If no text query, fall back to recent photos
|
||||
if not q:
|
||||
if tag_ids:
|
||||
from app.models.tags import photo_tags
|
||||
# Subquery to get distinct photo_ids matching the tag filter
|
||||
sub = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.tag_id.in_(tag_ids)
|
||||
).distinct().subquery()
|
||||
stmt = select(Photo.id).join(sub, Photo.id == sub.c.photo_id)
|
||||
else:
|
||||
stmt = select(Photo.id)
|
||||
stmt = stmt.order_by(Photo.added_at.desc())
|
||||
if date_from:
|
||||
stmt = stmt.where(Photo.taken_at >= date_from)
|
||||
if date_to:
|
||||
stmt = stmt.where(Photo.taken_at <= date_to)
|
||||
stmt = stmt.offset(offset).limit(limit)
|
||||
rows = (await db.execute(stmt)).fetchall()
|
||||
return [{"photo_id": row[0], "score": 0.0} for row in rows]
|
||||
|
||||
# Apply filters to scored results
|
||||
photo_ids = [pid for pid, _ in scored]
|
||||
if not photo_ids:
|
||||
return []
|
||||
|
||||
# Filter by tags if requested
|
||||
if tag_ids:
|
||||
from app.models.tags import photo_tags
|
||||
stmt = select(photo_tags.c.photo_id).where(
|
||||
photo_tags.c.photo_id.in_(photo_ids),
|
||||
photo_tags.c.tag_id.in_(tag_ids),
|
||||
).distinct()
|
||||
valid_ids = {row[0] for row in (await db.execute(stmt)).fetchall()}
|
||||
scored = [(pid, s) for pid, s in scored if pid in valid_ids]
|
||||
|
||||
# Paginate
|
||||
page = scored[offset : offset + limit]
|
||||
return [{"photo_id": pid, "score": score} for pid, score in page]
|
||||
7
backend/app/services/vision/__init__.py
Normal file
7
backend/app/services/vision/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
Vision pipeline services — embedding, OCR, object detection, face recognition.
|
||||
|
||||
All inference is done through the ModelRegistry singleton, which lazy-loads
|
||||
ONNX Runtime sessions on first use and caches them for the lifetime of the
|
||||
worker process.
|
||||
"""
|
||||
105
backend/app/services/vision/base.py
Normal file
105
backend/app/services/vision/base.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
Abstract base classes for vision backends.
|
||||
|
||||
Each ABC defines the contract a backend must satisfy. The default
|
||||
implementation is ONNXBackend (onnx_backend.py). A ROCm backend can be
|
||||
added later by subclassing these ABCs and registering via
|
||||
settings.vision.backend.
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectionBox:
|
||||
"""A single object detection result."""
|
||||
label: str
|
||||
confidence: float
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
|
||||
|
||||
@dataclass
|
||||
class OCRResult:
|
||||
"""A single OCR text region."""
|
||||
text: str
|
||||
confidence: float
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
language: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FaceDetection:
|
||||
"""A detected face with its recognition embedding."""
|
||||
bbox: list[float] # [x1, y1, x2, y2] normalized 0-1
|
||||
embedding: np.ndarray # float32 vector (128-d for SFace)
|
||||
quality: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassificationResult:
|
||||
"""A content-type classification."""
|
||||
label: str
|
||||
confidence: float
|
||||
|
||||
|
||||
class Embedder(ABC):
|
||||
"""Generates image and text embeddings (e.g. OpenCLIP ViT-B/32)."""
|
||||
|
||||
@abstractmethod
|
||||
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
||||
"""Return a normalized float32 embedding vector for an RGB image."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def embed_text(self, text: str) -> np.ndarray:
|
||||
"""Return a normalized float32 embedding vector for a text query."""
|
||||
...
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def dim(self) -> int:
|
||||
"""Dimensionality of the output embedding."""
|
||||
...
|
||||
|
||||
|
||||
class OCREngine(ABC):
|
||||
"""Extracts text from images (e.g. rapidocr-onnxruntime)."""
|
||||
|
||||
@abstractmethod
|
||||
def run(self, image: np.ndarray) -> list[OCRResult]:
|
||||
"""Return OCR results for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class ObjectDetector(ABC):
|
||||
"""Detects objects in images (e.g. YOLOv8n)."""
|
||||
|
||||
@abstractmethod
|
||||
def detect(self, image: np.ndarray) -> list[DetectionBox]:
|
||||
"""Return detections for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class ContentClassifier(ABC):
|
||||
"""Classifies images into content types (screenshot, document, etc.)."""
|
||||
|
||||
@abstractmethod
|
||||
def classify(self, image: np.ndarray) -> list[ClassificationResult]:
|
||||
"""Return content type classifications for an RGB image."""
|
||||
...
|
||||
|
||||
|
||||
class FaceProcessor(ABC):
|
||||
"""Detects faces and extracts recognition embeddings (e.g. YuNet + SFace)."""
|
||||
|
||||
@abstractmethod
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
"""Return face detections with embeddings for an RGB image."""
|
||||
...
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def embedding_dim(self) -> int:
|
||||
"""Dimensionality of face embedding vectors."""
|
||||
...
|
||||
85
backend/app/services/vision/bootstrap_models.py
Normal file
85
backend/app/services/vision/bootstrap_models.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
Download vision model weights on first worker boot.
|
||||
|
||||
Run as: python -m app.services.vision.bootstrap_models
|
||||
|
||||
Or called from the vision worker entrypoint before Celery starts.
|
||||
Downloads are idempotent — existing files are skipped.
|
||||
|
||||
For models that require export (OpenCLIP, YOLOv8n), see export_models.py.
|
||||
Those must be exported once on any machine with pip, then placed in
|
||||
the models volume before the worker starts.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# (relative_path, url, description)
|
||||
# Models with url=None must be pre-exported via export_models.py.
|
||||
# InsightFace (RetinaFace + ArcFace) auto-downloads via the insightface
|
||||
# package on first use — no manual download entries needed.
|
||||
DOWNLOADS = []
|
||||
|
||||
# Models that need manual export via export_models.py
|
||||
EXPORTS = [
|
||||
("embed/visual.onnx", "OpenCLIP ViT-B/32 visual encoder"),
|
||||
("embed/textual.onnx", "OpenCLIP ViT-B/32 textual encoder"),
|
||||
("detect/yolov8n.onnx", "YOLOv8n object detector"),
|
||||
]
|
||||
|
||||
|
||||
def bootstrap(models_dir: str | None = None):
|
||||
"""Ensure all model files are present. Download what we can, warn about
|
||||
files that need manual export."""
|
||||
base = Path(models_dir or settings.vision.models_dir)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Download auto-downloadable models
|
||||
for rel_path, url, desc in DOWNLOADS:
|
||||
dest = base / rel_path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if dest.exists():
|
||||
logger.debug("Already exists: %s (%s)", dest, desc)
|
||||
continue
|
||||
|
||||
logger.info("Downloading %s → %s", desc, dest)
|
||||
try:
|
||||
urlretrieve(url, str(dest))
|
||||
size_kb = dest.stat().st_size / 1024
|
||||
logger.info("Downloaded %s (%.0f KB)", desc, size_kb)
|
||||
except Exception as e:
|
||||
logger.error("Failed to download %s: %s", desc, e)
|
||||
if dest.exists():
|
||||
dest.unlink()
|
||||
|
||||
# Check for manually-exported models
|
||||
missing = []
|
||||
for rel_path, desc in EXPORTS:
|
||||
dest = base / rel_path
|
||||
if not dest.exists():
|
||||
missing.append((rel_path, desc))
|
||||
|
||||
if missing:
|
||||
logger.warning(
|
||||
"Missing %d model(s) that require manual export via export_models.py:",
|
||||
len(missing),
|
||||
)
|
||||
for rel_path, desc in missing:
|
||||
logger.warning(" %s — %s", base / rel_path, desc)
|
||||
logger.warning(
|
||||
"Run: python -m app.services.vision.export_models --models-dir %s",
|
||||
base,
|
||||
)
|
||||
else:
|
||||
logger.info("All model files present in %s", base)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
bootstrap()
|
||||
106
backend/app/services/vision/classify.py
Normal file
106
backend/app/services/vision/classify.py
Normal file
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
CLIP zero-shot content-type classifier.
|
||||
|
||||
Uses the native OpenCLIP PyTorch text encoder for high-quality text
|
||||
embeddings (the ONNX text encoder has degraded quality due to the
|
||||
eot_indices workaround). Image embeddings use the ONNX visual encoder
|
||||
which works well.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import ContentClassifier, ClassificationResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CATEGORY_PROMPTS = {
|
||||
"screenshot": [
|
||||
"a screenshot of a computer screen",
|
||||
"a screenshot of a phone screen",
|
||||
"a screen capture of a user interface",
|
||||
],
|
||||
"document": [
|
||||
"a scanned document",
|
||||
"a photo of a document with printed text",
|
||||
"a photo of a page of text on paper",
|
||||
],
|
||||
"receipt": [
|
||||
"a photo of a receipt",
|
||||
"a photo of a bill or invoice",
|
||||
],
|
||||
"meme": [
|
||||
"an internet meme with text overlay",
|
||||
"a funny image with caption text",
|
||||
],
|
||||
"artwork": [
|
||||
"a painting or drawing",
|
||||
"a sketch or illustration",
|
||||
"digital art or graphic design",
|
||||
],
|
||||
"photograph": [
|
||||
"a photograph taken with a camera",
|
||||
"a real photo of a real scene or person",
|
||||
"a candid photograph",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class CLIPContentClassifier(ContentClassifier):
|
||||
"""Zero-shot content classifier using CLIP text-image similarity.
|
||||
Uses native PyTorch for text encoding, ONNX for image encoding."""
|
||||
|
||||
def __init__(self, settings: VisionSettings):
|
||||
import open_clip
|
||||
|
||||
self._min_confidence = settings.classifier.min_confidence
|
||||
|
||||
# Load native model for text encoding only
|
||||
logger.info("Loading OpenCLIP text encoder for content classification")
|
||||
model, _, _ = open_clip.create_model_and_transforms(
|
||||
"ViT-B-32", pretrained="laion2b_s34b_b79k"
|
||||
)
|
||||
model.eval()
|
||||
self._model = model
|
||||
self._tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
||||
|
||||
# Get the ONNX image embedder from the registry
|
||||
from app.services.vision.registry import registry
|
||||
self._embedder = registry.get_embedder()
|
||||
|
||||
# Pre-compute text embeddings for each category
|
||||
self._category_embeddings: dict[str, np.ndarray] = {}
|
||||
for category, prompts in CATEGORY_PROMPTS.items():
|
||||
tokens = self._tokenizer(prompts)
|
||||
with torch.no_grad():
|
||||
text_features = model.encode_text(tokens)
|
||||
text_features /= text_features.norm(dim=-1, keepdim=True)
|
||||
avg = text_features.mean(dim=0)
|
||||
avg /= avg.norm()
|
||||
self._category_embeddings[category] = avg.numpy().astype(np.float32)
|
||||
|
||||
logger.info("Content classifier ready with %d categories", len(self._category_embeddings))
|
||||
|
||||
def classify(self, image: np.ndarray) -> list[ClassificationResult]:
|
||||
img_vec = self._embedder.embed_image(image)
|
||||
|
||||
# Cosine similarity against each category
|
||||
scores = {}
|
||||
for category, cat_vec in self._category_embeddings.items():
|
||||
scores[category] = float(np.dot(img_vec, cat_vec))
|
||||
|
||||
# Sort by score descending
|
||||
ranked = sorted(scores.items(), key=lambda x: -x[1])
|
||||
best_cat, best_score = ranked[0]
|
||||
second_score = ranked[1][1]
|
||||
|
||||
margin = best_score - second_score
|
||||
# Normalize: 0.01 margin → ~0.5 confidence, 0.03+ → ~1.0
|
||||
confidence = min(1.0, margin * 30)
|
||||
|
||||
if confidence >= self._min_confidence:
|
||||
return [ClassificationResult(label=best_cat, confidence=confidence)]
|
||||
|
||||
return []
|
||||
42
backend/app/services/vision/clustering.py
Normal file
42
backend/app/services/vision/clustering.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Face embedding clustering using DBSCAN with cosine distance.
|
||||
|
||||
Called by the periodic `recluster_faces` Celery task (PR7).
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
from sklearn.cluster import DBSCAN
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cluster_faces(
|
||||
embeddings: np.ndarray,
|
||||
eps: float = 0.35,
|
||||
min_samples: int = 2,
|
||||
) -> np.ndarray:
|
||||
"""Cluster face embeddings using DBSCAN with cosine metric.
|
||||
|
||||
Args:
|
||||
embeddings: (N, D) float32 array of L2-normalized face embeddings.
|
||||
eps: Maximum cosine distance between two samples to be in the
|
||||
same neighborhood. Lower = tighter clusters.
|
||||
min_samples: Minimum cluster size.
|
||||
|
||||
Returns:
|
||||
(N,) int array of cluster labels. -1 = noise / unclustered.
|
||||
"""
|
||||
if len(embeddings) < min_samples:
|
||||
return np.full(len(embeddings), -1, dtype=int)
|
||||
|
||||
db = DBSCAN(eps=eps, min_samples=min_samples, metric="cosine")
|
||||
labels = db.fit_predict(embeddings)
|
||||
|
||||
n_clusters = len(set(labels) - {-1})
|
||||
n_noise = (labels == -1).sum()
|
||||
logger.info(
|
||||
"Face clustering: %d embeddings → %d clusters, %d noise",
|
||||
len(embeddings), n_clusters, n_noise,
|
||||
)
|
||||
return labels
|
||||
140
backend/app/services/vision/detect.py
Normal file
140
backend/app/services/vision/detect.py
Normal file
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
YOLOv8n object detector using raw ONNX Runtime.
|
||||
|
||||
Expects {models_dir}/detect/yolov8n.onnx, exported from ultralytics
|
||||
via bootstrap_models.py. We do NOT ship ultralytics at runtime to
|
||||
avoid dragging in torch.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import ObjectDetector, DetectionBox
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_INPUT_SIZE = 640
|
||||
|
||||
# COCO class names (80 classes)
|
||||
COCO_LABELS = [
|
||||
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
|
||||
"truck", "boat", "traffic light", "fire hydrant", "stop sign",
|
||||
"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep",
|
||||
"cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella",
|
||||
"handbag", "tie", "suitcase", "frisbee", "skis", "snowboard",
|
||||
"sports ball", "kite", "baseball bat", "baseball glove", "skateboard",
|
||||
"surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork",
|
||||
"knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
|
||||
"broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
|
||||
"couch", "potted plant", "bed", "dining table", "toilet", "tv",
|
||||
"laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
|
||||
"oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
|
||||
"scissors", "teddy bear", "hair drier", "toothbrush",
|
||||
]
|
||||
|
||||
|
||||
def _preprocess(image: np.ndarray) -> tuple[np.ndarray, float, float]:
|
||||
"""Letterbox-resize + normalize to NCHW float32. Returns input tensor
|
||||
and scale factors for mapping boxes back to original coords."""
|
||||
from PIL import Image
|
||||
|
||||
img = Image.fromarray(image).convert("RGB")
|
||||
orig_w, orig_h = img.size
|
||||
|
||||
scale = min(_INPUT_SIZE / orig_w, _INPUT_SIZE / orig_h)
|
||||
new_w = int(orig_w * scale)
|
||||
new_h = int(orig_h * scale)
|
||||
img = img.resize((new_w, new_h), Image.BICUBIC)
|
||||
|
||||
# Paste onto gray canvas
|
||||
canvas = np.full((_INPUT_SIZE, _INPUT_SIZE, 3), 114, dtype=np.uint8)
|
||||
pad_x = (_INPUT_SIZE - new_w) // 2
|
||||
pad_y = (_INPUT_SIZE - new_h) // 2
|
||||
canvas[pad_y : pad_y + new_h, pad_x : pad_x + new_w] = np.array(img)
|
||||
|
||||
blob = canvas.astype(np.float32) / 255.0
|
||||
blob = blob.transpose(2, 0, 1)[np.newaxis] # NCHW
|
||||
return blob, scale, pad_x, pad_y
|
||||
|
||||
|
||||
def _postprocess(
|
||||
outputs: np.ndarray,
|
||||
scale: float,
|
||||
pad_x: int,
|
||||
pad_y: int,
|
||||
orig_w: int,
|
||||
orig_h: int,
|
||||
conf_threshold: float,
|
||||
max_detections: int,
|
||||
) -> list[DetectionBox]:
|
||||
"""Parse YOLOv8 output (1, 84, N) → list of DetectionBox."""
|
||||
# outputs shape: (1, 84, N) where 84 = 4 box coords + 80 class scores
|
||||
preds = outputs[0] # (84, N)
|
||||
preds = preds.T # (N, 84)
|
||||
|
||||
boxes_xywh = preds[:, :4]
|
||||
scores = preds[:, 4:]
|
||||
|
||||
class_ids = np.argmax(scores, axis=1)
|
||||
confidences = scores[np.arange(len(scores)), class_ids]
|
||||
|
||||
mask = confidences >= conf_threshold
|
||||
boxes_xywh = boxes_xywh[mask]
|
||||
class_ids = class_ids[mask]
|
||||
confidences = confidences[mask]
|
||||
|
||||
if len(confidences) == 0:
|
||||
return []
|
||||
|
||||
# Sort by confidence, take top N
|
||||
order = np.argsort(-confidences)[:max_detections]
|
||||
boxes_xywh = boxes_xywh[order]
|
||||
class_ids = class_ids[order]
|
||||
confidences = confidences[order]
|
||||
|
||||
results = []
|
||||
for i in range(len(confidences)):
|
||||
cx, cy, w, h = boxes_xywh[i]
|
||||
# Remove letterbox padding and rescale to original image
|
||||
x1 = (cx - w / 2 - pad_x) / scale
|
||||
y1 = (cy - h / 2 - pad_y) / scale
|
||||
x2 = (cx + w / 2 - pad_x) / scale
|
||||
y2 = (cy + h / 2 - pad_y) / scale
|
||||
# Normalize to 0-1
|
||||
bbox = [
|
||||
max(0, x1 / orig_w),
|
||||
max(0, y1 / orig_h),
|
||||
min(1, x2 / orig_w),
|
||||
min(1, y2 / orig_h),
|
||||
]
|
||||
label = COCO_LABELS[class_ids[i]] if class_ids[i] < len(COCO_LABELS) else f"class_{class_ids[i]}"
|
||||
results.append(DetectionBox(label=label, confidence=float(confidences[i]), bbox=bbox))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class YOLOv8Detector(ObjectDetector):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
model_path = Path(settings.models_dir) / "detect" / "yolov8n.onnx"
|
||||
|
||||
opts = ort.SessionOptions()
|
||||
opts.inter_op_num_threads = 2
|
||||
opts.intra_op_num_threads = 2
|
||||
|
||||
logger.info("Loading YOLOv8n from %s", model_path)
|
||||
self._session = ort.InferenceSession(str(model_path), opts, providers=["CPUExecutionProvider"])
|
||||
self._conf_threshold = settings.detector.min_confidence
|
||||
self._max_detections = settings.detector.max_detections
|
||||
|
||||
def detect(self, image: np.ndarray) -> list[DetectionBox]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
blob, scale, pad_x, pad_y = _preprocess(image)
|
||||
input_name = self._session.get_inputs()[0].name
|
||||
outputs = self._session.run(None, {input_name: blob})[0]
|
||||
return _postprocess(
|
||||
outputs, scale, pad_x, pad_y, orig_w, orig_h,
|
||||
self._conf_threshold, self._max_detections,
|
||||
)
|
||||
86
backend/app/services/vision/embed.py
Normal file
86
backend/app/services/vision/embed.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""
|
||||
OpenCLIP ViT-B/32 embedder using ONNX Runtime.
|
||||
|
||||
Expects two ONNX files under {models_dir}/embed/:
|
||||
- visual.onnx (image encoder)
|
||||
- textual.onnx (text encoder)
|
||||
|
||||
These are exported from open_clip via bootstrap_models.py.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# OpenCLIP ViT-B/32 preprocessing constants (ImageNet norm)
|
||||
_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
|
||||
_STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32)
|
||||
_INPUT_SIZE = 224
|
||||
|
||||
|
||||
def _preprocess_image(image: np.ndarray) -> np.ndarray:
|
||||
"""Resize, center-crop, normalize an RGB uint8 image to NCHW float32."""
|
||||
from PIL import Image
|
||||
|
||||
img = Image.fromarray(image).convert("RGB")
|
||||
# Resize shortest edge to _INPUT_SIZE, then center crop
|
||||
w, h = img.size
|
||||
scale = _INPUT_SIZE / min(w, h)
|
||||
img = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC)
|
||||
w, h = img.size
|
||||
left = (w - _INPUT_SIZE) // 2
|
||||
top = (h - _INPUT_SIZE) // 2
|
||||
img = img.crop((left, top, left + _INPUT_SIZE, top + _INPUT_SIZE))
|
||||
|
||||
arr = np.array(img, dtype=np.float32) / 255.0
|
||||
arr = (arr - _MEAN) / _STD
|
||||
arr = arr.transpose(2, 0, 1) # HWC → CHW
|
||||
return arr[np.newaxis] # NCHW
|
||||
|
||||
|
||||
class OpenCLIPEmbedder(Embedder):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
model_dir = Path(settings.models_dir) / "embed"
|
||||
visual_path = model_dir / "visual.onnx"
|
||||
textual_path = model_dir / "textual.onnx"
|
||||
|
||||
opts = ort.SessionOptions()
|
||||
opts.inter_op_num_threads = 2
|
||||
opts.intra_op_num_threads = 2
|
||||
|
||||
logger.info("Loading visual encoder from %s", visual_path)
|
||||
self._visual = ort.InferenceSession(str(visual_path), opts, providers=["CPUExecutionProvider"])
|
||||
|
||||
logger.info("Loading textual encoder from %s", textual_path)
|
||||
self._textual = ort.InferenceSession(str(textual_path), opts, providers=["CPUExecutionProvider"])
|
||||
|
||||
def embed_image(self, image: np.ndarray) -> np.ndarray:
|
||||
inp = _preprocess_image(image)
|
||||
input_name = self._visual.get_inputs()[0].name
|
||||
out = self._visual.run(None, {input_name: inp})[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
def embed_text(self, text: str) -> np.ndarray:
|
||||
import open_clip
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
||||
tokens = tokenizer([text]).numpy().astype(np.int64)
|
||||
# Compute EOT indices outside ONNX (avoids ArgMax(13) op)
|
||||
eot_indices = tokens.argmax(axis=-1).astype(np.int64)
|
||||
inputs = self._textual.get_inputs()
|
||||
out = self._textual.run(None, {
|
||||
inputs[0].name: tokens,
|
||||
inputs[1].name: eot_indices,
|
||||
})[0][0]
|
||||
out = out / np.linalg.norm(out)
|
||||
return out.astype(np.float32)
|
||||
|
||||
@property
|
||||
def dim(self) -> int:
|
||||
return 512
|
||||
177
backend/app/services/vision/export_models.py
Normal file
177
backend/app/services/vision/export_models.py
Normal file
@@ -0,0 +1,177 @@
|
||||
"""
|
||||
Export / download all vision model weights to ONNX format.
|
||||
|
||||
Run ONCE on any machine with Python + pip (doesn't need GPU):
|
||||
|
||||
pip install open-clip-torch ultralytics onnx
|
||||
python -m app.services.vision.export_models [--models-dir /data/models]
|
||||
|
||||
This produces:
|
||||
embed/visual.onnx (~350 MB)
|
||||
embed/textual.onnx (~250 MB)
|
||||
detect/yolov8n.onnx (~12 MB)
|
||||
|
||||
YuNet and SFace are downloaded by bootstrap_models.py at worker boot
|
||||
(Apache 2.0, lightweight, no export step needed).
|
||||
|
||||
After export, copy the /data/models directory into your Docker volume:
|
||||
docker cp /data/models mulita-worker:/data/models
|
||||
Or mount a host path in docker-compose.yml.
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def export_openclip(models_dir: Path):
|
||||
"""Export OpenCLIP ViT-B/32 to two ONNX files (visual + textual)."""
|
||||
import torch
|
||||
import open_clip
|
||||
|
||||
out_dir = models_dir / "embed"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
visual_path = out_dir / "visual.onnx"
|
||||
textual_path = out_dir / "textual.onnx"
|
||||
|
||||
if visual_path.exists() and textual_path.exists():
|
||||
logger.info("OpenCLIP ONNX files already exist, skipping export")
|
||||
return
|
||||
|
||||
logger.info("Loading OpenCLIP ViT-B-32 laion2b_s34b_b79k...")
|
||||
model, _, preprocess = open_clip.create_model_and_transforms(
|
||||
"ViT-B-32", pretrained="laion2b_s34b_b79k"
|
||||
)
|
||||
model.eval()
|
||||
|
||||
# Use dynamo=False to get the legacy TorchScript exporter which
|
||||
# produces IR version 9 (compatible with onnxruntime 1.17.x).
|
||||
# The new torch.onnx.export default (dynamo=True) emits IR 10.
|
||||
export_kwargs = dict(opset_version=14, dynamo=False)
|
||||
|
||||
# ── Visual encoder ────────────────────────────────────────────────
|
||||
if not visual_path.exists():
|
||||
logger.info("Exporting visual encoder → %s", visual_path)
|
||||
dummy_image = torch.randn(1, 3, 224, 224)
|
||||
torch.onnx.export(
|
||||
model.visual,
|
||||
dummy_image,
|
||||
str(visual_path),
|
||||
input_names=["image"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"image": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = visual_path.stat().st_size / 1e6
|
||||
logger.info("Visual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
# ── Textual encoder ───────────────────────────────────────────────
|
||||
if not textual_path.exists():
|
||||
logger.info("Exporting textual encoder → %s", textual_path)
|
||||
tokenizer = open_clip.get_tokenizer("ViT-B-32")
|
||||
dummy_text = tokenizer(["a photo"]).to(torch.int64)
|
||||
|
||||
class TextEncoder(torch.nn.Module):
|
||||
"""Wrap the CLIP text encoder to avoid argmax in the ONNX graph.
|
||||
OpenCLIP uses argmax to find the EOT token position, but ORT
|
||||
ARM64 doesn't support ArgMax(13). We pre-compute the EOT index
|
||||
from the token sequence and pass it directly."""
|
||||
def __init__(self, clip_model):
|
||||
super().__init__()
|
||||
self.transformer = clip_model.transformer
|
||||
self.token_embedding = clip_model.token_embedding
|
||||
self.positional_embedding = clip_model.positional_embedding
|
||||
self.ln_final = clip_model.ln_final
|
||||
self.text_projection = clip_model.text_projection
|
||||
|
||||
def forward(self, text, eot_indices):
|
||||
x = self.token_embedding(text)
|
||||
x = x + self.positional_embedding
|
||||
x = x.permute(1, 0, 2) # NLD -> LND
|
||||
x = self.transformer(x)
|
||||
x = x.permute(1, 0, 2) # LND -> NLD
|
||||
x = self.ln_final(x)
|
||||
# Take the feature at the EOT token. The EOT index is
|
||||
# passed in as a separate input (computed outside ONNX)
|
||||
# to avoid ArgMax(13) which ORT ARM64 doesn't support.
|
||||
x = x[torch.arange(x.shape[0]), eot_indices]
|
||||
x = x @ self.text_projection
|
||||
return x
|
||||
|
||||
text_enc = TextEncoder(model)
|
||||
text_enc.eval()
|
||||
|
||||
# Compute EOT indices from dummy tokens (argmax of token ids)
|
||||
dummy_eot = dummy_text.argmax(dim=-1)
|
||||
|
||||
torch.onnx.export(
|
||||
text_enc,
|
||||
(dummy_text, dummy_eot),
|
||||
str(textual_path),
|
||||
input_names=["text", "eot_indices"],
|
||||
output_names=["embedding"],
|
||||
dynamic_axes={"text": {0: "batch"}, "eot_indices": {0: "batch"}},
|
||||
**export_kwargs,
|
||||
)
|
||||
size_mb = textual_path.stat().st_size / 1e6
|
||||
logger.info("Textual encoder exported (%.1f MB)", size_mb)
|
||||
|
||||
|
||||
def export_yolov8n(models_dir: Path):
|
||||
"""Export YOLOv8n to ONNX."""
|
||||
out_dir = models_dir / "detect"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
onnx_path = out_dir / "yolov8n.onnx"
|
||||
|
||||
if onnx_path.exists():
|
||||
logger.info("YOLOv8n ONNX already exists, skipping export")
|
||||
return
|
||||
|
||||
logger.info("Exporting YOLOv8n → %s", onnx_path)
|
||||
|
||||
from ultralytics import YOLO
|
||||
|
||||
model = YOLO("yolov8n.pt")
|
||||
model.export(format="onnx", imgsz=640, simplify=True)
|
||||
|
||||
# ultralytics exports to cwd as yolov8n.onnx — move to target
|
||||
exported = Path("yolov8n.onnx")
|
||||
if exported.exists():
|
||||
exported.rename(onnx_path)
|
||||
|
||||
size_mb = onnx_path.stat().st_size / 1e6
|
||||
logger.info("YOLOv8n exported (%.1f MB)", size_mb)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Export vision model weights to ONNX")
|
||||
parser.add_argument(
|
||||
"--models-dir",
|
||||
type=Path,
|
||||
default=Path("/data/models"),
|
||||
help="Directory to write model files (default: /data/models)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
|
||||
models_dir = args.models_dir
|
||||
models_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info("Exporting models to %s", models_dir)
|
||||
|
||||
export_openclip(models_dir)
|
||||
export_yolov8n(models_dir)
|
||||
|
||||
logger.info("Done. Run bootstrap_models.py next to download YuNet + SFace.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
148
backend/app/services/vision/faces.py
Normal file
148
backend/app/services/vision/faces.py
Normal file
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
Face detection (YuNet) + recognition (SFace) using OpenCV DNN.
|
||||
|
||||
YuNet is loaded via cv2.FaceDetectorYN which handles the multi-scale
|
||||
anchor decoding and NMS internally. SFace recognition uses raw ONNX
|
||||
Runtime for the 128-d embedding.
|
||||
|
||||
Both models are from opencv_zoo (Apache 2.0 license).
|
||||
Expects {models_dir}/face/:
|
||||
- yunet.onnx (~233 KB)
|
||||
- sface.onnx (~37 MB, 128-d embeddings)
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import cv2
|
||||
import onnxruntime as ort
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import FaceProcessor, FaceDetection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _align_face(image: np.ndarray, landmarks: np.ndarray) -> np.ndarray:
|
||||
"""Align and crop a 112x112 face patch using 5-point landmarks."""
|
||||
left_eye = landmarks[0]
|
||||
right_eye = landmarks[1]
|
||||
|
||||
dx = right_eye[0] - left_eye[0]
|
||||
dy = right_eye[1] - left_eye[1]
|
||||
angle = np.degrees(np.arctan2(dy, dx))
|
||||
eye_center = ((left_eye[0] + right_eye[0]) / 2, (left_eye[1] + right_eye[1]) / 2)
|
||||
eye_dist = np.sqrt(dx * dx + dy * dy)
|
||||
|
||||
M = cv2.getRotationMatrix2D(eye_center, angle, 1.0)
|
||||
rotated = cv2.warpAffine(image, M, (image.shape[1], image.shape[0]))
|
||||
|
||||
# Crop around face center
|
||||
scale = 64.0 / max(eye_dist, 1e-6)
|
||||
cx, cy = eye_center
|
||||
half = 56.0 / scale
|
||||
x1 = max(0, int(cx - half))
|
||||
y1 = max(0, int(cy - half * 0.8))
|
||||
x2 = min(rotated.shape[1], int(cx + half))
|
||||
y2 = min(rotated.shape[0], int(cy + half * 1.2))
|
||||
crop = rotated[y1:y2, x1:x2]
|
||||
|
||||
if crop.size == 0:
|
||||
return np.zeros((112, 112, 3), dtype=np.float32)
|
||||
|
||||
return cv2.resize(crop, (112, 112)).astype(np.float32)
|
||||
|
||||
|
||||
class YuNetSFaceProcessor(FaceProcessor):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
face_dir = Path(settings.models_dir) / "face"
|
||||
yunet_path = str(face_dir / "yunet.onnx")
|
||||
sface_path = str(face_dir / "sface.onnx")
|
||||
|
||||
# YuNet via OpenCV's FaceDetectorYN — handles anchor decoding + NMS
|
||||
self._detector = cv2.FaceDetectorYN.create(
|
||||
yunet_path,
|
||||
"",
|
||||
(640, 640),
|
||||
settings.faces.recognition_threshold,
|
||||
0.3, # NMS threshold
|
||||
5000, # top_k
|
||||
)
|
||||
logger.info("YuNet face detector loaded via OpenCV")
|
||||
|
||||
# SFace via ONNX Runtime
|
||||
opts = ort.SessionOptions()
|
||||
opts.inter_op_num_threads = 2
|
||||
opts.intra_op_num_threads = 2
|
||||
ort.set_default_logger_severity(3)
|
||||
self._recognizer = ort.InferenceSession(sface_path, opts, providers=["CPUExecutionProvider"])
|
||||
logger.info("SFace recognizer loaded via ONNX Runtime")
|
||||
|
||||
self._min_face_size = settings.faces.min_face_size
|
||||
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
|
||||
# Convert RGB → BGR for OpenCV
|
||||
bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
|
||||
# Set input size to actual image dimensions
|
||||
self._detector.setInputSize((orig_w, orig_h))
|
||||
|
||||
# Detect faces
|
||||
_, faces_raw = self._detector.detect(bgr)
|
||||
|
||||
if faces_raw is None or len(faces_raw) == 0:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for face in faces_raw:
|
||||
# face: [x, y, w, h, right_eye_x, right_eye_y, left_eye_x, left_eye_y,
|
||||
# nose_x, nose_y, right_mouth_x, right_mouth_y, left_mouth_x, left_mouth_y, score]
|
||||
x, y, w, h = int(face[0]), int(face[1]), int(face[2]), int(face[3])
|
||||
score = float(face[14])
|
||||
|
||||
# Filter small faces
|
||||
face_size = max(w, h)
|
||||
if face_size < self._min_face_size:
|
||||
continue
|
||||
|
||||
# Normalized bbox
|
||||
bbox = [
|
||||
max(0, x / orig_w),
|
||||
max(0, y / orig_h),
|
||||
min(1, (x + w) / orig_w),
|
||||
min(1, (y + h) / orig_h),
|
||||
]
|
||||
|
||||
# Extract 5-point landmarks for alignment
|
||||
landmarks = np.array([
|
||||
[face[4], face[5]], # right eye
|
||||
[face[6], face[7]], # left eye
|
||||
[face[8], face[9]], # nose
|
||||
[face[10], face[11]], # right mouth
|
||||
[face[12], face[13]], # left mouth
|
||||
], dtype=np.float32)
|
||||
|
||||
# Align face for recognition
|
||||
face_crop = _align_face(image, landmarks)
|
||||
|
||||
# SFace expects (1, 3, 112, 112) float32, BGR
|
||||
face_bgr = cv2.cvtColor(face_crop.astype(np.uint8), cv2.COLOR_RGB2BGR)
|
||||
face_blob = (face_bgr.astype(np.float32) / 255.0).transpose(2, 0, 1)[np.newaxis]
|
||||
|
||||
rec_input = self._recognizer.get_inputs()[0].name
|
||||
embedding = self._recognizer.run(None, {rec_input: face_blob})[0][0]
|
||||
embedding = embedding / np.linalg.norm(embedding)
|
||||
|
||||
results.append(FaceDetection(
|
||||
bbox=bbox,
|
||||
embedding=embedding.astype(np.float32),
|
||||
quality=score,
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return 128
|
||||
70
backend/app/services/vision/insightface_processor.py
Normal file
70
backend/app/services/vision/insightface_processor.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Face detection + recognition using InsightFace (RetinaFace + ArcFace).
|
||||
|
||||
Uses the buffalo_l model pack which auto-downloads on first use (~300MB).
|
||||
Produces 512-d ArcFace embeddings. Non-commercial research license —
|
||||
fine for homelab self-hosting.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import FaceProcessor, FaceDetection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InsightFaceProcessor(FaceProcessor):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
from insightface.app import FaceAnalysis
|
||||
|
||||
model_root = str(Path(settings.models_dir) / "face" / "insightface")
|
||||
logger.info("Loading InsightFace buffalo_l from %s", model_root)
|
||||
|
||||
self._app = FaceAnalysis(
|
||||
name="buffalo_l",
|
||||
root=model_root,
|
||||
providers=["CPUExecutionProvider"],
|
||||
)
|
||||
self._app.prepare(ctx_id=-1, det_size=(640, 640))
|
||||
self._min_det_score = settings.faces.recognition_threshold
|
||||
|
||||
def process(self, image: np.ndarray) -> list[FaceDetection]:
|
||||
orig_h, orig_w = image.shape[:2]
|
||||
|
||||
# InsightFace expects BGR
|
||||
bgr = image[:, :, ::-1].copy()
|
||||
|
||||
faces = self._app.get(bgr)
|
||||
|
||||
if not faces:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for face in faces:
|
||||
if face.det_score < self._min_det_score:
|
||||
continue
|
||||
|
||||
# face.bbox is [x1, y1, x2, y2] in pixel coords
|
||||
x1, y1, x2, y2 = face.bbox
|
||||
bbox = [
|
||||
max(0, float(x1) / orig_w),
|
||||
max(0, float(y1) / orig_h),
|
||||
min(1, float(x2) / orig_w),
|
||||
min(1, float(y2) / orig_h),
|
||||
]
|
||||
|
||||
embedding = face.normed_embedding # already L2-normalized, 512-d
|
||||
results.append(FaceDetection(
|
||||
bbox=bbox,
|
||||
embedding=embedding.astype(np.float32),
|
||||
quality=float(face.det_score),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return 512
|
||||
45
backend/app/services/vision/ocr.py
Normal file
45
backend/app/services/vision/ocr.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
OCR engine using rapidocr-onnxruntime (PP-OCRv4 weights).
|
||||
|
||||
No PaddlePaddle dependency — pure ONNX Runtime. Language packs are
|
||||
downloaded automatically by rapidocr on first use.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import OCREngine, OCRResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RapidOCREngine(OCREngine):
|
||||
def __init__(self, settings: VisionSettings):
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
self._min_confidence = settings.ocr.min_confidence
|
||||
self._engine = RapidOCR()
|
||||
logger.info("RapidOCR engine initialized")
|
||||
|
||||
def run(self, image: np.ndarray) -> list[OCRResult]:
|
||||
result, _ = self._engine(image)
|
||||
if not result:
|
||||
return []
|
||||
|
||||
out = []
|
||||
for box, text, score in result:
|
||||
if score < self._min_confidence:
|
||||
continue
|
||||
# box is [[x1,y1],[x2,y2],[x3,y3],[x4,y4]] — take bounding rect
|
||||
xs = [p[0] for p in box]
|
||||
ys = [p[1] for p in box]
|
||||
h, w = image.shape[:2]
|
||||
bbox = [
|
||||
min(xs) / w,
|
||||
min(ys) / h,
|
||||
max(xs) / w,
|
||||
max(ys) / h,
|
||||
]
|
||||
out.append(OCRResult(text=text, confidence=float(score), bbox=bbox))
|
||||
return out
|
||||
41
backend/app/services/vision/onnx_backend.py
Normal file
41
backend/app/services/vision/onnx_backend.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
ONNX Runtime backend — default CPU inference for all vision models.
|
||||
|
||||
Each create_* method returns a concrete implementation of the
|
||||
corresponding ABC from base.py. Models are loaded from ONNX files
|
||||
under settings.vision.models_dir, downloaded on first boot by
|
||||
bootstrap_models.py.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor, ContentClassifier
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ONNXBackend:
|
||||
"""Factory for ONNX Runtime-based vision model instances."""
|
||||
|
||||
def __init__(self, vision_settings: VisionSettings):
|
||||
self._settings = vision_settings
|
||||
|
||||
def create_embedder(self) -> Embedder:
|
||||
from app.services.vision.embed import OpenCLIPEmbedder
|
||||
return OpenCLIPEmbedder(self._settings)
|
||||
|
||||
def create_ocr(self) -> OCREngine:
|
||||
from app.services.vision.ocr import RapidOCREngine
|
||||
return RapidOCREngine(self._settings)
|
||||
|
||||
def create_detector(self) -> ObjectDetector:
|
||||
from app.services.vision.detect import YOLOv8Detector
|
||||
return YOLOv8Detector(self._settings)
|
||||
|
||||
def create_face_processor(self) -> FaceProcessor:
|
||||
from app.services.vision.insightface_processor import InsightFaceProcessor
|
||||
return InsightFaceProcessor(self._settings)
|
||||
|
||||
def create_classifier(self) -> ContentClassifier:
|
||||
from app.services.vision.classify import CLIPContentClassifier
|
||||
return CLIPContentClassifier(self._settings)
|
||||
84
backend/app/services/vision/registry.py
Normal file
84
backend/app/services/vision/registry.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
ModelRegistry — singleton that lazy-loads vision models per worker process.
|
||||
|
||||
Usage from Celery tasks:
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
vec = embedder.embed_image(img)
|
||||
|
||||
Models are created on first access and cached for the worker's lifetime.
|
||||
The registry reads settings.vision to decide which backend to use and
|
||||
where model weights live.
|
||||
"""
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
|
||||
from app.config import settings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor, ContentClassifier
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ModelRegistry:
|
||||
"""Central access point for all vision models."""
|
||||
|
||||
def __init__(self):
|
||||
self._vision = settings.vision
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_embedder(self) -> Embedder:
|
||||
logger.info("Loading embedder: %s (backend=%s)", self._vision.embedder.name, self._vision.backend)
|
||||
return self._load_backend().create_embedder()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_ocr(self) -> OCREngine:
|
||||
logger.info("Loading OCR engine (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_ocr()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_detector(self) -> ObjectDetector:
|
||||
logger.info("Loading object detector (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_detector()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_face_processor(self) -> FaceProcessor:
|
||||
logger.info("Loading face processor (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_face_processor()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_classifier(self) -> ContentClassifier:
|
||||
logger.info("Loading content classifier (backend=%s)", self._vision.backend)
|
||||
return self._load_backend().create_classifier()
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_backend(self):
|
||||
"""Import and instantiate the configured backend."""
|
||||
backend_name = self._vision.backend
|
||||
if backend_name == "onnx":
|
||||
from app.services.vision.onnx_backend import ONNXBackend
|
||||
return ONNXBackend(self._vision)
|
||||
elif backend_name == "rocm":
|
||||
from app.services.vision.rocm_backend import ROCmBackend
|
||||
return ROCmBackend(self._vision)
|
||||
else:
|
||||
raise ValueError(f"Unknown vision backend: {backend_name}")
|
||||
|
||||
def warmup(self):
|
||||
"""Pre-load all enabled models. Called from Celery worker_process_init
|
||||
on the vision queue to avoid cold-start latency on the first task."""
|
||||
logger.info("Warming up vision models...")
|
||||
self.get_embedder()
|
||||
if self._vision.ocr.enabled:
|
||||
self.get_ocr()
|
||||
if self._vision.detector.enabled:
|
||||
self.get_detector()
|
||||
if self._vision.faces.enabled:
|
||||
self.get_face_processor()
|
||||
if self._vision.classifier.enabled:
|
||||
self.get_classifier()
|
||||
logger.info("Vision model warmup complete")
|
||||
|
||||
|
||||
# Module-level singleton. Import this from tasks.
|
||||
registry = ModelRegistry()
|
||||
25
backend/app/services/vision/rocm_backend.py
Normal file
25
backend/app/services/vision/rocm_backend.py
Normal file
@@ -0,0 +1,25 @@
|
||||
"""
|
||||
ROCm backend — GPU-accelerated inference for Radeon 760M-class hardware.
|
||||
|
||||
Stub: raises NotImplementedError on all factory methods. To enable,
|
||||
set `vision.backend: rocm` in mulita.yml once ROCm support is implemented.
|
||||
"""
|
||||
from app.config import VisionSettings
|
||||
from app.services.vision.base import Embedder, OCREngine, ObjectDetector, FaceProcessor
|
||||
|
||||
|
||||
class ROCmBackend:
|
||||
def __init__(self, vision_settings: VisionSettings):
|
||||
self._settings = vision_settings
|
||||
|
||||
def create_embedder(self) -> Embedder:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_ocr(self) -> OCREngine:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_detector(self) -> ObjectDetector:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
|
||||
def create_face_processor(self) -> FaceProcessor:
|
||||
raise NotImplementedError("ROCm backend not yet implemented — use 'onnx'")
|
||||
@@ -9,7 +9,7 @@ celery_app = Celery(
|
||||
'mulita',
|
||||
broker=settings.celery_broker_url,
|
||||
backend=settings.celery_result_backend,
|
||||
include=['app.tasks.scan', 'app.tasks.thumbs']
|
||||
include=['app.tasks.scan', 'app.tasks.thumbs', 'app.tasks.vision']
|
||||
)
|
||||
|
||||
# Configure Celery
|
||||
@@ -22,6 +22,13 @@ celery_app.conf.update(
|
||||
task_routes={
|
||||
'app.tasks.thumbs.*': {'queue': 'high'},
|
||||
'app.tasks.scan.*': {'queue': 'low'},
|
||||
'app.tasks.vision.*': {'queue': 'vision'},
|
||||
'embed_photo': {'queue': 'vision'},
|
||||
'ocr_photo': {'queue': 'vision'},
|
||||
'detect_objects': {'queue': 'vision'},
|
||||
'extract_faces': {'queue': 'vision'},
|
||||
'classify_content': {'queue': 'vision'},
|
||||
'vision_fanout': {'queue': 'vision'},
|
||||
},
|
||||
task_default_queue='default',
|
||||
task_default_exchange='default',
|
||||
|
||||
@@ -317,6 +317,15 @@ async def _generate_thumbnails_async(photo_id: str, task):
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"Thumbnails generated for photo {photo_id}")
|
||||
|
||||
# Dispatch vision pipeline (embedding, OCR, detection, faces)
|
||||
# after thumbs are ready so vision tasks have images to read.
|
||||
try:
|
||||
from app.tasks.vision import vision_fanout
|
||||
vision_fanout.delay(photo_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not dispatch vision_fanout for {photo_id}: {e}")
|
||||
|
||||
return {'status': 'success', 'photo_id': photo_id}
|
||||
|
||||
except Exception as e:
|
||||
|
||||
479
backend/app/tasks/vision.py
Normal file
479
backend/app/tasks/vision.py
Normal file
@@ -0,0 +1,479 @@
|
||||
"""
|
||||
Celery tasks for the vision pipeline — embedding, OCR, object detection,
|
||||
face recognition.
|
||||
|
||||
All tasks run on the dedicated `vision` queue with limited concurrency
|
||||
(memory-bound CPU inference). They read thumbnails generated by
|
||||
generate_thumbnails, so they MUST run after thumbs complete.
|
||||
|
||||
DB access uses sync psycopg2 sessions (not asyncpg) because Celery
|
||||
forks workers and asyncpg connections can't be shared across forks.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from celery import shared_task
|
||||
from sqlalchemy import create_engine, text as sa_text, select, delete
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from PIL import Image
|
||||
|
||||
from app.models.embeddings import Embedding
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_sync_session() -> Session:
|
||||
"""Create a sync DB session for use in Celery workers."""
|
||||
sync_url = settings.database_url.replace("+asyncpg", "+psycopg2").replace("+aiosqlite", "")
|
||||
engine = create_engine(sync_url, pool_pre_ping=True)
|
||||
return sessionmaker(bind=engine)()
|
||||
|
||||
|
||||
def _load_thumb(photo_id: str, size: str = "medium") -> np.ndarray | None:
|
||||
"""Load a thumbnail as an RGB numpy array."""
|
||||
thumb_path = Path(f"/data/thumbs/{photo_id}/{size}.webp")
|
||||
if not thumb_path.exists():
|
||||
logger.warning("Thumbnail not found: %s", thumb_path)
|
||||
return None
|
||||
img = Image.open(thumb_path).convert("RGB")
|
||||
return np.array(img)
|
||||
|
||||
|
||||
@shared_task(name='embed_photo', queue='vision')
|
||||
def embed_photo(photo_id: str):
|
||||
"""Generate CLIP embedding for a photo and store in pgvector."""
|
||||
if not settings.vision.enabled:
|
||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium") # 640px
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
embedder = registry.get_embedder()
|
||||
vector = embedder.embed_image(image)
|
||||
|
||||
model_name = settings.vision.embedder.name
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(
|
||||
delete(Embedding).where(
|
||||
Embedding.photo_id == photo_id,
|
||||
Embedding.model == model_name,
|
||||
)
|
||||
)
|
||||
emb = Embedding(
|
||||
photo_id=photo_id,
|
||||
model=model_name,
|
||||
vector=vector.tolist(),
|
||||
)
|
||||
session.add(emb)
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("Embedded photo %s with %s", photo_id, model_name)
|
||||
return {'status': 'success', 'photo_id': photo_id}
|
||||
|
||||
|
||||
@shared_task(name='vision_fanout', queue='vision')
|
||||
def vision_fanout(photo_id: str):
|
||||
"""Dispatch all enabled vision tasks for a photo."""
|
||||
if not settings.vision.enabled:
|
||||
return {'status': 'skipped', 'reason': 'vision disabled'}
|
||||
|
||||
embed_photo.delay(photo_id)
|
||||
|
||||
if settings.vision.ocr.enabled:
|
||||
ocr_photo.delay(photo_id)
|
||||
if settings.vision.detector.enabled:
|
||||
detect_objects.delay(photo_id)
|
||||
if settings.vision.faces.enabled:
|
||||
extract_faces.delay(photo_id)
|
||||
if settings.vision.classifier.enabled:
|
||||
classify_content.delay(photo_id)
|
||||
|
||||
return {'status': 'dispatched', 'photo_id': photo_id}
|
||||
|
||||
|
||||
@shared_task(name='ocr_photo', queue='vision')
|
||||
def ocr_photo(photo_id: str):
|
||||
"""Run OCR on a photo and store text regions."""
|
||||
if not settings.vision.enabled or not settings.vision.ocr.enabled:
|
||||
return {'status': 'skipped', 'reason': 'OCR disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "large") # 1280px for better OCR accuracy
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
ocr_engine = registry.get_ocr()
|
||||
results = ocr_engine.run(image)
|
||||
|
||||
if not results:
|
||||
logger.info("No OCR text found for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'regions': 0}
|
||||
|
||||
from app.models.ocr_text import OCRText
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(delete(OCRText).where(OCRText.photo_id == photo_id))
|
||||
for r in results:
|
||||
session.add(OCRText(
|
||||
photo_id=photo_id,
|
||||
text=r.text,
|
||||
language=r.language,
|
||||
confidence=r.confidence,
|
||||
bbox=r.bbox,
|
||||
))
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("OCR: %d text regions for photo %s", len(results), photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'regions': len(results)}
|
||||
|
||||
|
||||
@shared_task(name='detect_objects', queue='vision')
|
||||
def detect_objects(photo_id: str):
|
||||
"""Detect objects in a photo, create Tag(kind=object) rows, and
|
||||
link via photo_tags with confidence/bbox/source."""
|
||||
if not settings.vision.enabled or not settings.vision.detector.enabled:
|
||||
return {'status': 'skipped', 'reason': 'detection disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium") # 640px
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
detector = registry.get_detector()
|
||||
detections = detector.detect(image)
|
||||
|
||||
if not detections:
|
||||
logger.info("No objects detected for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'objects': 0}
|
||||
|
||||
from app.models.tags import Tag, photo_tags
|
||||
|
||||
source_name = "vision:yolov8n"
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
# Wipe previous detection results for this photo from this model
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.photo_id == photo_id,
|
||||
photo_tags.c.source == source_name,
|
||||
)
|
||||
)
|
||||
|
||||
# Group detections by label, keep highest confidence per label
|
||||
best_per_label: dict[str, tuple[float, list]] = {}
|
||||
for det in detections:
|
||||
if det.label not in best_per_label or det.confidence > best_per_label[det.label][0]:
|
||||
best_per_label[det.label] = (det.confidence, det.bbox)
|
||||
|
||||
for label, (confidence, bbox) in best_per_label.items():
|
||||
# Find or create the object tag
|
||||
tag = session.execute(
|
||||
select(Tag).where(Tag.name == label, Tag.kind == 'object')
|
||||
).scalar_one_or_none()
|
||||
|
||||
if not tag:
|
||||
tag = Tag(name=label, kind='object', source=source_name)
|
||||
session.add(tag)
|
||||
session.flush() # get tag.id
|
||||
|
||||
# Insert photo_tags association with ML metadata
|
||||
session.execute(
|
||||
photo_tags.insert().values(
|
||||
photo_id=photo_id,
|
||||
tag_id=tag.id,
|
||||
confidence=confidence,
|
||||
bbox=bbox,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
labels = [d.label for d in detections]
|
||||
logger.info("Detected %d objects in photo %s: %s", len(detections), photo_id, labels)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'objects': len(detections)}
|
||||
|
||||
|
||||
@shared_task(name='classify_content', queue='vision')
|
||||
def classify_content(photo_id: str):
|
||||
"""Classify image content type (screenshot, document, artwork, etc.)
|
||||
using CLIP zero-shot classification. Writes Tag(kind=content_type)."""
|
||||
if not settings.vision.enabled or not settings.vision.classifier.enabled:
|
||||
return {'status': 'skipped', 'reason': 'classifier disabled'}
|
||||
|
||||
image = _load_thumb(photo_id, "medium")
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'thumbnail not found'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
classifier = registry.get_classifier()
|
||||
results = classifier.classify(image)
|
||||
|
||||
if not results:
|
||||
logger.info("No confident classification for photo %s", photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'content_type': None}
|
||||
|
||||
from app.models.tags import Tag, photo_tags
|
||||
|
||||
source_name = "vision:clip_classifier"
|
||||
best = results[0]
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
# Wipe previous classification for this photo
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.photo_id == photo_id,
|
||||
photo_tags.c.source == source_name,
|
||||
)
|
||||
)
|
||||
|
||||
# Find or create content_type tag
|
||||
tag = session.execute(
|
||||
select(Tag).where(Tag.name == best.label, Tag.kind == 'content_type')
|
||||
).scalar_one_or_none()
|
||||
|
||||
if not tag:
|
||||
tag = Tag(name=best.label, kind='content_type', source=source_name)
|
||||
session.add(tag)
|
||||
session.flush()
|
||||
|
||||
session.execute(
|
||||
photo_tags.insert().values(
|
||||
photo_id=photo_id,
|
||||
tag_id=tag.id,
|
||||
confidence=best.confidence,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
logger.info("Classified photo %s as '%s' (%.2f)", photo_id, best.label, best.confidence)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'content_type': best.label}
|
||||
|
||||
|
||||
def _load_original(photo_id: str) -> np.ndarray | None:
|
||||
"""Load the original photo file as an RGB numpy array, resized to
|
||||
max 1280px on the longest edge for face detection."""
|
||||
from sqlalchemy import create_engine, select as sa_select, text as sa_text
|
||||
from app.models import Photo
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
photo = session.execute(
|
||||
sa_select(Photo).where(Photo.id == photo_id)
|
||||
).scalar_one_or_none()
|
||||
if not photo or not photo.filepath:
|
||||
return None
|
||||
filepath = photo.filepath
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if not Path(filepath).exists():
|
||||
logger.warning("Original file not found: %s", filepath)
|
||||
return None
|
||||
|
||||
try:
|
||||
img = Image.open(filepath).convert("RGB")
|
||||
# Cap at 4000px on longest edge to avoid OOM, but keep as large
|
||||
# as possible for face detection accuracy
|
||||
max_dim = 4000
|
||||
w, h = img.size
|
||||
if max(w, h) > max_dim:
|
||||
scale = max_dim / max(w, h)
|
||||
img = img.resize((int(w * scale), int(h * scale)), Image.BICUBIC)
|
||||
return np.array(img)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load original %s: %s", filepath, e)
|
||||
return None
|
||||
|
||||
|
||||
@shared_task(name='extract_faces', queue='vision')
|
||||
def extract_faces(photo_id: str):
|
||||
"""Detect faces and store recognition embeddings using InsightFace
|
||||
(RetinaFace + ArcFace). No YOLO workaround needed — RetinaFace has
|
||||
strong human-vs-non-human precision on its own."""
|
||||
if not settings.vision.enabled or not settings.vision.faces.enabled:
|
||||
return {'status': 'skipped', 'reason': 'faces disabled'}
|
||||
|
||||
image = _load_original(photo_id)
|
||||
if image is None:
|
||||
image = _load_thumb(photo_id, "large")
|
||||
if image is None:
|
||||
return {'status': 'error', 'message': 'no image available'}
|
||||
|
||||
from app.services.vision.registry import registry
|
||||
face_proc = registry.get_face_processor()
|
||||
faces = face_proc.process(image)
|
||||
|
||||
if not faces:
|
||||
logger.info("No faces detected for photo %s", photo_id)
|
||||
|
||||
return _save_faces(photo_id, faces)
|
||||
|
||||
|
||||
def _save_faces(photo_id: str, faces) -> dict:
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
session.execute(delete(FaceEmbedding).where(FaceEmbedding.photo_id == photo_id))
|
||||
for face in faces:
|
||||
session.add(FaceEmbedding(
|
||||
photo_id=photo_id,
|
||||
bbox=face.bbox,
|
||||
vector=face.embedding.tolist(),
|
||||
quality=face.quality,
|
||||
cluster_id=None,
|
||||
))
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if faces:
|
||||
logger.info("Extracted %d verified face(s) from photo %s", len(faces), photo_id)
|
||||
return {'status': 'success', 'photo_id': photo_id, 'faces': len(faces)}
|
||||
|
||||
|
||||
@shared_task(name='recluster_faces', queue='vision')
|
||||
def recluster_faces():
|
||||
"""Run DBSCAN clustering over all face embeddings and assign/create
|
||||
Tag(kind=face_cluster) entries."""
|
||||
if not settings.vision.enabled or not settings.vision.faces.enabled:
|
||||
return {'status': 'skipped', 'reason': 'faces disabled'}
|
||||
|
||||
from app.models.face_embedding import FaceEmbedding
|
||||
from app.models.tags import Tag, photo_tags
|
||||
from app.services.vision.clustering import cluster_faces
|
||||
|
||||
source_name = "vision:sface"
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
face_rows = session.execute(
|
||||
select(FaceEmbedding).order_by(FaceEmbedding.created_at)
|
||||
).scalars().all()
|
||||
|
||||
if len(face_rows) < 2:
|
||||
logger.info("Not enough faces for clustering (%d)", len(face_rows))
|
||||
return {'status': 'success', 'clusters': 0}
|
||||
|
||||
embeddings = np.array([f.vector for f in face_rows], dtype=np.float32)
|
||||
labels = cluster_faces(embeddings, eps=settings.vision.faces.cluster_eps)
|
||||
|
||||
# Clean up old face_cluster tags and their photo_tags
|
||||
old_cluster_tags = session.execute(
|
||||
select(Tag).where(Tag.kind == 'face_cluster', Tag.source == source_name)
|
||||
).scalars().all()
|
||||
for old_tag in old_cluster_tags:
|
||||
session.execute(
|
||||
delete(photo_tags).where(
|
||||
photo_tags.c.tag_id == old_tag.id,
|
||||
photo_tags.c.source == source_name,
|
||||
)
|
||||
)
|
||||
session.delete(old_tag)
|
||||
session.flush()
|
||||
|
||||
# Build new clusters
|
||||
cluster_tag_map: dict[int, str] = {}
|
||||
# Track which photos belong to which cluster
|
||||
cluster_photos: dict[int, set[str]] = {}
|
||||
|
||||
for i, label in enumerate(labels):
|
||||
if label == -1:
|
||||
face_rows[i].cluster_id = None
|
||||
continue
|
||||
|
||||
if label not in cluster_photos:
|
||||
cluster_photos[label] = set()
|
||||
cluster_photos[label].add(face_rows[i].photo_id)
|
||||
|
||||
if label not in cluster_tag_map:
|
||||
cluster_name = f"Person {label + 1}"
|
||||
tag = Tag(
|
||||
name=cluster_name,
|
||||
kind='face_cluster',
|
||||
source=source_name,
|
||||
representative_photo_id=face_rows[i].photo_id,
|
||||
)
|
||||
session.add(tag)
|
||||
session.flush()
|
||||
cluster_tag_map[label] = tag.id
|
||||
|
||||
face_rows[i].cluster_id = cluster_tag_map[label]
|
||||
|
||||
# Write photo_tags associations so the tag count and tag_ids
|
||||
# filter work for face clusters
|
||||
for label, photo_ids in cluster_photos.items():
|
||||
tag_id = cluster_tag_map[label]
|
||||
for pid in photo_ids:
|
||||
session.execute(
|
||||
photo_tags.insert().values(
|
||||
photo_id=pid,
|
||||
tag_id=tag_id,
|
||||
source=source_name,
|
||||
)
|
||||
)
|
||||
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
n_clusters = len(cluster_tag_map)
|
||||
logger.info("Face clustering: %d clusters from %d faces", n_clusters, len(face_rows))
|
||||
return {'status': 'success', 'clusters': n_clusters, 'faces': len(face_rows)}
|
||||
|
||||
|
||||
@shared_task(name='backfill_vision')
|
||||
def backfill_vision(task: str | None = None, limit: int | None = None):
|
||||
"""Queue vision tasks for photos that haven't been processed yet.
|
||||
Uses a sync DB connection to avoid asyncpg conflicts in Celery."""
|
||||
model_name = settings.vision.embedder.name
|
||||
sql = """
|
||||
SELECT p.id FROM photos p
|
||||
LEFT JOIN embeddings e ON e.photo_id = p.id AND e.model = :model
|
||||
WHERE e.photo_id IS NULL
|
||||
AND p.processing_status = 'completed'
|
||||
ORDER BY p.added_at DESC
|
||||
"""
|
||||
if limit:
|
||||
sql += f" LIMIT {limit}"
|
||||
|
||||
session = _get_sync_session()
|
||||
try:
|
||||
result = session.execute(sa_text(sql), {"model": model_name})
|
||||
photo_ids = [row[0] for row in result.fetchall()]
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
count = 0
|
||||
for pid in photo_ids:
|
||||
if task == 'embed' or task is None:
|
||||
embed_photo.delay(pid)
|
||||
if task == 'ocr' or task is None:
|
||||
ocr_photo.delay(pid)
|
||||
if task == 'detect' or task is None:
|
||||
detect_objects.delay(pid)
|
||||
if task == 'faces' or task is None:
|
||||
extract_faces.delay(pid)
|
||||
count += 1
|
||||
|
||||
logger.info("Backfill queued %d photos for vision processing", count)
|
||||
return {'status': 'queued', 'count': count}
|
||||
@@ -5,7 +5,10 @@ python-multipart==0.0.6
|
||||
|
||||
# Database
|
||||
sqlalchemy[asyncio]==2.0.25
|
||||
aiosqlite==0.19.0
|
||||
aiosqlite==0.19.0 # SQLite escape hatch (docker-compose.sqlite.yml override)
|
||||
asyncpg==0.29.0 # async Postgres driver (default)
|
||||
psycopg2-binary==2.9.9 # sync Postgres driver, used by Alembic CLI
|
||||
pgvector==0.2.5 # pgvector SQLAlchemy types
|
||||
alembic==1.13.1
|
||||
|
||||
# Redis and Celery
|
||||
@@ -31,6 +34,14 @@ pyexiftool==0.5.6
|
||||
# File watching
|
||||
watchfiles==0.21.0
|
||||
|
||||
# Vision pipeline (ONNX Runtime CPU inference)
|
||||
onnxruntime==1.18.1
|
||||
open-clip-torch==2.24.0 # tokenizer + export helper; inference via ONNX
|
||||
rapidocr-onnxruntime==1.3.22
|
||||
scikit-learn==1.4.0 # DBSCAN for face clustering
|
||||
insightface>=0.7.3 # RetinaFace + ArcFace face detection/recognition
|
||||
numpy>=1.26.0,<2.0
|
||||
|
||||
# Utilities
|
||||
pyyaml==6.0.1
|
||||
pydantic==2.5.3
|
||||
|
||||
47
docker-compose.sqlite.yml
Normal file
47
docker-compose.sqlite.yml
Normal file
@@ -0,0 +1,47 @@
|
||||
# SQLite escape hatch override.
|
||||
#
|
||||
# Usage (omit the `db` service from the up command):
|
||||
#
|
||||
# docker compose -f docker-compose.yml -f docker-compose.sqlite.yml \
|
||||
# up frontend backend worker redis
|
||||
#
|
||||
# This pins the backend and worker to the legacy SQLite database file at
|
||||
# /data/db/mulita.db (in the existing db_data volume), drops the dependency
|
||||
# on Postgres, and skips Alembic — the SQLite schema is still managed by
|
||||
# the inline ALTERs in app/database.py:init_db.
|
||||
#
|
||||
# Vision features that depend on pgvector (PR4 onward) will refuse to enable
|
||||
# in this mode; the search/embedding endpoints will return 503 with a clear
|
||||
# error pointing back at the default Postgres setup.
|
||||
|
||||
services:
|
||||
backend:
|
||||
command: sh -c "uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload"
|
||||
environment:
|
||||
- DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-*}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
|
||||
worker:
|
||||
environment:
|
||||
- DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
- PHOTO_DIRS=${PHOTO_DIRS:-/photos}
|
||||
- CELERYD_CONCURRENCY=${CELERYD_CONCURRENCY:-4}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_started
|
||||
backend:
|
||||
condition: service_started
|
||||
@@ -1,5 +1,3 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
frontend:
|
||||
build:
|
||||
@@ -36,9 +34,13 @@ services:
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
- thumbs_data:/data/thumbs
|
||||
- proxies_data:/data/proxies
|
||||
- db_data:/data/db
|
||||
- db_data:/data/db # retained so the docker-compose.sqlite.yml override has somewhere to put mulita.db
|
||||
# Run Alembic migrations before starting uvicorn. On a fresh Postgres
|
||||
# the empty 0001 baseline is a no-op stamp; create_all in init_db then
|
||||
# builds the schema.
|
||||
command: sh -c "alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload"
|
||||
environment:
|
||||
- DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
@@ -47,7 +49,10 @@ services:
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
- redis
|
||||
redis:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
@@ -57,15 +62,16 @@ services:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
container_name: mulita-worker
|
||||
command: celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERYD_CONCURRENCY:-4}
|
||||
command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERYD_CONCURRENCY:-4} -Q default,high,low,vision"
|
||||
volumes:
|
||||
- ./mulita.yml:/app/config/mulita.yml:ro
|
||||
- ${PHOTO_DIRS:-./photos}:/photos:rw
|
||||
- thumbs_data:/data/thumbs
|
||||
- proxies_data:/data/proxies
|
||||
- db_data:/data/db
|
||||
- models_data:/data/models
|
||||
environment:
|
||||
- DATABASE_URL=sqlite+aiosqlite:////data/db/mulita.db
|
||||
- DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- CELERY_BROKER_URL=redis://redis:6379
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379
|
||||
@@ -74,12 +80,34 @@ services:
|
||||
- LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
- TZ=${TZ:-UTC}
|
||||
depends_on:
|
||||
- redis
|
||||
- backend
|
||||
redis:
|
||||
condition: service_started
|
||||
backend:
|
||||
condition: service_started
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
|
||||
db:
|
||||
image: pgvector/pgvector:pg16
|
||||
container_name: mulita-db
|
||||
environment:
|
||||
POSTGRES_USER: mulita
|
||||
POSTGRES_PASSWORD: mulita
|
||||
POSTGRES_DB: mulita
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- mulita-network
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U mulita -d mulita"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: mulita-redis
|
||||
@@ -102,4 +130,6 @@ volumes:
|
||||
thumbs_data:
|
||||
proxies_data:
|
||||
db_data:
|
||||
redis_data:
|
||||
redis_data:
|
||||
pg_data:
|
||||
models_data:
|
||||
@@ -2,6 +2,10 @@ import { useState } from 'react'
|
||||
import { Timeline } from './components/timeline/Timeline'
|
||||
import { DuplicatesView } from './components/duplicates/DuplicatesView'
|
||||
import { MapView } from './components/map/MapView'
|
||||
import { PeopleView } from './components/people/PeopleView'
|
||||
import { TagsView } from './components/tags/TagsView'
|
||||
import { ColorsView } from './components/colors/ColorsView'
|
||||
import { RatedView } from './components/rated/RatedView'
|
||||
import { LeftSidebar } from './components/layout/LeftSidebar'
|
||||
import { RightSidebar } from './components/layout/RightSidebar'
|
||||
import { TopBar } from './components/layout/TopBar'
|
||||
@@ -86,6 +90,14 @@ function App() {
|
||||
<MapView />
|
||||
) : currentSection === 'duplicates' ? (
|
||||
<DuplicatesView />
|
||||
) : currentSection === 'people' ? (
|
||||
<PeopleView />
|
||||
) : currentSection === 'tags' ? (
|
||||
<TagsView />
|
||||
) : currentSection === 'colors' ? (
|
||||
<ColorsView />
|
||||
) : currentSection === 'rated' ? (
|
||||
<RatedView />
|
||||
) : (
|
||||
<Timeline />
|
||||
)}
|
||||
|
||||
183
frontend/src/components/colors/ColorsView.tsx
Normal file
183
frontend/src/components/colors/ColorsView.tsx
Normal file
@@ -0,0 +1,183 @@
|
||||
import { useState, useMemo, useCallback } from 'react'
|
||||
import { Palette, ArrowLeft, Loader2 } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { photos as photosApi } from '../../services/api'
|
||||
import { useFilterStore } from '../../store/filterStore'
|
||||
import { usePhotosQuery } from '../../hooks/usePhotosQuery'
|
||||
import { COLOR_LABEL_OPTIONS, type ColorLabel } from '../../constants/colorLabels'
|
||||
import { useCardGridNav } from '../../hooks/useCardGridNav'
|
||||
import { Timeline } from '../timeline/Timeline'
|
||||
import type { Photo } from '../../types/photo'
|
||||
|
||||
interface ColorGroup {
|
||||
label: string
|
||||
value: ColorLabel | null
|
||||
className: string
|
||||
count: number
|
||||
representative: Photo | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Colors view — two states:
|
||||
* 1. Grid of color label cards (default) — arrow keys + Enter to browse
|
||||
* 2. Detail view showing a color's photos in the full Timeline — Esc to go back
|
||||
*/
|
||||
export function ColorsView() {
|
||||
const { data: allPhotos = [], isLoading } = usePhotosQuery()
|
||||
const setColorLabel = useFilterStore((s) => s.setColorLabel)
|
||||
const [selectedGroup, setSelectedGroup] = useState<ColorGroup | null>(null)
|
||||
|
||||
const groups = useMemo(() => {
|
||||
const buckets = new Map<string, Photo[]>()
|
||||
const uncolored: Photo[] = []
|
||||
|
||||
for (const photo of allPhotos) {
|
||||
if (photo.color_label) {
|
||||
const arr = buckets.get(photo.color_label) ?? []
|
||||
arr.push(photo)
|
||||
buckets.set(photo.color_label, arr)
|
||||
} else {
|
||||
uncolored.push(photo)
|
||||
}
|
||||
}
|
||||
|
||||
const result: ColorGroup[] = []
|
||||
for (const { value, className } of COLOR_LABEL_OPTIONS) {
|
||||
const photos = buckets.get(value) ?? []
|
||||
if (photos.length === 0) continue
|
||||
result.push({
|
||||
label: value.charAt(0).toUpperCase() + value.slice(1),
|
||||
value,
|
||||
className,
|
||||
count: photos.length,
|
||||
representative: photos[0],
|
||||
})
|
||||
}
|
||||
if (uncolored.length > 0) {
|
||||
result.push({
|
||||
label: 'Uncolored',
|
||||
value: null,
|
||||
className: 'bg-neutral-400',
|
||||
count: uncolored.length,
|
||||
representative: uncolored[0],
|
||||
})
|
||||
}
|
||||
return result
|
||||
}, [allPhotos])
|
||||
|
||||
const enterDetail = useCallback(
|
||||
(group: ColorGroup) => {
|
||||
setColorLabel((group.value ?? 'none') as ColorLabel)
|
||||
setSelectedGroup(group)
|
||||
},
|
||||
[setColorLabel]
|
||||
)
|
||||
|
||||
const exitDetail = useCallback(() => {
|
||||
setColorLabel(null)
|
||||
setSelectedGroup(null)
|
||||
}, [setColorLabel])
|
||||
|
||||
const { activeIndex, gridRef } = useCardGridNav({
|
||||
items: groups,
|
||||
inDetail: selectedGroup !== null,
|
||||
onEnter: enterDetail,
|
||||
onExit: exitDetail,
|
||||
})
|
||||
|
||||
if (selectedGroup) {
|
||||
return (
|
||||
<div className="flex h-full flex-col overflow-hidden">
|
||||
<div className="flex items-center gap-3 border-b border-border px-4 py-3">
|
||||
<button
|
||||
onClick={exitDetail}
|
||||
className="rounded p-1 text-text-muted transition-colors hover:bg-surface-2 hover:text-text"
|
||||
title="Back to colors"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</button>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className={`inline-block h-3 w-3 rounded-full ${selectedGroup.className}`} />
|
||||
<h2 className="text-sm font-semibold text-text">{selectedGroup.label}</h2>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex-1 overflow-hidden">
|
||||
<Timeline />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center text-text-muted">
|
||||
<Loader2 className="mr-2 h-5 w-5 animate-spin" />
|
||||
Loading colors...
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (groups.length === 0) {
|
||||
return (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-3 text-text-muted">
|
||||
<Palette className="h-12 w-12 opacity-40" />
|
||||
<p className="text-sm">No color labels assigned yet</p>
|
||||
<p className="max-w-xs text-center text-xs opacity-70">
|
||||
Color labels will appear here once you assign them to photos.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto p-4">
|
||||
<div className="mb-4 flex items-center gap-2 text-text-muted">
|
||||
<Palette className="h-4 w-4" />
|
||||
<span className="text-sm font-medium">
|
||||
{groups.length} {groups.length === 1 ? 'color' : 'colors'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
ref={gridRef}
|
||||
className="grid grid-cols-[repeat(auto-fill,minmax(140px,1fr))] gap-3"
|
||||
>
|
||||
{groups.map((group, i) => (
|
||||
<div
|
||||
key={group.label}
|
||||
className={clsx(
|
||||
'group cursor-pointer overflow-hidden rounded-lg border bg-surface transition-all hover:border-primary/50 hover:shadow-md',
|
||||
i === activeIndex
|
||||
? 'border-primary ring-1 ring-primary/30'
|
||||
: 'border-border'
|
||||
)}
|
||||
onClick={() => enterDetail(group)}
|
||||
>
|
||||
<div className="relative aspect-square overflow-hidden bg-surface-2">
|
||||
{group.representative ? (
|
||||
<img
|
||||
src={photosApi.getThumbnailUrl(group.representative.id, 'small')}
|
||||
alt={group.label}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center">
|
||||
<Palette className="h-10 w-10 text-text-muted/30" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span className="absolute bottom-1.5 right-1.5 rounded-full bg-black/60 px-2 py-0.5 text-[11px] font-medium text-white backdrop-blur-sm">
|
||||
{group.count}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-1.5 px-2 py-1.5">
|
||||
<span className={`inline-block h-2.5 w-2.5 rounded-full ${group.className}`} />
|
||||
<p className="truncate text-xs font-medium text-text">{group.label}</p>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -195,7 +195,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
|
||||
if (!isOpen) return null
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50">
|
||||
<div className="fixed inset-0 z-[2000]">
|
||||
<div
|
||||
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
|
||||
onClick={onClose}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
Pencil,
|
||||
PanelLeftClose,
|
||||
Settings,
|
||||
Users,
|
||||
} from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { sourceFolders, photos as photosApi, type FolderTreeNode } from '../../services/api'
|
||||
@@ -63,6 +64,7 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
|
||||
const navigateToSection = useFilterStore((s) => s.navigateToSection)
|
||||
const currentSection = useFilterStore((s) => s.currentSection)
|
||||
const { data: allTags = [] } = useTagsQuery()
|
||||
const { data: faceClusters = [] } = useTagsQuery('face_cluster')
|
||||
const { data: stats } = useLibraryStatsQuery()
|
||||
const [dropTargetId, setDropTargetId] = useState<string | null>(null)
|
||||
|
||||
@@ -255,6 +257,9 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
|
||||
case 'tags':
|
||||
navigateToSection('tags', { groupBy: 'tag' })
|
||||
break
|
||||
case 'people':
|
||||
navigateToSection('people', { groupBy: 'tag' })
|
||||
break
|
||||
case 'colors':
|
||||
navigateToSection('colors', { groupBy: 'color' })
|
||||
break
|
||||
@@ -349,8 +354,10 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
|
||||
: undefined,
|
||||
})
|
||||
|
||||
// Total tag count for the badge on the Tags entry.
|
||||
const tagsTotalCount = allTags.reduce((sum, t) => sum + (t.photo_count || 0), 0)
|
||||
// Total tag count for the badge on the Tags entry (user tags only).
|
||||
const userTags = allTags.filter((t) => t.kind === 'user')
|
||||
const tagsTotalCount = userTags.reduce((sum, t) => sum + (t.photo_count || 0), 0)
|
||||
const peopleTotalCount = faceClusters.reduce((sum, t) => sum + (t.photo_count || 0), 0)
|
||||
|
||||
const libraryTree: TreeItem[] = [
|
||||
{
|
||||
@@ -361,6 +368,7 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
|
||||
{ id: 'all-photos', label: 'All Photos', icon: <Image className="h-4 w-4" />, count: stats?.all_photos ?? 0 },
|
||||
{ id: 'rated', label: 'Rated', icon: <Star className="h-4 w-4" />, count: stats?.rated ?? 0 },
|
||||
{ id: 'tags', label: 'Tags', icon: <TagIcon className="h-4 w-4" />, count: tagsTotalCount },
|
||||
{ id: 'people', label: 'People', icon: <Users className="h-4 w-4" />, count: peopleTotalCount },
|
||||
{ id: 'colors', label: 'Colors', icon: <Palette className="h-4 w-4" />, count: stats?.colored ?? 0 },
|
||||
{ id: 'map', label: 'Map', icon: <MapPin className="h-4 w-4" />, count: stats?.with_gps ?? 0 },
|
||||
{ id: 'duplicates', label: 'Duplicates', icon: <Copy className="h-4 w-4" />, count: stats?.duplicates ?? 0 },
|
||||
|
||||
237
frontend/src/components/people/PeopleView.tsx
Normal file
237
frontend/src/components/people/PeopleView.tsx
Normal file
@@ -0,0 +1,237 @@
|
||||
import { useState, useCallback } from 'react'
|
||||
import { Users, Pencil, Check, X, Loader2, ArrowLeft } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { useTagsQuery } from '../../hooks/useTagsQuery'
|
||||
import {
|
||||
tags as tagsApi,
|
||||
photos as photosApi,
|
||||
type Tag,
|
||||
} from '../../services/api'
|
||||
import { useFilterStore } from '../../store/filterStore'
|
||||
import { useCardGridNav } from '../../hooks/useCardGridNav'
|
||||
import { Timeline } from '../timeline/Timeline'
|
||||
import { toast } from '../ToastContainer'
|
||||
|
||||
/**
|
||||
* People view — two states:
|
||||
* 1. Grid of face cluster cards (default) — arrow keys + Enter to browse
|
||||
* 2. Detail view showing a person's photos in the full Timeline — Esc to go back
|
||||
*/
|
||||
export function PeopleView() {
|
||||
const { data: clusters = [], isLoading } = useTagsQuery('face_cluster')
|
||||
const queryClient = useQueryClient()
|
||||
const setTagIds = useFilterStore((s) => s.setTagIds)
|
||||
|
||||
const [selectedPerson, setSelectedPerson] = useState<Tag | null>(null)
|
||||
const [editingId, setEditingId] = useState<string | null>(null)
|
||||
const [editName, setEditName] = useState('')
|
||||
|
||||
const renameMutation = useMutation({
|
||||
mutationFn: ({ id, name }: { id: string; name: string }) =>
|
||||
tagsApi.update(id, { name }),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ['tags'] })
|
||||
setEditingId(null)
|
||||
if (selectedPerson && editingId === selectedPerson.id) {
|
||||
setSelectedPerson({ ...selectedPerson, name: editName.trim() })
|
||||
}
|
||||
toast.success('Renamed')
|
||||
},
|
||||
onError: (e: any) =>
|
||||
toast.error('Rename failed', e?.response?.data?.detail || e.message),
|
||||
})
|
||||
|
||||
const startEditing = (tag: Tag) => {
|
||||
setEditingId(tag.id)
|
||||
setEditName(tag.name)
|
||||
}
|
||||
|
||||
const submitRename = () => {
|
||||
if (!editingId || !editName.trim()) return
|
||||
renameMutation.mutate({ id: editingId, name: editName.trim() })
|
||||
}
|
||||
|
||||
const enterDetail = useCallback(
|
||||
(person: Tag) => {
|
||||
setTagIds([person.id])
|
||||
setSelectedPerson(person)
|
||||
},
|
||||
[setTagIds]
|
||||
)
|
||||
|
||||
const exitDetail = useCallback(() => {
|
||||
setTagIds([])
|
||||
setSelectedPerson(null)
|
||||
}, [setTagIds])
|
||||
|
||||
const { activeIndex, gridRef } = useCardGridNav({
|
||||
items: clusters,
|
||||
inDetail: selectedPerson !== null,
|
||||
onEnter: enterDetail,
|
||||
onExit: exitDetail,
|
||||
})
|
||||
|
||||
// ── Detail view: a person's photos ─────────────────────────────────
|
||||
if (selectedPerson) {
|
||||
const isEditing = editingId === selectedPerson.id
|
||||
return (
|
||||
<div className="flex h-full flex-col overflow-hidden">
|
||||
<div className="flex items-center gap-3 border-b border-border px-4 py-3">
|
||||
<button
|
||||
onClick={exitDetail}
|
||||
className="rounded p-1 text-text-muted transition-colors hover:bg-surface-2 hover:text-text"
|
||||
title="Back to people"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</button>
|
||||
|
||||
{isEditing ? (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<input
|
||||
autoFocus
|
||||
value={editName}
|
||||
onChange={(e) => setEditName(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') submitRename()
|
||||
if (e.key === 'Escape') setEditingId(null)
|
||||
}}
|
||||
className="rounded border border-border bg-bg px-2 py-1 text-sm text-text focus:border-primary focus:outline-none"
|
||||
/>
|
||||
<button onClick={submitRename} className="rounded p-1 text-green-500 hover:bg-green-500/10">
|
||||
<Check className="h-4 w-4" />
|
||||
</button>
|
||||
<button onClick={() => setEditingId(null)} className="rounded p-1 text-text-muted hover:bg-surface-2">
|
||||
<X className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex items-center gap-2">
|
||||
<h2 className="text-sm font-semibold text-text">{selectedPerson.name}</h2>
|
||||
<button
|
||||
onClick={() => startEditing(selectedPerson)}
|
||||
className="rounded p-0.5 text-text-muted transition-colors hover:text-text"
|
||||
title="Rename"
|
||||
>
|
||||
<Pencil className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex-1 overflow-hidden">
|
||||
<Timeline />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ── Card grid ──────────────────────────────────────────────────────
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center text-text-muted">
|
||||
<Loader2 className="mr-2 h-5 w-5 animate-spin" />
|
||||
Loading people...
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (clusters.length === 0) {
|
||||
return (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-3 text-text-muted">
|
||||
<Users className="h-12 w-12 opacity-40" />
|
||||
<p className="text-sm">No people identified yet</p>
|
||||
<p className="max-w-xs text-center text-xs opacity-70">
|
||||
Face detection runs automatically when photos are scanned.
|
||||
People will appear here once faces are found and clustered.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto p-4">
|
||||
<div className="mb-4 flex items-center gap-2 text-text-muted">
|
||||
<Users className="h-4 w-4" />
|
||||
<span className="text-sm font-medium">
|
||||
{clusters.length} {clusters.length === 1 ? 'person' : 'people'} identified
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
ref={gridRef}
|
||||
className="grid grid-cols-[repeat(auto-fill,minmax(140px,1fr))] gap-3"
|
||||
>
|
||||
{clusters.map((tag, i) => (
|
||||
<div
|
||||
key={tag.id}
|
||||
className={clsx(
|
||||
'group cursor-pointer overflow-hidden rounded-lg border bg-surface transition-all hover:border-primary/50 hover:shadow-md',
|
||||
i === activeIndex
|
||||
? 'border-primary ring-1 ring-primary/30'
|
||||
: editingId === tag.id
|
||||
? 'border-primary ring-1 ring-primary/30'
|
||||
: 'border-border'
|
||||
)}
|
||||
onClick={() => {
|
||||
if (editingId !== tag.id) enterDetail(tag)
|
||||
}}
|
||||
>
|
||||
<div className="relative aspect-square overflow-hidden bg-surface-2">
|
||||
{tag.representative_photo_id ? (
|
||||
<img
|
||||
src={photosApi.getThumbnailUrl(tag.representative_photo_id, 'small')}
|
||||
alt={tag.name}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center">
|
||||
<Users className="h-10 w-10 text-text-muted/30" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span className="absolute bottom-1.5 right-1.5 rounded-full bg-black/60 px-2 py-0.5 text-[11px] font-medium text-white backdrop-blur-sm">
|
||||
{tag.photo_count}
|
||||
</span>
|
||||
|
||||
<button
|
||||
className="absolute right-1.5 top-1.5 rounded-full bg-black/50 p-1 text-white opacity-0 transition-opacity group-hover:opacity-100"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
startEditing(tag)
|
||||
}}
|
||||
title="Rename"
|
||||
>
|
||||
<Pencil className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="px-2 py-1.5">
|
||||
{editingId === tag.id ? (
|
||||
<div className="flex items-center gap-1" onClick={(e) => e.stopPropagation()}>
|
||||
<input
|
||||
autoFocus
|
||||
value={editName}
|
||||
onChange={(e) => setEditName(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') submitRename()
|
||||
if (e.key === 'Escape') setEditingId(null)
|
||||
}}
|
||||
className="min-w-0 flex-1 rounded border border-border bg-bg px-1.5 py-0.5 text-xs text-text focus:border-primary focus:outline-none"
|
||||
/>
|
||||
<button onClick={submitRename} className="rounded p-0.5 text-green-500 hover:bg-green-500/10">
|
||||
<Check className="h-3 w-3" />
|
||||
</button>
|
||||
<button onClick={() => setEditingId(null)} className="rounded p-0.5 text-text-muted hover:bg-surface-2">
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<p className="truncate text-xs font-medium text-text">{tag.name}</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
170
frontend/src/components/rated/RatedView.tsx
Normal file
170
frontend/src/components/rated/RatedView.tsx
Normal file
@@ -0,0 +1,170 @@
|
||||
import { useState, useMemo, useCallback } from 'react'
|
||||
import { Star, ArrowLeft, Loader2 } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { photos as photosApi } from '../../services/api'
|
||||
import { useFilterStore } from '../../store/filterStore'
|
||||
import { usePhotosQuery } from '../../hooks/usePhotosQuery'
|
||||
import { useCardGridNav } from '../../hooks/useCardGridNav'
|
||||
import { Timeline } from '../timeline/Timeline'
|
||||
import type { Photo } from '../../types/photo'
|
||||
|
||||
interface RatingGroup {
|
||||
rating: number
|
||||
label: string
|
||||
count: number
|
||||
representative: Photo | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Rated view — two states:
|
||||
* 1. Grid of rating-level cards (default) — arrow keys + Enter to browse
|
||||
* 2. Detail view showing a rating level's photos in the full Timeline — Esc to go back
|
||||
*/
|
||||
export function RatedView() {
|
||||
const { data: allPhotos = [], isLoading } = usePhotosQuery()
|
||||
const setRatingMin = useFilterStore((s) => s.setRatingMin)
|
||||
const setRatingMax = useFilterStore((s) => s.setRatingMax)
|
||||
const [selectedGroup, setSelectedGroup] = useState<RatingGroup | null>(null)
|
||||
|
||||
const groups = useMemo(() => {
|
||||
const buckets = new Map<number, Photo[]>()
|
||||
|
||||
for (const photo of allPhotos) {
|
||||
if (photo.rating > 0) {
|
||||
const arr = buckets.get(photo.rating) ?? []
|
||||
arr.push(photo)
|
||||
buckets.set(photo.rating, arr)
|
||||
}
|
||||
}
|
||||
|
||||
// Highest rating first
|
||||
const result: RatingGroup[] = []
|
||||
for (let r = 5; r >= 1; r--) {
|
||||
const photos = buckets.get(r) ?? []
|
||||
if (photos.length === 0) continue
|
||||
result.push({
|
||||
rating: r,
|
||||
label: '★'.repeat(r),
|
||||
count: photos.length,
|
||||
representative: photos[0],
|
||||
})
|
||||
}
|
||||
return result
|
||||
}, [allPhotos])
|
||||
|
||||
const enterDetail = useCallback(
|
||||
(group: RatingGroup) => {
|
||||
setRatingMin(group.rating)
|
||||
setRatingMax(group.rating)
|
||||
setSelectedGroup(group)
|
||||
},
|
||||
[setRatingMin, setRatingMax]
|
||||
)
|
||||
|
||||
const exitDetail = useCallback(() => {
|
||||
// Restore the section preset: ratingMin=1 (all rated), no max
|
||||
setRatingMin(1)
|
||||
setRatingMax(0)
|
||||
setSelectedGroup(null)
|
||||
}, [setRatingMin, setRatingMax])
|
||||
|
||||
const { activeIndex, gridRef } = useCardGridNav({
|
||||
items: groups,
|
||||
inDetail: selectedGroup !== null,
|
||||
onEnter: enterDetail,
|
||||
onExit: exitDetail,
|
||||
})
|
||||
|
||||
if (selectedGroup) {
|
||||
return (
|
||||
<div className="flex h-full flex-col overflow-hidden">
|
||||
<div className="flex items-center gap-3 border-b border-border px-4 py-3">
|
||||
<button
|
||||
onClick={exitDetail}
|
||||
className="rounded p-1 text-text-muted transition-colors hover:bg-surface-2 hover:text-text"
|
||||
title="Back to ratings"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</button>
|
||||
<h2 className="text-sm font-semibold text-amber-400">{selectedGroup.label}</h2>
|
||||
</div>
|
||||
<div className="flex-1 overflow-hidden">
|
||||
<Timeline />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center text-text-muted">
|
||||
<Loader2 className="mr-2 h-5 w-5 animate-spin" />
|
||||
Loading ratings...
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (groups.length === 0) {
|
||||
return (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-3 text-text-muted">
|
||||
<Star className="h-12 w-12 opacity-40" />
|
||||
<p className="text-sm">No rated photos yet</p>
|
||||
<p className="max-w-xs text-center text-xs opacity-70">
|
||||
Rate photos with 1–5 stars and they will appear here grouped by
|
||||
rating.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto p-4">
|
||||
<div className="mb-4 flex items-center gap-2 text-text-muted">
|
||||
<Star className="h-4 w-4" />
|
||||
<span className="text-sm font-medium">
|
||||
{groups.length} rating {groups.length === 1 ? 'level' : 'levels'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
ref={gridRef}
|
||||
className="grid grid-cols-[repeat(auto-fill,minmax(140px,1fr))] gap-3"
|
||||
>
|
||||
{groups.map((group, i) => (
|
||||
<div
|
||||
key={group.rating}
|
||||
className={clsx(
|
||||
'group cursor-pointer overflow-hidden rounded-lg border bg-surface transition-all hover:border-primary/50 hover:shadow-md',
|
||||
i === activeIndex
|
||||
? 'border-primary ring-1 ring-primary/30'
|
||||
: 'border-border'
|
||||
)}
|
||||
onClick={() => enterDetail(group)}
|
||||
>
|
||||
<div className="relative aspect-square overflow-hidden bg-surface-2">
|
||||
{group.representative ? (
|
||||
<img
|
||||
src={photosApi.getThumbnailUrl(group.representative.id, 'small')}
|
||||
alt={group.label}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center">
|
||||
<Star className="h-10 w-10 text-text-muted/30" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span className="absolute bottom-1.5 right-1.5 rounded-full bg-black/60 px-2 py-0.5 text-[11px] font-medium text-white backdrop-blur-sm">
|
||||
{group.count}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="px-2 py-1.5">
|
||||
<p className="truncate text-xs font-medium text-amber-400">{group.label}</p>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
138
frontend/src/components/tags/TagsView.tsx
Normal file
138
frontend/src/components/tags/TagsView.tsx
Normal file
@@ -0,0 +1,138 @@
|
||||
import { useState, useMemo, useCallback } from 'react'
|
||||
import { Tag as TagIcon, ArrowLeft, Loader2 } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { useTagsQuery } from '../../hooks/useTagsQuery'
|
||||
import { photos as photosApi, type Tag } from '../../services/api'
|
||||
import { useFilterStore } from '../../store/filterStore'
|
||||
import { useCardGridNav } from '../../hooks/useCardGridNav'
|
||||
import { Timeline } from '../timeline/Timeline'
|
||||
|
||||
/**
|
||||
* Tags view — two states:
|
||||
* 1. Grid of tag cards (default) — arrow keys + Enter to browse
|
||||
* 2. Detail view showing a tag's photos in the full Timeline — Esc to go back
|
||||
*/
|
||||
export function TagsView() {
|
||||
const { data: allTags = [], isLoading } = useTagsQuery()
|
||||
const setTagIds = useFilterStore((s) => s.setTagIds)
|
||||
const [selectedTag, setSelectedTag] = useState<Tag | null>(null)
|
||||
|
||||
// Exclude face_cluster tags (those live in PeopleView)
|
||||
const tags = useMemo(
|
||||
() => allTags.filter((t) => t.kind !== 'face_cluster'),
|
||||
[allTags]
|
||||
)
|
||||
|
||||
const enterDetail = useCallback(
|
||||
(tag: Tag) => {
|
||||
setTagIds([tag.id])
|
||||
setSelectedTag(tag)
|
||||
},
|
||||
[setTagIds]
|
||||
)
|
||||
|
||||
const exitDetail = useCallback(() => {
|
||||
setTagIds([])
|
||||
setSelectedTag(null)
|
||||
}, [setTagIds])
|
||||
|
||||
const { activeIndex, gridRef } = useCardGridNav({
|
||||
items: tags,
|
||||
inDetail: selectedTag !== null,
|
||||
onEnter: enterDetail,
|
||||
onExit: exitDetail,
|
||||
})
|
||||
|
||||
if (selectedTag) {
|
||||
return (
|
||||
<div className="flex h-full flex-col overflow-hidden">
|
||||
<div className="flex items-center gap-3 border-b border-border px-4 py-3">
|
||||
<button
|
||||
onClick={exitDetail}
|
||||
className="rounded p-1 text-text-muted transition-colors hover:bg-surface-2 hover:text-text"
|
||||
title="Back to tags"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</button>
|
||||
<h2 className="text-sm font-semibold text-text">{selectedTag.name}</h2>
|
||||
</div>
|
||||
<div className="flex-1 overflow-hidden">
|
||||
<Timeline />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center text-text-muted">
|
||||
<Loader2 className="mr-2 h-5 w-5 animate-spin" />
|
||||
Loading tags...
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (tags.length === 0) {
|
||||
return (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-3 text-text-muted">
|
||||
<TagIcon className="h-12 w-12 opacity-40" />
|
||||
<p className="text-sm">No tags yet</p>
|
||||
<p className="max-w-xs text-center text-xs opacity-70">
|
||||
Tags will appear here once photos are tagged — either manually or by
|
||||
the auto-tagger.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto p-4">
|
||||
<div className="mb-4 flex items-center gap-2 text-text-muted">
|
||||
<TagIcon className="h-4 w-4" />
|
||||
<span className="text-sm font-medium">
|
||||
{tags.length} {tags.length === 1 ? 'tag' : 'tags'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
ref={gridRef}
|
||||
className="grid grid-cols-[repeat(auto-fill,minmax(140px,1fr))] gap-3"
|
||||
>
|
||||
{tags.map((tag, i) => (
|
||||
<div
|
||||
key={tag.id}
|
||||
className={clsx(
|
||||
'group cursor-pointer overflow-hidden rounded-lg border bg-surface transition-all hover:border-primary/50 hover:shadow-md',
|
||||
i === activeIndex
|
||||
? 'border-primary ring-1 ring-primary/30'
|
||||
: 'border-border'
|
||||
)}
|
||||
onClick={() => enterDetail(tag)}
|
||||
>
|
||||
<div className="relative aspect-square overflow-hidden bg-surface-2">
|
||||
{tag.representative_photo_id ? (
|
||||
<img
|
||||
src={photosApi.getThumbnailUrl(tag.representative_photo_id, 'small')}
|
||||
alt={tag.name}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center">
|
||||
<TagIcon className="h-10 w-10 text-text-muted/30" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span className="absolute bottom-1.5 right-1.5 rounded-full bg-black/60 px-2 py-0.5 text-[11px] font-medium text-white backdrop-blur-sm">
|
||||
{tag.photo_count}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="px-2 py-1.5">
|
||||
<p className="truncate text-xs font-medium text-text">{tag.name}</p>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import { useFilterStore } from '../../store/filterStore'
|
||||
import { PhotoThumbnail } from './PhotoThumbnail'
|
||||
import { usePhotosQuery } from '../../hooks/usePhotosQuery'
|
||||
import { useActiveHeapMembers } from '../../hooks/useActiveHeapMembersQuery'
|
||||
import { COLOR_LABEL_OPTIONS } from '../../constants/colorLabels'
|
||||
import type { Photo } from '../../types/photo'
|
||||
|
||||
// Layout constants for the grid + grouped headers.
|
||||
@@ -27,22 +26,19 @@ type TimelineItem =
|
||||
/**
|
||||
* Build the flat header|row item array the virtualizer renders.
|
||||
*
|
||||
* Five modes:
|
||||
* - groupBy='tag': one bucket per unique tag (plus an "Untagged" bucket
|
||||
* for photos with no tags). A photo with N tags appears in N buckets.
|
||||
* - groupBy='rating': one bucket per star rating 5..1 (plus "Unrated"
|
||||
* for rating 0). Each photo lands in exactly one bucket.
|
||||
* - groupBy='color': one bucket per color label, in canonical order
|
||||
* (plus an "Uncolored" bucket for photos with no label).
|
||||
* - groupBy='date' AND sortBy is a date field: month buckets (existing).
|
||||
* Two modes:
|
||||
* - sortBy is a date field: month buckets.
|
||||
* - otherwise: one un-headered stream.
|
||||
*
|
||||
* Tag, rating, and color grouping now live in their own dedicated views
|
||||
* (TagsView, RatedView, ColorsView) instead of being handled here.
|
||||
*/
|
||||
function buildItems(
|
||||
photos: Photo[],
|
||||
columns: number,
|
||||
rowHeight: number,
|
||||
sortBy: string,
|
||||
groupBy: 'date' | 'tag' | 'rating' | 'color'
|
||||
groupBy: string,
|
||||
): TimelineItem[] {
|
||||
if (photos.length === 0) return []
|
||||
|
||||
@@ -61,155 +57,13 @@ function buildItems(
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tag grouping ──────────────────────────────────────────────────────
|
||||
if (groupBy === 'tag') {
|
||||
// Bucket by tag name. A photo with multiple tags lands in multiple
|
||||
// buckets. Photos with no tags go into "Untagged".
|
||||
const tagBuckets = new Map<string, PhotoCell[]>()
|
||||
const untagged: PhotoCell[] = []
|
||||
|
||||
photos.forEach((photo, globalIndex) => {
|
||||
const cell: PhotoCell = { photo, globalIndex }
|
||||
const tags = photo.tags ?? []
|
||||
if (tags.length === 0) {
|
||||
untagged.push(cell)
|
||||
} else {
|
||||
for (const t of tags) {
|
||||
const arr = tagBuckets.get(t.name) ?? []
|
||||
arr.push(cell)
|
||||
tagBuckets.set(t.name, arr)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Sort tag groups alphabetically; Untagged goes at the end.
|
||||
const sortedTagNames = Array.from(tagBuckets.keys()).sort((a, b) =>
|
||||
a.localeCompare(b)
|
||||
)
|
||||
|
||||
let bucketIndex = 0
|
||||
for (const name of sortedTagNames) {
|
||||
items.push({
|
||||
type: 'header',
|
||||
key: `tag::${bucketIndex}::${name}`,
|
||||
label: name,
|
||||
height: HEADER_HEIGHT,
|
||||
})
|
||||
pushRowsForGroup(`tag::${bucketIndex}::${name}`, tagBuckets.get(name)!)
|
||||
bucketIndex++
|
||||
}
|
||||
if (untagged.length > 0) {
|
||||
items.push({
|
||||
type: 'header',
|
||||
key: `tag::${bucketIndex}::__untagged`,
|
||||
label: 'Untagged',
|
||||
height: HEADER_HEIGHT,
|
||||
})
|
||||
pushRowsForGroup(`tag::${bucketIndex}::untagged`, untagged)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// ── Rating grouping ───────────────────────────────────────────────────
|
||||
if (groupBy === 'rating') {
|
||||
// Bucket by star rating. Each photo lands in exactly one bucket;
|
||||
// rating 0 goes into "Unrated".
|
||||
const ratingBuckets = new Map<number, PhotoCell[]>()
|
||||
const unrated: PhotoCell[] = []
|
||||
|
||||
photos.forEach((photo, globalIndex) => {
|
||||
const cell: PhotoCell = { photo, globalIndex }
|
||||
if (photo.rating > 0) {
|
||||
const arr = ratingBuckets.get(photo.rating) ?? []
|
||||
arr.push(cell)
|
||||
ratingBuckets.set(photo.rating, arr)
|
||||
} else {
|
||||
unrated.push(cell)
|
||||
}
|
||||
})
|
||||
|
||||
// Highest rating first; Unrated goes at the end.
|
||||
const sortedRatings = Array.from(ratingBuckets.keys()).sort((a, b) => b - a)
|
||||
|
||||
let bucketIndex = 0
|
||||
for (const rating of sortedRatings) {
|
||||
items.push({
|
||||
type: 'header',
|
||||
key: `rating::${bucketIndex}::${rating}`,
|
||||
label: '★'.repeat(rating),
|
||||
height: HEADER_HEIGHT,
|
||||
})
|
||||
pushRowsForGroup(
|
||||
`rating::${bucketIndex}::${rating}`,
|
||||
ratingBuckets.get(rating)!
|
||||
)
|
||||
bucketIndex++
|
||||
}
|
||||
if (unrated.length > 0) {
|
||||
items.push({
|
||||
type: 'header',
|
||||
key: `rating::${bucketIndex}::__unrated`,
|
||||
label: 'Unrated',
|
||||
height: HEADER_HEIGHT,
|
||||
})
|
||||
pushRowsForGroup(`rating::${bucketIndex}::unrated`, unrated)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// ── Color label grouping ──────────────────────────────────────────────
|
||||
if (groupBy === 'color') {
|
||||
// Bucket by color_label. Each photo lands in exactly one bucket;
|
||||
// photos with no label go into "Uncolored".
|
||||
const colorBuckets = new Map<string, PhotoCell[]>()
|
||||
const uncolored: PhotoCell[] = []
|
||||
|
||||
photos.forEach((photo, globalIndex) => {
|
||||
const cell: PhotoCell = { photo, globalIndex }
|
||||
const label = photo.color_label
|
||||
if (label) {
|
||||
const arr = colorBuckets.get(label) ?? []
|
||||
arr.push(cell)
|
||||
colorBuckets.set(label, arr)
|
||||
} else {
|
||||
uncolored.push(cell)
|
||||
}
|
||||
})
|
||||
|
||||
// Walk the canonical color order so headers always read R-O-Y-G-B-P,
|
||||
// matching every other color UI in the app. Skip empty buckets and
|
||||
// ignore any unexpected label values that aren't in the canonical
|
||||
// list (they'd be invalid backend state).
|
||||
let bucketIndex = 0
|
||||
for (const { value } of COLOR_LABEL_OPTIONS) {
|
||||
const cells = colorBuckets.get(value)
|
||||
if (!cells || cells.length === 0) continue
|
||||
const label = value.charAt(0).toUpperCase() + value.slice(1)
|
||||
items.push({
|
||||
type: 'header',
|
||||
key: `color::${bucketIndex}::${value}`,
|
||||
label,
|
||||
height: HEADER_HEIGHT,
|
||||
})
|
||||
pushRowsForGroup(`color::${bucketIndex}::${value}`, cells)
|
||||
bucketIndex++
|
||||
}
|
||||
if (uncolored.length > 0) {
|
||||
items.push({
|
||||
type: 'header',
|
||||
key: `color::${bucketIndex}::__uncolored`,
|
||||
label: 'Uncolored',
|
||||
height: HEADER_HEIGHT,
|
||||
})
|
||||
pushRowsForGroup(`color::${bucketIndex}::uncolored`, uncolored)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// ── Date grouping (existing) ──────────────────────────────────────────
|
||||
// Date grouping only applies when groupBy is explicitly 'date' and
|
||||
// the sort field is a date column. Other sections (tags, colors,
|
||||
// rated, people) reuse Timeline for their detail views and should
|
||||
// render a flat grid without month headers.
|
||||
const isDateSort = sortBy === 'taken_at' || sortBy === 'added_at'
|
||||
|
||||
if (!isDateSort) {
|
||||
if (!isDateSort || groupBy !== 'date') {
|
||||
// No grouping — one row stream.
|
||||
const cells: PhotoCell[] = photos.map((photo, globalIndex) => ({
|
||||
photo,
|
||||
@@ -344,8 +198,7 @@ export function Timeline() {
|
||||
const activeHeapName = activeHeap?.name ?? null
|
||||
|
||||
// Build the flat virtualizer items: a mix of group headers and rows of
|
||||
// photos. Date headers appear when sorted by a date field; tag headers
|
||||
// appear when groupBy === 'tag' (overrides date grouping).
|
||||
// photos. Date headers appear only in the main timeline (groupBy='date').
|
||||
const items = useMemo(
|
||||
() => buildItems(photos, columns, cellSize, sortBy, groupBy),
|
||||
[photos, columns, cellSize, sortBy, groupBy]
|
||||
|
||||
115
frontend/src/hooks/useCardGridNav.ts
Normal file
115
frontend/src/hooks/useCardGridNav.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import { useState, useEffect, useCallback, useRef } from 'react'
|
||||
import { usePhotoStore } from '../store/photoStore'
|
||||
|
||||
/**
|
||||
* Keyboard navigation for card grids (tags, colors, ratings, people).
|
||||
*
|
||||
* Arrow keys move the active index through the grid (wrapping at row
|
||||
* boundaries based on the actual CSS column count), Enter opens the
|
||||
* selected card, and Escape / Backspace exits the detail view.
|
||||
*
|
||||
* In detail mode, Escape only exits back to the card grid when the
|
||||
* preview is closed and no photos are selected — otherwise it defers
|
||||
* to Timeline's own Escape handler (clear selection / close preview).
|
||||
*
|
||||
* The grid container ref is used to measure the rendered column count
|
||||
* so up/down navigation stays column-aligned.
|
||||
*/
|
||||
export function useCardGridNav<T>(opts: {
|
||||
items: T[]
|
||||
/** True when the detail view is showing (disables grid nav, enables Esc) */
|
||||
inDetail: boolean
|
||||
onEnter: (item: T, index: number) => void
|
||||
onExit: () => void
|
||||
}) {
|
||||
const { items, inDetail, onEnter, onExit } = opts
|
||||
const [activeIndex, setActiveIndex] = useState(0)
|
||||
const gridRef = useRef<HTMLDivElement>(null)
|
||||
const viewMode = usePhotoStore((s) => s.viewMode)
|
||||
const selectedPhotos = usePhotoStore((s) => s.selectedPhotos)
|
||||
|
||||
// Clamp active index when the item list shrinks
|
||||
useEffect(() => {
|
||||
if (items.length > 0 && activeIndex >= items.length) {
|
||||
setActiveIndex(items.length - 1)
|
||||
}
|
||||
}, [items.length, activeIndex])
|
||||
|
||||
// Measure column count from the grid container
|
||||
const getColumns = useCallback(() => {
|
||||
const el = gridRef.current
|
||||
if (!el) return 1
|
||||
return getComputedStyle(el).gridTemplateColumns.split(' ').length
|
||||
}, [])
|
||||
|
||||
// Scroll the active card into view
|
||||
const scrollIntoView = useCallback((index: number) => {
|
||||
const el = gridRef.current
|
||||
if (!el) return
|
||||
const card = el.children[index] as HTMLElement | undefined
|
||||
card?.scrollIntoView({ block: 'nearest' })
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (items.length === 0) return
|
||||
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
const target = e.target as HTMLElement | null
|
||||
if (target && (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA')) return
|
||||
|
||||
// Detail view: Escape or Backspace exits back to card grid, but
|
||||
// only when the preview is closed and no photos are selected —
|
||||
// otherwise defer to Timeline's own Escape handler.
|
||||
if (inDetail) {
|
||||
if (e.key === 'Backspace') {
|
||||
e.preventDefault()
|
||||
onExit()
|
||||
} else if (e.key === 'Escape' && viewMode === 'grid' && selectedPhotos.length === 0) {
|
||||
e.preventDefault()
|
||||
onExit()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Card grid navigation
|
||||
const cols = getColumns()
|
||||
const count = items.length
|
||||
let next = activeIndex
|
||||
|
||||
switch (e.key) {
|
||||
case 'ArrowRight':
|
||||
e.preventDefault()
|
||||
next = Math.min(activeIndex + 1, count - 1)
|
||||
break
|
||||
case 'ArrowLeft':
|
||||
e.preventDefault()
|
||||
next = Math.max(activeIndex - 1, 0)
|
||||
break
|
||||
case 'ArrowDown':
|
||||
e.preventDefault()
|
||||
next = Math.min(activeIndex + cols, count - 1)
|
||||
break
|
||||
case 'ArrowUp':
|
||||
e.preventDefault()
|
||||
next = Math.max(activeIndex - cols, 0)
|
||||
break
|
||||
case 'Enter':
|
||||
e.preventDefault()
|
||||
if (items[activeIndex]) onEnter(items[activeIndex], activeIndex)
|
||||
return
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
if (next !== activeIndex) {
|
||||
setActiveIndex(next)
|
||||
scrollIntoView(next)
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener('keydown', handleKeyDown)
|
||||
return () => window.removeEventListener('keydown', handleKeyDown)
|
||||
}, [items, activeIndex, inDetail, onEnter, onExit, getColumns, scrollIntoView, viewMode, selectedPhotos])
|
||||
|
||||
return { activeIndex, setActiveIndex, gridRef }
|
||||
}
|
||||
@@ -60,6 +60,12 @@ function parseUrl(): HydratePayload {
|
||||
if (Number.isFinite(n) && n >= 0 && n <= 5) out.ratingMin = n
|
||||
}
|
||||
|
||||
const rx = sp.get('rating_max')
|
||||
if (rx) {
|
||||
const n = parseInt(rx, 10)
|
||||
if (Number.isFinite(n) && n >= 0 && n <= 5) out.ratingMax = n
|
||||
}
|
||||
|
||||
const cl = sp.get('color_label')
|
||||
if (cl && ALLOWED_COLORS.includes(cl as ColorLabel)) {
|
||||
out.colorLabel = cl as ColorLabel
|
||||
@@ -110,6 +116,7 @@ function writeUrl(f: FilterState & { currentSection?: string }) {
|
||||
if (f.dateTo) sp.set('date_to', f.dateTo)
|
||||
if (f.mediaTypes.length > 0) sp.set('media_type', f.mediaTypes.join(','))
|
||||
if (f.ratingMin > 0) sp.set('rating_min', String(f.ratingMin))
|
||||
if (f.ratingMax > 0) sp.set('rating_max', String(f.ratingMax))
|
||||
if (f.colorLabel) sp.set('color_label', f.colorLabel)
|
||||
if (f.flag !== 'any') sp.set('flag', f.flag)
|
||||
if (f.heapId) sp.set('heap_id', f.heapId)
|
||||
|
||||
@@ -32,6 +32,7 @@ export function usePhotosQuery() {
|
||||
const dateTo = useFilterStore((s) => s.dateTo)
|
||||
const mediaTypes = useFilterStore((s) => s.mediaTypes)
|
||||
const ratingMin = useFilterStore((s) => s.ratingMin)
|
||||
const ratingMax = useFilterStore((s) => s.ratingMax)
|
||||
const colorLabel = useFilterStore((s) => s.colorLabel)
|
||||
const flag = useFilterStore((s) => s.flag)
|
||||
const heapId = useFilterStore((s) => s.heapId)
|
||||
@@ -50,6 +51,7 @@ export function usePhotosQuery() {
|
||||
dateTo,
|
||||
mediaTypes,
|
||||
ratingMin,
|
||||
ratingMax,
|
||||
colorLabel,
|
||||
flag,
|
||||
heapId,
|
||||
@@ -60,7 +62,7 @@ export function usePhotosQuery() {
|
||||
sortBy,
|
||||
sortOrder,
|
||||
}),
|
||||
[q, dateFrom, dateTo, mediaTypes, ratingMin, colorLabel, flag, heapId, folderId, tagIds, duplicates, groupBy, sortBy, sortOrder]
|
||||
[q, dateFrom, dateTo, mediaTypes, ratingMin, ratingMax, colorLabel, flag, heapId, folderId, tagIds, duplicates, groupBy, sortBy, sortOrder]
|
||||
)
|
||||
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
37
frontend/src/hooks/useSearchQuery.ts
Normal file
37
frontend/src/hooks/useSearchQuery.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { search, type SearchResult } from '../services/api'
|
||||
import { useFilterStore } from '../store/filterStore'
|
||||
|
||||
/**
|
||||
* Hybrid search hook — fires POST /photos/search when the user has a
|
||||
* non-empty search query. Returns results ranked by RRF (FTS + semantic).
|
||||
*
|
||||
* When `q` is empty, this hook is disabled and returns no data — the
|
||||
* normal usePhotosQuery takes over for browse mode.
|
||||
*/
|
||||
export function useSearchQuery() {
|
||||
const q = useFilterStore((s) => s.q)
|
||||
const tagIds = useFilterStore((s) => s.tagIds)
|
||||
const dateFrom = useFilterStore((s) => s.dateFrom)
|
||||
const dateTo = useFilterStore((s) => s.dateTo)
|
||||
|
||||
const hasQuery = q.trim().length > 0
|
||||
|
||||
return useQuery<SearchResult[]>({
|
||||
queryKey: ['search', q, tagIds, dateFrom, dateTo],
|
||||
queryFn: async () => {
|
||||
const resp = await search.query({
|
||||
q: q.trim(),
|
||||
filters: {
|
||||
tag_ids: tagIds.length > 0 ? tagIds : undefined,
|
||||
date_from: dateFrom ?? undefined,
|
||||
date_to: dateTo ?? undefined,
|
||||
},
|
||||
limit: 200,
|
||||
})
|
||||
return resp.results
|
||||
},
|
||||
enabled: hasQuery,
|
||||
staleTime: 30_000,
|
||||
})
|
||||
}
|
||||
@@ -1,12 +1,12 @@
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { tags as tagsApi, type Tag } from '../services/api'
|
||||
import { tags as tagsApi, type Tag, type TagKind } from '../services/api'
|
||||
|
||||
export const TAGS_QUERY_KEY = ['tags'] as const
|
||||
|
||||
export function useTagsQuery() {
|
||||
export function useTagsQuery(kind?: TagKind) {
|
||||
return useQuery<Tag[]>({
|
||||
queryKey: TAGS_QUERY_KEY,
|
||||
queryFn: tagsApi.list,
|
||||
queryKey: kind ? ['tags', kind] : TAGS_QUERY_KEY,
|
||||
queryFn: () => tagsApi.list(kind),
|
||||
staleTime: 30_000,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -519,21 +519,27 @@ export const heaps = {
|
||||
}
|
||||
|
||||
// Tags API
|
||||
export type TagKind = 'user' | 'object' | 'scene' | 'face_cluster'
|
||||
|
||||
export interface Tag {
|
||||
id: string
|
||||
name: string
|
||||
color: string | null
|
||||
kind: TagKind
|
||||
source: string | null
|
||||
representative_photo_id: string | null
|
||||
photo_count: number
|
||||
}
|
||||
|
||||
export const tags = {
|
||||
list: async (): Promise<Tag[]> => {
|
||||
const response = await api.get('/tags')
|
||||
list: async (kind?: TagKind): Promise<Tag[]> => {
|
||||
const params = kind ? { kind } : undefined
|
||||
const response = await api.get('/tags', { params })
|
||||
return response.data
|
||||
},
|
||||
|
||||
create: async (name: string, color?: string): Promise<Tag> => {
|
||||
const response = await api.post('/tags', { name, color })
|
||||
create: async (name: string, color?: string, kind: TagKind = 'user'): Promise<Tag> => {
|
||||
const response = await api.post('/tags', { name, color, kind })
|
||||
return response.data
|
||||
},
|
||||
|
||||
@@ -546,6 +552,11 @@ export const tags = {
|
||||
await api.delete(`/tags/${tagId}`)
|
||||
},
|
||||
|
||||
merge: async (sourceId: string, targetId: string): Promise<{ merged_into: string; target_name: string }> => {
|
||||
const response = await api.post(`/tags/${sourceId}/merge`, { target_id: targetId })
|
||||
return response.data
|
||||
},
|
||||
|
||||
/** Add one or more tags to a photo. */
|
||||
addToPhoto: async (photoId: string, tagIds: string[]) => {
|
||||
const response = await api.post(`/photos/${photoId}/tags`, { tag_ids: tagIds })
|
||||
@@ -558,6 +569,38 @@ export const tags = {
|
||||
},
|
||||
}
|
||||
|
||||
// Search API — hybrid FTS + semantic search
|
||||
export interface SearchResult {
|
||||
id: string
|
||||
filename: string
|
||||
filepath: string
|
||||
media_type: string
|
||||
width: number
|
||||
height: number
|
||||
taken_at: string | null
|
||||
rating: number
|
||||
color_label: string | null
|
||||
thumb_small: string
|
||||
thumb_medium: string
|
||||
score: number
|
||||
}
|
||||
|
||||
export const search = {
|
||||
query: async (params: {
|
||||
q?: string
|
||||
filters?: {
|
||||
tag_ids?: string[]
|
||||
date_from?: string
|
||||
date_to?: string
|
||||
}
|
||||
limit?: number
|
||||
offset?: number
|
||||
}): Promise<{ results: SearchResult[]; total: number }> => {
|
||||
const response = await api.post('/photos/search', params)
|
||||
return response.data
|
||||
},
|
||||
}
|
||||
|
||||
// Discard API
|
||||
export const discard = {
|
||||
list: async () => {
|
||||
|
||||
@@ -19,6 +19,7 @@ export interface FilterState {
|
||||
dateTo: string | null
|
||||
mediaTypes: MediaType[]
|
||||
ratingMin: number // 0-5; 0 means no filter
|
||||
ratingMax: number // 0-5; 0 means no filter
|
||||
colorLabel: ColorLabel | null
|
||||
flag: FlagFilter
|
||||
/** When set, restrict to photos in this heap. Independent of `activeHeapId`
|
||||
@@ -59,6 +60,7 @@ interface FilterStore extends FilterState {
|
||||
setDateTo: (date: string | null) => void
|
||||
toggleMediaType: (t: MediaType) => void
|
||||
setRatingMin: (rating: number) => void
|
||||
setRatingMax: (rating: number) => void
|
||||
setColorLabel: (label: ColorLabel | null) => void
|
||||
setFlag: (flag: FlagFilter) => void
|
||||
setHeapId: (id: string | null) => void
|
||||
@@ -93,6 +95,7 @@ export const INITIAL_FILTERS: FilterState = {
|
||||
dateTo: null,
|
||||
mediaTypes: [],
|
||||
ratingMin: 0,
|
||||
ratingMax: 0,
|
||||
colorLabel: null,
|
||||
flag: 'any',
|
||||
heapId: null,
|
||||
@@ -114,6 +117,7 @@ function snapshotFilters(s: FilterState): FilterState {
|
||||
dateTo: s.dateTo,
|
||||
mediaTypes: [...s.mediaTypes],
|
||||
ratingMin: s.ratingMin,
|
||||
ratingMax: s.ratingMax,
|
||||
colorLabel: s.colorLabel,
|
||||
flag: s.flag,
|
||||
heapId: s.heapId,
|
||||
@@ -142,6 +146,7 @@ export const useFilterStore = create<FilterStore>((set) => ({
|
||||
: [...s.mediaTypes, t],
|
||||
})),
|
||||
setRatingMin: (ratingMin) => set({ ratingMin }),
|
||||
setRatingMax: (ratingMax) => set({ ratingMax }),
|
||||
setColorLabel: (colorLabel) => set({ colorLabel }),
|
||||
setFlag: (flag) => set({ flag }),
|
||||
setHeapId: (heapId) => set({ heapId }),
|
||||
@@ -205,6 +210,7 @@ export function filtersToParams(f: FilterState): Record<string, string | number>
|
||||
if (f.dateTo) params.date_to = f.dateTo
|
||||
if (f.mediaTypes.length > 0) params.media_type = f.mediaTypes.join(',')
|
||||
if (f.ratingMin > 0) params.rating_min = f.ratingMin
|
||||
if (f.ratingMax > 0) params.rating_max = f.ratingMax
|
||||
if (f.colorLabel) params.color_label = f.colorLabel
|
||||
if (f.flag === 'discarded') params.is_discarded = 'true'
|
||||
if (f.heapId) params.heap_id = f.heapId
|
||||
@@ -224,6 +230,7 @@ export function hasActiveFilters(f: FilterState): boolean {
|
||||
f.dateTo !== null ||
|
||||
f.mediaTypes.length > 0 ||
|
||||
f.ratingMin > 0 ||
|
||||
f.ratingMax > 0 ||
|
||||
f.colorLabel !== null ||
|
||||
f.flag !== 'any' ||
|
||||
f.heapId !== null ||
|
||||
|
||||
28
mulita.yml
28
mulita.yml
@@ -22,3 +22,31 @@ performance:
|
||||
cache_ttl: 3600
|
||||
db_pool_size: 20
|
||||
db_pool_recycle: 3600
|
||||
|
||||
# AI vision pipeline — embedding, OCR, object detection, face recognition.
|
||||
# Runs on the dedicated `vision` Celery queue (PR4+). Set enabled: false
|
||||
# to disable all vision processing.
|
||||
vision:
|
||||
enabled: true
|
||||
backend: onnx # "onnx" (CPU) | "rocm" (future GPU)
|
||||
models_dir: /data/models
|
||||
embedder:
|
||||
name: openclip_vitb32
|
||||
batch_size: 8
|
||||
ocr:
|
||||
enabled: true
|
||||
languages: [en]
|
||||
min_confidence: 0.5
|
||||
detector:
|
||||
enabled: true
|
||||
min_confidence: 0.35
|
||||
max_detections: 50
|
||||
faces:
|
||||
enabled: true
|
||||
min_face_size: 40
|
||||
recognition_threshold: 0.65
|
||||
cluster_eps: 0.5
|
||||
classifier:
|
||||
enabled: true
|
||||
min_confidence: 0.3
|
||||
worker_concurrency: 2
|
||||
|
||||
Reference in New Issue
Block a user