dtoro
94c07b1d0d
Replace OpenCLIP ViT-B/32 (512-d, ~78% recall) with SigLIP2 ViT-B/16 (768-d, ~84% recall) as the default embedding model for significantly better image-text retrieval quality. - New SigLIP2Embedder class with 384px input and SigLIP normalization - ONNX export pipeline for SigLIP2 visual + textual encoders - Migration 0010: resize embeddings.vector from 512 to 768 dimensions - Config-driven model selection: "siglip2_vitb16" (default) or "openclip_vitb32" (legacy) — both models can coexist - Content classifier follows the configured embedder family - Existing embeddings cleared on migration; vision backfill regenerates Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
40 lines
1.3 KiB
Python
40 lines
1.3 KiB
Python
"""embeddings vector 512 -> 768
|
|
|
|
Revision ID: 0010_embeddings_768d
|
|
Revises: 0009_users_and_auth
|
|
Create Date: 2026-04-12
|
|
|
|
Resize embeddings.vector from Vector(512) to Vector(768) for
|
|
SigLIP2 ViT-B/16 embeddings. Drops existing data and HNSW index,
|
|
recreates with the new dimension. Existing embeddings will be
|
|
regenerated by the vision backfill task.
|
|
"""
|
|
from typing import Sequence, Union
|
|
|
|
from alembic import op
|
|
|
|
revision: str = "0010_embeddings_768d"
|
|
down_revision: Union[str, None] = "0009_users_and_auth"
|
|
branch_labels: Union[str, Sequence[str], None] = None
|
|
depends_on: Union[str, Sequence[str], None] = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.execute("DROP INDEX IF EXISTS ix_embeddings_vector_hnsw")
|
|
op.execute("DELETE FROM embeddings")
|
|
op.execute("ALTER TABLE embeddings ALTER COLUMN vector TYPE vector(768)")
|
|
op.execute("""
|
|
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
|
ON embeddings USING hnsw (vector vector_cosine_ops)
|
|
""")
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.execute("DROP INDEX IF EXISTS ix_embeddings_vector_hnsw")
|
|
op.execute("DELETE FROM embeddings")
|
|
op.execute("ALTER TABLE embeddings ALTER COLUMN vector TYPE vector(512)")
|
|
op.execute("""
|
|
CREATE INDEX IF NOT EXISTS ix_embeddings_vector_hnsw
|
|
ON embeddings USING hnsw (vector vector_cosine_ops)
|
|
""")
|