feat: add OCR text extraction and Postgres full-text search
- Create ocr_text table for storing per-region OCR results - Add tsvector search_vector column to photos with GIN index and auto-update trigger on filename/user_title/user_notes - Implement ocr_photo Celery task using rapidocr-onnxruntime - Add FTS leg to hybrid search: queries photos.search_vector and ocr_text via UNION, fused with semantic results via RRF (k=60) Migration 0004 backfills search_vector for existing rows. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
82
backend/alembic/versions/0004_ocr_text_and_fts.py
Normal file
82
backend/alembic/versions/0004_ocr_text_and_fts.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""ocr_text table and Postgres FTS
|
||||
|
||||
Revision ID: 0004_ocr_fts
|
||||
Revises: 0003_pgvector_embeddings
|
||||
Create Date: 2026-04-10
|
||||
|
||||
Create ocr_text table for storing OCR results. Add a tsvector column
|
||||
to photos for unified full-text search (filename + user_title +
|
||||
user_notes) with a GIN index. OCR text is rolled up into a materialized
|
||||
view or joined at query time.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0004_ocr_fts"
|
||||
down_revision: Union[str, None] = "0003_pgvector_embeddings"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ── ocr_text table ────────────────────────────────────────────────
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ocr_text (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
photo_id VARCHAR NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
language VARCHAR(8) DEFAULT '',
|
||||
confidence FLOAT,
|
||||
bbox JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT now()
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_ocr_text_photo_id ON ocr_text(photo_id)")
|
||||
|
||||
# ── tsvector column on photos ─────────────────────────────────────
|
||||
op.execute("ALTER TABLE photos ADD COLUMN IF NOT EXISTS search_vector tsvector")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_photos_search_vector ON photos USING GIN (search_vector)")
|
||||
|
||||
# Trigger to auto-update search_vector on INSERT/UPDATE
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION photos_search_vector_update() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_vector :=
|
||||
setweight(to_tsvector('english', coalesce(NEW.filename, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.user_title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.user_notes, '')), 'B');
|
||||
RETURN NEW;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
op.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_trigger WHERE tgname = 'photos_search_vector_trigger'
|
||||
) THEN
|
||||
CREATE TRIGGER photos_search_vector_trigger
|
||||
BEFORE INSERT OR UPDATE OF filename, user_title, user_notes
|
||||
ON photos
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION photos_search_vector_update();
|
||||
END IF;
|
||||
END $$;
|
||||
""")
|
||||
|
||||
# Backfill existing rows
|
||||
op.execute("""
|
||||
UPDATE photos SET search_vector =
|
||||
setweight(to_tsvector('english', coalesce(filename, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(user_title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(user_notes, '')), 'B')
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TRIGGER IF EXISTS photos_search_vector_trigger ON photos")
|
||||
op.execute("DROP FUNCTION IF EXISTS photos_search_vector_update()")
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_search_vector")
|
||||
op.execute("ALTER TABLE photos DROP COLUMN IF EXISTS search_vector")
|
||||
op.execute("DROP TABLE IF EXISTS ocr_text")
|
||||
Reference in New Issue
Block a user