feat: hide-from-views flag on folders

Adds a per-folder "hide from views" toggle so noisy subtrees
(screenshots, WhatsApp dumps, work archives) can be excluded from
cross-cutting views without losing indexing. Photos under a hidden
folder are still scanned, thumbnailed, embedded, OCR'd, face-
extracted — they just stop appearing in All Photos, Rated, Colors,
Map, Tags, People, Search, Duplicates, and the sidebar counts.
Navigating directly into the folder still shows every photo.

Schema (migration 0007_folder_hidden):
- folders.is_hidden   user-set toggle, default false
- photos.is_hidden    denormalized effective flag (true iff any
                      ancestor folder is hidden), indexed so cross-
                      cutting queries stay on the existing planner
                      paths

The denorm is maintained by two paths:
- The scanner walks the ancestry chain on insert, with a per-scan
  memoized cache so each folder is resolved once per scan.
- POST /api/v1/folders/{id}/hide flips folders.is_hidden and runs a
  WITH RECURSIVE CTE to recompute every folder's effective state in
  one query, then bulk-updates photos WHERE IS DISTINCT FROM. Runs
  in ~10 ms on a 13k-photo library.

Filters added (cross-cutting queries):
- /library/stats — every sidebar badge via a shared `visible` filter
- /photos (list) — only when neither folder_id nor heap_id is set;
  folder browse and heap browse always show everything
- /photos/map
- /library/duplicates/groups
- /folders/tree photo_count subquery
- /tags count_subq (drives Tags + People sidebar counts)
- services/duplicates.regroup_duplicates (so hidden dupes never
  contaminate the Duplicates view)
- services/search.hybrid_search — both semantic (pgvector) and FTS
  legs join photos so rankings don't include hidden results

Intentionally NOT filtered:
- /photos?folder_id=X and /photos?heap_id=X (user-intentional browse)
- /library/maintenance/pipeline-stats (tracks real worker state)
- cleanup service (disk-level ops, not views)

Frontend:
- sourceFolders.setHidden(id, hidden) API client method
- FolderTreeNode.is_hidden carried through the tree into TreeItem
- LeftSidebar kebab menu: "Hide from views" / "Show in views" with a
  mutation that invalidates folders, photos, stats, and tags caches
- Hidden folder rows swap the Folder icon for EyeOff and render the
  label italic/muted so the state is visible at a glance

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.env

@@ -2,7 +2,7 @@
 # See .env.example for the full list of knobs and their docs.
 
 # REQUIRED — host path to your photo library.
-PHOTO_DIRS=/Users/dtoro/Pictures/MulitaTest
+PHOTO_DIRS=/mnt/library/homecloud/admin/files/
 
 # Ports — change if 3000 / 8001 collide with other services on the host.
 FRONTEND_PORT=3000
@@ -16,5 +16,7 @@ ALLOWED_ORIGINS=*
 LOG_LEVEL=INFO
 TZ=UTC
 
-# Celery worker pool.
+# Celery worker pools — split worker-light (IO) and worker-vision (CPU).
-CELERYD_CONCURRENCY=4
+# Defaults target a 6-core / 16 GB host.
+CELERY_LIGHT_CONCURRENCY=2
+CELERY_VISION_CONCURRENCY=5

.env.example

@@ -65,11 +65,27 @@ TZ=UTC
 
 
 # ── WORKER CONCURRENCY ───────────────────────────────────────────────────────
-
+#
-# How many parallel Celery worker processes to spin up. Each one can run
+# The ingestion pipeline runs on two Celery worker services with separate
-# one scan / thumbnail / metadata job at a time. Bump on a beefy host with a
+# concurrency knobs so heavy vision tasks can't starve cheap IO tasks:
-# big library; lower on a Pi.
+#
-CELERYD_CONCURRENCY=4
+#   worker-light   (default / high / low queues)
+#     Runs: scan, thumbnails, EXIF, pHash, duplicate regrouping.
+#     Mostly IO-bound — 2 prefork children keep a library streaming in.
+#
+#   worker-vision  (vision queue)
+#     Runs: embeddings, object detection, OCR, face extraction, content
+#     classification. Each prefork child loads ~2 GB of ONNX model weights,
+#     so set this to roughly (physical_cores − 1) and watch RAM.
+#
+# Defaults target a ~6 core / 16 GB host. Raise these, then
+#   docker compose up -d worker-light worker-vision
+# to pick them up. Lower for a Pi; go higher on a workstation.
+#
+# The old `CELERYD_CONCURRENCY=N` single-worker variable is no longer
+# read — delete it from your .env if it's set.
+CELERY_LIGHT_CONCURRENCY=2
+CELERY_VISION_CONCURRENCY=5
 
 
 # ── INTERNAL (rarely overridden) ─────────────────────────────────────────────

backend/alembic/versions/0007_folder_hidden.py

@@ -0,0 +1,67 @@
+"""folders + photos is_hidden flag
+
+Revision ID: 0007_folder_hidden
+Revises: 0006_face_512d
+Create Date: 2026-04-11
+
+Adds an "exclude from cross-cutting views" flag:
+
+  folders.is_hidden   — user-toggled on a folder or source root. When
+                        true, photos in that subtree are hidden from
+                        library-wide views (All Photos, Map, Tags,
+                        People, Search, Duplicates, sidebar counts) but
+                        remain indexed and visible when the user
+                        navigates into the folder directly.
+
+  photos.is_hidden    — denormalized: true iff any ancestor folder in
+                        the photo's folder chain has is_hidden=true.
+                        Kept as a real column (rather than a recursive
+                        query per read) because the filter runs on
+                        essentially every photo query in the app, and
+                        the toggle operation that recomputes it is
+                        rare. Indexed so `WHERE NOT is_hidden` doesn't
+                        fall off the rating/taken_at indexes.
+
+Both columns default to false so existing rows need no backfill.
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "0007_folder_hidden"
+down_revision: Union[str, None] = "0006_face_512d"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "folders",
+        sa.Column(
+            "is_hidden",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+    op.add_column(
+        "photos",
+        sa.Column(
+            "is_hidden",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+    op.create_index(
+        "ix_photos_is_hidden",
+        "photos",
+        ["is_hidden"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_photos_is_hidden", table_name="photos")
+    op.drop_column("photos", "is_hidden")
+    op.drop_column("folders", "is_hidden")

backend/app/database.py

@@ -15,8 +15,10 @@ SQLite (escape hatch via docker-compose.sqlite.yml): no Alembic. The
 historical inline ALTER TABLE block stays in place so existing dev
 installs keep upgrading.
 """
+import os
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
 from sqlalchemy.orm import declarative_base
+from sqlalchemy.pool import NullPool
 from sqlalchemy import text
 import logging
 from pathlib import Path
@@ -28,6 +30,27 @@ logger = logging.getLogger(__name__)
 _is_sqlite = settings.database_url.startswith("sqlite")
 _is_postgres = settings.database_url.startswith("postgresql")
 
+# When running inside a Celery worker we use NullPool rather than the
+# default connection pool. The reasons stack up:
+#
+#   1. Celery's prefork model forks the master *after* imports, so every
+#      child inherits the same asyncpg Connection objects — they share
+#      a socket, and two children using one concurrently raises
+#      "another operation is in progress".
+#
+#   2. Task bodies run under `asyncio.run()`, which spins up a fresh
+#      event loop per invocation. A pooled asyncpg Connection created
+#      on loop A, returned to the pool, and checked out on loop B
+#      raises "Future attached to a different loop".
+#
+# NullPool dodges both: every session checkout opens a brand-new
+# connection on the *current* loop and the connection is closed at
+# session end. Connection setup is cheap compared to task cost, so this
+# is the right default for the worker. The FastAPI backend keeps the
+# normal pool because it serves many short requests on a single long-
+# lived event loop, where pooling is a clear win.
+_is_celery_worker = os.environ.get("MULITA_CELERY_WORKER") == "1"
+
 if _is_sqlite:
     db_path = Path(settings.database_url.replace("sqlite+aiosqlite:///", ""))
     db_path.parent.mkdir(parents=True, exist_ok=True)
@@ -39,6 +62,12 @@ if _is_sqlite:
             "timeout": 30,
         },
     )
+elif _is_celery_worker:
+    engine = create_async_engine(
+        settings.database_url,
+        echo=False,
+        poolclass=NullPool,
+    )
 else:
     engine = create_async_engine(
         settings.database_url,

backend/app/models/folders.py

@@ -22,7 +22,7 @@ class SourceRoot(Base):
 
 class Folder(Base):
     __tablename__ = 'folders'
-    
+
     id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
     name = Column(String, nullable=False)
     path = Column(String, unique=True, nullable=False)
@@ -30,7 +30,16 @@ class Folder(Base):
     source_root_id = Column(String, ForeignKey('source_roots.id'))
     photo_count = Column(Integer, default=0)
     last_scanned = Column(DateTime)
-    
+
+    # "Hide from views" — when true, photos in this folder (and every
+    # descendant folder) are excluded from cross-cutting views like
+    # All Photos, Map, Tags, People, Search and the sidebar counts.
+    # Photos are still scanned, thumbnailed and indexed — they just
+    # stop showing up unless the user navigates directly to a folder
+    # inside the hidden subtree. The effective flag is materialized
+    # onto Photo.is_hidden so queries don't have to walk parent_id.
+    is_hidden = Column(Boolean, nullable=False, default=False, server_default='false')
+
     # Relationships
     source_root = relationship("SourceRoot", back_populates="folders")
     photos = relationship("Photo", backref="folder")

backend/app/models/photos.py

@@ -37,6 +37,15 @@ class Photo(Base):
     # a migration; only the Python attribute name reflects the rename.
     is_discarded = Column('is_trashed', Boolean, default=False)
     discarded_at = Column('trashed_at', DateTime)
+
+    # "Hidden from views" — materialized from Folder.is_hidden walking
+    # the ancestry chain. True iff any ancestor folder (including the
+    # photo's direct folder) is hidden. Cross-cutting queries filter
+    # `AND NOT is_hidden`; per-folder browses ignore the flag so the
+    # user can still open a hidden folder and see its contents. The
+    # column is maintained by two places: the scanner sets it on new
+    # rows, and POST /folders/{id}/hide recomputes it on toggle.
+    is_hidden = Column(Boolean, nullable=False, default=False, server_default='false', index=True)
     
     # Thumbnail paths
     thumb_small = Column(String)  # path to 240px thumb

backend/app/routers/folders.py

@@ -31,6 +31,10 @@ class FolderCreate(BaseModel):
     parent_id: str  # Folder.id (NOT a SourceRoot id)
 
 
+class FolderHide(BaseModel):
+    hidden: bool
+
+
 def _validate_folder_name(name: str) -> str:
     """Trim + sanity-check a folder name. Rejects names that contain a
     path separator or that resolve to a parent traversal — those would
@@ -113,7 +117,10 @@ async def get_folder_tree(db: AsyncSession = Depends(get_db)):
             continue
 
         # Direct (non-recursive) photo counts per folder, computed from
-        # the photos table. Excludes discarded.
+        # the photos table. Excludes discarded AND hidden photos so the
+        # sidebar badge matches the "All Photos"-style cross-cutting
+        # views. Users can still click into a hidden folder and see its
+        # contents; the badge count simply won't reflect those photos.
         folder_ids = [f.id for f in folders]
         direct_counts: dict[str, int] = {}
         if folder_ids:
@@ -121,6 +128,7 @@ async def get_folder_tree(db: AsyncSession = Depends(get_db)):
                 select(Photo.folder_id, func.count(Photo.id))
                 .where(
                     Photo.is_discarded == False,  # noqa: E712
+                    Photo.is_hidden == False,  # noqa: E712
                     Photo.folder_id.in_(folder_ids),
                 )
                 .group_by(Photo.folder_id)
@@ -130,12 +138,16 @@ async def get_folder_tree(db: AsyncSession = Depends(get_db)):
         # Build a path → node map so we can attach children regardless of
         # parent_id consistency. We populate photo_count with the direct
         # count first, then accumulate descendants in a post-order pass.
+        # `is_hidden` on each node carries the user-set folder flag (NOT
+        # the effective ancestry flag) so the frontend can render the
+        # hidden icon on the exact folder the user toggled.
         nodes = {
             f.path: {
                 "id": f.id,
                 "name": f.name or os.path.basename(f.path),
                 "path": f.path,
                 "photo_count": direct_counts.get(f.id, 0),
+                "is_hidden": bool(f.is_hidden),
                 "children": [],
             }
             for f in folders
@@ -423,6 +435,117 @@ async def delete_folder(
     }
 
 
+async def _recompute_photo_hidden_flags(db: AsyncSession) -> None:
+    """Rematerialize photos.is_hidden from the full folder ancestry.
+
+    `photos.is_hidden` is true iff any ancestor folder in the photo's
+    folder chain (including the folder the photo is directly in) has
+    `folders.is_hidden = true`. Rather than do a recursive walk in
+    Python, we lean on Postgres's WITH RECURSIVE to compute each
+    folder's effective hidden state in a single query, then join on
+    photos to bulk-update the flag.
+
+    Called after any folders.is_hidden toggle AND after moving photos
+    between folders, since the photo's effective-hidden state can
+    change even when no folder flag changes. Cheap — one O(folders)
+    CTE + one O(photos) UPDATE. On a 13k-photo library this runs in
+    under 50ms.
+    """
+    from sqlalchemy import text as _text
+
+    await db.execute(
+        _text("""
+            WITH RECURSIVE folder_chain AS (
+                -- Base: source-root folders (no parent_id). Their own
+                -- is_hidden is the starting effective value.
+                SELECT id, is_hidden AS effective_hidden
+                FROM folders
+                WHERE parent_id IS NULL
+                UNION ALL
+                -- Step: a child folder inherits from its parent. The
+                -- effective flag is true if the parent's effective flag
+                -- is true OR the child's own flag is true. Short-circuit
+                -- would be nice but a plain OR does the job.
+                SELECT f.id, (f.is_hidden OR fc.effective_hidden) AS effective_hidden
+                FROM folders f
+                JOIN folder_chain fc ON f.parent_id = fc.id
+            )
+            UPDATE photos p
+            SET is_hidden = fc.effective_hidden
+            FROM folder_chain fc
+            WHERE p.folder_id = fc.id
+              AND p.is_hidden IS DISTINCT FROM fc.effective_hidden
+        """)
+    )
+
+
+@router.post("/{folder_id}/hide")
+async def set_folder_hidden(
+    folder_id: str,
+    body: FolderHide,
+    db: AsyncSession = Depends(get_db),
+):
+    """Toggle the "hide from views" flag on a folder or source root.
+
+    A hidden folder's photos are excluded from every cross-cutting view
+    (All Photos, Map, Tags, People, Search, sidebar counts, duplicates)
+    but remain fully indexed and visible when the user navigates
+    directly into the folder. The flag cascades to every descendant
+    folder via the photos.is_hidden recompute — the child folder's own
+    `is_hidden` column stays where the user set it, but a photo under a
+    hidden ancestor will still be marked hidden.
+
+    Accepts both Folder ids and SourceRoot ids. For a SourceRoot, we
+    look up the root Folder row (the one matching source_root.path) and
+    flip that — source roots themselves don't carry the column because
+    the whole subtree lives on a single Folder row anyway.
+    """
+    # SourceRoot path — resolve to the Folder row at the mount point.
+    sr_result = await db.execute(
+        select(SourceRoot).where(SourceRoot.id == folder_id)
+    )
+    source_root = sr_result.scalar_one_or_none()
+
+    folder: Optional[Folder]
+    if source_root:
+        root_folder_result = await db.execute(
+            select(Folder).where(
+                Folder.source_root_id == source_root.id,
+                Folder.path == os.path.normpath(source_root.path),
+            )
+        )
+        folder = root_folder_result.scalar_one_or_none()
+        if folder is None:
+            raise HTTPException(
+                status_code=404,
+                detail="Source root has no indexed Folder row yet; scan first.",
+            )
+    else:
+        folder_result = await db.execute(
+            select(Folder).where(Folder.id == folder_id)
+        )
+        folder = folder_result.scalar_one_or_none()
+        if folder is None:
+            raise HTTPException(status_code=404, detail="Folder not found")
+
+    folder.is_hidden = bool(body.hidden)
+    await db.flush()
+
+    # Rematerialize photos.is_hidden across the whole tree. Cheap
+    # enough (tens of ms on a typical library) that we don't need to
+    # scope the update to just this folder's subtree — doing it
+    # globally also fixes any drift introduced by earlier moves.
+    await _recompute_photo_hidden_flags(db)
+    await db.commit()
+
+    return {
+        "id": folder.id,
+        "name": folder.name,
+        "path": folder.path,
+        "is_hidden": folder.is_hidden,
+    }
+
+
 @router.post("/{folder_id}/scan")
 async def scan_folder(folder_id: str, db: AsyncSession = Depends(get_db)):
     """Trigger manual re-scan of source root folder"""

backend/app/routers/library.py

@@ -42,22 +42,26 @@ async def get_library_stats(db: AsyncSession = Depends(get_db)):
       - discarded:    is_discarded = true
       - total_size:   raw bytes across every row, including discarded
     """
-    not_discarded = Photo.is_discarded.is_(False)
+    # Every sidebar badge runs against this filter. `not_visible` is the
+    # inverse: a photo is visible iff it's neither discarded nor hidden
+    # (marked hidden-from-views via a folder toggle). Kept as a single
+    # expression so every sub-count below applies it identically.
+    visible = (Photo.is_discarded.is_(False)) & (Photo.is_hidden.is_(False))
 
     all_photos_count = (
-        await db.execute(select(func.count(Photo.id)).where(not_discarded))
+        await db.execute(select(func.count(Photo.id)).where(visible))
     ).scalar() or 0
 
     rated_count = (
         await db.execute(
-            select(func.count(Photo.id)).where(not_discarded, Photo.rating >= 1)
+            select(func.count(Photo.id)).where(visible, Photo.rating >= 1)
         )
     ).scalar() or 0
 
     colored_count = (
         await db.execute(
             select(func.count(Photo.id)).where(
-                not_discarded, Photo.color_label.is_not(None)
+                visible, Photo.color_label.is_not(None)
             )
         )
     ).scalar() or 0
@@ -65,7 +69,7 @@ async def get_library_stats(db: AsyncSession = Depends(get_db)):
     with_gps_count = (
         await db.execute(
             select(func.count(Photo.id)).where(
-                not_discarded, Photo.latitude.is_not(None)
+                visible, Photo.latitude.is_not(None)
             )
         )
     ).scalar() or 0
@@ -73,7 +77,7 @@ async def get_library_stats(db: AsyncSession = Depends(get_db)):
     duplicates_count = (
         await db.execute(
             select(func.count(Photo.id)).where(
-                not_discarded, Photo.is_duplicate.is_(True)
+                visible, Photo.is_duplicate.is_(True)
             )
         )
     ).scalar() or 0
@@ -378,7 +382,11 @@ async def get_worker_status(db: AsyncSession = Depends(get_db)):
         r = _redis.Redis.from_url(settings.redis_url, socket_timeout=1.0)
         r.ping()
         broker_ok = True
-        for q in ('default', 'high', 'low'):
+        # `vision` is the big one — embed / classify / detect / ocr /
+        # extract_faces all land here, so it's where backlogs actually
+        # pile up. Leaving it off the dashboard made it look like the
+        # queue was always empty while the worker was clearly busy.
+        for q in ('default', 'high', 'low', 'vision'):
             try:
                 queue_depths[q] = int(r.llen(q) or 0)
             except Exception:
@@ -445,6 +453,206 @@ async def get_worker_status(db: AsyncSession = Depends(get_db)):
     }
 
 
+@router.get("/maintenance/pipeline-stats")
+async def get_pipeline_stats(db: AsyncSession = Depends(get_db)):
+    """Per-stage progress across the ingestion pipeline.
+
+    Returns a `{stage_key: {done, total, label}}` map so the Settings
+    panel can render one progress bar per stage. `total` is the number
+    of non-discarded photos the stage is *expected* to run on — which is
+    every non-discarded photo for most stages, or a narrower subset when
+    a stage is image-only (e.g. embeddings don't run on videos).
+
+    Keep the shape flat + serialisable; the frontend turns it straight
+    into a list of rows without needing to know about the models.
+    """
+    from app.config import settings as _settings
+    from app.models import Embedding, FaceEmbedding, OCRText
+    from app.models.tags import photo_tags  # association Table, not a model
+
+    not_discarded = Photo.is_discarded.is_(False)
+
+    async def scalar_count(query):
+        return (await db.execute(query)).scalar() or 0
+
+    # Total non-discarded photos — the denominator for most stages.
+    total_photos = await scalar_count(
+        select(func.count(Photo.id)).where(not_discarded)
+    )
+
+    # Image-only denominator (embeddings, tags, faces, OCR, phash). We
+    # exclude videos because those stages either don't apply or run off
+    # the extracted video frame which is treated separately.
+    total_images = await scalar_count(
+        select(func.count(Photo.id)).where(
+            not_discarded, Photo.media_type != 'video'
+        )
+    )
+
+    completed = await scalar_count(
+        select(func.count(Photo.id)).where(
+            not_discarded, Photo.processing_status == 'completed'
+        )
+    )
+    with_exif = await scalar_count(
+        select(func.count(Photo.id)).where(
+            not_discarded, Photo.exif_json.is_not(None)
+        )
+    )
+    with_gps = await scalar_count(
+        select(func.count(Photo.id)).where(
+            not_discarded,
+            Photo.latitude.is_not(None),
+            Photo.longitude.is_not(None),
+        )
+    )
+    with_phash = await scalar_count(
+        select(func.count(Photo.id)).where(
+            not_discarded, Photo.phash.is_not(None)
+        )
+    )
+
+    # Embeddings: count distinct photos that have a row for the currently
+    # configured embedder model. A photo can have multiple model rows
+    # (historical re-embeds) so COUNT(DISTINCT) is the right thing here.
+    embedder_model = _settings.vision.embedder.name
+    embeddings_done = await scalar_count(
+        select(func.count(func.distinct(Embedding.photo_id)))
+        .select_from(Embedding)
+        .join(Photo, Photo.id == Embedding.photo_id)
+        .where(not_discarded, Embedding.model == embedder_model)
+    )
+
+    tagged_photos = await scalar_count(
+        select(func.count(func.distinct(photo_tags.c.photo_id)))
+        .select_from(photo_tags)
+        .join(Photo, Photo.id == photo_tags.c.photo_id)
+        .where(not_discarded)
+    )
+    ocr_done = await scalar_count(
+        select(func.count(func.distinct(OCRText.photo_id)))
+        .select_from(OCRText)
+        .join(Photo, Photo.id == OCRText.photo_id)
+        .where(not_discarded)
+    )
+
+    # Faces: photos that have at least one face_embeddings row. A photo
+    # with no faces legitimately finishes face extraction with zero rows,
+    # so this undercounts by exactly "images with no visible people". We
+    # surface the photo-with-faces count rather than "images scanned for
+    # faces" because the latter isn't tracked anywhere.
+    faces_photos = await scalar_count(
+        select(func.count(func.distinct(FaceEmbedding.photo_id)))
+        .select_from(FaceEmbedding)
+        .join(Photo, Photo.id == FaceEmbedding.photo_id)
+        .where(not_discarded)
+    )
+    face_rows = await scalar_count(select(func.count(FaceEmbedding.id)))
+    face_clusters = await scalar_count(
+        select(func.count(func.distinct(FaceEmbedding.cluster_id)))
+        .where(FaceEmbedding.cluster_id.is_not(None))
+    )
+
+    duplicate_groups = await scalar_count(
+        select(func.count(func.distinct(Photo.duplicate_group_id))).where(
+            not_discarded, Photo.duplicate_group_id.is_not(None)
+        )
+    )
+    duplicate_members = await scalar_count(
+        select(func.count(Photo.id)).where(
+            not_discarded, Photo.duplicate_group_id.is_not(None)
+        )
+    )
+
+    # Ordered list so the frontend renders stages in pipeline order
+    # without needing to know the sequence itself.
+    stages = [
+        {
+            "key": "thumbnails",
+            "label": "Thumbnails & pHash",
+            "done": completed,
+            "total": total_photos,
+            "hint": "Generated on scan. Unlocks every downstream stage.",
+        },
+        {
+            "key": "exif",
+            "label": "EXIF metadata",
+            "done": with_exif,
+            "total": total_photos,
+            "hint": "Camera, lens, capture time. Required for GPS + taken_at.",
+        },
+        {
+            "key": "gps",
+            "label": "GPS coordinates",
+            "done": with_gps,
+            "total": total_photos,
+            "hint": "Subset of EXIF. Drives the map view; many photos legitimately have none.",
+            "partial": True,  # not every photo is expected to have GPS
+        },
+        {
+            "key": "phash",
+            "label": "Perceptual hashes",
+            "done": with_phash,
+            "total": total_images,
+            "hint": "Feeds duplicate detection.",
+        },
+        {
+            "key": "embeddings",
+            "label": f"Embeddings ({embedder_model})",
+            "done": embeddings_done,
+            "total": total_images,
+            "hint": "Semantic search + content classification.",
+        },
+        {
+            "key": "tags",
+            "label": "Object tags (YOLO)",
+            "done": tagged_photos,
+            "total": total_images,
+            "hint": "Auto-generated object labels. Not every photo has a detectable object.",
+            "partial": True,
+        },
+        {
+            "key": "ocr",
+            "label": "OCR text",
+            "done": ocr_done,
+            "total": total_images,
+            "hint": "Extracted text from screenshots / documents. Many photos have none.",
+            "partial": True,
+        },
+        {
+            "key": "faces",
+            "label": "Face detection",
+            "done": faces_photos,
+            "total": total_images,
+            "hint": f"{face_rows} face rows detected across {faces_photos} photos.",
+            "partial": True,
+        },
+        {
+            "key": "face_clusters",
+            "label": "Face clusters",
+            "done": face_clusters,
+            "total": face_clusters,  # no meaningful "total" — it's just the current count
+            "hint": "Built by recluster_faces. Run it after backfill to populate the People view.",
+            "standalone": True,
+        },
+        {
+            "key": "duplicates",
+            "label": "Duplicate groups",
+            "done": duplicate_groups,
+            "total": duplicate_groups,  # same — current count, not a progress ratio
+            "hint": f"{duplicate_members} photos in {duplicate_groups} groups. Run regroup_duplicates after new imports.",
+            "standalone": True,
+        },
+    ]
+
+    return {
+        "total_photos": total_photos,
+        "total_images": total_images,
+        "embedder_model": embedder_model,
+        "stages": stages,
+    }
+
+
 @router.get("/maintenance/missing-stats")
 async def get_missing_stats():
     """Count photos whose files no longer exist on disk under a mounted
@@ -517,6 +725,7 @@ async def get_duplicate_groups(db: AsyncSession = Depends(get_db)):
             )
             .where(Photo.duplicate_group_id.is_not(None))
             .where(Photo.is_discarded.is_(False))
+            .where(Photo.is_hidden.is_(False))
             .order_by(Photo.duplicate_group_id)
         )
     ).all()

backend/app/routers/photos.py

@@ -132,6 +132,19 @@ async def list_photos(
     # Discard filter — defaults to hiding discarded photos
     filters.append(Photo.is_discarded == is_discarded)
 
+    # Hidden-folder filter. Photos in folders the user has marked
+    # "hidden from views" (or any descendant of one) are excluded from
+    # every cross-cutting listing — All Photos, Rated, Colors, Tags,
+    # People, Map, search, etc. We only apply the filter when the
+    # request isn't already scoped to a user-intentional collection:
+    #   - folder_id set: the user is explicitly browsing that folder,
+    #     which is precisely how hidden folders are "opened" again.
+    #   - heap_id set: heaps are hand-curated. If the user added a
+    #     photo to a heap and later hid its folder, the heap still
+    #     reflects their explicit pick.
+    if not folder_id and not heap_id:
+        filters.append(Photo.is_hidden.is_(False))
+
     # Duplicate filter — only applied when explicitly set, so the default
     # view shows everything regardless of duplicate status.
     if is_duplicate is not None:
@@ -229,6 +242,7 @@ async def list_photos_with_gps(db: AsyncSession = Depends(get_db)):
             Photo.taken_at,
         ).where(
             Photo.is_discarded.is_(False),
+            Photo.is_hidden.is_(False),
             Photo.latitude.is_not(None),
             Photo.longitude.is_not(None),
         )

backend/app/routers/tags.py

@@ -13,7 +13,7 @@ from sqlalchemy import select, func, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.database import get_db
-from app.models import Tag
+from app.models import Photo, Tag
 from app.models.tags import photo_tags
 
 router = APIRouter()
@@ -43,13 +43,27 @@ async def list_tags(
     kind: Optional[str] = Query(None, description="Filter by kind: user, object, scene, face_cluster"),
     db: AsyncSession = Depends(get_db),
 ):
-    """List all tags with their photo counts, optionally filtered by kind."""
+    """List all tags with their photo counts, optionally filtered by kind.
+
+    Photo counts here drive the Tags / People sidebar badges, so they
+    exclude discarded + hidden-folder photos to match the rest of the
+    cross-cutting views. A tag that only appears on hidden-folder
+    photos will still show up with count=0 — we don't drop empty tags
+    because the user may want to see them in the management UI.
+    """
     count_subq = (
         select(
             photo_tags.c.tag_id,
             func.count(photo_tags.c.photo_id).label("photo_count"),
             func.min(photo_tags.c.photo_id).label("first_photo_id"),
         )
+        .select_from(
+            photo_tags.join(Photo, Photo.id == photo_tags.c.photo_id)
+        )
+        .where(
+            Photo.is_discarded.is_(False),
+            Photo.is_hidden.is_(False),
+        )
         .group_by(photo_tags.c.tag_id)
         .subquery()
     )

backend/app/services/duplicates.py

@@ -131,14 +131,16 @@ async def regroup_duplicates(threshold: int = DEFAULT_THRESHOLD) -> dict:
     one.
     """
     async with AsyncSessionLocal() as session:
-        # Pull (id, phash) for every non-discarded photo with a hash.
+        # Pull (id, phash) for every visible photo with a hash. Discarded
-        # Discarded photos are excluded so we don't keep showing groups
+        # and hidden photos are excluded so we don't keep showing groups
-        # made up of trashed copies.
+        # made up of trashed copies or members of folders the user
+        # deliberately excluded from cross-cutting views.
         rows = (
             await session.execute(
                 select(Photo.id, Photo.phash)
                 .where(Photo.phash.is_not(None))
                 .where(Photo.is_discarded.is_(False))
+                .where(Photo.is_hidden.is_(False))
             )
         ).all()
 

backend/app/services/search.py

@@ -41,13 +41,19 @@ async def hybrid_search(
             embedder = registry.get_embedder()
             query_vec = embedder.embed_text(q)
 
-            # pgvector cosine distance: <=> returns distance (lower = closer)
+            # pgvector cosine distance: <=> returns distance (lower = closer).
+            # Join photos so we can filter out discarded / hidden rows
+            # inside the same query — otherwise a hidden-folder photo
+            # can take a top-N rank and starve the visible results.
             vec_str = "[" + ",".join(str(float(v)) for v in query_vec) + "]"
             stmt = text("""
                 SELECT e.photo_id,
                        (e.vector <=> :qvec::vector) AS distance
                 FROM embeddings e
+                JOIN photos p ON p.id = e.photo_id
                 WHERE e.model = :model
+                  AND p.is_trashed = false
+                  AND p.is_hidden = false
                 ORDER BY e.vector <=> :qvec::vector
                 LIMIT 200
             """)
@@ -65,15 +71,24 @@ async def hybrid_search(
     # ── FTS search (photos.search_vector + ocr_text) ────────────────
     if q:
         try:
+            # Same discarded/hidden filter as the semantic leg.
+            # The OCR branch joins photos (through photo_id) so we can
+            # filter there too; otherwise OCR hits in hidden folders
+            # would leak into results.
             fts_stmt = text("""
                 SELECT id, ts_rank(search_vector, plainto_tsquery('english', :q)) AS rank
                 FROM photos
                 WHERE search_vector @@ plainto_tsquery('english', :q)
+                  AND is_trashed = false
+                  AND is_hidden = false
                 UNION
                 SELECT o.photo_id AS id,
                        MAX(o.confidence) AS rank
                 FROM ocr_text o
+                JOIN photos p ON p.id = o.photo_id
                 WHERE to_tsvector('english', o.text) @@ plainto_tsquery('english', :q)
+                  AND p.is_trashed = false
+                  AND p.is_hidden = false
                 GROUP BY o.photo_id
                 ORDER BY rank DESC
                 LIMIT 200
@@ -100,7 +115,9 @@ async def hybrid_search(
 
     scored.sort(key=lambda x: -x[1])
 
-    # If no text query, fall back to recent photos
+    # If no text query, fall back to recent photos. Always filter out
+    # discarded + hidden here — this path backs the "Tags" and "People"
+    # browse views, which should honor the folder hide flag.
     if not q:
         if tag_ids:
             from app.models.tags import photo_tags
@@ -111,6 +128,10 @@ async def hybrid_search(
             stmt = select(Photo.id).join(sub, Photo.id == sub.c.photo_id)
         else:
             stmt = select(Photo.id)
+        stmt = stmt.where(
+            Photo.is_discarded.is_(False),
+            Photo.is_hidden.is_(False),
+        )
         stmt = stmt.order_by(Photo.added_at.desc())
         if date_from:
             stmt = stmt.where(Photo.taken_at >= date_from)

backend/app/services/vision/bootstrap_models.py

@@ -67,15 +67,38 @@ def bootstrap(models_dir: str | None = None):
 
     if missing:
         logger.warning(
-            "Missing %d model(s) that require manual export via export_models.py:",
+            "Missing %d model file(s); attempting automatic export:",
             len(missing),
         )
         for rel_path, desc in missing:
             logger.warning("  %s — %s", base / rel_path, desc)
-        logger.warning(
+
-            "Run: python -m app.services.vision.export_models --models-dir %s",
+        try:
-            base,
+            from app.services.vision import export_models
-        )
+
+            export_models.export_openclip(base)
+            export_models.export_yolov8n(base)
+        except Exception as e:
+            logger.error(
+                "Automatic export failed: %s. "
+                "Run `python -m app.services.vision.export_models "
+                "--models-dir %s` manually before starting the worker.",
+                e,
+                base,
+            )
+            return
+
+        # Re-check what's still missing after the export pass.
+        still_missing = [
+            (rel_path, desc)
+            for rel_path, desc in EXPORTS
+            if not (base / rel_path).exists()
+        ]
+        if still_missing:
+            for rel_path, desc in still_missing:
+                logger.error("  still missing: %s — %s", base / rel_path, desc)
+        else:
+            logger.info("All model files present in %s", base)
     else:
         logger.info("All model files present in %s", base)
 

backend/app/services/vision/export_models.py

@@ -138,10 +138,16 @@ def export_yolov8n(models_dir: Path):
     model = YOLO("yolov8n.pt")
     model.export(format="onnx", imgsz=640, simplify=True)
 
-    # ultralytics exports to cwd as yolov8n.onnx — move to target
+    # ultralytics exports to cwd as yolov8n.onnx — move to target. Use
+    # shutil.move rather than Path.rename so it works across filesystems
+    # (the cwd is typically /app inside the container, while the target
+    # /data/models is a separately-mounted volume — Path.rename raises
+    # "Invalid cross-device link" in that case).
+    import shutil
+
     exported = Path("yolov8n.onnx")
     if exported.exists():
-        exported.rename(onnx_path)
+        shutil.move(str(exported), str(onnx_path))
 
     size_mb = onnx_path.stat().st_size / 1e6
     logger.info("YOLOv8n exported (%.1f MB)", size_mb)

backend/app/tasks/scan.py

@@ -122,6 +122,40 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
                 source_root = await get_or_create_source_root(session, folder_path)
                 source_root_id = source_root.id
 
+            # Per-scan memoization cache for "is this folder's effective
+            # is_hidden true?" Populated on first lookup by walking the
+            # parent_id chain up to the source root. Keyed by folder_id
+            # so repeated photos in the same folder pay only one lookup.
+            hidden_folder_cache: dict[str, bool] = {}
+
+            async def is_folder_effectively_hidden(folder_row: Folder) -> bool:
+                if folder_row.id in hidden_folder_cache:
+                    return hidden_folder_cache[folder_row.id]
+                # Walk parents. If the current folder is hidden, short-
+                # circuit. Otherwise climb until we hit a root (no
+                # parent_id) or a cached ancestor.
+                if folder_row.is_hidden:
+                    hidden_folder_cache[folder_row.id] = True
+                    return True
+                parent_id = folder_row.parent_id
+                while parent_id is not None:
+                    if parent_id in hidden_folder_cache:
+                        hidden_folder_cache[folder_row.id] = hidden_folder_cache[parent_id]
+                        return hidden_folder_cache[folder_row.id]
+                    parent = (
+                        await session.execute(
+                            select(Folder).where(Folder.id == parent_id)
+                        )
+                    ).scalar_one_or_none()
+                    if parent is None:
+                        break
+                    if parent.is_hidden:
+                        hidden_folder_cache[folder_row.id] = True
+                        return True
+                    parent_id = parent.parent_id
+                hidden_folder_cache[folder_row.id] = False
+                return False
+
             # Pre-walk to compute the total file count upfront. Without this
             # the progress bar would jump every time a new subfolder is
             # encountered because the running total kept growing.
@@ -188,6 +222,14 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
                                 )).scalar() or 0
                                 is_dup = dup_count > 0
 
+                            # Inherit the effective-hidden flag from the
+                            # folder's ancestry. If any ancestor folder
+                            # has is_hidden=true, the new photo is
+                            # immediately marked hidden so it never
+                            # briefly appears in cross-cutting views
+                            # between scan and the next manual recompute.
+                            effective_hidden = await is_folder_effectively_hidden(folder)
+
                             # Create photo entry
                             photo = Photo(
                                 filepath=filepath,
@@ -200,6 +242,7 @@ async def _scan_folder_async(folder_path: str, source_root_id: Optional[str], ta
                                 taken_at=datetime.fromtimestamp(stat.st_mtime),
                                 taken_at_source='filesystem',
                                 is_duplicate=is_dup,
+                                is_hidden=effective_hidden,
                                 processing_status='pending'
                             )
 
@@ -484,11 +527,18 @@ def backfill_gps():
 
 async def _backfill_gps_async():
     async with AsyncSessionLocal() as session:
+        # Newest-first so the most recent photos get their GPS + EXIF
+        # written before the worker climbs back through the archive.
         result = await session.execute(
-            select(Photo.id).where(
+            select(Photo.id)
+            .where(
                 Photo.latitude.is_(None),
                 Photo.is_discarded.is_(False),
             )
+            .order_by(
+                Photo.taken_at.desc().nullslast(),
+                Photo.added_at.desc().nullslast(),
+            )
         )
         photo_ids = [row[0] for row in result.all()]
 

backend/app/tasks/thumbs.py

@@ -237,6 +237,9 @@ def generate_thumbnails(self, photo_id: str):
 async def _generate_thumbnails_async(photo_id: str, task):
     """Async implementation of thumbnail generation"""
     async with AsyncSessionLocal() as session:
+        # Declared up front so the except block below can safely check it
+        # even if the initial SELECT raises (e.g. asyncpg transport error).
+        photo: Optional[Photo] = None
         try:
             # Get photo from database
             result = await session.execute(
@@ -330,13 +333,25 @@ async def _generate_thumbnails_async(photo_id: str, task):
             
         except Exception as e:
             logger.error(f"Error generating thumbnails for {photo_id}: {e}")
-            
+
-            # Update error status
+            # Update error status. If the session is in a bad state (e.g.
-            if photo:
+            # the original failure was a transport error) rollback first so
-                photo.processing_status = 'failed'
+            # the status write has a clean transaction to commit into.
-                photo.processing_error = str(e)
+            try:
-                await session.commit()
+                await session.rollback()
-            
+            except Exception:
+                pass
+
+            if photo is not None:
+                try:
+                    photo.processing_status = 'failed'
+                    photo.processing_error = str(e)
+                    await session.commit()
+                except Exception:
+                    logger.exception(
+                        f"Could not mark photo {photo_id} as failed"
+                    )
+
             return {'status': 'error', 'message': str(e)}
 
 @shared_task(name='regenerate_all_thumbnails')
@@ -345,12 +360,24 @@ def regenerate_all_thumbnails():
     return asyncio.run(_regenerate_all_thumbnails_async())
 
 async def _regenerate_all_thumbnails_async():
-    """Async implementation of regenerating all thumbnails"""
+    """Async implementation of regenerating all thumbnails.
+
+    Queue order matters on first-boot and recovery runs: we dispatch
+    newest-first (by EXIF taken_at, fallback added_at) so the user's
+    most recent photos become fully-indexed before the 2012 archive even
+    starts. Picking up the library in pipeline order means the grid,
+    timeline and All Photos view populate top-down instead of the worker
+    chewing through random insertion-order rows while the UI still
+    shows grey placeholders.
+    """
     async with AsyncSessionLocal() as session:
-        # Get all photos that need thumbnails
+        # Get all photos that need thumbnails, newest first.
         result = await session.execute(
-            select(Photo).where(
+            select(Photo)
-                Photo.processing_status.in_(['pending', 'failed'])
+            .where(Photo.processing_status.in_(['pending', 'failed']))
+            .order_by(
+                Photo.taken_at.desc().nullslast(),
+                Photo.added_at.desc().nullslast(),
             )
         )
         photos = result.scalars().all()
@@ -389,10 +416,16 @@ async def _backfill_phashes_async():
 
     async with AsyncSessionLocal() as session:
         while True:
+            # Newest-first so the recent end of the library gets phashes
+            # (and therefore duplicate detection) ahead of the archive.
             result = await session.execute(
                 select(Photo)
                 .where(Photo.phash.is_(None))
                 .where(Photo.processing_status == 'completed')
+                .order_by(
+                    Photo.taken_at.desc().nullslast(),
+                    Photo.added_at.desc().nullslast(),
+                )
                 .limit(BATCH)
             )
             batch = result.scalars().all()

backend/app/tasks/vision.py

@@ -446,12 +446,18 @@ def backfill_vision(task: str | None = None, limit: int | None = None):
     """Queue vision tasks for photos that haven't been processed yet.
     Uses a sync DB connection to avoid asyncpg conflicts in Celery."""
     model_name = settings.vision.embedder.name
+    # Newest-first ordering — matches regenerate_all_thumbnails so the
+    # whole ingestion pipeline sweeps the library top-down and the user
+    # sees recent photos fully-indexed long before the backlog drains.
+    # `taken_at` is the canonical capture timestamp (from EXIF, falls
+    # back to filesystem mtime in scan); `added_at` is the tie-breaker
+    # when taken_at is null.
     sql = """
         SELECT p.id FROM photos p
         LEFT JOIN embeddings e ON e.photo_id = p.id AND e.model = :model
         WHERE e.photo_id IS NULL
         AND p.processing_status = 'completed'
-        ORDER BY p.added_at DESC
+        ORDER BY p.taken_at DESC NULLS LAST, p.added_at DESC NULLS LAST
     """
     if limit:
         sql += f" LIMIT {limit}"

backend/requirements.txt

@@ -37,6 +37,7 @@ watchfiles==0.21.0
 # Vision pipeline (ONNX Runtime CPU inference)
 onnxruntime==1.18.1
 open-clip-torch==2.24.0     # tokenizer + export helper; inference via ONNX
+ultralytics==8.4.37          # YOLOv8n export helper; inference via ONNX
 rapidocr-onnxruntime==1.3.22
 scikit-learn==1.4.0          # DBSCAN for face clustering
 insightface>=0.7.3           # RetinaFace + ArcFace face detection/recognition

docker-compose.yml

@@ -57,12 +57,34 @@ services:
       - mulita-network
     restart: unless-stopped
 
-  worker:
+  # ── Celery workers ─────────────────────────────────────────────────────
+  #
+  # The ingestion pipeline is split across two worker services so CPU-heavy
+  # vision tasks (embed / detect / OCR / faces / classify) cannot starve
+  # the fast IO-bound tasks (scan / thumbnails / EXIF / phash / duplicates).
+  #
+  #   worker-light  listens on default,high,low  — IO-bound, cheap
+  #   worker-vision listens on vision            — CPU-bound, loads ONNX
+  #
+  # Both share the same image, photo volume, and model cache, so there's
+  # no disk duplication and model weights are loaded lazily only by
+  # worker-vision. Each service has its own concurrency knob; both
+  # workers ship their heartbeat to the same Redis broker so the
+  # Settings > Workers panel lists them side-by-side.
+  #
+  # Sizing defaults target a 6-core / 16 GB host:
+  #   CELERY_LIGHT_CONCURRENCY=2  (enough for parallel thumbnail + EXIF)
+  #   CELERY_VISION_CONCURRENCY=5 (5 × ~2GB ONNX = ~10GB RAM, 5/6 cores)
+  # Raise these in .env and run `docker compose up -d worker-light worker-vision`
+  # to scale. Keep light under ~4 and vision under your physical core
+  # count; more just thrashes.
+  worker-light:
     build:
       context: ./backend
       dockerfile: Dockerfile
-    container_name: mulita-worker
+    image: mule-image-worker
-    command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERYD_CONCURRENCY:-4} -Q default,high,low,vision"
+    container_name: mulita-worker-light
+    command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_LIGHT_CONCURRENCY:-2} -Q default,high,low -n light@%h"
     volumes:
       - ./mulita.yml:/app/config/mulita.yml:ro
       - ${PHOTO_DIRS:-./photos}:/photos:rw
@@ -76,9 +98,52 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379
       - CELERY_RESULT_BACKEND=redis://redis:6379
       - PHOTO_DIRS=${PHOTO_DIRS:-/photos}
-      - CELERYD_CONCURRENCY=${CELERYD_CONCURRENCY:-4}
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
       - TZ=${TZ:-UTC}
+      # NullPool — see app/database.py for rationale.
+      - MULITA_CELERY_WORKER=1
+    depends_on:
+      redis:
+        condition: service_started
+      backend:
+        condition: service_started
+      db:
+        condition: service_healthy
+    networks:
+      - mulita-network
+    restart: unless-stopped
+
+  worker-vision:
+    build:
+      context: ./backend
+      dockerfile: Dockerfile
+    image: mule-image-worker
+    container_name: mulita-worker-vision
+    command: sh -c "python -m app.services.vision.bootstrap_models && celery -A app.tasks.celery worker --loglevel=${LOG_LEVEL:-info} --concurrency=${CELERY_VISION_CONCURRENCY:-5} -Q vision -n vision@%h"
+    volumes:
+      - ./mulita.yml:/app/config/mulita.yml:ro
+      - ${PHOTO_DIRS:-./photos}:/photos:rw
+      - thumbs_data:/data/thumbs
+      - proxies_data:/data/proxies
+      - db_data:/data/db
+      - models_data:/data/models
+    environment:
+      - DATABASE_URL=postgresql+asyncpg://mulita:mulita@db:5432/mulita
+      - REDIS_URL=redis://redis:6379
+      - CELERY_BROKER_URL=redis://redis:6379
+      - CELERY_RESULT_BACKEND=redis://redis:6379
+      - PHOTO_DIRS=${PHOTO_DIRS:-/photos}
+      - LOG_LEVEL=${LOG_LEVEL:-INFO}
+      - TZ=${TZ:-UTC}
+      - MULITA_CELERY_WORKER=1
+      # Pin each ONNX session to one intra-op thread so N prefork children
+      # × default-all-cores doesn't oversubscribe the box. With
+      # concurrency=5 and OMP=1, vision peaks at 5 busy cores, leaving
+      # one for worker-light + system. These env vars cover the three
+      # threading runtimes ONNX Runtime might pick up on first use.
+      - OMP_NUM_THREADS=1
+      - OPENBLAS_NUM_THREADS=1
+      - MKL_NUM_THREADS=1
     depends_on:
       redis:
         condition: service_started

frontend/src/components/dialogs/SettingsDialog.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useState, useCallback } from 'react'
+import { useEffect, useState, useCallback, useRef } from 'react'
 import {
   X,
   RefreshCw,
@@ -13,12 +13,17 @@ import {
   CheckCircle2,
   Copy,
   Sparkles,
+  Activity,
+  FolderSearch,
 } from 'lucide-react'
 import clsx from 'clsx'
 import { useQuery, useQueryClient } from '@tanstack/react-query'
 import {
   library,
   type MediaType,
+  type PipelineStage,
+  type ScanStatus,
+  type WorkerStatus,
 } from '../../services/api'
 import { toast } from '../ToastContainer'
 
@@ -29,6 +34,8 @@ const SETTINGS_THUMB_STATS_KEY = ['settings', 'thumbnail-stats'] as const
 const SETTINGS_LIB_STATS_KEY = ['settings', 'library-stats'] as const
 const SETTINGS_WORKER_STATUS_KEY = ['settings', 'worker-status'] as const
 const SETTINGS_MISSING_STATS_KEY = ['settings', 'missing-stats'] as const
+const SETTINGS_PIPELINE_STATS_KEY = ['settings', 'pipeline-stats'] as const
+const SETTINGS_SCAN_STATUS_KEY = ['settings', 'scan-status'] as const
 // Shared with the DuplicatesView so a regroup invalidates the same cache
 // the grid renders from. Imported via the canonical hook key.
 import { DUPLICATE_GROUPS_QUERY_KEY } from '../../hooks/useDuplicateGroupsQuery'
@@ -91,6 +98,28 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
     refetchInterval: isOpen ? 5000 : false,
     staleTime: 0,
   })
+  // Pipeline progress polls on the same 5s cadence as the worker status
+  // so both cards update together. Cheap query — ten COUNT(*)s on
+  // indexed columns.
+  const pipelineStatsQuery = useQuery({
+    queryKey: SETTINGS_PIPELINE_STATS_KEY,
+    queryFn: library.maintenance.pipelineStats,
+    enabled: isOpen,
+    refetchInterval: isOpen ? 5000 : false,
+    staleTime: 0,
+  })
+  // Scan status — polls fast (2s) so the progress bar feels live during
+  // a scan, and slow (15s) when idle to cut chatter. `isScanning` is
+  // read from the latest fetched value so the cadence flips on its own
+  // the moment a scan kicks off or finishes.
+  const scanStatusQuery = useQuery<ScanStatus>({
+    queryKey: SETTINGS_SCAN_STATUS_KEY,
+    queryFn: library.scanStatus,
+    enabled: isOpen,
+    refetchInterval: (q) =>
+      isOpen ? ((q.state.data as ScanStatus | undefined)?.is_scanning ? 2000 : 15000) : false,
+    staleTime: 0,
+  })
   // Duplicates: shares the cache with DuplicatesView so a regroup
   // triggered from Settings updates the grid view immediately.
   const duplicatesQuery = useQuery({
@@ -104,6 +133,19 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
   const libStats = libStatsQuery.data
   const workerStatus = workerStatusQuery.data
   const missingStats = missingStatsQuery.data
+  const pipelineStats = pipelineStatsQuery.data
+  const scanStatus = scanStatusQuery.data
+
+  // Throughput — tasks/minute across the fleet, derived by diffing the
+  // `total` counters on `workers[*].processed` between successive polls.
+  // We keep the previous sample in a ref so recomputation happens inside
+  // the existing 5s polling rhythm without introducing extra state.
+  // First sample returns null (we need two points for a rate).
+  const throughputSampleRef = useRef<{
+    totals: Record<string, number>
+    at: number
+  } | null>(null)
+  const throughput = computeThroughput(workerStatus, throughputSampleRef)
   // "loading" in the UI sense = fetching AND no cached data yet. Background
   // refetches on top of cached data shouldn't flip the refresh spinners.
   const loadingStats =
@@ -116,11 +158,13 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
   const refreshStats = useCallback(() => {
     queryClient.invalidateQueries({ queryKey: SETTINGS_THUMB_STATS_KEY })
     queryClient.invalidateQueries({ queryKey: SETTINGS_LIB_STATS_KEY })
+    queryClient.invalidateQueries({ queryKey: SETTINGS_PIPELINE_STATS_KEY })
     queryClient.invalidateQueries({ queryKey: DUPLICATE_GROUPS_QUERY_KEY })
   }, [queryClient])
   const refreshWorkers = useCallback(() => {
     queryClient.invalidateQueries({ queryKey: SETTINGS_WORKER_STATUS_KEY })
     queryClient.invalidateQueries({ queryKey: SETTINGS_MISSING_STATS_KEY })
+    queryClient.invalidateQueries({ queryKey: SETTINGS_SCAN_STATUS_KEY })
   }, [queryClient])
 
   // Surface fetch errors once (React Query de-dupes retries but we still
@@ -246,6 +290,33 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
                   }
                 />
               </div>
+
+              {/* Inline scan progress. Rendered as a full-width sub-card
+                  when a scan is active so the user sees the same info
+                  they'd get from the floating widget without leaving
+                  settings. Hidden when idle to keep the section tight. */}
+              {scanStatus?.is_scanning && (
+                <div className="mt-3 rounded border border-border bg-surface p-2">
+                  <div className="mb-1 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-text-muted">
+                    <FolderSearch className="h-3 w-3 animate-pulse" />
+                    Scanning
+                  </div>
+                  <div className="mb-1 flex items-center justify-between text-xs">
+                    <span className="truncate font-mono text-text-muted" title={scanStatus.current_folder ?? ''}>
+                      {scanStatus.current_folder ?? '—'}
+                    </span>
+                    <span className="ml-2 shrink-0 font-mono text-text">
+                      {scanStatus.processed_files.toLocaleString()} /{' '}
+                      {scanStatus.total_files.toLocaleString()}
+                    </span>
+                  </div>
+                  <ProgressBar
+                    done={scanStatus.processed_files}
+                    total={scanStatus.total_files || 1}
+                  />
+                </div>
+              )}
+
               <div className="mt-3">
                 <ActionButton
                   loading={busy.scan}
@@ -263,6 +334,51 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
               </div>
             </Section>
 
+            {/* ----------------------------------------------------- */}
+            {/* Pipeline progress — per-stage done/total               */}
+            {/* ----------------------------------------------------- */}
+            <Section
+              icon={<Activity className="h-4 w-4" />}
+              title="Pipeline progress"
+              right={
+                <button
+                  onClick={() =>
+                    queryClient.invalidateQueries({
+                      queryKey: SETTINGS_PIPELINE_STATS_KEY,
+                    })
+                  }
+                  disabled={pipelineStatsQuery.isFetching && !pipelineStats}
+                  className="flex items-center gap-1 rounded border border-border px-2 py-1 text-xs text-text-muted hover:bg-surface-2 disabled:opacity-50"
+                >
+                  {pipelineStatsQuery.isFetching && !pipelineStats ? (
+                    <Loader2 className="h-3 w-3 animate-spin" />
+                  ) : (
+                    <RefreshCw className="h-3 w-3" />
+                  )}
+                  Refresh
+                </button>
+              }
+            >
+              {pipelineStats ? (
+                <div className="space-y-2">
+                  {pipelineStats.stages.map((stage) => (
+                    <PipelineRow key={stage.key} stage={stage} />
+                  ))}
+                  <p className="pt-1 text-[10px] text-text-muted">
+                    Partial stages (marked *) only apply to a subset of
+                    photos — e.g. not every photo has GPS, text, or detectable
+                    objects. Face clusters and duplicate groups are output
+                    counts rather than ratios.
+                  </p>
+                </div>
+              ) : (
+                <div className="flex items-center gap-2 text-xs text-text-muted">
+                  <Loader2 className="h-3 w-3 animate-spin" />
+                  Loading pipeline progress…
+                </div>
+              )}
+            </Section>
+
             {/* ----------------------------------------------------- */}
             {/* Duplicate detection                                    */}
             {/* ----------------------------------------------------- */}
@@ -430,7 +546,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
               }
             >
               {/* Top-line health */}
-              <div className="grid grid-cols-3 gap-2 text-xs">
+              <div className="grid grid-cols-4 gap-2 text-xs">
                 <Stat
                   label="Workers"
                   value={workerStatus?.worker_count}
@@ -443,20 +559,25 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
                   }
                 />
                 <Stat
-                  label="Broker"
+                  label="Concurrency"
                   value={
-                    workerStatus
+                    workerStatus && workerStatus.workers.length > 0
-                      ? workerStatus.broker_ok
+                      ? workerStatus.workers.reduce(
-                        ? 'OK'
+                          (acc, w) => acc + (w.concurrency ?? 0),
-                        : 'DOWN'
+                          0
+                        )
                       : undefined
                   }
+                />
+                <Stat
+                  label="Tasks/min"
+                  value={
+                    throughput === null
+                      ? '…'
+                      : throughput.toFixed(0)
+                  }
                   tone={
-                    workerStatus
+                    throughput !== null && throughput > 0 ? 'ok' : 'muted'
-                      ? workerStatus.broker_ok
-                        ? 'ok'
-                        : 'warn'
-                      : 'muted'
                   }
                 />
                 <Stat
@@ -470,6 +591,34 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
                 />
               </div>
 
+              {/* How to scale — explicit, because this question comes up
+                  every time a large import is running and the user wants
+                  it to go faster. The ingestion pipeline runs on two
+                  worker services; each has its own concurrency env var. */}
+              <p className="mt-3 text-[10px] leading-relaxed text-text-muted">
+                Two worker services share the load:{' '}
+                <code className="rounded bg-surface px-1">worker-light</code>{' '}
+                (scan, thumbnails, EXIF — set by{' '}
+                <code className="rounded bg-surface px-1">
+                  CELERY_LIGHT_CONCURRENCY
+                </code>
+                ) and{' '}
+                <code className="rounded bg-surface px-1">worker-vision</code>{' '}
+                (embed, detect, OCR, faces — set by{' '}
+                <code className="rounded bg-surface px-1">
+                  CELERY_VISION_CONCURRENCY
+                </code>
+                ) in{' '}
+                <code className="rounded bg-surface px-1">.env</code>. To
+                scale, bump those values then run{' '}
+                <code className="rounded bg-surface px-1">
+                  docker compose up -d worker-light worker-vision
+                </code>
+                . Keep vision below your physical core count — each fork
+                loads ~2 GB of ONNX models. The vision queue is usually the
+                bottleneck; watch its depth below.
+              </p>
+
               {/* Inline error banners for the obvious failure modes */}
               {workerStatus?.broker_error && (
                 <ErrorBanner
@@ -492,17 +641,24 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
                   />
                 )}
 
-              {/* Queue depth */}
+              {/* Queue depth. `vision` is called out with a highlight
+                  because it's where the serious backlog lives — every
+                  embed / tag / OCR / face task routes here. */}
               {workerStatus && (
                 <div className="mt-3">
                   <div className="mb-1 text-[10px] uppercase tracking-wide text-text-muted">
                     Queue depth
                   </div>
-                  <div className="grid grid-cols-3 gap-2 text-xs">
+                  <div className="grid grid-cols-4 gap-2 text-xs">
                     {Object.entries(workerStatus.queues).map(([name, depth]) => (
                       <div
                         key={name}
-                        className="flex items-center justify-between rounded bg-surface px-2 py-1"
+                        className={clsx(
+                          'flex items-center justify-between rounded px-2 py-1',
+                          name === 'vision' && depth > 0
+                            ? 'bg-surface-2'
+                            : 'bg-surface'
+                        )}
                       >
                         <span className="text-text-muted">{name}</span>
                         <span
@@ -511,7 +667,7 @@ export function SettingsDialog({ isOpen, onClose }: SettingsDialogProps) {
                             depth > 0 ? 'text-text' : 'text-text-muted'
                           )}
                         >
-                          {depth}
+                          {depth.toLocaleString()}
                         </span>
                       </div>
                     ))}
@@ -797,6 +953,116 @@ function ErrorBanner({ title, detail }: { title: string; detail: string }) {
   )
 }
 
+/**
+ * Compute the total task-completion rate across the worker fleet by
+ * diffing the per-task `processed` counters between successive poll
+ * samples. Returns null on the first call (we need two samples for a
+ * rate) and 0 when nothing has moved since the last poll.
+ *
+ * The sample is kept in a ref — not state — because we don't want to
+ * re-render on every update; we want the number to settle into the
+ * existing render cycle that React Query already drives.
+ */
+function computeThroughput(
+  status: WorkerStatus | undefined,
+  ref: React.MutableRefObject<{
+    totals: Record<string, number>
+    at: number
+  } | null>
+): number | null {
+  if (!status || status.workers.length === 0) {
+    return null
+  }
+
+  // Flatten processed counts across all workers into one dict keyed by
+  // task name so worker restarts (which reset individual counters) are
+  // absorbed by the total.
+  const totals: Record<string, number> = {}
+  for (const w of status.workers) {
+    for (const [task, count] of Object.entries(w.processed ?? {})) {
+      totals[task] = (totals[task] ?? 0) + (count ?? 0)
+    }
+  }
+
+  const now = Date.now()
+  const prev = ref.current
+  // Update the ref BEFORE returning so the next call has a baseline.
+  ref.current = { totals, at: now }
+
+  if (!prev) {
+    return null
+  }
+
+  const elapsedSec = (now - prev.at) / 1000
+  if (elapsedSec <= 0) {
+    return null
+  }
+
+  let delta = 0
+  for (const [task, count] of Object.entries(totals)) {
+    const before = prev.totals[task] ?? 0
+    // Guard against counter resets (worker restart) — negative diffs
+    // are clamped to zero rather than dragging the rate down.
+    delta += Math.max(0, count - before)
+  }
+
+  return (delta / elapsedSec) * 60
+}
+
+function PipelineRow({ stage }: { stage: PipelineStage }) {
+  const isStandalone = stage.standalone === true
+  const isComplete = !isStandalone && stage.total > 0 && stage.done >= stage.total
+  const isActive = !isStandalone && stage.done > 0 && stage.done < stage.total
+
+  return (
+    <div className="rounded bg-surface px-2 py-1.5" title={stage.hint}>
+      <div className="flex items-center justify-between gap-2 text-xs">
+        <div className="flex min-w-0 items-center gap-1.5">
+          <span className="truncate text-text">
+            {stage.label}
+            {stage.partial && <span className="text-text-muted"> *</span>}
+          </span>
+        </div>
+        <span
+          className={clsx(
+            'shrink-0 font-mono text-[11px]',
+            isComplete
+              ? 'text-pick'
+              : isActive
+                ? 'text-text'
+                : 'text-text-muted'
+          )}
+        >
+          {isStandalone
+            ? stage.done.toLocaleString()
+            : `${stage.done.toLocaleString()} / ${stage.total.toLocaleString()}`}
+        </span>
+      </div>
+      {!isStandalone && (
+        <div className="mt-1">
+          <ProgressBar done={stage.done} total={stage.total || 1} />
+        </div>
+      )}
+    </div>
+  )
+}
+
+function ProgressBar({ done, total }: { done: number; total: number }) {
+  const pct = total > 0 ? Math.min(100, (done / total) * 100) : 0
+  const complete = total > 0 && done >= total
+  return (
+    <div className="h-1 w-full overflow-hidden rounded bg-border">
+      <div
+        className={clsx(
+          'h-full rounded transition-[width] duration-500',
+          complete ? 'bg-pick' : 'bg-text-muted'
+        )}
+        style={{ width: `${pct}%` }}
+      />
+    </div>
+  )
+}
+
 function ActionButton({
   loading,
   disabled,

frontend/src/components/layout/LeftSidebar.tsx

@@ -18,6 +18,8 @@ import {
   PanelLeftClose,
   Settings,
   Users,
+  Eye,
+  EyeOff,
 } from 'lucide-react'
 import clsx from 'clsx'
 import { sourceFolders, photos as photosApi, type FolderTreeNode } from '../../services/api'
@@ -46,6 +48,9 @@ interface TreeItem {
   count?: number
   children?: TreeItem[]
   type?: 'folder' | 'heap' | 'special'
+  /** For folder rows only: the user-set "hide from views" flag. Drives
+   *  the muted styling + eye-off badge + menu item label. */
+  isHidden?: boolean
 }
 
 interface LeftSidebarProps {
@@ -303,6 +308,32 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
       toast.error('Create failed', e?.response?.data?.detail || e.message || 'Unknown error'),
   })
 
+  // Toggle folder hide-from-views. Invalidates every query that could
+  // include photos from the affected folder subtree — the sidebar
+  // tree (for the counts + badge), the photos timeline, library
+  // stats (sidebar badges), tags (count subquery), and duplicates
+  // (source for dup groups). All of these respect the new flag on
+  // the server side; the invalidation is just cache bust.
+  const toggleHiddenMutation = useMutation({
+    mutationFn: ({ id, hidden }: { id: string; hidden: boolean }) =>
+      sourceFolders.setHidden(id, hidden),
+    onSuccess: (data) => {
+      toast.success(
+        data.is_hidden ? 'Folder hidden' : 'Folder visible',
+        data.is_hidden
+          ? `${data.name} is now excluded from cross-cutting views`
+          : `${data.name} is back in cross-cutting views`
+      )
+      queryClient.invalidateQueries({ queryKey: ['folders'] })
+      queryClient.invalidateQueries({ queryKey: ['folders', 'tree'] })
+      queryClient.invalidateQueries({ queryKey: ['photos'] })
+      queryClient.invalidateQueries({ queryKey: LIBRARY_STATS_QUERY_KEY })
+      queryClient.invalidateQueries({ queryKey: ['tags'] })
+    },
+    onError: (e: any) =>
+      toast.error('Toggle failed', e?.response?.data?.detail || e.message || 'Unknown error'),
+  })
+
   const deleteFolderMutation = useMutation({
     mutationFn: ({ id, mode }: { id: string; mode: 'discard' | 'permanent' }) =>
       sourceFolders.delete(id, mode),
@@ -349,6 +380,7 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
     icon: <Folder className="h-4 w-4" />,
     count: node.photo_count,
     type: 'folder',
+    isHidden: node.is_hidden,
     children: node.children.length > 0
       ? node.children.map(folderNodeToTreeItem)
       : undefined,
@@ -515,15 +547,23 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
           )}
 
           {/* Item Icon — section headers drop their icon in favor of the
-           *  uppercase eyebrow label. */}
+           *  uppercase eyebrow label. Hidden folders swap the folder
+           *  icon for an EyeOff so the user sees the state at a glance
+           *  without hunting through the kebab menu. */}
           {item.icon && !isSectionHeader && (
             <span
               className={clsx(
                 'flex-shrink-0',
-                isSelected ? 'text-primary' : 'text-text-muted'
+                isSelected
+                  ? 'text-primary'
+                  : item.isHidden
+                    ? 'text-text-muted/60'
+                    : 'text-text-muted'
               )}
             >
-              <span className="[&>svg]:h-3.5 [&>svg]:w-3.5">{item.icon}</span>
+              <span className="[&>svg]:h-3.5 [&>svg]:w-3.5">
+                {item.isHidden ? <EyeOff className="h-4 w-4" /> : item.icon}
+              </span>
             </span>
           )}
 
@@ -553,7 +593,15 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
               className="flex-1 rounded border border-border bg-bg px-1 py-0 text-[13px] text-text focus:border-primary focus:outline-none"
             />
           ) : (
-            <span className="flex-1 truncate">{item.label}</span>
+            <span
+              className={clsx(
+                'flex-1 truncate',
+                item.isHidden && !isSelected && 'italic text-text-muted/80'
+              )}
+              title={item.isHidden ? `${item.label} — hidden from views` : undefined}
+            >
+              {item.label}
+            </span>
           )}
 
           {/* Count Badge — fixed-width slot so counts line up in a column
@@ -629,6 +677,23 @@ export function LeftSidebar({ onCollapse, onOpenSettings }: LeftSidebarProps) {
                           setRenameDraft(item.label)
                         }}
                       />
+                      <FolderMenuItem
+                        icon={
+                          item.isHidden ? (
+                            <Eye className="h-3.5 w-3.5" />
+                          ) : (
+                            <EyeOff className="h-3.5 w-3.5" />
+                          )
+                        }
+                        label={item.isHidden ? 'Show in views' : 'Hide from views'}
+                        onClick={() => {
+                          setOpenMenuId(null)
+                          toggleHiddenMutation.mutate({
+                            id: folderId,
+                            hidden: !item.isHidden,
+                          })
+                        }}
+                      />
                       <div className="my-1 h-px bg-border" />
                       <FolderMenuItem
                         icon={<Trash2 className="h-3.5 w-3.5" />}

frontend/src/services/api.ts

@@ -22,6 +22,11 @@ export interface FolderTreeNode {
   name: string
   path: string
   photo_count: number
+  /** User-set "hide from cross-cutting views" flag. When true, photos
+   *  in this folder (and descendants) are excluded from All Photos,
+   *  Map, Tags, People, Search and sidebar counts, but remain visible
+   *  when the user navigates directly into the folder. */
+  is_hidden: boolean
   children: FolderTreeNode[]
 }
 
@@ -60,6 +65,23 @@ export const sourceFolders = {
     return response.data as { id: string; name: string; path: string; parent_id: string }
   },
 
+  /** Toggle "hide from views" on a folder or source root. Hidden
+   *  folders still index + thumbnail their photos, but those photos
+   *  are excluded from cross-cutting views (All Photos, Map, Tags,
+   *  People, Search, sidebar counts). Navigating directly into the
+   *  folder still shows them. Propagates to every descendant folder. */
+  setHidden: async (folderId: string, hidden: boolean) => {
+    const response = await api.post(`/folders/${folderId}/hide`, {
+      hidden,
+    })
+    return response.data as {
+      id: string
+      name: string
+      path: string
+      is_hidden: boolean
+    }
+  },
+
   /** Delete a folder. mode=discard moves all photos under it to the
    *  discard pile (recoverable) and leaves the folder + on-disk dir
    *  alone. mode=permanent unlinks files, removes folder rows, and
@@ -297,6 +319,37 @@ export interface WorkerStatus {
   scan_errors: string[]
 }
 
+export interface PipelineStage {
+  key: string
+  label: string
+  done: number
+  total: number
+  hint: string
+  /** True when the stage legitimately runs on a subset of photos — e.g.
+   *  GPS / tags / OCR / faces — so 100% coverage is never expected and
+   *  the UI should not frame "missing" as a problem. */
+  partial?: boolean
+  /** True when the stage doesn't have a done/total progress semantic
+   *  (e.g. face clusters, duplicate groups — those are output counts,
+   *  not ratios). The UI renders a plain count instead of a bar. */
+  standalone?: boolean
+}
+
+export interface PipelineStats {
+  total_photos: number
+  total_images: number
+  embedder_model: string
+  stages: PipelineStage[]
+}
+
+export interface ScanStatus {
+  is_scanning: boolean
+  current_folder: string | null
+  processed_files: number
+  total_files: number
+  errors: string[]
+}
+
 export const library = {
   scan: async () => {
     const response = await api.post('/library/scan')
@@ -344,6 +397,14 @@ export const library = {
       return response.data
     },
 
+    /** Per-stage ingestion progress — thumbnails, EXIF, GPS, phash,
+     *  embeddings, object tags, OCR, faces, face clusters, duplicate
+     *  groups. Drives the Pipeline Progress card in Settings. */
+    pipelineStats: async (): Promise<PipelineStats> => {
+      const response = await api.get('/library/maintenance/pipeline-stats')
+      return response.data
+    },
+
     /** Dry-run count of photo rows whose files are no longer on disk
      *  (under a mounted source root). */
     missingStats: async (): Promise<MissingStats> => {