Compare commits
3 Commits
758fda619e
...
1408ec3fa3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1408ec3fa3 | ||
|
|
b1c3ee68dd | ||
|
|
4bb2c959a8 |
40
backend/alembic/versions/0017_photos_list_index.py
Normal file
40
backend/alembic/versions/0017_photos_list_index.py
Normal file
@@ -0,0 +1,40 @@
|
||||
"""Partial index for the photos list query
|
||||
|
||||
Revision ID: 0017_photos_list_index
|
||||
Revises: 0016_nextcloud_integration
|
||||
Create Date: 2026-05-10
|
||||
|
||||
The default photos-list query (GET /api/v1/photos?per_page=N&sort=taken_at&order=desc)
|
||||
filters `NOT is_trashed AND NOT is_hidden` and sorts by
|
||||
`(taken_at DESC NULLS LAST, id DESC)`. EXPLAIN ANALYZE on a 21k-row
|
||||
table showed a seq-scan + top-N heapsort (~20ms standalone, worse under
|
||||
concurrency) — Postgres can't use the single-column ix_photos_taken_at
|
||||
when the leading WHERE clause is two booleans.
|
||||
|
||||
A partial index on the sort key, scoped to the visible subset, lets the
|
||||
planner index-scan in reverse and stop at LIMIT N. Two booleans select
|
||||
~95% of rows, so the partial predicate is tighter than the full table
|
||||
without losing common queries.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0017_photos_list_index"
|
||||
down_revision: Union[str, None] = "0016_nextcloud_integration"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS ix_photos_list_visible
|
||||
ON photos (taken_at DESC NULLS LAST, id DESC)
|
||||
WHERE NOT is_trashed AND NOT is_hidden
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_photos_list_visible")
|
||||
@@ -119,22 +119,64 @@ async def _recompute_folder_counts(session: AsyncSession) -> None:
|
||||
f.photo_count = int(count_result.scalar() or 0)
|
||||
|
||||
|
||||
def _parent_is_accessible(path: str) -> bool:
|
||||
"""True if the parent directory of `path` is readable. Used to
|
||||
distinguish 'user renamed/deleted the source root folder' (parent
|
||||
mount fine, leaf gone) from 'drive unmounted' (whole subtree
|
||||
inaccessible). The former is safe to prune from; the latter is
|
||||
not."""
|
||||
parent = os.path.dirname(path.rstrip(os.sep))
|
||||
if not parent:
|
||||
return False
|
||||
try:
|
||||
os.listdir(parent)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _sr_state(sr_path: str) -> str:
|
||||
"""Classify a source root path as one of:
|
||||
'present' — directory exists, business as usual
|
||||
'renamed' — leaf missing but parent mount is accessible (user
|
||||
renamed/deleted the folder in their file manager)
|
||||
'unmounted'— parent itself inaccessible (drive not mounted)
|
||||
"""
|
||||
if os.path.isdir(sr_path):
|
||||
return 'present'
|
||||
if _parent_is_accessible(sr_path):
|
||||
return 'renamed'
|
||||
return 'unmounted'
|
||||
|
||||
|
||||
async def _warn_stale_source_roots(session: AsyncSession) -> int:
|
||||
"""Log a warning for any active source root whose path no longer exists
|
||||
on disk. Doesn't delete — a missing path could be a temporarily
|
||||
unmounted drive, and silently dropping user data is worse than
|
||||
surfacing a noisy log line.
|
||||
surfacing a noisy log line. Logs different hints for renamed-vs-
|
||||
unmounted so the user knows which knob to turn.
|
||||
"""
|
||||
result = await session.execute(select(SourceRoot))
|
||||
rows = result.scalars().all()
|
||||
stale = 0
|
||||
for sr in rows:
|
||||
if not os.path.isdir(sr.path):
|
||||
stale += 1
|
||||
state = _sr_state(sr.path)
|
||||
if state == 'present':
|
||||
continue
|
||||
stale += 1
|
||||
if state == 'renamed':
|
||||
logger.warning(
|
||||
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
||||
f"— is the docker mount still in place? "
|
||||
f"(Edit docker-compose.yml or PHOTO_DIRS in .env to fix.)"
|
||||
f"— parent mount is fine, looks like the folder was renamed "
|
||||
f"or deleted. Photos under it can be cleared via "
|
||||
f"POST /api/v1/library/maintenance/prune-missing."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Source root '{sr.name}' path is missing on disk: {sr.path} "
|
||||
f"— parent directory is also inaccessible; is the docker "
|
||||
f"mount still in place? (Edit docker-compose.yml or "
|
||||
f"PHOTO_DIRS in .env to fix.)"
|
||||
)
|
||||
return stale
|
||||
|
||||
@@ -146,14 +188,20 @@ async def find_missing(
|
||||
they still resolve on disk. Returns
|
||||
(deletable_photo_ids, deletable_folder_ids, skipped_photo_ids).
|
||||
|
||||
Skipped rows are photos/folders whose owning source_root path itself
|
||||
doesn't resolve — that's almost always an unmounted drive, and
|
||||
silently deleting those rows would be data loss. The caller can
|
||||
surface the skip count separately so the user knows the cleanup
|
||||
wasn't a no-op by accident.
|
||||
Skipped rows are photos/folders whose owning source_root is truly
|
||||
inaccessible (parent mount missing) — that's almost always an
|
||||
unmounted drive, and silently deleting those rows would be data
|
||||
loss. Photos under a source root whose leaf is missing but whose
|
||||
parent mount IS accessible (user renamed/deleted the folder) are
|
||||
treated as deletable, since their files are genuinely gone from
|
||||
the user's library.
|
||||
"""
|
||||
sr_rows = (await session.execute(select(SourceRoot))).scalars().all()
|
||||
sr_mounted: dict[str, bool] = {sr.id: os.path.isdir(sr.path) for sr in sr_rows}
|
||||
# "Available" = source root path exists OR parent mount is accessible.
|
||||
# Only truly-unmounted source roots skip pruning.
|
||||
sr_mounted: dict[str, bool] = {
|
||||
sr.id: _sr_state(sr.path) != 'unmounted' for sr in sr_rows
|
||||
}
|
||||
|
||||
photos = (await session.execute(
|
||||
select(Photo.id, Photo.filepath, Photo.folder_id)
|
||||
|
||||
@@ -28,20 +28,20 @@ export function ScanProgress() {
|
||||
queryKey: ['scan-status'],
|
||||
queryFn: () => library.scanStatus(),
|
||||
refetchInterval: (query) =>
|
||||
query.state.data?.is_scanning ? 2000 : 10000,
|
||||
query.state.data?.is_scanning ? 2000 : 30000,
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
const isScanning = scanStatus?.is_scanning ?? false
|
||||
|
||||
// Poll worker status to track vision queue activity.
|
||||
// Fast polling (3s) while processing, slow (15s) otherwise.
|
||||
// Fast polling (3s) while processing, slow (30s) otherwise.
|
||||
const { data: workerStatus } = useQuery<WorkerStatus>({
|
||||
queryKey: ['worker-status-progress'],
|
||||
queryFn: () => library.maintenance.workerStatus(),
|
||||
refetchInterval: (query) => {
|
||||
const q = totalQueued(query.state.data)
|
||||
return q > 0 ? 3000 : 15000
|
||||
return q > 0 ? 3000 : 30000
|
||||
},
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
@@ -81,17 +81,19 @@ export function usePhotosQuery() {
|
||||
// seeks directly to the next slice via an indexed range scan,
|
||||
// O(1) regardless of depth (no OFFSET skipping).
|
||||
//
|
||||
// MAX_PAGES is intentionally modest: 20 × 500 = 10 000 photos
|
||||
// covers almost every browsing session up-front without burning
|
||||
// through a 100k library on cold load. If a user scrolls past
|
||||
// that horizon we'll add an infinite-query trigger; for now the
|
||||
// cap keeps cold-load memory / network pressure sane.
|
||||
const PER_PAGE = 500
|
||||
// First page is small (~one viewport) so the grid paints fast;
|
||||
// background pages are larger so we still cover ~10k photos in
|
||||
// a few round-trips without burning through a 100k library on
|
||||
// cold load. If a user scrolls past that horizon we'll add an
|
||||
// infinite-query trigger; for now the cap keeps cold-load
|
||||
// memory / network pressure sane.
|
||||
const PER_PAGE_INITIAL = 100
|
||||
const PER_PAGE_BACKGROUND = 500
|
||||
const MAX_PAGES = 20
|
||||
const INTER_PAGE_DELAY_MS = 50
|
||||
|
||||
const first = await fetchCursorPage(
|
||||
{ per_page: PER_PAGE, ...filterParams },
|
||||
{ per_page: PER_PAGE_INITIAL, ...filterParams },
|
||||
signal,
|
||||
)
|
||||
const firstBatch = first.photos || []
|
||||
@@ -104,7 +106,7 @@ export function usePhotosQuery() {
|
||||
if (signal?.aborted) return
|
||||
try {
|
||||
const page = await fetchCursorPage(
|
||||
{ per_page: PER_PAGE, cursor: nextCursor, ...filterParams },
|
||||
{ per_page: PER_PAGE_BACKGROUND, cursor: nextCursor, ...filterParams },
|
||||
signal,
|
||||
)
|
||||
if (signal?.aborted) return
|
||||
@@ -114,7 +116,7 @@ export function usePhotosQuery() {
|
||||
['photos', filterParams],
|
||||
(prev) => (prev ? [...prev, ...more] : more)
|
||||
)
|
||||
if (!nextCursor || more.length < PER_PAGE) return
|
||||
if (!nextCursor || more.length < PER_PAGE_BACKGROUND) return
|
||||
// Yield a beat between pages so the main thread stays
|
||||
// responsive (thumbnail decode, scroll handling) while
|
||||
// we're back-filling in the background.
|
||||
|
||||
@@ -49,14 +49,14 @@ export function useScanActivity() {
|
||||
queryKey: ['scan-status'],
|
||||
queryFn: () => library.scanStatus(),
|
||||
refetchInterval: (query) =>
|
||||
query.state.data?.is_scanning ? 2000 : 10000,
|
||||
query.state.data?.is_scanning ? 2000 : 30000,
|
||||
})
|
||||
const { data: workerStatus } = useQuery<WorkerStatus>({
|
||||
queryKey: ['worker-status-progress'],
|
||||
queryFn: () => library.maintenance.workerStatus(),
|
||||
refetchInterval: (query) => {
|
||||
const data = query.state.data
|
||||
return pendingTasks(data) + activeTasks(data) > 0 ? 3000 : 15000
|
||||
return pendingTasks(data) + activeTasks(data) > 0 ? 3000 : 30000
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user