diff --git a/docker-compose.photoprism.yml b/docker-compose.photoprism.yml index 8ba1618..7d87488 100644 --- a/docker-compose.photoprism.yml +++ b/docker-compose.photoprism.yml @@ -125,6 +125,49 @@ services: - "./pp/import:/photoprism/import:Z" networks: [photoprism-network] + # mule-sidecar — Go + Gin + GORM service for endpoints PhotoPrism's API + # does not expose (file rename, folder mutations, heap convert, duplicate + # scan, per-photo marks). Same wire contract as the M3 Node prototype; + # the SvelteKit dev server proxies /api/sidecar/* here. + sidecar: + build: + context: ./sidecar + container_name: pp-sidecar + restart: unless-stopped + depends_on: + mariadb: + condition: service_healthy + photoprism: + condition: service_started + # Match PhotoPrism's UID/GID so renames/folder mutations preserve the + # ownership the indexer expects on the bind-mounted originals. + user: "${PP_UID:-1000}:${PP_GID:-1000}" + ports: + # Loopback only — Vite (host) proxies /api/sidecar/* to this port. + # Behind a reverse proxy in production; never published beyond the + # host. + - "127.0.0.1:${SIDECAR_PORT:-8000}:8000" + environment: + ORIGINALS_ROOT: /photoprism/originals + PHOTOPRISM_BASE_URL: http://photoprism:2342 + # Bind on all interfaces inside the container so the host-side + # 127.0.0.1:8000 port mapping can reach the listener. The Go + # binary defaults to 127.0.0.1 for the host-mode dev loop. + SIDECAR_LISTEN_ADDR: 0.0.0.0 + SIDECAR_PORT: "8000" + SIDECAR_DB_HOST: mariadb + SIDECAR_DB_PORT: "3306" + SIDECAR_DB_USER: sidecar + # Rotate before any non-local deployment. Provisioned by + # mariadb/init/01-sidecar.sql on first boot of the mariadb volume. + SIDECAR_DB_PASSWORD: ${SIDECAR_DB_PASSWORD:-replace-at-m4-bringup} + SIDECAR_DB_NAME: mule_sidecar + volumes: + # Sidecar mutates originals (rename, folder mutations, heap + # convert) — always rw regardless of PhotoPrism's mount mode. + - "${PHOTO_DIRS:?set PHOTO_DIRS in .env.photoprism}:/photoprism/originals:rw,Z" + networks: [photoprism-network] + networks: photoprism-network: driver: bridge diff --git a/sidecar/.dockerignore b/sidecar/.dockerignore new file mode 100644 index 0000000..5102ca3 --- /dev/null +++ b/sidecar/.dockerignore @@ -0,0 +1,13 @@ +# Files that have no business shipping into the build context. +# Anything not listed here gets COPY'd into /src so keep this tight. + +# Local host-mode build output — re-built inside the image. +mule-sidecar + +# Runtime state from the M3 Node prototype. +data/ + +# Docs + git noise. +README.md +.git/ +.gitignore diff --git a/sidecar/Dockerfile b/sidecar/Dockerfile new file mode 100644 index 0000000..c77c0eb --- /dev/null +++ b/sidecar/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1.6 +# +# mule-sidecar — Go service for endpoints PhotoPrism does not expose. +# Multi-stage build: a Go toolchain image compiles a static binary, +# then we copy it onto a distroless base so the runtime image is ~12 MB +# with no shell, package manager, or libc. + +FROM docker.io/library/golang:1.25-alpine AS build +WORKDIR /src + +# Cache deps separately from source so a one-line code change doesn't +# re-download the whole module graph. +COPY go.mod go.sum ./ +RUN go mod download + +COPY . ./ +# CGO disabled → fully static binary that runs on the distroless base +# (no libc resolution at startup). -trimpath strips local paths from +# debug info; -s -w drop the symbol table to keep the binary small. +RUN CGO_ENABLED=0 GOOS=linux go build \ + -trimpath \ + -ldflags='-s -w' \ + -o /out/mule-sidecar . + +FROM gcr.io/distroless/static-debian12:latest +COPY --from=build /out/mule-sidecar /mule-sidecar +EXPOSE 8000 +ENTRYPOINT ["/mule-sidecar"] diff --git a/sidecar/README.md b/sidecar/README.md index 57d6335..598a444 100644 --- a/sidecar/README.md +++ b/sidecar/README.md @@ -31,7 +31,27 @@ Marks persist to **MariaDB** (`mule_sidecar.marks`); everything else operates on the filesystem under `${ORIGINALS_ROOT}` and triggers a PhotoPrism reindex of the affected parent in the background. -## Build & run +## Run + +The sidecar is a `sidecar` service in the PhotoPrism compose stack. +Bringing the whole stack up brings it up too: + +```sh +podman-compose --env-file .env.photoprism \ + -f docker-compose.photoprism.yml \ + -f docker-compose.photoprism.podman.yml \ + up -d +``` + +This builds [Dockerfile](Dockerfile) (multi-stage `golang:1.25-alpine` → +`gcr.io/distroless/static`, ~12 MB final image), starts the container, +and binds `127.0.0.1:8000` to the service. The SvelteKit dev server +proxies `/api/sidecar/*` to that port transparently. + +### Dev-iteration loop (host build) + +For tight iteration without rebuilding the image on every change you +can run it as a host process — Go is already on the dev machine: ```sh cd sidecar @@ -43,8 +63,8 @@ SIDECAR_PORT=8000 \ ./mule-sidecar ``` -The SvelteKit dev server proxies `/api/sidecar/*` to -`http://localhost:8000`. +The host build connects to `mariadb` via the loopback port the compose +file publishes; stop `pp-sidecar` first so they don't fight for 8000. ## Env @@ -81,6 +101,7 @@ state. Heap-sharing tables (M4) will land in subsequent migrations. ```text sidecar/ +├── Dockerfile multi-stage golang:1.25 → distroless/static ├── main.go entrypoint, route wiring, graceful shutdown ├── config.go env-driven Config ├── db.go GORM open + Mark model + AutoMigrate @@ -91,6 +112,5 @@ sidecar/ ├── handlers_folders.go ├── handlers_marks.go ├── handlers_heap.go -├── handlers_dups.go -└── legacy/server.mjs Node prototype, retained for one cycle as a reference. +└── handlers_dups.go ``` diff --git a/sidecar/config.go b/sidecar/config.go index b14c1dc..fdb9939 100644 --- a/sidecar/config.go +++ b/sidecar/config.go @@ -12,7 +12,8 @@ import ( type Config struct { OriginalsRoot string // absolute path to PhotoPrism's originals dir PhotoprismBaseURL string // e.g. http://localhost:2342 - Port int // HTTP listen port (loopback only) + ListenAddr string // bind interface — 127.0.0.1 for host mode, 0.0.0.0 in containers + Port int // HTTP listen port DSN string // GORM/MySQL connection string for mule_sidecar } @@ -47,6 +48,7 @@ func loadConfig() (*Config, error) { return &Config{ OriginalsRoot: abs, PhotoprismBaseURL: envOr("PHOTOPRISM_BASE_URL", "http://localhost:2342"), + ListenAddr: envOr("SIDECAR_LISTEN_ADDR", "127.0.0.1"), Port: port, DSN: dsn, }, nil diff --git a/sidecar/legacy/server.mjs b/sidecar/legacy/server.mjs deleted file mode 100644 index 3bdde2f..0000000 --- a/sidecar/legacy/server.mjs +++ /dev/null @@ -1,845 +0,0 @@ -// mule-sidecar — Node.js prototype. -// -// Endpoints PhotoPrism does not expose. Today: file rename on disk. -// Future Go rewrite (per plan) will keep the same wire contract. -// -// Auth: forwards the caller's `X-Auth-Token` to PhotoPrism's session check -// before doing anything destructive. The token belongs to the end user; the -// sidecar does not hold its own credentials in this prototype. - -import http from 'node:http'; -import { URL, fileURLToPath } from 'node:url'; -import { promises as fs } from 'node:fs'; -import { createHash } from 'node:crypto'; -import { createReadStream } from 'node:fs'; -import path from 'node:path'; - -const ORIGINALS_ROOT = path.resolve( - process.env.ORIGINALS_ROOT ?? '/photoprism/originals' -); -const PHOTOPRISM_BASE_URL = - process.env.PHOTOPRISM_BASE_URL ?? 'http://localhost:2342'; -const PORT = Number(process.env.SIDECAR_PORT ?? 8000); - -// JSON-backed marks store. Holds the mule-image extras PhotoPrism doesn't: -// per-photo rating (0..5) and color label (red/orange/yellow/green/''). -// Lives next to server.mjs so the Go rewrite can migrate it into MariaDB -// without touching ORIGINALS_ROOT. -const SIDECAR_DIR = path.dirname(fileURLToPath(import.meta.url)); -const MARKS_FILE = path.join(SIDECAR_DIR, 'data', 'marks.json'); - -// ── helpers ────────────────────────────────────────────────────────────────── - -function json(res, status, body) { - res.writeHead(status, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(body)); -} - -async function readJson(req) { - const chunks = []; - for await (const c of req) chunks.push(c); - const raw = Buffer.concat(chunks).toString('utf8'); - return raw ? JSON.parse(raw) : {}; -} - -async function pp(method, urlPath, token, body) { - const url = new URL(urlPath, PHOTOPRISM_BASE_URL); - const init = { - method, - headers: { - 'X-Auth-Token': token, - 'Content-Type': 'application/json' - } - }; - if (body !== undefined) init.body = JSON.stringify(body); - const r = await fetch(url, init); - const text = await r.text(); - let data = null; - try { - data = text ? JSON.parse(text) : null; - } catch { - data = text; - } - return { ok: r.ok, status: r.status, data }; -} - -async function validateSession(token) { - if (!token) return false; - // Cheapest auth probe: list 1 photo. 401 if the token is bad. - const r = await pp('GET', '/api/v1/photos?count=1', token); - return r.ok; -} - -/** - * Enforce: NewName is a bare filename (no path separators, no leading dot, - * no surprising chars). PhotoPrism's index works fine with most filename - * shapes, but we lock down the obvious dangerous ones. - */ -function sanitizeFilename(name) { - if (typeof name !== 'string') return null; - const trimmed = name.trim(); - if (!trimmed) return null; - if (trimmed.length > 240) return null; - if (trimmed.startsWith('.')) return null; - if (/[\\/\x00]/.test(trimmed)) return null; - if (trimmed === '..' || trimmed === '.') return null; - return trimmed; -} - -// ── Marks store (rating + color label) ─────────────────────────────────────── -// In-memory cache backed by an atomic JSON file write. Single-threaded Node -// means we don't need an external lock — sequential awaits serialize writes. - -/** @type {Record} */ -let MARKS_CACHE = null; -let marksLoadPromise = null; - -async function loadMarks() { - if (MARKS_CACHE !== null) return MARKS_CACHE; - if (marksLoadPromise) return marksLoadPromise; - marksLoadPromise = (async () => { - await fs.mkdir(path.dirname(MARKS_FILE), { recursive: true }); - try { - const raw = await fs.readFile(MARKS_FILE, 'utf8'); - MARKS_CACHE = JSON.parse(raw); - } catch (err) { - if (err.code === 'ENOENT') MARKS_CACHE = {}; - else throw err; - } - return MARKS_CACHE; - })(); - return marksLoadPromise; -} - -async function persistMarks() { - const tmp = MARKS_FILE + '.tmp'; - await fs.writeFile(tmp, JSON.stringify(MARKS_CACHE ?? {}, null, 2)); - await fs.rename(tmp, MARKS_FILE); -} - -/** Normalize partial input. Strips unknown fields, clamps rating to 0..5, - * whitelists colors to mule-image's four-color palette. */ -function sanitizeMarkPatch(patch) { - if (!patch || typeof patch !== 'object') return null; - const out = {}; - if ('rating' in patch) { - const r = Math.round(Number(patch.rating)); - if (!Number.isFinite(r) || r < 0 || r > 5) return null; - out.rating = r; - } - if ('color' in patch) { - const c = typeof patch.color === 'string' ? patch.color.toLowerCase() : ''; - if (c !== '' && !['red', 'orange', 'yellow', 'green'].includes(c)) return null; - out.color = c; - } - return out; -} - -/** Merge a patch onto an existing mark, dropping zero/empty so the JSON - * stays sparse — never write `rating: 0` or `color: ''` to disk, just - * delete the field. */ -function mergeMark(prev, patch) { - const merged = { ...(prev ?? {}) }; - if ('rating' in patch) { - if (patch.rating > 0) merged.rating = patch.rating; - else delete merged.rating; - } - if ('color' in patch) { - if (patch.color) merged.color = patch.color; - else delete merged.color; - } - const hasAny = 'rating' in merged || 'color' in merged; - if (!hasAny) return null; - merged.updatedAt = new Date().toISOString(); - return merged; -} - -async function ensureWithinOriginals(absPath) { - const real = await fs.realpath(path.dirname(absPath)).catch(() => null); - if (!real) return false; - return real === ORIGINALS_ROOT || real.startsWith(ORIGINALS_ROOT + path.sep); -} - -/** - * Resolve a user-supplied relative path under ORIGINALS_ROOT. - * Returns the absolute resolved path on success, or null if it escapes - * the root, contains traversal sequences, or its parent is missing. - * - * `mustExist=false` is used for the *target* of a rename/create where - * the path itself doesn't yet exist; we still ensure the parent does. - */ -async function resolveUnderRoot(rel, { mustExist = true } = {}) { - if (typeof rel !== 'string') return null; - const clean = rel.replace(/^\/+/, ''); - if (!clean || clean === '.' || clean.split('/').some((seg) => seg === '..' || seg === '')) { - return null; - } - const abs = path.resolve(ORIGINALS_ROOT, clean); - const parent = path.dirname(abs); - // Confirm both the abs and its parent resolve back under ORIGINALS_ROOT - // (defends against symlinks pointing out of the library). - const parentReal = await fs.realpath(parent).catch(() => null); - if (!parentReal) return null; - if ( - parentReal !== ORIGINALS_ROOT && - !parentReal.startsWith(ORIGINALS_ROOT + path.sep) - ) { - return null; - } - if (mustExist) { - const stat = await fs.stat(abs).catch(() => null); - if (!stat) return null; - } - return abs; -} - -// ── handlers ───────────────────────────────────────────────────────────────── - -async function handleRename(req, res, photoUid) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - - const body = await readJson(req).catch(() => null); - if (!body) return json(res, 400, { error: 'invalid json' }); - - const newName = sanitizeFilename(body.newName); - if (!newName) return json(res, 400, { error: 'newName must be a plain filename' }); - - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - - // Fetch the photo to discover the file's path on disk. The single-photo - // endpoint nests Files[0] with Root + Name; the file lookup by UID - // (/api/v1/files/:uid) doesn't exist in this build. - const photoResp = await pp('GET', `/api/v1/photos/${photoUid}`, token); - if (!photoResp.ok) return json(res, photoResp.status, { error: 'photo not found' }); - - const photo = photoResp.data; - const file = - (photo.Files || []).find((f) => f.Primary) || (photo.Files || [])[0]; - if (!file) return json(res, 404, { error: 'no files on this photo' }); - - // Build the absolute current path. - const root = (file.Root && file.Root !== '/') ? file.Root : ''; - const relPath = path.posix.join(root, file.Name); - const oldAbs = path.resolve(path.join(ORIGINALS_ROOT, relPath)); - - if (!(await ensureWithinOriginals(oldAbs))) { - return json(res, 400, { error: 'path escapes originals root' }); - } - const stat = await fs.stat(oldAbs).catch(() => null); - if (!stat || !stat.isFile()) return json(res, 404, { error: 'file missing on disk' }); - - const newAbs = path.resolve(path.join(path.dirname(oldAbs), newName)); - if (!(await ensureWithinOriginals(newAbs))) { - return json(res, 400, { error: 'new path escapes originals root' }); - } - - // Refuse to clobber an existing file. - if (await fs.stat(newAbs).then(() => true, () => false)) { - return json(res, 409, { error: 'target filename already exists' }); - } - - const oldName = file.Name; - console.log(`[rename] ${relPath} -> ${path.posix.join(root, newName)}`); - await fs.rename(oldAbs, newAbs); - - // Tell PhotoPrism to re-index the parent so the DB picks up the new path. - // `cleanup: true` removes orphan rows for the old filename. - const indexResp = await pp( - 'POST', - '/api/v1/index', - token, - { path: root || '/', rescan: false, cleanup: true } - ); - if (!indexResp.ok) { - // Best-effort: file is renamed, index will catch up eventually. - console.warn('[rename] reindex returned', indexResp.status); - } - - json(res, 200, { - ok: true, - oldName, - newName, - oldRelPath: relPath, - newRelPath: path.posix.join(root, newName) - }); -} - -function handleHealth(_req, res) { - json(res, 200, { ok: true, originalsRoot: ORIGINALS_ROOT }); -} - -// ── Marks handlers (rating + color) ────────────────────────────────────────── - -async function handleMarksListAll(req, res) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - const all = await loadMarks(); - json(res, 200, all); -} - -async function handleMarkGet(req, res, uid) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - const all = await loadMarks(); - json(res, 200, all[uid] ?? {}); -} - -async function handleMarkPut(req, res, uid) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - const body = await readJson(req).catch(() => null); - const patch = sanitizeMarkPatch(body); - if (patch === null) return json(res, 400, { error: 'invalid patch' }); - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - - const all = await loadMarks(); - const next = mergeMark(all[uid], patch); - if (next === null) delete all[uid]; - else all[uid] = next; - await persistMarks(); - json(res, 200, all[uid] ?? {}); -} - -async function handleMarkBulk(req, res) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - const body = await readJson(req).catch(() => null); - if (!body || !Array.isArray(body.ids)) { - return json(res, 400, { error: 'ids[] required' }); - } - const patch = sanitizeMarkPatch(body.patch); - if (patch === null) return json(res, 400, { error: 'invalid patch' }); - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - - const all = await loadMarks(); - const applied = {}; - for (const uid of body.ids) { - if (typeof uid !== 'string' || !uid) continue; - const next = mergeMark(all[uid], patch); - if (next === null) delete all[uid]; - else all[uid] = next; - applied[uid] = all[uid] ?? {}; - } - await persistMarks(); - json(res, 200, { count: Object.keys(applied).length, marks: applied }); -} - -// ── Folder mutations ──────────────────────────────────────────────────────── -// -// Each endpoint operates on a relative path under ORIGINALS_ROOT, then -// triggers PhotoPrism's re-index of the parent so the DB picks up the -// change. PhotoPrism's "folder" concept is just a directory on disk — -// there's no DB-side folder entity to mutate. - -async function reindex(parentRel, token) { - const r = await pp( - 'POST', - '/api/v1/index', - token, - { path: parentRel || '/', rescan: false, cleanup: true } - ); - if (!r.ok) console.warn('[folder] reindex returned', r.status); -} - -async function handleFolderCreate(req, res) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - const body = await readJson(req).catch(() => null); - if (!body) return json(res, 400, { error: 'invalid json' }); - if (typeof body.path !== 'string') return json(res, 400, { error: 'path required' }); - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - - const abs = await resolveUnderRoot(body.path, { mustExist: false }); - if (!abs) return json(res, 400, { error: 'invalid path' }); - if (await fs.stat(abs).then(() => true, () => false)) { - return json(res, 409, { error: 'already exists' }); - } - await fs.mkdir(abs, { recursive: false }); - const rel = path.relative(ORIGINALS_ROOT, abs); - console.log('[folder.create]', rel); - void reindex(path.posix.dirname('/' + rel), token); - json(res, 200, { ok: true, path: rel }); -} - -async function handleFolderRename(req, res, rel) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - const body = await readJson(req).catch(() => null); - if (!body) return json(res, 400, { error: 'invalid json' }); - const newName = sanitizeFilename(body.newName); - if (!newName) return json(res, 400, { error: 'newName must be a plain dirname' }); - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - - const oldAbs = await resolveUnderRoot(rel); - if (!oldAbs) return json(res, 400, { error: 'invalid path' }); - const stat = await fs.stat(oldAbs); - if (!stat.isDirectory()) return json(res, 400, { error: 'not a directory' }); - - const newAbs = path.join(path.dirname(oldAbs), newName); - if (await fs.stat(newAbs).then(() => true, () => false)) { - return json(res, 409, { error: 'target already exists' }); - } - if ( - !newAbs.startsWith(ORIGINALS_ROOT + path.sep) && - newAbs !== ORIGINALS_ROOT - ) { - return json(res, 400, { error: 'target escapes root' }); - } - - await fs.rename(oldAbs, newAbs); - const oldRel = path.relative(ORIGINALS_ROOT, oldAbs); - const newRel = path.relative(ORIGINALS_ROOT, newAbs); - console.log('[folder.rename]', oldRel, '→', newRel); - void reindex(path.posix.dirname('/' + oldRel), token); - json(res, 200, { ok: true, oldPath: oldRel, newPath: newRel }); -} - -// ── Heap convert (move/copy heap photos into a folder) ───────────────────── -// PhotoPrism has no native "move all album photos into folder X" operation -// — it can't because the file-on-disk layout is its source of truth. The -// flow: list members via q=album:, fs.rename / fs.copyFile each primary -// file into the destination, optionally delete the album, then reindex -// both source and destination so PhotoPrism's DB catches up. - -/** Find a non-clobbering destination for `basename` inside `destDir`. If - * `foo.jpg` exists, try `foo-1.jpg`, `foo-2.jpg`, … up to a sane cap. */ -async function uniqueName(destDir, basename) { - const ext = path.extname(basename); - const stem = basename.slice(0, basename.length - ext.length); - for (let i = 0; i < 1000; i++) { - const candidate = i === 0 ? basename : `${stem}-${i}${ext}`; - const abs = path.join(destDir, candidate); - const exists = await fs.stat(abs).then(() => true, () => false); - if (!exists) return { abs, name: candidate }; - } - return null; -} - -async function handleHeapConvert(req, res, albumUid) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - const body = await readJson(req).catch(() => null); - if (!body) return json(res, 400, { error: 'invalid json' }); - - const mode = body.mode === 'copy' ? 'copy' : 'move'; - const deleteHeap = mode === 'move' && body.deleteHeap === true; - const subfolderRaw = typeof body.subfolder === 'string' ? body.subfolder.trim() : ''; - const subfolder = subfolderRaw ? sanitizeFilename(subfolderRaw) : null; - if (subfolderRaw && !subfolder) { - return json(res, 400, { error: 'invalid subfolder name' }); - } - - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - - // Resolve target folder. Picker passes a relative path under ORIGINALS_ROOT. - const targetAbs = await resolveUnderRoot(body.targetFolder); - if (!targetAbs) return json(res, 400, { error: 'invalid targetFolder' }); - - // Create the optional subfolder (idempotent). - let destAbs = targetAbs; - if (subfolder) { - destAbs = path.join(targetAbs, subfolder); - await fs.mkdir(destAbs, { recursive: true }); - } - - // Fetch the album's photos. PhotoPrism's q-DSL lets us filter by album - // UID; merged=true expands to one row per file (we need every variant - // in a stack to follow the primary). 1000 covers any realistic heap. - const listResp = await pp( - 'GET', - `/api/v1/photos?q=${encodeURIComponent(`album:${albumUid}`)}&count=1000&merged=true`, - token - ); - if (!listResp.ok) return json(res, listResp.status, { error: 'list photos failed' }); - const photos = Array.isArray(listResp.data) ? listResp.data : []; - - // Track source parents so we know which paths to reindex once we're done. - const sourceParents = new Set(); - const errors = []; - let moved = 0; - let copied = 0; - - for (const photo of photos) { - const file = - (photo.Files || []).find((f) => f.Primary) || (photo.Files || [])[0]; - if (!file || typeof file.Name !== 'string') { - errors.push({ uid: photo.UID, reason: 'no primary file' }); - continue; - } - // PhotoPrism Files[].Name is already originals-relative. - const srcRel = file.Name; - const srcAbs = path.resolve(path.join(ORIGINALS_ROOT, srcRel)); - if (!srcAbs.startsWith(ORIGINALS_ROOT + path.sep) && srcAbs !== ORIGINALS_ROOT) { - errors.push({ uid: photo.UID, reason: 'path escapes originals' }); - continue; - } - const stat = await fs.stat(srcAbs).catch(() => null); - if (!stat || !stat.isFile()) { - errors.push({ uid: photo.UID, reason: 'file missing on disk' }); - continue; - } - // Avoid no-op moves (file already lives in dest). - if (path.dirname(srcAbs) === destAbs) { - errors.push({ uid: photo.UID, reason: 'already in target' }); - continue; - } - - const basename = path.basename(srcAbs); - const target = await uniqueName(destAbs, basename); - if (!target) { - errors.push({ uid: photo.UID, reason: 'too many collisions' }); - continue; - } - - try { - if (mode === 'move') { - await fs.rename(srcAbs, target.abs); - moved += 1; - } else { - await fs.copyFile(srcAbs, target.abs); - copied += 1; - } - sourceParents.add(path.dirname(srcRel)); - } catch (err) { - errors.push({ - uid: photo.UID, - reason: err instanceof Error ? err.message : String(err) - }); - } - } - - // Reindex destination + every distinct source parent so PhotoPrism's - // DB catches up to the new on-disk layout. - const destRel = path.relative(ORIGINALS_ROOT, destAbs); - const reindexPaths = new Set(sourceParents); - reindexPaths.add(destRel); - if (subfolder) reindexPaths.add(path.relative(ORIGINALS_ROOT, targetAbs)); - for (const p of reindexPaths) { - void reindex(p ? '/' + p : '/', token); - } - - // Optionally delete the album after a successful move. We don't gate on - // `errors.length === 0` — partial successes still warrant heap cleanup - // if the user explicitly opted in. - let heap_deleted = false; - if (deleteHeap) { - const delResp = await pp('DELETE', `/api/v1/albums/${albumUid}`, token); - heap_deleted = delResp.ok; - if (!delResp.ok) { - errors.push({ uid: albumUid, reason: `album delete: HTTP ${delResp.status}` }); - } - } - - console.log( - `[heap.convert] album=${albumUid} mode=${mode} moved=${moved} copied=${copied} errors=${errors.length} heap_deleted=${heap_deleted}` - ); - json(res, 200, { moved, copied, errors, heap_deleted }); -} - -async function handleFolderDelete(req, res, rel) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - - const abs = await resolveUnderRoot(rel); - if (!abs) return json(res, 400, { error: 'invalid path' }); - if (abs === ORIGINALS_ROOT) return json(res, 400, { error: 'refuse to delete root' }); - const stat = await fs.stat(abs); - if (!stat.isDirectory()) return json(res, 400, { error: 'not a directory' }); - - // Safety: only remove if empty. Recursive rm is left to the user via shell - // until M5 lands proper file moves. - const entries = await fs.readdir(abs); - if (entries.length > 0) { - return json(res, 409, { error: 'directory not empty' }); - } - await fs.rmdir(abs); - console.log('[folder.delete]', rel); - void reindex(path.posix.dirname('/' + rel), token); - json(res, 200, { ok: true, path: rel }); -} - -// ── Cross-folder duplicate scan ───────────────────────────────────────────── -// -// PhotoPrism silently drops byte-identical files at index time, so duplicates -// across folders never enter its DB. This sidecar scans ORIGINALS_ROOT -// directly: walk every file, pre-filter by size (files with unique sizes -// can't be hash-duplicates so we skip hashing them), sha1 the rest, and -// return groups of ≥2 files that share a hash. -// -// Resolution archives the unwanted copies into a `.duplicates/` quarantine -// folder under ORIGINALS_ROOT — PhotoPrism's indexer ignores dotfile dirs, -// so the moved files stop appearing in the index. Recoverable by moving -// them back to a regular subfolder + reindex. - -const QUARANTINE_DIR = '.duplicates'; -const SUPPORTED_EXTS = new Set([ - '.jpg', '.jpeg', '.png', '.heic', '.heif', '.tiff', '.tif', - '.gif', '.bmp', '.webp', '.avif', - '.mov', '.mp4', '.m4v', '.avi', '.mkv', '.webm', - '.dng', '.cr2', '.cr3', '.nef', '.arw', '.orf', '.rw2', '.raw' -]); - -async function walkFiles(rootAbs) { - /** @type {{ relPath: string, absPath: string, size: number }[]} */ - const out = []; - async function recurse(dir) { - let entries; - try { - entries = await fs.readdir(dir, { withFileTypes: true }); - } catch { - return; - } - for (const ent of entries) { - // Skip dotfile/dotdir (matches PhotoPrism's indexer behaviour - // + our own quarantine dir). - if (ent.name.startsWith('.')) continue; - const abs = path.join(dir, ent.name); - if (ent.isDirectory()) { - await recurse(abs); - continue; - } - if (!ent.isFile()) continue; - const ext = path.extname(ent.name).toLowerCase(); - if (!SUPPORTED_EXTS.has(ext)) continue; - let st; - try { - st = await fs.stat(abs); - } catch { - continue; - } - out.push({ - relPath: path.relative(rootAbs, abs), - absPath: abs, - size: st.size - }); - } - } - await recurse(rootAbs); - return out; -} - -async function sha1File(absPath) { - return new Promise((resolve, reject) => { - const hash = createHash('sha1'); - createReadStream(absPath) - .on('data', (chunk) => hash.update(chunk)) - .on('end', () => resolve(hash.digest('hex'))) - .on('error', reject); - }); -} - -async function scanDuplicates(token) { - const all = await walkFiles(ORIGINALS_ROOT); - - // Group by size first; hashes only fire for size collisions. For a - // typical library this skips ~95% of the IO+CPU. - /** @type {Map} */ - const bySize = new Map(); - for (const f of all) { - const arr = bySize.get(f.size); - if (arr) arr.push(f); - else bySize.set(f.size, [f]); - } - - /** @type {Map} */ - const groups = new Map(); - for (const [size, group] of bySize) { - if (group.length < 2) continue; - // Concurrently hash each file in this size bucket. - const hashes = await Promise.all(group.map((f) => sha1File(f.absPath))); - for (let i = 0; i < group.length; i++) { - const h = hashes[i]; - const g = groups.get(h); - if (g) g.files.push(group[i]); - else groups.set(h, { hash: h, size, files: [group[i]] }); - } - } - - // Filter to groups with ≥2 files (size collisions where hashes differed - // produce singleton entries we drop here). For each surviving group, - // ask PhotoPrism which path it has indexed — that becomes the default - // "keep" candidate so the user doesn't accidentally archive the only - // indexed copy. - const out = []; - for (const g of groups.values()) { - if (g.files.length < 2) continue; - let indexedPath = null; - try { - const r = await pp( - 'GET', - `/api/v1/photos?q=hash:${g.hash}&count=1&merged=true`, - token - ); - if (r.ok && Array.isArray(r.data) && r.data[0]) { - const photo = r.data[0]; - const primary = - (photo.Files || []).find((f) => f.Primary) || - (photo.Files || [])[0]; - if (primary?.Name) indexedPath = primary.Name; - } - } catch { - /* swallow — best-effort hint */ - } - out.push({ - hash: g.hash, - size: g.size, - indexedPath, - files: g.files.map((f) => ({ path: f.relPath, size: f.size })) - }); - } - // Sort groups by size descending so the biggest disk wins float to top. - out.sort((a, b) => b.size * (b.files.length - 1) - a.size * (a.files.length - 1)); - return out; -} - -async function handleDupScan(req, res) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - const started = Date.now(); - console.log('[dup.scan] starting walk under', ORIGINALS_ROOT); - const groups = await scanDuplicates(token); - const ms = Date.now() - started; - console.log(`[dup.scan] ${groups.length} groups in ${ms} ms`); - json(res, 200, { groups, scannedMs: ms }); -} - -async function handleDupArchive(req, res) { - const token = req.headers['x-auth-token']; - if (!token || Array.isArray(token)) return json(res, 401, { error: 'no token' }); - const body = await readJson(req).catch(() => null); - if (!body || !Array.isArray(body.paths)) { - return json(res, 400, { error: 'paths[] required' }); - } - if (!(await validateSession(token))) return json(res, 401, { error: 'invalid session' }); - - const moved = []; - const errors = []; - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const targetDir = path.join(ORIGINALS_ROOT, QUARANTINE_DIR, stamp); - await fs.mkdir(targetDir, { recursive: true }); - - for (const rel of body.paths) { - try { - const abs = await resolveUnderRoot(rel); - if (!abs) { - errors.push({ path: rel, error: 'invalid path' }); - continue; - } - // Use the file's basename as the quarantine name; prepend a - // short slice of its parent dir if a collision would happen so - // two `IMG_0001.jpg` from different folders don't overwrite - // each other in the same quarantine batch. - let dest = path.join(targetDir, path.basename(abs)); - let i = 1; - while (await fs.stat(dest).then(() => true, () => false)) { - const parsed = path.parse(path.basename(abs)); - dest = path.join(targetDir, `${parsed.name}__${i}${parsed.ext}`); - i++; - } - await fs.rename(abs, dest); - moved.push({ - from: rel, - to: path.relative(ORIGINALS_ROOT, dest) - }); - console.log('[dup.archive]', rel, '→', path.relative(ORIGINALS_ROOT, dest)); - } catch (err) { - errors.push({ - path: rel, - error: err instanceof Error ? err.message : String(err) - }); - } - } - - // Trigger a cleanup reindex so PhotoPrism drops any photo entries whose - // underlying file is now in the quarantine dir (out of its scan path). - if (moved.length > 0) { - void pp('POST', '/api/v1/index', token, { - path: '/', - rescan: false, - cleanup: true - }); - } - - json(res, 200, { moved, errors }); -} - -// ── router ─────────────────────────────────────────────────────────────────── - -const RENAME_RE = /^\/api\/sidecar\/files\/([^/]+)\/rename$/; -const FOLDER_CREATE_RE = /^\/api\/sidecar\/folders$/; -const FOLDER_RENAME_RE = /^\/api\/sidecar\/folders\/(.+)\/rename$/; -const FOLDER_DELETE_RE = /^\/api\/sidecar\/folders\/(.+)$/; -const MARKS_BULK_RE = /^\/api\/sidecar\/photos\/marks\/bulk$/; -const MARKS_ALL_RE = /^\/api\/sidecar\/photos\/marks$/; -const MARKS_ONE_RE = /^\/api\/sidecar\/photos\/([^/]+)\/marks$/; -const HEAP_CONVERT_RE = /^\/api\/sidecar\/albums\/([^/]+)\/convert$/; -const DUP_SCAN_RE = /^\/api\/sidecar\/duplicates\/scan$/; -const DUP_ARCHIVE_RE = /^\/api\/sidecar\/duplicates\/archive$/; - -const server = http.createServer(async (req, res) => { - try { - const url = new URL(req.url ?? '/', 'http://localhost'); - const p = url.pathname; - - if (p === '/api/sidecar/healthz' && req.method === 'GET') { - return handleHealth(req, res); - } - // Cross-folder duplicate detection. - if (DUP_SCAN_RE.test(p) && req.method === 'GET') { - return handleDupScan(req, res); - } - if (DUP_ARCHIVE_RE.test(p) && req.method === 'POST') { - return handleDupArchive(req, res); - } - // Marks routes — bulk match first so "marks/bulk" doesn't get - // captured by the "{uid}/marks" pattern. - if (MARKS_BULK_RE.test(p) && req.method === 'POST') { - return handleMarkBulk(req, res); - } - if (MARKS_ALL_RE.test(p) && req.method === 'GET') { - return handleMarksListAll(req, res); - } - const markM = MARKS_ONE_RE.exec(p); - if (markM) { - if (req.method === 'GET') return handleMarkGet(req, res, markM[1]); - if (req.method === 'PUT') return handleMarkPut(req, res, markM[1]); - } - const fileM = RENAME_RE.exec(p); - if (fileM && req.method === 'POST') { - return handleRename(req, res, fileM[1]); - } - if (FOLDER_CREATE_RE.test(p) && req.method === 'POST') { - return handleFolderCreate(req, res); - } - const renameFolderM = FOLDER_RENAME_RE.exec(p); - if (renameFolderM && req.method === 'POST') { - return handleFolderRename(req, res, decodeURIComponent(renameFolderM[1])); - } - const deleteFolderM = FOLDER_DELETE_RE.exec(p); - if (deleteFolderM && req.method === 'DELETE') { - // Avoid matching the rename URL which has a trailing /rename. - if (!p.endsWith('/rename')) { - return handleFolderDelete(req, res, decodeURIComponent(deleteFolderM[1])); - } - } - const heapConvertM = HEAP_CONVERT_RE.exec(p); - if (heapConvertM && req.method === 'POST') { - return handleHeapConvert(req, res, heapConvertM[1]); - } - json(res, 404, { error: 'no route' }); - } catch (err) { - console.error(err); - json(res, 500, { error: err instanceof Error ? err.message : String(err) }); - } -}); - -server.listen(PORT, '127.0.0.1', () => { - console.log( - `mule-sidecar listening on http://127.0.0.1:${PORT} (originals=${ORIGINALS_ROOT})` - ); -}); diff --git a/sidecar/main.go b/sidecar/main.go index 0b04d2c..dd8c286 100644 --- a/sidecar/main.go +++ b/sidecar/main.go @@ -79,7 +79,7 @@ func main() { auth.POST("/duplicates/archive", handleDupArchive(cfg, pp)) } - addr := "127.0.0.1:" + itoa(cfg.Port) + addr := cfg.ListenAddr + ":" + itoa(cfg.Port) srv := &http.Server{ Addr: addr, Handler: r,