feat(sidecar): port Node prototype to Go + Gin + GORM + MariaDB
Replace the Node prototype (server.mjs) with the stack the merge plan calls for: Go 1.25, Gin for routing, GORM + MariaDB for persistence. Same wire contract on /api/sidecar/* so the SvelteKit client doesn't change. - Marks move from a JSON file on disk to mule_sidecar.marks (auto- migrated by GORM on first boot). The Node prototype's marks.json was dev-only; not migrated. - Folder/rename/heap-convert/duplicates handlers reproduce the prototype's behaviour, including the path-traversal defence (resolveUnderRoot + EvalSymlinks), the size-bucket prefilter for the duplicate hasher, and the background reindex fire-and-forget pattern. - Auth model unchanged: requireSession middleware proxies the caller's X-Auth-Token to PhotoPrism's /api/v1/photos?count=1 before any destructive op. - Expose pp-mariadb on 127.0.0.1:3306 in docker-compose so the host Go process can reach mule_sidecar.* without joining the container network. - Archive the Node prototype under sidecar/legacy/server.mjs for one cycle as reference. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
230
sidecar/handlers_dups.go
Normal file
230
sidecar/handlers_dups.go
Normal file
@@ -0,0 +1,230 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const quarantineDir = ".duplicates"
|
||||
|
||||
type dupFileLite struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type dupGroup struct {
|
||||
Hash string `json:"hash"`
|
||||
Size int64 `json:"size"`
|
||||
IndexedPath *string `json:"indexedPath"`
|
||||
Files []dupFileLite `json:"files"`
|
||||
}
|
||||
|
||||
// dupListPhoto is the partial photo shape we pull from PhotoPrism when
|
||||
// looking up "which file path has this hash already indexed", used to
|
||||
// hint the UI which copy to keep.
|
||||
type dupListPhoto struct {
|
||||
Files []ppFile `json:"Files"`
|
||||
}
|
||||
|
||||
func handleDupScan(cfg *Config, pp *ppClient) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
token := ctxToken(c)
|
||||
start := time.Now()
|
||||
slog.Info("dup.scan starting", "root", cfg.OriginalsRoot)
|
||||
|
||||
all, err := walkFiles(cfg.OriginalsRoot)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Group by size first: byte-identical files necessarily share size,
|
||||
// so size-collision is a cheap O(N) prefilter that lets us skip
|
||||
// hashing >95% of a typical library.
|
||||
bySize := map[int64][]fileEntry{}
|
||||
for _, f := range all {
|
||||
bySize[f.Size] = append(bySize[f.Size], f)
|
||||
}
|
||||
|
||||
// Hash size-collision buckets concurrently. Cap fan-out to GOMAXPROCS
|
||||
// so we don't drown the disk with parallel reads on a spinning HDD.
|
||||
type hashOut struct {
|
||||
hash string
|
||||
f fileEntry
|
||||
}
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
sem = make(chan struct{}, 4)
|
||||
outMu sync.Mutex
|
||||
byHash = map[string][]fileEntry{}
|
||||
hashSize = map[string]int64{}
|
||||
)
|
||||
for size, group := range bySize {
|
||||
if len(group) < 2 {
|
||||
continue
|
||||
}
|
||||
for _, f := range group {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(f fileEntry, sz int64) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
h, err := sha1File(f.AbsPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
outMu.Lock()
|
||||
byHash[h] = append(byHash[h], f)
|
||||
hashSize[h] = sz
|
||||
outMu.Unlock()
|
||||
}(f, size)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Drop singletons (size collision but different hashes), then ask
|
||||
// PhotoPrism which of the duplicates it has indexed so the UI can
|
||||
// default the "keep" selection to that one.
|
||||
groups := make([]dupGroup, 0)
|
||||
for h, files := range byHash {
|
||||
if len(files) < 2 {
|
||||
continue
|
||||
}
|
||||
g := dupGroup{Hash: h, Size: hashSize[h]}
|
||||
for _, f := range files {
|
||||
g.Files = append(g.Files, dupFileLite{Path: f.RelPath, Size: f.Size})
|
||||
}
|
||||
// Best-effort lookup; swallow errors. The hash query is cheap on
|
||||
// PhotoPrism's side (indexed column).
|
||||
resp, err := pp.call(c.Request.Context(), http.MethodGet,
|
||||
"/api/v1/photos?q=hash:"+h+"&count=1&merged=true", token, nil)
|
||||
if err == nil && resp.OK {
|
||||
var photos []dupListPhoto
|
||||
if err := json.Unmarshal(resp.Body, &photos); err == nil && len(photos) > 0 {
|
||||
if pf, ok := primaryFileOf(&ppPhoto{Files: photos[0].Files}); ok && pf.Name != "" {
|
||||
p := pf.Name
|
||||
g.IndexedPath = &p
|
||||
}
|
||||
}
|
||||
}
|
||||
groups = append(groups, g)
|
||||
}
|
||||
// Sort by reclaimable bytes descending (size × duplicate-count) so
|
||||
// the biggest wins float to the top of the UI.
|
||||
sort.Slice(groups, func(i, j int) bool {
|
||||
return groups[i].Size*int64(len(groups[i].Files)-1) >
|
||||
groups[j].Size*int64(len(groups[j].Files)-1)
|
||||
})
|
||||
|
||||
ms := time.Since(start).Milliseconds()
|
||||
slog.Info("dup.scan done", "groups", len(groups), "ms", ms)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"groups": groups,
|
||||
"scannedMs": ms,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type dupArchiveBody struct {
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
type dupMoved struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
}
|
||||
|
||||
type dupArchiveErr struct {
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
func handleDupArchive(cfg *Config, pp *ppClient) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
token := ctxToken(c)
|
||||
var body dupArchiveBody
|
||||
if err := c.ShouldBindJSON(&body); err != nil || len(body.Paths) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "paths[] required"})
|
||||
return
|
||||
}
|
||||
|
||||
// Each archive batch lands in its own timestamped subdir so the
|
||||
// user can browse what was quarantined when (and recover by hand
|
||||
// if they change their mind).
|
||||
stamp := time.Now().UTC().Format("2006-01-02T15-04-05.000Z")
|
||||
targetDir := filepath.Join(cfg.OriginalsRoot, quarantineDir, stamp)
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// archiveOne moves a single file into the quarantine batch dir
|
||||
// and returns the new relative path. Disambiguates same-basename
|
||||
// collisions within the batch so two `IMG_0001.jpg` from
|
||||
// different folders don't clobber each other.
|
||||
archiveOne := func(rel string) (string, error) {
|
||||
abs, err := resolveUnderRoot(cfg.OriginalsRoot, rel, true)
|
||||
if err != nil {
|
||||
return "", errors.New("invalid path")
|
||||
}
|
||||
base := filepath.Base(abs)
|
||||
dest := filepath.Join(targetDir, base)
|
||||
for i := 1; ; i++ {
|
||||
if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) {
|
||||
break
|
||||
} else if err != nil {
|
||||
return "", err
|
||||
}
|
||||
stem := base[:len(base)-len(filepath.Ext(base))]
|
||||
dest = filepath.Join(targetDir, stem+"__"+itoa(i)+filepath.Ext(base))
|
||||
}
|
||||
if err := os.Rename(abs, dest); err != nil {
|
||||
// EXDEV fallback — copy+remove for libraries that span
|
||||
// filesystems (e.g. originals on a different mount).
|
||||
if err2 := copyFile(abs, dest); err2 != nil {
|
||||
return "", err
|
||||
}
|
||||
if err2 := os.Remove(abs); err2 != nil {
|
||||
return "", errors.New("moved but source remove failed: " + err2.Error())
|
||||
}
|
||||
}
|
||||
relDest, _ := filepath.Rel(cfg.OriginalsRoot, dest)
|
||||
return relDest, nil
|
||||
}
|
||||
|
||||
moved := []dupMoved{}
|
||||
errs := []dupArchiveErr{}
|
||||
for _, rel := range body.Paths {
|
||||
relDest, err := archiveOne(rel)
|
||||
if err != nil {
|
||||
errs = append(errs, dupArchiveErr{Path: rel, Error: err.Error()})
|
||||
continue
|
||||
}
|
||||
moved = append(moved, dupMoved{From: rel, To: relDest})
|
||||
slog.Info("dup.archive", "from", rel, "to", relDest)
|
||||
}
|
||||
|
||||
// Reindex the entire library so PhotoPrism drops rows for the
|
||||
// archived files. cleanup:true is critical — the files still
|
||||
// exist on disk, just under .duplicates/ which the indexer
|
||||
// ignores.
|
||||
if len(moved) > 0 {
|
||||
go func() {
|
||||
if err := pp.reindex(context.Background(), token, "/"); err != nil {
|
||||
slog.Warn("dup.archive reindex failed", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"moved": moved, "errors": errs})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user