mule-image/sidecar/handlers_dups.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"log/slog"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/gin-gonic/gin"
	"gorm.io/gorm"
)

const quarantineDir = ".duplicates"

type dupFileLite struct {
	Path string `json:"path"`
	Size int64  `json:"size"`
}

type dupGroup struct {
	Hash         string        `json:"hash"`
	Size         int64         `json:"size"`
	IndexedPath  *string       `json:"indexedPath"`
	Files        []dupFileLite `json:"files"`
}

// dupListPhoto is the partial photo shape we pull from PhotoPrism when
// looking up "which file path has this hash already indexed", used to
// hint the UI which copy to keep.
type dupListPhoto struct {
	Files []ppFile `json:"Files"`
}

func handleDupScan(cfg *Config, pp *ppClient, db *gorm.DB) gin.HandlerFunc {
	return func(c *gin.Context) {
		token := ctxToken(c)
		start := time.Now()

		// Scope the walk to the user's effective library root (BasePath +
		// chosen index sub-path), same as the folders/timeline/reindex scope —
		// otherwise a narrowed root would still surface every other user's
		// files in the cross-folder duplicate scan. "" means whole library
		// (today's admin-without-BasePath default).
		root := effectiveLibraryRoot(c, db)
		scanRoot := cfg.OriginalsRoot
		if root != "" {
			abs, err := resolveUnderRoot(cfg.OriginalsRoot, root, true)
			if err != nil {
				c.JSON(http.StatusBadRequest, gin.H{"error": "invalid library root"})
				return
			}
			scanRoot = abs
		}
		slog.Info("dup.scan starting", "root", scanRoot)

		all, err := walkFiles(scanRoot)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		// walkFiles computes RelPath relative to scanRoot; re-prefix with the
		// scoped sub-path so RelPath stays originals-root-relative, matching
		// what handleDupArchive (and the rest of the API) expects.
		if root != "" {
			for i := range all {
				all[i].RelPath = root + "/" + all[i].RelPath
			}
		}

		// Group by size first: byte-identical files necessarily share size,
		// so size-collision is a cheap O(N) prefilter that lets us skip
		// hashing >95% of a typical library.
		bySize := map[int64][]fileEntry{}
		for _, f := range all {
			bySize[f.Size] = append(bySize[f.Size], f)
		}

		// Hash size-collision buckets concurrently. Cap fan-out to GOMAXPROCS
		// so we don't drown the disk with parallel reads on a spinning HDD.
		type hashOut struct {
			hash string
			f    fileEntry
		}
		var (
			wg       sync.WaitGroup
			sem      = make(chan struct{}, 4)
			outMu    sync.Mutex
			byHash   = map[string][]fileEntry{}
			hashSize = map[string]int64{}
		)
		for size, group := range bySize {
			if len(group) < 2 {
				continue
			}
			for _, f := range group {
				wg.Add(1)
				sem <- struct{}{}
				go func(f fileEntry, sz int64) {
					defer wg.Done()
					defer func() { <-sem }()
					h, err := sha1File(f.AbsPath)
					if err != nil {
						return
					}
					outMu.Lock()
					byHash[h] = append(byHash[h], f)
					hashSize[h] = sz
					outMu.Unlock()
				}(f, size)
			}
		}
		wg.Wait()

		// Drop singletons (size collision but different hashes), then ask
		// PhotoPrism which of the duplicates it has indexed so the UI can
		// default the "keep" selection to that one.
		groups := make([]dupGroup, 0)
		for h, files := range byHash {
			if len(files) < 2 {
				continue
			}
			g := dupGroup{Hash: h, Size: hashSize[h]}
			for _, f := range files {
				g.Files = append(g.Files, dupFileLite{Path: f.RelPath, Size: f.Size})
			}
			// Best-effort lookup; swallow errors. The hash query is cheap on
			// PhotoPrism's side (indexed column).
			resp, err := pp.call(c.Request.Context(), http.MethodGet,
				"/api/v1/photos?q=hash:"+h+"&count=1&merged=true", token, nil)
			if err == nil && resp.OK {
				var photos []dupListPhoto
				if err := json.Unmarshal(resp.Body, &photos); err == nil && len(photos) > 0 {
					if pf, ok := primaryFileOf(&ppPhoto{Files: photos[0].Files}); ok && pf.Name != "" {
						p := pf.Name
						g.IndexedPath = &p
					}
				}
			}
			groups = append(groups, g)
		}
		// Sort by reclaimable bytes descending (size × duplicate-count) so
		// the biggest wins float to the top of the UI.
		sort.Slice(groups, func(i, j int) bool {
			return groups[i].Size*int64(len(groups[i].Files)-1) >
				groups[j].Size*int64(len(groups[j].Files)-1)
		})

		ms := time.Since(start).Milliseconds()
		slog.Info("dup.scan done", "groups", len(groups), "ms", ms)
		c.JSON(http.StatusOK, gin.H{
			"groups":    groups,
			"scannedMs": ms,
		})
	}
}

type dupArchiveBody struct {
	Paths []string `json:"paths"`
}

type dupMoved struct {
	From string `json:"from"`
	To   string `json:"to"`
}

type dupArchiveErr struct {
	Path  string `json:"path"`
	Error string `json:"error"`
}

func handleDupArchive(cfg *Config, pp *ppClient, db *gorm.DB) gin.HandlerFunc {
	return func(c *gin.Context) {
		token := ctxToken(c)
		var body dupArchiveBody
		if err := c.ShouldBindJSON(&body); err != nil || len(body.Paths) == 0 {
			c.JSON(http.StatusBadRequest, gin.H{"error": "paths[] required"})
			return
		}

		// Authz: every path must live under the caller's effective library
		// root. The scan above already only ever returns paths from there,
		// but this endpoint takes paths straight from the request body, so a
		// scoped (non-admin, or admin-with-sub-path) user could otherwise
		// pass an arbitrary originals-relative path and archive (move) files
		// outside their own folder.
		root := effectiveLibraryRoot(c, db)
		if root != "" {
			for _, p := range body.Paths {
				clean := strings.Trim(p, "/")
				if clean != root && !strings.HasPrefix(clean, root+"/") {
					c.JSON(http.StatusForbidden, gin.H{"error": "path outside your library root"})
					return
				}
			}
		}

		// Each archive batch lands in its own timestamped subdir so the
		// user can browse what was quarantined when (and recover by hand
		// if they change their mind).
		stamp := time.Now().UTC().Format("2006-01-02T15-04-05.000Z")
		targetDir := filepath.Join(cfg.OriginalsRoot, quarantineDir, stamp)
		if err := os.MkdirAll(targetDir, 0o755); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		// archiveOne moves a single file into the quarantine batch dir
		// and returns the new relative path. Disambiguates same-basename
		// collisions within the batch so two `IMG_0001.jpg` from
		// different folders don't clobber each other.
		archiveOne := func(rel string) (string, error) {
			abs, err := resolveUnderRoot(cfg.OriginalsRoot, rel, true)
			if err != nil {
				return "", errors.New("invalid path")
			}
			base := filepath.Base(abs)
			dest := filepath.Join(targetDir, base)
			for i := 1; ; i++ {
				if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) {
					break
				} else if err != nil {
					return "", err
				}
				stem := base[:len(base)-len(filepath.Ext(base))]
				dest = filepath.Join(targetDir, stem+"__"+itoa(i)+filepath.Ext(base))
			}
			if err := os.Rename(abs, dest); err != nil {
				// EXDEV fallback — copy+remove for libraries that span
				// filesystems (e.g. originals on a different mount).
				if err2 := copyFile(abs, dest); err2 != nil {
					return "", err
				}
				if err2 := os.Remove(abs); err2 != nil {
					return "", errors.New("moved but source remove failed: " + err2.Error())
				}
			}
			relDest, _ := filepath.Rel(cfg.OriginalsRoot, dest)
			return relDest, nil
		}

		moved := []dupMoved{}
		errs := []dupArchiveErr{}
		for _, rel := range body.Paths {
			relDest, err := archiveOne(rel)
			if err != nil {
				errs = append(errs, dupArchiveErr{Path: rel, Error: err.Error()})
				continue
			}
			moved = append(moved, dupMoved{From: rel, To: relDest})
			slog.Info("dup.archive", "from", rel, "to", relDest)
		}

		// Reindex the entire library so PhotoPrism drops rows for the
		// archived files. cleanup:true is critical — the files still
		// exist on disk, just under .duplicates/ which the indexer
		// ignores.
		if len(moved) > 0 {
			go func() {
				if err := pp.reindex(context.Background(), token, "/"); err != nil {
					slog.Warn("dup.archive reindex failed", "err", err)
				}
			}()
		}
		c.JSON(http.StatusOK, gin.H{"moved": moved, "errors": errs})
	}
}