package main import ( "context" "encoding/json" "errors" "log/slog" "net/http" "os" "path/filepath" "sort" "strings" "sync" "time" "github.com/gin-gonic/gin" "gorm.io/gorm" ) const quarantineDir = ".duplicates" type dupFileLite struct { Path string `json:"path"` Size int64 `json:"size"` } type dupGroup struct { Hash string `json:"hash"` Size int64 `json:"size"` IndexedPath *string `json:"indexedPath"` Files []dupFileLite `json:"files"` } // dupListPhoto is the partial photo shape we pull from PhotoPrism when // looking up "which file path has this hash already indexed", used to // hint the UI which copy to keep. type dupListPhoto struct { Files []ppFile `json:"Files"` } func handleDupScan(cfg *Config, pp *ppClient, db *gorm.DB) gin.HandlerFunc { return func(c *gin.Context) { token := ctxToken(c) start := time.Now() // Scope the walk to the user's effective library root (BasePath + // chosen index sub-path), same as the folders/timeline/reindex scope — // otherwise a narrowed root would still surface every other user's // files in the cross-folder duplicate scan. "" means whole library // (today's admin-without-BasePath default). root := effectiveLibraryRoot(c, db) scanRoot := cfg.OriginalsRoot if root != "" { abs, err := resolveUnderRoot(cfg.OriginalsRoot, root, true) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "invalid library root"}) return } scanRoot = abs } slog.Info("dup.scan starting", "root", scanRoot) all, err := walkFiles(scanRoot) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } // walkFiles computes RelPath relative to scanRoot; re-prefix with the // scoped sub-path so RelPath stays originals-root-relative, matching // what handleDupArchive (and the rest of the API) expects. if root != "" { for i := range all { all[i].RelPath = root + "/" + all[i].RelPath } } // Group by size first: byte-identical files necessarily share size, // so size-collision is a cheap O(N) prefilter that lets us skip // hashing >95% of a typical library. bySize := map[int64][]fileEntry{} for _, f := range all { bySize[f.Size] = append(bySize[f.Size], f) } // Hash size-collision buckets concurrently. Cap fan-out to GOMAXPROCS // so we don't drown the disk with parallel reads on a spinning HDD. type hashOut struct { hash string f fileEntry } var ( wg sync.WaitGroup sem = make(chan struct{}, 4) outMu sync.Mutex byHash = map[string][]fileEntry{} hashSize = map[string]int64{} ) for size, group := range bySize { if len(group) < 2 { continue } for _, f := range group { wg.Add(1) sem <- struct{}{} go func(f fileEntry, sz int64) { defer wg.Done() defer func() { <-sem }() h, err := sha1File(f.AbsPath) if err != nil { return } outMu.Lock() byHash[h] = append(byHash[h], f) hashSize[h] = sz outMu.Unlock() }(f, size) } } wg.Wait() // Drop singletons (size collision but different hashes), then ask // PhotoPrism which of the duplicates it has indexed so the UI can // default the "keep" selection to that one. groups := make([]dupGroup, 0) for h, files := range byHash { if len(files) < 2 { continue } g := dupGroup{Hash: h, Size: hashSize[h]} for _, f := range files { g.Files = append(g.Files, dupFileLite{Path: f.RelPath, Size: f.Size}) } // Best-effort lookup; swallow errors. The hash query is cheap on // PhotoPrism's side (indexed column). resp, err := pp.call(c.Request.Context(), http.MethodGet, "/api/v1/photos?q=hash:"+h+"&count=1&merged=true", token, nil) if err == nil && resp.OK { var photos []dupListPhoto if err := json.Unmarshal(resp.Body, &photos); err == nil && len(photos) > 0 { if pf, ok := primaryFileOf(&ppPhoto{Files: photos[0].Files}); ok && pf.Name != "" { p := pf.Name g.IndexedPath = &p } } } groups = append(groups, g) } // Sort by reclaimable bytes descending (size × duplicate-count) so // the biggest wins float to the top of the UI. sort.Slice(groups, func(i, j int) bool { return groups[i].Size*int64(len(groups[i].Files)-1) > groups[j].Size*int64(len(groups[j].Files)-1) }) ms := time.Since(start).Milliseconds() slog.Info("dup.scan done", "groups", len(groups), "ms", ms) c.JSON(http.StatusOK, gin.H{ "groups": groups, "scannedMs": ms, }) } } type dupArchiveBody struct { Paths []string `json:"paths"` } type dupMoved struct { From string `json:"from"` To string `json:"to"` } type dupArchiveErr struct { Path string `json:"path"` Error string `json:"error"` } func handleDupArchive(cfg *Config, pp *ppClient, db *gorm.DB) gin.HandlerFunc { return func(c *gin.Context) { token := ctxToken(c) var body dupArchiveBody if err := c.ShouldBindJSON(&body); err != nil || len(body.Paths) == 0 { c.JSON(http.StatusBadRequest, gin.H{"error": "paths[] required"}) return } // Authz: every path must live under the caller's effective library // root. The scan above already only ever returns paths from there, // but this endpoint takes paths straight from the request body, so a // scoped (non-admin, or admin-with-sub-path) user could otherwise // pass an arbitrary originals-relative path and archive (move) files // outside their own folder. root := effectiveLibraryRoot(c, db) if root != "" { for _, p := range body.Paths { clean := strings.Trim(p, "/") if clean != root && !strings.HasPrefix(clean, root+"/") { c.JSON(http.StatusForbidden, gin.H{"error": "path outside your library root"}) return } } } // Each archive batch lands in its own timestamped subdir so the // user can browse what was quarantined when (and recover by hand // if they change their mind). stamp := time.Now().UTC().Format("2006-01-02T15-04-05.000Z") targetDir := filepath.Join(cfg.OriginalsRoot, quarantineDir, stamp) if err := os.MkdirAll(targetDir, 0o755); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } // archiveOne moves a single file into the quarantine batch dir // and returns the new relative path. Disambiguates same-basename // collisions within the batch so two `IMG_0001.jpg` from // different folders don't clobber each other. archiveOne := func(rel string) (string, error) { abs, err := resolveUnderRoot(cfg.OriginalsRoot, rel, true) if err != nil { return "", errors.New("invalid path") } base := filepath.Base(abs) dest := filepath.Join(targetDir, base) for i := 1; ; i++ { if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) { break } else if err != nil { return "", err } stem := base[:len(base)-len(filepath.Ext(base))] dest = filepath.Join(targetDir, stem+"__"+itoa(i)+filepath.Ext(base)) } if err := os.Rename(abs, dest); err != nil { // EXDEV fallback — copy+remove for libraries that span // filesystems (e.g. originals on a different mount). if err2 := copyFile(abs, dest); err2 != nil { return "", err } if err2 := os.Remove(abs); err2 != nil { return "", errors.New("moved but source remove failed: " + err2.Error()) } } relDest, _ := filepath.Rel(cfg.OriginalsRoot, dest) return relDest, nil } moved := []dupMoved{} errs := []dupArchiveErr{} for _, rel := range body.Paths { relDest, err := archiveOne(rel) if err != nil { errs = append(errs, dupArchiveErr{Path: rel, Error: err.Error()}) continue } moved = append(moved, dupMoved{From: rel, To: relDest}) slog.Info("dup.archive", "from", rel, "to", relDest) } // Reindex the entire library so PhotoPrism drops rows for the // archived files. cleanup:true is critical — the files still // exist on disk, just under .duplicates/ which the indexer // ignores. if len(moved) > 0 { go func() { if err := pp.reindex(context.Background(), token, "/"); err != nil { slog.Warn("dup.archive reindex failed", "err", err) } }() } c.JSON(http.StatusOK, gin.H{"moved": moved, "errors": errs}) } }