oikos/internal/httpapi/knowledge_drift.go

package httpapi

import (
	"encoding/json"
	"fmt"
	"log/slog"
	"net/http"
	"sort"
	"strconv"
	"strings"
)

// Drift tooling for the knowledge base — the maintenance half of the wiki.
//
// These endpoints exist because the knowledge base measurably rots on its
// own. Two failure modes are already present in live data:
//
//   - **Duplicate pileup.** upsert_knowledge keys on exact title, so a note
//     titled "rclone backup live inspection — 2026-07-15 10:08 UTC" and one
//     titled "... 11:18 UTC" are different notes. A single day of agent
//     activity produced eight near-identical investigations that should have
//     been one living page. Nothing surfaced that, so it kept happening.
//   - **Tag drift.** `oom` and `OOM` were separate tags; so were `422` and
//     `proton-422`. Each split halves the usefulness of tag navigation, and
//     neither is visible from any single note.
//
// normalizeTags (knowledge_write.go) stops new casing splits at the door;
// these endpoints clean up what's already there and make the rot visible.

// serveKnowledgeTags returns the tag index: every tag with its usage count,
// plus the distinct casings actually stored. `variants` is the interesting
// column — it's how the operator discovers that `oom` and `OOM` are the same
// idea filed twice, which no individual note reveals.
func (s *Server) serveKnowledgeTags(w http.ResponseWriter, req *http.Request) {
	ctx := req.Context()

	rows, err := s.pool.Query(ctx, `
		SELECT lower(tag) AS norm,
		       count(*) AS uses,
		       array_agg(DISTINCT tag ORDER BY tag) AS variants
		FROM knowledge_entities ke, unnest(ke.tags) AS tag
		WHERE ke.deleted_at IS NULL
		GROUP BY lower(tag)
		ORDER BY uses DESC, norm`)
	if err != nil {
		writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
		return
	}
	defer rows.Close()

	type tagRow struct {
		Tag      string   `json:"tag"`
		Uses     int      `json:"uses"`
		Variants []string `json:"variants"`
		// True when the same tag is stored under more than one casing —
		// the UI badges these as needing a normalize.
		Split bool `json:"split"`
	}
	items := []tagRow{}
	for rows.Next() {
		var t tagRow
		if err := rows.Scan(&t.Tag, &t.Uses, &t.Variants); err != nil {
			slog.Error("httpapi: knowledge/tags row scan failed", "error", err)
			continue
		}
		t.Split = len(t.Variants) > 1
		items = append(items, t)
	}

	writeJSON(w, map[string]any{"items": items})
}

// serveRenameKnowledgeTag rewrites one or more tags to a single target across
// every live note — the merge/rename/normalize action behind the tag manager.
// Passing several `from` values into one `to` is the merge case
// (`{"from":["422","proton-422"],"to":"proton-422"}`); passing one is a plain
// rename; passing the mixed-case variants is the normalize case.
func (s *Server) serveRenameKnowledgeTag(w http.ResponseWriter, req *http.Request) {
	ctx := req.Context()

	var body struct {
		From []string `json:"from"`
		To   string   `json:"to"`
	}
	if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
		writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error())
		return
	}
	to := strings.ToLower(strings.TrimSpace(body.To))
	from := []string{}
	for _, f := range body.From {
		if f = strings.TrimSpace(f); f != "" {
			from = append(from, f)
		}
	}
	if to == "" || len(from) == 0 {
		writeProblem(w, req, http.StatusBadRequest, "from and to are required", "")
		return
	}

	// Rebuild each affected note's tag array: map every `from` member to
	// `to`, leave everything else alone, then de-duplicate. The dedupe
	// matters for the merge case — a note tagged both `422` and
	// `proton-422` would otherwise end up with `proton-422` twice.
	//
	// This is a plain UPDATE on knowledge_entities, so trg_knowledge_revision
	// fires and every affected note gets a revision. A tag merge across 17
	// notes is exactly the kind of bulk edit worth being able to inspect
	// afterwards.
	tag, err := s.pool.Exec(ctx, `
		UPDATE knowledge_entities ke
		SET tags = sub.new_tags, updated_at = now()
		FROM (
			SELECT k.entity_id,
			       ARRAY(SELECT DISTINCT CASE WHEN lower(t) = ANY($1) THEN $2 ELSE t END
			             FROM unnest(k.tags) AS t) AS new_tags
			FROM knowledge_entities k
			WHERE k.deleted_at IS NULL
			  AND EXISTS (SELECT 1 FROM unnest(k.tags) AS t WHERE lower(t) = ANY($1))
		) AS sub
		WHERE ke.entity_id = sub.entity_id`,
		lowerAll(from), to)
	if err != nil {
		writeProblem(w, req, http.StatusInternalServerError, "rename failed", err.Error())
		return
	}

	_, actorLabel := actorInfo(ctx)
	slog.Info("knowledge tags renamed", "from", from, "to", to,
		"notes", tag.RowsAffected(), "actor", actorLabel)
	writeJSON(w, map[string]any{"ok": true, "notes_updated": tag.RowsAffected()})
}

// serveKnowledgeDuplicates clusters notes whose titles are near-identical.
//
// Pairwise trigram similarity is computed in SQL (indexed, and the whole
// point of pulling in pg_trgm); the grouping is done here in Go. Returning
// clusters rather than pairs matters for the real data: the rclone pileup
// produces dozens of pairs, which is unreadable, versus one cluster, which
// is the actionable unit.
//
// The grouping uses **complete linkage** — a note joins a cluster only if it
// is similar to every member already in it. The obvious implementation
// (union-find over the pairs) is single linkage, and on this data it chains
// badly: "A~B, B~C" merged notes that were not remotely alike, collapsing
// fifteen distinct backup events into one unusable blob. Requiring mutual
// similarity keeps clusters tight enough to act on.
//
// Even so, these are *candidates for review*, never a verdict. The five
// "Lifecycle: <verb> a node" runbooks are mutually similar by title and are
// five deliberately distinct documents — no threshold distinguishes them
// from a genuine duplicate, so merging stays a manual, previewed action.
func (s *Server) serveKnowledgeDuplicates(w http.ResponseWriter, req *http.Request) {
	ctx := req.Context()

	// 0.6, tuned against the live data: at 0.45 the "Lifecycle: <verb> a
	// node" runbooks (five deliberately distinct documents that happen to
	// share a naming template) formed a false-positive cluster; 0.6 clears
	// that down to a single borderline pair while keeping every genuine
	// duplicate cluster (the rclone/apt-audit/uptime pileups) intact.
	// Tunable per request — the UI exposes this as the review net widens.
	threshold := 0.6
	if t := req.URL.Query().Get("threshold"); t != "" {
		if v, err := strconv.ParseFloat(t, 64); err == nil && v > 0 && v <= 1 {
			threshold = v
		}
	}

	rows, err := s.pool.Query(ctx, `
		SELECT a.slug, b.slug, similarity(ka.title, kb.title) AS sim
		FROM knowledge_entities ka
		JOIN knowledge_entities kb ON ka.entity_id < kb.entity_id
		JOIN entities a ON a.id = ka.entity_id
		JOIN entities b ON b.id = kb.entity_id
		WHERE ka.deleted_at IS NULL AND kb.deleted_at IS NULL
		  AND similarity(ka.title, kb.title) > $1
		ORDER BY sim DESC`, threshold)
	if err != nil {
		writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
		return
	}
	defer rows.Close()

	type pair struct {
		A, B string
		Sim  float64
	}
	pairs := []pair{}
	for rows.Next() {
		var p pair
		if err := rows.Scan(&p.A, &p.B, &p.Sim); err != nil {
			slog.Error("httpapi: knowledge/duplicates row scan failed", "error", err)
			continue
		}
		pairs = append(pairs, p)
	}

	// Complete-linkage grouping. `pairs` arrives sorted by similarity
	// descending, so each new cluster is seeded from the strongest remaining
	// pair and then only grows with notes that are similar to *everything*
	// already inside it.
	sim := make(map[string]float64, len(pairs)*2)
	key := func(a, b string) string {
		if a > b {
			a, b = b, a
		}
		return a + "\x00" + b
	}
	for _, p := range pairs {
		sim[key(p.A, p.B)] = p.Sim
	}
	linked := func(a, b string) bool { return sim[key(a, b)] > 0 }

	assigned := map[string]bool{}
	type rawCluster struct {
		members []string
		top     float64
	}
	raw := []rawCluster{}

	for _, p := range pairs {
		if assigned[p.A] || assigned[p.B] {
			continue
		}
		c := rawCluster{members: []string{p.A, p.B}, top: p.Sim}
		assigned[p.A], assigned[p.B] = true, true

		// Sweep the remaining pairs for candidates that connect to every
		// current member. Repeat until a full pass adds nothing, since
		// admitting one member can qualify another.
		for grew := true; grew; {
			grew = false
			for _, q := range pairs {
				for _, cand := range []string{q.A, q.B} {
					if assigned[cand] {
						continue
					}
					ok := true
					for _, m := range c.members {
						if !linked(cand, m) {
							ok = false
							break
						}
					}
					if ok {
						c.members = append(c.members, cand)
						assigned[cand] = true
						grew = true
					}
				}
			}
		}
		raw = append(raw, c)
	}

	groups := map[string][]string{}
	best := map[string]float64{}
	for _, c := range raw {
		root := c.members[0]
		groups[root] = c.members
		best[root] = c.top
	}

	// Re-fetch display detail for the clustered slugs only.
	type member struct {
		Slug      string `json:"slug"`
		Title     string `json:"title"`
		Kind      string `json:"kind"`
		Size      int    `json:"size"`
		UpdatedAt string `json:"updated_at"`
		EditedBy  string `json:"edited_by"`
	}
	detail := map[string]member{}
	if len(groups) > 0 {
		all := []string{}
		for _, g := range groups {
			all = append(all, g...)
		}
		drows, derr := s.pool.Query(ctx, `
			SELECT e.slug, ke.title, e.type, length(ke.content),
			       ke.updated_at::text, COALESCE(ke.edited_by,'')
			FROM knowledge_entities ke
			JOIN entities e ON e.id = ke.entity_id
			WHERE e.slug = ANY($1) AND ke.deleted_at IS NULL`, all)
		if derr != nil {
			writeProblem(w, req, http.StatusInternalServerError, "detail query failed", derr.Error())
			return
		}
		defer drows.Close()
		for drows.Next() {
			var m member
			if err := drows.Scan(&m.Slug, &m.Title, &m.Kind, &m.Size, &m.UpdatedAt, &m.EditedBy); err != nil {
				slog.Error("httpapi: knowledge/duplicates detail scan failed", "error", err)
				continue
			}
			detail[m.Slug] = m
		}
	}

	type cluster struct {
		Members   []member `json:"members"`
		TopSim    float64  `json:"top_similarity"`
		TotalSize int      `json:"total_size"`
	}
	out := []cluster{}
	for root, slugs := range groups {
		c := cluster{TopSim: best[root]}
		for _, sl := range slugs {
			if m, ok := detail[sl]; ok {
				c.Members = append(c.Members, m)
				c.TotalSize += m.Size
			}
		}
		if len(c.Members) < 2 {
			continue
		}
		// Newest first inside a cluster — the most recent note is usually
		// the one worth keeping as the merge target.
		sort.Slice(c.Members, func(i, j int) bool {
			return c.Members[i].UpdatedAt > c.Members[j].UpdatedAt
		})
		out = append(out, c)
	}
	// Biggest clusters first: an eight-note pileup deserves attention before
	// a two-note coincidence.
	sort.Slice(out, func(i, j int) bool {
		if len(out[i].Members) != len(out[j].Members) {
			return len(out[i].Members) > len(out[j].Members)
		}
		return out[i].TopSim > out[j].TopSim
	})

	writeJSON(w, map[string]any{"clusters": out, "threshold": threshold})
}

// serveKnowledgeOrphans surfaces notes that have fallen out of every
// navigation path — the ones that are technically present but effectively
// unreachable, and so quietly stop being maintained.
//
// Three independent reasons, reported per note (a note can have several):
//   - untagged: invisible to tag navigation
//   - unlinked: not `about` any entity, so it never appears on a machine's page
//   - stale: untouched for 90+ days
func (s *Server) serveKnowledgeOrphans(w http.ResponseWriter, req *http.Request) {
	ctx := req.Context()

	staleDays := 90
	if d := req.URL.Query().Get("stale_days"); d != "" {
		if v, err := strconv.Atoi(d); err == nil && v > 0 && v <= 3650 {
			staleDays = v
		}
	}

	rows, err := s.pool.Query(ctx, fmt.Sprintf(`
		SELECT e.slug, ke.title, e.type, COALESCE(ke.edited_by,''),
		       ke.updated_at::text,
		       (ke.tags IS NULL OR cardinality(ke.tags) = 0) AS untagged,
		       NOT EXISTS (
		         SELECT 1 FROM relationships r
		         WHERE r.source_id = ke.entity_id AND r.valid_to IS NULL
		           AND r.type IN ('documents', 'about')
		       ) AS unlinked,
		       (ke.updated_at < now() - interval '%d days') AS stale
		FROM knowledge_entities ke
		JOIN entities e ON e.id = ke.entity_id
		WHERE ke.deleted_at IS NULL
		ORDER BY ke.updated_at ASC`, staleDays))
	if err != nil {
		writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
		return
	}
	defer rows.Close()

	type orphan struct {
		Slug      string   `json:"slug"`
		Title     string   `json:"title"`
		Kind      string   `json:"kind"`
		EditedBy  string   `json:"edited_by"`
		UpdatedAt string   `json:"updated_at"`
		Reasons   []string `json:"reasons"`
	}
	items := []orphan{}
	counts := map[string]int{"untagged": 0, "unlinked": 0, "stale": 0}
	for rows.Next() {
		var o orphan
		var untagged, unlinked, stale bool
		if err := rows.Scan(&o.Slug, &o.Title, &o.Kind, &o.EditedBy, &o.UpdatedAt,
			&untagged, &unlinked, &stale); err != nil {
			slog.Error("httpapi: knowledge/orphans row scan failed", "error", err)
			continue
		}
		o.Reasons = []string{}
		if untagged {
			o.Reasons = append(o.Reasons, "untagged")
			counts["untagged"]++
		}
		if unlinked {
			o.Reasons = append(o.Reasons, "unlinked")
			counts["unlinked"]++
		}
		if stale {
			o.Reasons = append(o.Reasons, "stale")
			counts["stale"]++
		}
		if len(o.Reasons) > 0 {
			items = append(items, o)
		}
	}

	writeJSON(w, map[string]any{
		"items":      items,
		"counts":     counts,
		"stale_days": staleDays,
	})
}

// serveMergeKnowledge folds several notes into one: each source's body is
// appended to the target under a provenance heading, the union of all tags is
// kept, and the sources are soft-deleted.
//
// Append rather than discard, and soft-delete rather than hard: a merge is a
// judgement call made from a similarity score, and the operator needs to be
// able to walk it back. The target's pre-merge state is captured by the
// revision trigger, so the merge itself is undoable from the History tab.
func (s *Server) serveMergeKnowledge(w http.ResponseWriter, req *http.Request) {
	ctx := req.Context()

	var body struct {
		Target  string   `json:"target"`
		Sources []string `json:"sources"`
	}
	if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
		writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error())
		return
	}
	if strings.TrimSpace(body.Target) == "" || len(body.Sources) == 0 {
		writeProblem(w, req, http.StatusBadRequest, "target and sources are required", "")
		return
	}

	targetID, err := s.resolveKnowledgeEntity(ctx, body.Target)
	if err != nil {
		writeProblem(w, req, http.StatusNotFound, "target note not found", body.Target)
		return
	}
	_, actorLabel := actorInfo(ctx)

	tx, err := s.pool.Begin(ctx)
	if err != nil {
		writeProblem(w, req, http.StatusInternalServerError, "begin failed", err.Error())
		return
	}
	defer tx.Rollback(ctx)

	var merged []string
	var appended strings.Builder
	tagSet := map[string]bool{}

	for _, srcSlug := range body.Sources {
		if srcSlug == body.Target {
			continue // merging a note into itself would duplicate its body
		}
		var srcTitle, srcContent, srcUpdated string
		var srcTags []string
		err := tx.QueryRow(ctx, `
			SELECT ke.title, ke.content, COALESCE(ke.tags,'{}'), ke.updated_at::text
			FROM knowledge_entities ke
			JOIN entities e ON e.id = ke.entity_id
			WHERE (e.slug = $1 OR e.id::text = $1) AND ke.deleted_at IS NULL`,
			srcSlug).Scan(&srcTitle, &srcContent, &srcTags, &srcUpdated)
		if err != nil {
			slog.Warn("knowledge merge: source not found, skipping", "slug", srcSlug)
			continue
		}
		appended.WriteString("\n\n---\n\n## Merged: ")
		appended.WriteString(srcTitle)
		appended.WriteString("\n\n*Originally ")
		appended.WriteString(srcSlug)
		appended.WriteString(", last updated ")
		appended.WriteString(srcUpdated)
		appended.WriteString("*\n\n")
		appended.WriteString(srcContent)
		for _, t := range srcTags {
			tagSet[strings.ToLower(strings.TrimSpace(t))] = true
		}
		merged = append(merged, srcSlug)
	}

	if len(merged) == 0 {
		writeProblem(w, req, http.StatusBadRequest, "no valid source notes to merge", "")
		return
	}

	extraTags := make([]string, 0, len(tagSet))
	for t := range tagSet {
		if t != "" {
			extraTags = append(extraTags, t)
		}
	}
	sort.Strings(extraTags)

	// The array concat + DISTINCT keeps the target's own tags first and adds
	// only what the sources contribute.
	if _, err := tx.Exec(ctx, `
		UPDATE knowledge_entities
		SET content   = content || $2,
		    tags      = ARRAY(SELECT DISTINCT unnest(COALESCE(tags,'{}') || $3::text[])),
		    edited_by = $4,
		    updated_at = now()
		WHERE entity_id = $1`,
		targetID, appended.String(), extraTags, actorLabel); err != nil {
		writeProblem(w, req, http.StatusInternalServerError, "merge write failed", err.Error())
		return
	}

	for _, srcSlug := range merged {
		if _, err := tx.Exec(ctx, `
			UPDATE knowledge_entities ke
			SET deleted_at = now(), edited_by = $2
			FROM entities e
			WHERE e.id = ke.entity_id AND (e.slug = $1 OR e.id::text = $1)`,
			srcSlug, actorLabel); err != nil {
			writeProblem(w, req, http.StatusInternalServerError, "source delete failed", err.Error())
			return
		}
	}

	if err := tx.Commit(ctx); err != nil {
		writeProblem(w, req, http.StatusInternalServerError, "commit failed", err.Error())
		return
	}

	slog.Info("knowledge merged", "target", body.Target, "sources", merged, "actor", actorLabel)
	writeJSON(w, map[string]any{"ok": true, "merged": merged, "tags_added": extraTags})
}

// lowerAll is the case-folding helper the tag queries compare against.
func lowerAll(in []string) []string {
	out := make([]string, len(in))
	for i, s := range in {
		out[i] = strings.ToLower(strings.TrimSpace(s))
	}
	return out
}