diff --git a/internal/httpapi/knowledge.go b/internal/httpapi/knowledge.go index c990609..aea8e5a 100644 --- a/internal/httpapi/knowledge.go +++ b/internal/httpapi/knowledge.go @@ -50,6 +50,7 @@ func (s *Server) serveRecentKnowledge(w http.ResponseWriter, req *http.Request) FROM knowledge_entities ke JOIN entities e ON e.id = ke.entity_id WHERE ($1 = '' OR ke.source = $1) + AND ke.deleted_at IS NULL ORDER BY ke.updated_at DESC LIMIT $2`, source, limit) if err != nil { @@ -82,6 +83,7 @@ func (s *Server) serveRecentKnowledge(w http.ResponseWriter, req *http.Request) COUNT(*) FILTER (WHERE ke.source = 'nomos-agent'), COUNT(*) FILTER (WHERE ke.updated_at > now() - interval '7 days') FROM knowledge_entities ke JOIN entities e ON e.id = ke.entity_id + WHERE ke.deleted_at IS NULL GROUP BY e.type`) if err == nil { defer srows.Close() @@ -128,15 +130,19 @@ func (s *Server) serveKnowledgeContent(w http.ResponseWriter, req *http.Request) return } - var title, content, source string + var title, content, source, editedBy string var tags []string var updatedAt string + var revisions int err = s.pool.QueryRow(ctx, ` - SELECT ke.title, ke.content, COALESCE(ke.source,''), ke.tags, ke.updated_at::text + SELECT ke.title, ke.content, COALESCE(ke.source,''), COALESCE(ke.edited_by,''), + ke.tags, ke.updated_at::text, + (SELECT count(*) FROM knowledge_revisions kr WHERE kr.entity_id = ke.entity_id) FROM knowledge_entities ke JOIN entities e ON e.id = ke.entity_id - WHERE e.slug = $1 OR e.id::text = $1`, idOrSlug). - Scan(&title, &content, &source, &tags, &updatedAt) + WHERE (e.slug = $1 OR e.id::text = $1) + AND ke.deleted_at IS NULL`, idOrSlug). + Scan(&title, &content, &source, &editedBy, &tags, &updatedAt, &revisions) if err != nil { writeProblem(w, req, http.StatusNotFound, "no knowledge content for entity", "") return @@ -150,8 +156,10 @@ func (s *Server) serveKnowledgeContent(w http.ResponseWriter, req *http.Request) "title": title, "content": content, "source": source, + "edited_by": editedBy, "tags": tags, "updated_at": updatedAt, + "revisions": revisions, }) } @@ -169,6 +177,7 @@ func (s *Server) SearchKnowledge(ctx context.Context, request gen.SearchKnowledg JOIN entities e ON e.id = ke.entity_id JOIN entity_types et ON et.name = e.type WHERE ke.search @@ plainto_tsquery('english', $1) + AND ke.deleted_at IS NULL ORDER BY rank DESC LIMIT $2`, q, limit) @@ -232,6 +241,7 @@ func (s *Server) GetEntityKnowledge(ctx context.Context, request gen.GetEntityKn WHERE target.slug = $1 AND r.valid_to IS NULL AND r.type IN ('documents', 'about') + AND ke.deleted_at IS NULL UNION SELECT e.id, e.slug, COALESCE(et.name,''), ke.title, ke.source, ke.tags FROM knowledge_entities ke @@ -242,6 +252,7 @@ func (s *Server) GetEntityKnowledge(ctx context.Context, request gen.GetEntityKn JOIN entities ent ON ent.type = target_type.name AND ent.slug = $1 WHERE r.valid_to IS NULL AND r.type = 'procedure-for' + AND ke.deleted_at IS NULL ORDER BY 2`, entitySlug) if err != nil { diff --git a/internal/httpapi/knowledge_drift.go b/internal/httpapi/knowledge_drift.go new file mode 100644 index 0000000..8b4086e --- /dev/null +++ b/internal/httpapi/knowledge_drift.go @@ -0,0 +1,544 @@ +package httpapi + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "sort" + "strconv" + "strings" +) + +// Drift tooling for the knowledge base — the maintenance half of the wiki. +// +// These endpoints exist because the knowledge base measurably rots on its +// own. Two failure modes are already present in live data: +// +// - **Duplicate pileup.** upsert_knowledge keys on exact title, so a note +// titled "rclone backup live inspection — 2026-07-15 10:08 UTC" and one +// titled "... 11:18 UTC" are different notes. A single day of agent +// activity produced eight near-identical investigations that should have +// been one living page. Nothing surfaced that, so it kept happening. +// - **Tag drift.** `oom` and `OOM` were separate tags; so were `422` and +// `proton-422`. Each split halves the usefulness of tag navigation, and +// neither is visible from any single note. +// +// normalizeTags (knowledge_write.go) stops new casing splits at the door; +// these endpoints clean up what's already there and make the rot visible. + +// serveKnowledgeTags returns the tag index: every tag with its usage count, +// plus the distinct casings actually stored. `variants` is the interesting +// column — it's how the operator discovers that `oom` and `OOM` are the same +// idea filed twice, which no individual note reveals. +func (s *Server) serveKnowledgeTags(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + rows, err := s.pool.Query(ctx, ` + SELECT lower(tag) AS norm, + count(*) AS uses, + array_agg(DISTINCT tag ORDER BY tag) AS variants + FROM knowledge_entities ke, unnest(ke.tags) AS tag + WHERE ke.deleted_at IS NULL + GROUP BY lower(tag) + ORDER BY uses DESC, norm`) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error()) + return + } + defer rows.Close() + + type tagRow struct { + Tag string `json:"tag"` + Uses int `json:"uses"` + Variants []string `json:"variants"` + // True when the same tag is stored under more than one casing — + // the UI badges these as needing a normalize. + Split bool `json:"split"` + } + items := []tagRow{} + for rows.Next() { + var t tagRow + if err := rows.Scan(&t.Tag, &t.Uses, &t.Variants); err != nil { + slog.Error("httpapi: knowledge/tags row scan failed", "error", err) + continue + } + t.Split = len(t.Variants) > 1 + items = append(items, t) + } + + writeJSON(w, map[string]any{"items": items}) +} + +// serveRenameKnowledgeTag rewrites one or more tags to a single target across +// every live note — the merge/rename/normalize action behind the tag manager. +// Passing several `from` values into one `to` is the merge case +// (`{"from":["422","proton-422"],"to":"proton-422"}`); passing one is a plain +// rename; passing the mixed-case variants is the normalize case. +func (s *Server) serveRenameKnowledgeTag(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + var body struct { + From []string `json:"from"` + To string `json:"to"` + } + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { + writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error()) + return + } + to := strings.ToLower(strings.TrimSpace(body.To)) + from := []string{} + for _, f := range body.From { + if f = strings.TrimSpace(f); f != "" { + from = append(from, f) + } + } + if to == "" || len(from) == 0 { + writeProblem(w, req, http.StatusBadRequest, "from and to are required", "") + return + } + + // Rebuild each affected note's tag array: map every `from` member to + // `to`, leave everything else alone, then de-duplicate. The dedupe + // matters for the merge case — a note tagged both `422` and + // `proton-422` would otherwise end up with `proton-422` twice. + // + // This is a plain UPDATE on knowledge_entities, so trg_knowledge_revision + // fires and every affected note gets a revision. A tag merge across 17 + // notes is exactly the kind of bulk edit worth being able to inspect + // afterwards. + tag, err := s.pool.Exec(ctx, ` + UPDATE knowledge_entities ke + SET tags = sub.new_tags, updated_at = now() + FROM ( + SELECT k.entity_id, + ARRAY(SELECT DISTINCT CASE WHEN lower(t) = ANY($1) THEN $2 ELSE t END + FROM unnest(k.tags) AS t) AS new_tags + FROM knowledge_entities k + WHERE k.deleted_at IS NULL + AND EXISTS (SELECT 1 FROM unnest(k.tags) AS t WHERE lower(t) = ANY($1)) + ) AS sub + WHERE ke.entity_id = sub.entity_id`, + lowerAll(from), to) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "rename failed", err.Error()) + return + } + + _, actorLabel := actorInfo(ctx) + slog.Info("knowledge tags renamed", "from", from, "to", to, + "notes", tag.RowsAffected(), "actor", actorLabel) + writeJSON(w, map[string]any{"ok": true, "notes_updated": tag.RowsAffected()}) +} + +// serveKnowledgeDuplicates clusters notes whose titles are near-identical. +// +// Pairwise trigram similarity is computed in SQL (indexed, and the whole +// point of pulling in pg_trgm); the grouping is done here in Go. Returning +// clusters rather than pairs matters for the real data: the rclone pileup +// produces dozens of pairs, which is unreadable, versus one cluster, which +// is the actionable unit. +// +// The grouping uses **complete linkage** — a note joins a cluster only if it +// is similar to every member already in it. The obvious implementation +// (union-find over the pairs) is single linkage, and on this data it chains +// badly: "A~B, B~C" merged notes that were not remotely alike, collapsing +// fifteen distinct backup events into one unusable blob. Requiring mutual +// similarity keeps clusters tight enough to act on. +// +// Even so, these are *candidates for review*, never a verdict. The five +// "Lifecycle: a node" runbooks are mutually similar by title and are +// five deliberately distinct documents — no threshold distinguishes them +// from a genuine duplicate, so merging stays a manual, previewed action. +func (s *Server) serveKnowledgeDuplicates(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + // 0.6, tuned against the live data: at 0.45 the "Lifecycle: a + // node" runbooks (five deliberately distinct documents that happen to + // share a naming template) formed a false-positive cluster; 0.6 clears + // that down to a single borderline pair while keeping every genuine + // duplicate cluster (the rclone/apt-audit/uptime pileups) intact. + // Tunable per request — the UI exposes this as the review net widens. + threshold := 0.6 + if t := req.URL.Query().Get("threshold"); t != "" { + if v, err := strconv.ParseFloat(t, 64); err == nil && v > 0 && v <= 1 { + threshold = v + } + } + + rows, err := s.pool.Query(ctx, ` + SELECT a.slug, b.slug, similarity(ka.title, kb.title) AS sim + FROM knowledge_entities ka + JOIN knowledge_entities kb ON ka.entity_id < kb.entity_id + JOIN entities a ON a.id = ka.entity_id + JOIN entities b ON b.id = kb.entity_id + WHERE ka.deleted_at IS NULL AND kb.deleted_at IS NULL + AND similarity(ka.title, kb.title) > $1 + ORDER BY sim DESC`, threshold) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error()) + return + } + defer rows.Close() + + type pair struct { + A, B string + Sim float64 + } + pairs := []pair{} + for rows.Next() { + var p pair + if err := rows.Scan(&p.A, &p.B, &p.Sim); err != nil { + slog.Error("httpapi: knowledge/duplicates row scan failed", "error", err) + continue + } + pairs = append(pairs, p) + } + + // Complete-linkage grouping. `pairs` arrives sorted by similarity + // descending, so each new cluster is seeded from the strongest remaining + // pair and then only grows with notes that are similar to *everything* + // already inside it. + sim := make(map[string]float64, len(pairs)*2) + key := func(a, b string) string { + if a > b { + a, b = b, a + } + return a + "\x00" + b + } + for _, p := range pairs { + sim[key(p.A, p.B)] = p.Sim + } + linked := func(a, b string) bool { return sim[key(a, b)] > 0 } + + assigned := map[string]bool{} + type rawCluster struct { + members []string + top float64 + } + raw := []rawCluster{} + + for _, p := range pairs { + if assigned[p.A] || assigned[p.B] { + continue + } + c := rawCluster{members: []string{p.A, p.B}, top: p.Sim} + assigned[p.A], assigned[p.B] = true, true + + // Sweep the remaining pairs for candidates that connect to every + // current member. Repeat until a full pass adds nothing, since + // admitting one member can qualify another. + for grew := true; grew; { + grew = false + for _, q := range pairs { + for _, cand := range []string{q.A, q.B} { + if assigned[cand] { + continue + } + ok := true + for _, m := range c.members { + if !linked(cand, m) { + ok = false + break + } + } + if ok { + c.members = append(c.members, cand) + assigned[cand] = true + grew = true + } + } + } + } + raw = append(raw, c) + } + + groups := map[string][]string{} + best := map[string]float64{} + for _, c := range raw { + root := c.members[0] + groups[root] = c.members + best[root] = c.top + } + + // Re-fetch display detail for the clustered slugs only. + type member struct { + Slug string `json:"slug"` + Title string `json:"title"` + Kind string `json:"kind"` + Size int `json:"size"` + UpdatedAt string `json:"updated_at"` + EditedBy string `json:"edited_by"` + } + detail := map[string]member{} + if len(groups) > 0 { + all := []string{} + for _, g := range groups { + all = append(all, g...) + } + drows, derr := s.pool.Query(ctx, ` + SELECT e.slug, ke.title, e.type, length(ke.content), + ke.updated_at::text, COALESCE(ke.edited_by,'') + FROM knowledge_entities ke + JOIN entities e ON e.id = ke.entity_id + WHERE e.slug = ANY($1) AND ke.deleted_at IS NULL`, all) + if derr != nil { + writeProblem(w, req, http.StatusInternalServerError, "detail query failed", derr.Error()) + return + } + defer drows.Close() + for drows.Next() { + var m member + if err := drows.Scan(&m.Slug, &m.Title, &m.Kind, &m.Size, &m.UpdatedAt, &m.EditedBy); err != nil { + slog.Error("httpapi: knowledge/duplicates detail scan failed", "error", err) + continue + } + detail[m.Slug] = m + } + } + + type cluster struct { + Members []member `json:"members"` + TopSim float64 `json:"top_similarity"` + TotalSize int `json:"total_size"` + } + out := []cluster{} + for root, slugs := range groups { + c := cluster{TopSim: best[root]} + for _, sl := range slugs { + if m, ok := detail[sl]; ok { + c.Members = append(c.Members, m) + c.TotalSize += m.Size + } + } + if len(c.Members) < 2 { + continue + } + // Newest first inside a cluster — the most recent note is usually + // the one worth keeping as the merge target. + sort.Slice(c.Members, func(i, j int) bool { + return c.Members[i].UpdatedAt > c.Members[j].UpdatedAt + }) + out = append(out, c) + } + // Biggest clusters first: an eight-note pileup deserves attention before + // a two-note coincidence. + sort.Slice(out, func(i, j int) bool { + if len(out[i].Members) != len(out[j].Members) { + return len(out[i].Members) > len(out[j].Members) + } + return out[i].TopSim > out[j].TopSim + }) + + writeJSON(w, map[string]any{"clusters": out, "threshold": threshold}) +} + +// serveKnowledgeOrphans surfaces notes that have fallen out of every +// navigation path — the ones that are technically present but effectively +// unreachable, and so quietly stop being maintained. +// +// Three independent reasons, reported per note (a note can have several): +// - untagged: invisible to tag navigation +// - unlinked: not `about` any entity, so it never appears on a machine's page +// - stale: untouched for 90+ days +func (s *Server) serveKnowledgeOrphans(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + staleDays := 90 + if d := req.URL.Query().Get("stale_days"); d != "" { + if v, err := strconv.Atoi(d); err == nil && v > 0 && v <= 3650 { + staleDays = v + } + } + + rows, err := s.pool.Query(ctx, fmt.Sprintf(` + SELECT e.slug, ke.title, e.type, COALESCE(ke.edited_by,''), + ke.updated_at::text, + (ke.tags IS NULL OR cardinality(ke.tags) = 0) AS untagged, + NOT EXISTS ( + SELECT 1 FROM relationships r + WHERE r.source_id = ke.entity_id AND r.valid_to IS NULL + AND r.type IN ('documents', 'about') + ) AS unlinked, + (ke.updated_at < now() - interval '%d days') AS stale + FROM knowledge_entities ke + JOIN entities e ON e.id = ke.entity_id + WHERE ke.deleted_at IS NULL + ORDER BY ke.updated_at ASC`, staleDays)) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error()) + return + } + defer rows.Close() + + type orphan struct { + Slug string `json:"slug"` + Title string `json:"title"` + Kind string `json:"kind"` + EditedBy string `json:"edited_by"` + UpdatedAt string `json:"updated_at"` + Reasons []string `json:"reasons"` + } + items := []orphan{} + counts := map[string]int{"untagged": 0, "unlinked": 0, "stale": 0} + for rows.Next() { + var o orphan + var untagged, unlinked, stale bool + if err := rows.Scan(&o.Slug, &o.Title, &o.Kind, &o.EditedBy, &o.UpdatedAt, + &untagged, &unlinked, &stale); err != nil { + slog.Error("httpapi: knowledge/orphans row scan failed", "error", err) + continue + } + o.Reasons = []string{} + if untagged { + o.Reasons = append(o.Reasons, "untagged") + counts["untagged"]++ + } + if unlinked { + o.Reasons = append(o.Reasons, "unlinked") + counts["unlinked"]++ + } + if stale { + o.Reasons = append(o.Reasons, "stale") + counts["stale"]++ + } + if len(o.Reasons) > 0 { + items = append(items, o) + } + } + + writeJSON(w, map[string]any{ + "items": items, + "counts": counts, + "stale_days": staleDays, + }) +} + +// serveMergeKnowledge folds several notes into one: each source's body is +// appended to the target under a provenance heading, the union of all tags is +// kept, and the sources are soft-deleted. +// +// Append rather than discard, and soft-delete rather than hard: a merge is a +// judgement call made from a similarity score, and the operator needs to be +// able to walk it back. The target's pre-merge state is captured by the +// revision trigger, so the merge itself is undoable from the History tab. +func (s *Server) serveMergeKnowledge(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + var body struct { + Target string `json:"target"` + Sources []string `json:"sources"` + } + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { + writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error()) + return + } + if strings.TrimSpace(body.Target) == "" || len(body.Sources) == 0 { + writeProblem(w, req, http.StatusBadRequest, "target and sources are required", "") + return + } + + targetID, err := s.resolveKnowledgeEntity(ctx, body.Target) + if err != nil { + writeProblem(w, req, http.StatusNotFound, "target note not found", body.Target) + return + } + _, actorLabel := actorInfo(ctx) + + tx, err := s.pool.Begin(ctx) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "begin failed", err.Error()) + return + } + defer tx.Rollback(ctx) + + var merged []string + var appended strings.Builder + tagSet := map[string]bool{} + + for _, srcSlug := range body.Sources { + if srcSlug == body.Target { + continue // merging a note into itself would duplicate its body + } + var srcTitle, srcContent, srcUpdated string + var srcTags []string + err := tx.QueryRow(ctx, ` + SELECT ke.title, ke.content, COALESCE(ke.tags,'{}'), ke.updated_at::text + FROM knowledge_entities ke + JOIN entities e ON e.id = ke.entity_id + WHERE (e.slug = $1 OR e.id::text = $1) AND ke.deleted_at IS NULL`, + srcSlug).Scan(&srcTitle, &srcContent, &srcTags, &srcUpdated) + if err != nil { + slog.Warn("knowledge merge: source not found, skipping", "slug", srcSlug) + continue + } + appended.WriteString("\n\n---\n\n## Merged: ") + appended.WriteString(srcTitle) + appended.WriteString("\n\n*Originally ") + appended.WriteString(srcSlug) + appended.WriteString(", last updated ") + appended.WriteString(srcUpdated) + appended.WriteString("*\n\n") + appended.WriteString(srcContent) + for _, t := range srcTags { + tagSet[strings.ToLower(strings.TrimSpace(t))] = true + } + merged = append(merged, srcSlug) + } + + if len(merged) == 0 { + writeProblem(w, req, http.StatusBadRequest, "no valid source notes to merge", "") + return + } + + extraTags := make([]string, 0, len(tagSet)) + for t := range tagSet { + if t != "" { + extraTags = append(extraTags, t) + } + } + sort.Strings(extraTags) + + // The array concat + DISTINCT keeps the target's own tags first and adds + // only what the sources contribute. + if _, err := tx.Exec(ctx, ` + UPDATE knowledge_entities + SET content = content || $2, + tags = ARRAY(SELECT DISTINCT unnest(COALESCE(tags,'{}') || $3::text[])), + edited_by = $4, + updated_at = now() + WHERE entity_id = $1`, + targetID, appended.String(), extraTags, actorLabel); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "merge write failed", err.Error()) + return + } + + for _, srcSlug := range merged { + if _, err := tx.Exec(ctx, ` + UPDATE knowledge_entities ke + SET deleted_at = now(), edited_by = $2 + FROM entities e + WHERE e.id = ke.entity_id AND (e.slug = $1 OR e.id::text = $1)`, + srcSlug, actorLabel); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "source delete failed", err.Error()) + return + } + } + + if err := tx.Commit(ctx); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "commit failed", err.Error()) + return + } + + slog.Info("knowledge merged", "target", body.Target, "sources", merged, "actor", actorLabel) + writeJSON(w, map[string]any{"ok": true, "merged": merged, "tags_added": extraTags}) +} + +// lowerAll is the case-folding helper the tag queries compare against. +func lowerAll(in []string) []string { + out := make([]string, len(in)) + for i, s := range in { + out[i] = strings.ToLower(strings.TrimSpace(s)) + } + return out +} diff --git a/internal/httpapi/knowledge_write.go b/internal/httpapi/knowledge_write.go new file mode 100644 index 0000000..72f45d2 --- /dev/null +++ b/internal/httpapi/knowledge_write.go @@ -0,0 +1,659 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/go-chi/chi/v5" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// Operator-facing write path for the knowledge base. Until this file, the +// only way anything reached knowledge_entities was the MCP tool +// upsert_knowledge (internal/mcp/server.go) — an agent-only surface. The web +// UI could search and read but never create, correct, or remove a note, so +// the operator's own knowledge had nowhere to go and an agent mistake had no +// fix short of psql. +// +// All routes here are non-OpenAPI custom routes, consistent with the existing +// knowledge read routes (see the carve-out block in server.go): they trade in +// raw markdown and ad-hoc aggregates rather than generated schema types. +// +// Deletion is soft (deleted_at) — see migrations/022_knowledge_revisions.up.sql +// for why — so every read path in this file filters on `ke.deleted_at IS NULL`. + +// knowledgeSlugSegmentRe strips a title down to a single slug segment. +// Mirrors knowledgeSlugRe in internal/mcp/server.go; duplicated rather than +// exported across the package boundary because the two callers namespace +// their output differently (see knowledgeSlugFor). +var knowledgeSlugSegmentRe = regexp.MustCompile(`[^a-z0-9]+`) + +// knowledgeSlugFor builds `:/`. The MCP tool's +// equivalent hardcodes the `nomos/` folder; operator-created notes need to +// land somewhere else so the navigator tree can tell at a glance who wrote +// what, and so an operator note can never collide with an agent note that +// happens to share a title. +func knowledgeSlugFor(kind, folder, title string) string { + s := strings.ToLower(strings.TrimSpace(title)) + s = knowledgeSlugSegmentRe.ReplaceAllString(s, "-") + s = strings.Trim(s, "-") + if s == "" { + s = "note" + } + if len(s) > 80 { + s = s[:80] + } + folder = strings.Trim(strings.ToLower(strings.TrimSpace(folder)), "/") + folder = knowledgeSlugSegmentRe.ReplaceAllString(folder, "-") + folder = strings.Trim(folder, "-") + if folder == "" { + folder = "operator" + } + return kind + ":" + folder + "/" + s +} + +// validKnowledgeKind mirrors the three entity types that knowledge_entities +// rows are allowed to hang off (see upsert_knowledge's own check). +func validKnowledgeKind(kind string) bool { + switch kind { + case "document", "investigation", "runbook": + return true + } + return false +} + +// resolveKnowledgeEntity maps an id-or-slug path segment to the entity id of +// a live (non-deleted) knowledge note. Returns pgx.ErrNoRows when there's no +// such note, which callers turn into a 404. +func (s *Server) resolveKnowledgeEntity(ctx context.Context, idOrSlug string) (uuid.UUID, error) { + var id uuid.UUID + err := s.pool.QueryRow(ctx, ` + SELECT ke.entity_id + FROM knowledge_entities ke + JOIN entities e ON e.id = ke.entity_id + WHERE (e.slug = $1 OR e.id::text = $1) + AND ke.deleted_at IS NULL`, idOrSlug).Scan(&id) + return id, err +} + +// resolveKnowledgeEntityAny is resolveKnowledgeEntity without the +// deleted_at filter — for the one read path (revisions) that must still work +// on a deleted note. The whole point of soft-delete is that a note's history +// stays inspectable after removal (e.g. to confirm what was lost before +// restoring it); requiring the note to be live first would defeat that. +func (s *Server) resolveKnowledgeEntityAny(ctx context.Context, idOrSlug string) (uuid.UUID, error) { + var id uuid.UUID + err := s.pool.QueryRow(ctx, ` + SELECT ke.entity_id + FROM knowledge_entities ke + JOIN entities e ON e.id = ke.entity_id + WHERE e.slug = $1 OR e.id::text = $1`, idOrSlug).Scan(&id) + return id, err +} + +// pathParam pulls a chi URL param and percent-decodes it. Knowledge slugs +// contain both ':' and '/' (e.g. "document:containers/101-jellyfin"), so they +// reach the handler still encoded — chi.URLParam does no decoding of its own +// on manually-registered routes (unlike the OpenAPI-generated ones, which +// decode via runtime.BindStyledParameterWithOptions). +func pathParam(req *http.Request, name string) (string, error) { + return url.PathUnescape(chi.URLParam(req, name)) +} + +// serveKnowledgeList returns every live note without its body — the backing +// data for the wiki navigator tree. Distinct from /knowledge/recent, which +// caps at 200 and exists to answer "what changed lately" for the stats view: +// the tree needs the complete set, and needs the linked-entity slugs so it +// can offer a group-by-entity arrangement without N+1 fetches. +// +// Body text is deliberately excluded — with ~100 notes averaging ~1 KB the +// full payload would be ~100 KB per app open, to render a list that shows +// only titles. +func (s *Server) serveKnowledgeList(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + type item struct { + ID string `json:"id"` + Slug string `json:"slug"` + Title string `json:"title"` + Kind string `json:"kind"` + Source string `json:"source"` + EditedBy string `json:"edited_by"` + Tags []string `json:"tags"` + About []string `json:"about"` + Size int `json:"size"` + UpdatedAt string `json:"updated_at"` + CreatedAt string `json:"created_at"` + Revisions int `json:"revisions"` + } + + // The `about` aggregate mirrors GetEntityKnowledge's first UNION branch + // (documents/about edges) — the 'procedure-for' branch is left out here + // because it joins against entity *types* rather than entities and can't + // produce a per-note slug list. + rows, err := s.pool.Query(ctx, ` + SELECT e.id::text, e.slug, ke.title, e.type, COALESCE(ke.source,''), + COALESCE(ke.edited_by,''), COALESCE(ke.tags, '{}'), + COALESCE(( + SELECT array_agg(DISTINCT t.slug) + FROM relationships r + JOIN entities t ON t.id = r.target_id + WHERE r.source_id = ke.entity_id + AND r.valid_to IS NULL + AND r.type IN ('documents', 'about') + ), '{}'), + length(ke.content), + ke.updated_at::text, ke.created_at::text, + (SELECT count(*) FROM knowledge_revisions kr WHERE kr.entity_id = ke.entity_id) + FROM knowledge_entities ke + JOIN entities e ON e.id = ke.entity_id + WHERE ke.deleted_at IS NULL + ORDER BY ke.updated_at DESC`) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error()) + return + } + defer rows.Close() + + items := []item{} + for rows.Next() { + var it item + if err := rows.Scan(&it.ID, &it.Slug, &it.Title, &it.Kind, &it.Source, + &it.EditedBy, &it.Tags, &it.About, &it.Size, + &it.UpdatedAt, &it.CreatedAt, &it.Revisions); err != nil { + slog.Error("httpapi: knowledge/list row scan failed", "error", err) + continue + } + items = append(items, it) + } + + writeJSON(w, map[string]any{"items": items}) +} + +// serveKnowledgeTrash lists soft-deleted notes — the counterpart to +// serveKnowledgeList, and what the "restore" affordance in the UI browses. +// Without this, a deleted note is invisible from every list endpoint +// (correctly — they all filter deleted_at) with no way to even discover it +// exists to restore. +func (s *Server) serveKnowledgeTrash(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + rows, err := s.pool.Query(ctx, ` + SELECT e.slug, ke.title, e.type, COALESCE(ke.edited_by,''), ke.deleted_at::text + FROM knowledge_entities ke + JOIN entities e ON e.id = ke.entity_id + WHERE ke.deleted_at IS NOT NULL + ORDER BY ke.deleted_at DESC`) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error()) + return + } + defer rows.Close() + + type item struct { + Slug string `json:"slug"` + Title string `json:"title"` + Kind string `json:"kind"` + DeletedBy string `json:"deleted_by"` + DeletedAt string `json:"deleted_at"` + } + items := []item{} + for rows.Next() { + var it item + if err := rows.Scan(&it.Slug, &it.Title, &it.Kind, &it.DeletedBy, &it.DeletedAt); err != nil { + slog.Error("httpapi: knowledge/trash row scan failed", "error", err) + continue + } + items = append(items, it) + } + + writeJSON(w, map[string]any{"items": items}) +} + +// knowledgeWriteBody is the shared request shape for create and update. +// Every field is a pointer so update can distinguish "not supplied" (leave +// alone) from "supplied empty" (clear it) — a PUT that only changes tags +// must not blank the body. +type knowledgeWriteBody struct { + Title *string `json:"title"` + Content *string `json:"content"` + Kind *string `json:"kind"` + Tags *[]string `json:"tags"` + Folder *string `json:"folder"` + About *[]string `json:"about"` +} + +// serveCreateKnowledge creates a note plus its backing entity, and links it +// to whatever entities it's about. +func (s *Server) serveCreateKnowledge(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + var body knowledgeWriteBody + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { + writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error()) + return + } + + title := strings.TrimSpace(deref(body.Title)) + content := strings.TrimSpace(deref(body.Content)) + if title == "" || content == "" { + writeProblem(w, req, http.StatusBadRequest, "title and content are required", "") + return + } + kind := deref(body.Kind) + if kind == "" { + kind = "document" + } + if !validKnowledgeKind(kind) { + writeProblem(w, req, http.StatusBadRequest, "invalid kind", + "kind must be document, investigation, or runbook") + return + } + tags := normalizeTags(derefSlice(body.Tags)) + slug := knowledgeSlugFor(kind, deref(body.Folder), title) + _, actorLabel := actorInfo(ctx) + + tx, err := s.pool.Begin(ctx) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "begin failed", err.Error()) + return + } + defer tx.Rollback(ctx) + + docID, _ := uuid.NewV7() + // ON CONFLICT covers the soft-deleted case: the entity row survives a + // delete, so recreating a note under the same slug must reuse it rather + // than fail the unique constraint. + if err := tx.QueryRow(ctx, ` + INSERT INTO entities (id, slug, type, name, attributes) + VALUES ($1, $2, $3, $4, '{}') + ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, updated_at = now() + RETURNING id`, docID, slug, kind, title).Scan(&docID); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "create entity failed", err.Error()) + return + } + + // Refuse to silently overwrite an existing LIVE note — upsert_knowledge + // (the MCP tool) deliberately upserts by title (the agent re-records the + // same finding as it learns more), but an operator hitting "create" with + // a colliding title almost certainly means to write something new. + // + // The `WHERE knowledge_entities.deleted_at IS NOT NULL` guard makes this + // check atomic with the write, rather than a separate SELECT before it: + // a plain pre-check has a TOCTOU race where two concurrent creates of + // the same title can both pass the check and then both proceed to + // INSERT ON CONFLICT DO UPDATE, silently clobbering each other. Here, + // the UPDATE branch only actually applies when the conflicting row is + // soft-deleted (a legitimate "resurrect" case). When it isn't, the row + // is left untouched, RETURNING yields no row, and pgx.ErrNoRows below + // becomes the 409 — the collision can never be missed, no matter how + // the two writers interleave. + var wroteID uuid.UUID + err = tx.QueryRow(ctx, ` + INSERT INTO knowledge_entities + (entity_id, title, content, source, tags, edited_by, updated_at, deleted_at) + VALUES ($1, $2, $3, $4, $5, $4, now(), NULL) + ON CONFLICT (entity_id) DO UPDATE + SET title = EXCLUDED.title, content = EXCLUDED.content, + tags = EXCLUDED.tags, edited_by = EXCLUDED.edited_by, + updated_at = now(), deleted_at = NULL + WHERE knowledge_entities.deleted_at IS NOT NULL + RETURNING entity_id`, + docID, title, content, actorLabel, tags).Scan(&wroteID) + if errors.Is(err, pgx.ErrNoRows) { + writeProblem(w, req, http.StatusConflict, "a note with this title already exists", slug) + return + } else if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "write knowledge failed", err.Error()) + return + } + + linked := s.linkKnowledgeAbout(ctx, tx, docID, derefSlice(body.About)) + + if err := tx.Commit(ctx); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "commit failed", err.Error()) + return + } + + slog.Info("knowledge created", "slug", slug, "kind", kind, "actor", actorLabel, "linked", linked) + // Content-Type before WriteHeader — setting it after is a no-op, the + // status line is already on the wire. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + if err := json.NewEncoder(w).Encode(map[string]any{ + "slug": slug, "id": docID.String(), "linked": linked, + }); err != nil { + slog.Error("httpapi: json encode failed", "error", err) + } +} + +// serveUpdateKnowledge edits a live note in place. The prior version is +// captured by the trg_knowledge_revision trigger, not by this handler — see +// the migration for why that lives in the database. +// +// Note the slug is intentionally NOT recomputed when the title changes: +// slugs are the wiki's stable link target ([[slug]] references, relationship +// rows, bookmarked window ids), and silently re-slugging on a typo fix would +// break every inbound link. +func (s *Server) serveUpdateKnowledge(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + idOrSlug, err := pathParam(req, "id") + if err != nil { + writeProblem(w, req, http.StatusBadRequest, "invalid id", err.Error()) + return + } + + var body knowledgeWriteBody + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { + writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error()) + return + } + if body.Title == nil && body.Content == nil && body.Tags == nil && body.About == nil { + writeProblem(w, req, http.StatusBadRequest, "nothing to update", + "supply at least one of title, content, tags, about") + return + } + if body.Title != nil && strings.TrimSpace(*body.Title) == "" { + writeProblem(w, req, http.StatusBadRequest, "title cannot be empty", "") + return + } + if body.Content != nil && strings.TrimSpace(*body.Content) == "" { + writeProblem(w, req, http.StatusBadRequest, "content cannot be empty", "") + return + } + + entityID, err := s.resolveKnowledgeEntity(ctx, idOrSlug) + if err != nil { + writeProblem(w, req, http.StatusNotFound, "no such knowledge note", "") + return + } + _, actorLabel := actorInfo(ctx) + + tx, err := s.pool.Begin(ctx) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "begin failed", err.Error()) + return + } + defer tx.Rollback(ctx) + + // COALESCE keeps unsupplied fields untouched; edited_by and updated_at + // always move so the UI can show who last touched it. The trigger only + // snapshots when title/content/tags actually differ, so a no-op save + // doesn't manufacture a revision. + var newTitle *string + if body.Title != nil { + t := strings.TrimSpace(*body.Title) + newTitle = &t + } + var newContent *string + if body.Content != nil { + c := strings.TrimSpace(*body.Content) + newContent = &c + } + var newTags *[]string + if body.Tags != nil { + t := normalizeTags(*body.Tags) + newTags = &t + } + + if _, err := tx.Exec(ctx, ` + UPDATE knowledge_entities + SET title = COALESCE($2, title), + content = COALESCE($3, content), + tags = COALESCE($4, tags), + edited_by = $5, + updated_at = now() + WHERE entity_id = $1`, + entityID, newTitle, newContent, newTags, actorLabel); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "update failed", err.Error()) + return + } + + // Keep the entity's display name in step with the note title — the graph + // and the fleet table read entities.name, and leaving it stale is exactly + // the drift this app exists to fight. + if newTitle != nil { + if _, err := tx.Exec(ctx, + `UPDATE entities SET name = $2, updated_at = now() WHERE id = $1`, + entityID, *newTitle); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "rename entity failed", err.Error()) + return + } + } + + // About is replace-semantics, not merge: the editor presents the full + // link set, so an absent slug means the operator removed it. Existing + // edges are closed (valid_to) rather than deleted, preserving history. + var linked []string + if body.About != nil { + if _, err := tx.Exec(ctx, ` + UPDATE relationships SET valid_to = now() + WHERE source_id = $1 AND valid_to IS NULL AND type IN ('documents', 'about')`, + entityID); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "unlink failed", err.Error()) + return + } + linked = s.linkKnowledgeAbout(ctx, tx, entityID, *body.About) + } + + if err := tx.Commit(ctx); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "commit failed", err.Error()) + return + } + + slog.Info("knowledge updated", "entity_id", entityID, "actor", actorLabel) + // `linked` lets the caller diff against what it submitted and warn about + // any slug that didn't resolve — see linkKnowledgeAbout: a typo'd entity + // slug otherwise fails with nothing but a server-side slog.Warn, so the + // operator gets no feedback that one of their About links didn't take. + writeJSON(w, map[string]any{"ok": true, "linked": linked}) +} + +// serveDeleteKnowledge soft-deletes a note. The row, its revision trail and +// its entity all survive; only the deleted_at stamp changes, and every read +// path filters on it. +func (s *Server) serveDeleteKnowledge(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + idOrSlug, err := pathParam(req, "id") + if err != nil { + writeProblem(w, req, http.StatusBadRequest, "invalid id", err.Error()) + return + } + entityID, err := s.resolveKnowledgeEntity(ctx, idOrSlug) + if err != nil { + writeProblem(w, req, http.StatusNotFound, "no such knowledge note", "") + return + } + _, actorLabel := actorInfo(ctx) + + // Snapshot the live version before tombstoning. The trigger fires on + // title/content/tags changes only, and a delete changes none of them — + // without this the most recent version would be the one version missing + // from the history if the note is later restored. + if _, err := s.pool.Exec(ctx, ` + INSERT INTO knowledge_revisions + (entity_id, title, content, source, tags, edited_by, version_at) + SELECT entity_id, title, content, source, tags, COALESCE(edited_by,''), updated_at + FROM knowledge_entities WHERE entity_id = $1`, entityID); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "snapshot failed", err.Error()) + return + } + if _, err := s.pool.Exec(ctx, ` + UPDATE knowledge_entities SET deleted_at = now(), edited_by = $2 + WHERE entity_id = $1`, entityID, actorLabel); err != nil { + writeProblem(w, req, http.StatusInternalServerError, "delete failed", err.Error()) + return + } + + slog.Info("knowledge deleted", "entity_id", entityID, "actor", actorLabel) + writeJSON(w, map[string]any{"ok": true}) +} + +// serveRestoreKnowledge undoes a soft delete. The counterpart to +// serveDeleteKnowledge — without it, "recoverable by clearing the column" +// (see the migration) would only be true via psql, which isn't a real +// recovery path for an operator using the wiki. +func (s *Server) serveRestoreKnowledge(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + idOrSlug, err := pathParam(req, "id") + if err != nil { + writeProblem(w, req, http.StatusBadRequest, "invalid id", err.Error()) + return + } + entityID, err := s.resolveKnowledgeEntityAny(ctx, idOrSlug) + if err != nil { + writeProblem(w, req, http.StatusNotFound, "no such knowledge note", "") + return + } + _, actorLabel := actorInfo(ctx) + + ct, err := s.pool.Exec(ctx, ` + UPDATE knowledge_entities SET deleted_at = NULL, edited_by = $2 + WHERE entity_id = $1 AND deleted_at IS NOT NULL`, entityID, actorLabel) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "restore failed", err.Error()) + return + } + if ct.RowsAffected() == 0 { + writeProblem(w, req, http.StatusConflict, "note is not deleted", "") + return + } + + slog.Info("knowledge restored", "entity_id", entityID, "actor", actorLabel) + writeJSON(w, map[string]any{"ok": true}) +} + +// serveKnowledgeRevisions returns the note's superseded versions, newest +// first. Bodies are included: revisions are small (~1 KB) and few, and the +// diff view needs both sides anyway — paginating would cost a round trip per +// comparison to save nothing. +func (s *Server) serveKnowledgeRevisions(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + idOrSlug, err := pathParam(req, "id") + if err != nil { + writeProblem(w, req, http.StatusBadRequest, "invalid id", err.Error()) + return + } + entityID, err := s.resolveKnowledgeEntityAny(ctx, idOrSlug) + if err != nil { + writeProblem(w, req, http.StatusNotFound, "no such knowledge note", "") + return + } + + rows, err := s.pool.Query(ctx, ` + SELECT id, title, content, COALESCE(edited_by,''), COALESCE(tags,'{}'), + version_at::text, revised_at::text + FROM knowledge_revisions + WHERE entity_id = $1 + ORDER BY version_at DESC`, entityID) + if err != nil { + writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error()) + return + } + defer rows.Close() + + type revision struct { + ID int64 `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + EditedBy string `json:"edited_by"` + Tags []string `json:"tags"` + VersionAt string `json:"version_at"` + RevisedAt string `json:"revised_at"` + } + items := []revision{} + for rows.Next() { + var r revision + if err := rows.Scan(&r.ID, &r.Title, &r.Content, &r.EditedBy, &r.Tags, + &r.VersionAt, &r.RevisedAt); err != nil { + slog.Error("httpapi: knowledge/revisions row scan failed", "error", err) + continue + } + items = append(items, r) + } + + writeJSON(w, map[string]any{"items": items}) +} + +// linkKnowledgeAbout points a note at the entities it concerns, skipping +// slugs that don't resolve and edges that already exist. Returns the slugs +// actually linked so the caller can report what stuck — a typo'd slug is a +// silent no-op otherwise. +func (s *Server) linkKnowledgeAbout(ctx context.Context, tx pgx.Tx, docID uuid.UUID, slugs []string) []string { + linked := []string{} + for _, raw := range slugs { + slug := strings.TrimSpace(raw) + if slug == "" { + continue + } + var targetID uuid.UUID + if err := tx.QueryRow(ctx, `SELECT id FROM entities WHERE slug = $1`, slug).Scan(&targetID); err != nil { + slog.Warn("knowledge: about slug not found, skipping", "slug", slug) + continue + } + if _, err := tx.Exec(ctx, ` + INSERT INTO relationships (source_id, target_id, type, attributes, valid_from) + SELECT $1, $2, 'about', '{"by":"operator"}'::jsonb, now() + WHERE NOT EXISTS ( + SELECT 1 FROM relationships + WHERE source_id = $1 AND target_id = $2 AND type = 'about' AND valid_to IS NULL)`, + docID, targetID); err != nil { + slog.Warn("knowledge: link failed", "slug", slug, "error", err) + continue + } + linked = append(linked, slug) + } + return linked +} + +// normalizeTags trims, lowercases and de-duplicates while preserving order. +// Lowercasing is the fix for the casing drift already in the data — `oom` +// and `OOM` were separate tags on separate notes, so neither tag page showed +// the full set. Applied on every write so the split can't reopen. +func normalizeTags(in []string) []string { + seen := map[string]bool{} + out := []string{} + for _, t := range in { + t = strings.ToLower(strings.TrimSpace(t)) + if t == "" || seen[t] { + continue + } + seen[t] = true + out = append(out, t) + } + return out +} + +func deref(p *string) string { + if p == nil { + return "" + } + return *p +} + +func derefSlice(p *[]string) []string { + if p == nil { + return nil + } + return *p +} + +// writeJSON is the success-path counterpart to writeProblem, so the handlers +// in this file don't each repeat the header/encode dance. +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(v); err != nil { + slog.Error("httpapi: json encode failed", "error", err) + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 08b53ef..6402f04 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -109,6 +109,16 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand // /api/v1/events/stream — in OpenAPI but re-registered for SSE Flush() // /api/v1/knowledge/recent — ad-hoc aggregation, no schema type yet // /api/v1/knowledge/content/{id} — returns raw markdown, not a gen type + // /api/v1/knowledge/list — full tree listing, ad-hoc aggregate + // /api/v1/knowledge (POST) — markdown in, no gen type + // /api/v1/knowledge/content/{id} (PUT/DELETE) — markdown in, soft delete + // /api/v1/knowledge/trash — soft-deleted notes, ad-hoc + // /api/v1/knowledge/restore/{id} — undo a soft delete, no gen type + // /api/v1/knowledge/revisions/{id} — version history, no schema type + // /api/v1/knowledge/tags{,/rename} — tag index + bulk rewrite + // /api/v1/knowledge/duplicates — trigram clustering, ad-hoc + // /api/v1/knowledge/orphans — derived maintenance view + // /api/v1/knowledge/merge — bulk fold-in, ad-hoc // /api/v1/activity/recent — recency-ordered, not paginated // /api/v1/activity/session/{id} — session-scoped aggregation // /api/v1/executions/{id}/logs — streamed command output, no schema type @@ -203,6 +213,30 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand // (See "Non-OpenAPI routes" carve-out block above.) r.With(combinedAuth(cfg, false)).Get("/api/v1/knowledge/content/{id}", s.serveKnowledgeContent) + // Custom (non-OpenAPI) routes: the operator-facing knowledge CRUD surface + // (see internal/httpapi/knowledge_write.go) and the drift tooling (see + // knowledge_drift.go). Before these, knowledge could only be written by + // the agent through the MCP upsert_knowledge tool — the web UI had no way + // to create, correct or retire a note. + // + // Registered on the base router rather than through the OpenAPI codegen + // for the same reason as the read routes above: they trade in raw + // markdown and ad-hoc aggregates, not generated schema types. + // (See "Non-OpenAPI routes" carve-out block above.) + r.With(combinedAuth(cfg, false)).Get("/api/v1/knowledge/list", s.serveKnowledgeList) + r.With(combinedAuth(cfg, false)).Post("/api/v1/knowledge", s.serveCreateKnowledge) + r.With(combinedAuth(cfg, false)).Put("/api/v1/knowledge/content/{id}", s.serveUpdateKnowledge) + r.With(combinedAuth(cfg, false)).Delete("/api/v1/knowledge/content/{id}", s.serveDeleteKnowledge) + r.With(combinedAuth(cfg, false)).Get("/api/v1/knowledge/trash", s.serveKnowledgeTrash) + r.With(combinedAuth(cfg, false)).Post("/api/v1/knowledge/restore/{id}", s.serveRestoreKnowledge) + r.With(combinedAuth(cfg, false)).Get("/api/v1/knowledge/revisions/{id}", s.serveKnowledgeRevisions) + + r.With(combinedAuth(cfg, false)).Get("/api/v1/knowledge/tags", s.serveKnowledgeTags) + r.With(combinedAuth(cfg, false)).Post("/api/v1/knowledge/tags/rename", s.serveRenameKnowledgeTag) + r.With(combinedAuth(cfg, false)).Get("/api/v1/knowledge/duplicates", s.serveKnowledgeDuplicates) + r.With(combinedAuth(cfg, false)).Get("/api/v1/knowledge/orphans", s.serveKnowledgeOrphans) + r.With(combinedAuth(cfg, false)).Post("/api/v1/knowledge/merge", s.serveMergeKnowledge) + // Custom (non-OpenAPI) routes: the global activity feed (recency-ordered, // unlike ListExecutions which sorts by target for pagination) and the // per-session "what did this session do" digest. diff --git a/migrations/022_knowledge_revisions.up.sql b/migrations/022_knowledge_revisions.up.sql new file mode 100644 index 0000000..10e39bf --- /dev/null +++ b/migrations/022_knowledge_revisions.up.sql @@ -0,0 +1,119 @@ +-- 022_knowledge_revisions.up.sql +-- Version history for knowledge_entities, so an edit can never be silently lost. +-- +-- The concrete hazard this closes: the MCP tool `upsert_knowledge` +-- (internal/mcp/server.go) keys on title and does +-- `ON CONFLICT (entity_id) DO UPDATE SET content = EXCLUDED.content` — +-- unconditionally. Before this migration, an operator hand-editing a note in +-- the web UI would have that edit overwritten with no trace the next time +-- Nomos re-upserted a note with the same title. There was no history table +-- and no way to recover the prior body. +-- +-- The snapshot is a BEFORE UPDATE **trigger** rather than application-level +-- code in the HTTP handler, specifically because there are two independent +-- writers: the web API (new in this change) and the MCP tool the agent uses. +-- App-level snapshotting would only cover whichever path remembered to call +-- it. A trigger covers both, plus any future writer and any manual psql fix. +-- +-- Each row in knowledge_revisions is a *superseded* version: the state of the +-- note before the update that displaced it. The current version always lives +-- in knowledge_entities, never here, so "history" is +-- knowledge_entities + knowledge_revisions ordered by version_at DESC. + +-- Who authored the version currently in knowledge_entities. Distinct from +-- `source`, which is overloaded: it holds either 'nomos-agent' (written via +-- MCP) or a seed file path ('containers/101-jellyfin') and is NOT updated on +-- conflict, so a seeded doc later rewritten by the agent still reports its +-- original file path. edited_by answers the question the UI actually asks — +-- "did a human or the agent last touch this?" — without disturbing source, +-- which the seeding logic still relies on. +ALTER TABLE knowledge_entities + ADD COLUMN IF NOT EXISTS edited_by TEXT NOT NULL DEFAULT ''; + +-- Backfill: every existing row's last writer is whatever source says. For +-- agent-written notes that's exactly right; for seeded notes it records the +-- seed path, which is the honest answer (no human has edited them yet). +UPDATE knowledge_entities +SET edited_by = COALESCE(source, '') +WHERE edited_by = ''; + +-- Soft delete. A hard DELETE would cascade knowledge_revisions away with the +-- entity, which contradicts the point of this migration — removing a note is +-- exactly the moment its history matters most. Deleting sets deleted_at; all +-- read paths filter it out, the revision trail survives, and an accidental +-- delete is recoverable by clearing the column. +ALTER TABLE knowledge_entities + ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; + +-- Partial index: every list/search/read query carries `deleted_at IS NULL`, +-- and deleted notes are expected to stay a small minority. +CREATE INDEX IF NOT EXISTS idx_knowledge_live + ON knowledge_entities (updated_at DESC) + WHERE deleted_at IS NULL; + +CREATE TABLE IF NOT EXISTS knowledge_revisions ( + id BIGSERIAL PRIMARY KEY, + entity_id UUID NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + title TEXT NOT NULL, + content TEXT NOT NULL, + source TEXT, + tags TEXT[], + edited_by TEXT NOT NULL DEFAULT '', + -- When this version was written (the superseded row's updated_at). + version_at TIMESTAMPTZ NOT NULL, + -- When it was replaced. version_at of revision N and revised_at of + -- revision N-1 bracket how long that version was the live one. + revised_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- The only access pattern: "show me the history of this note, newest first." +CREATE INDEX IF NOT EXISTS idx_knowledge_revisions_entity + ON knowledge_revisions (entity_id, version_at DESC); + +-- Snapshot the outgoing row whenever the substance changes. Deliberately +-- ignores updated_at-only touches: upsert_knowledge sets `updated_at = now()` +-- on every call even when re-writing byte-identical content (it has no +-- change detection), and without this guard a re-run of the same agent task +-- would pile up identical revisions and bury the real edits. +-- +-- `search` is a GENERATED column and is intentionally not carried into +-- revisions — it is derived from title+content and would be dead weight. +CREATE OR REPLACE FUNCTION snapshot_knowledge_revision() RETURNS TRIGGER AS $$ +BEGIN + IF OLD.title IS DISTINCT FROM NEW.title + OR OLD.content IS DISTINCT FROM NEW.content + OR OLD.tags IS DISTINCT FROM NEW.tags THEN + INSERT INTO knowledge_revisions + (entity_id, title, content, source, tags, edited_by, version_at) + VALUES + (OLD.entity_id, OLD.title, OLD.content, OLD.source, OLD.tags, + OLD.edited_by, OLD.updated_at); + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- DROP + CREATE rather than CREATE OR REPLACE: Postgres 16 has no +-- CREATE OR REPLACE TRIGGER for this form, and the migration must stay +-- re-runnable. +DROP TRIGGER IF EXISTS trg_knowledge_revision ON knowledge_entities; + +CREATE TRIGGER trg_knowledge_revision + BEFORE UPDATE ON knowledge_entities + FOR EACH ROW EXECUTE FUNCTION snapshot_knowledge_revision(); + +-- Trigram similarity, for the duplicate-detection view. The knowledge base +-- has already accumulated near-duplicates that exact matching cannot catch — +-- four separate "rclone backup live inspection — " investigations, each +-- a fresh note where an update to the existing one was meant. upsert_knowledge +-- keys on exact title, so a date suffix is enough to fork a new note. +-- +-- similarity() over titles is what lets the UI cluster those and offer a +-- merge. fuzzystrmatch (levenshtein) was the alternative; trigram wins here +-- because these titles differ by whole appended words rather than typos, and +-- because it comes with a GIN index while levenshtein cannot be indexed. +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE INDEX IF NOT EXISTS idx_knowledge_title_trgm + ON knowledge_entities USING gin (title gin_trgm_ops) + WHERE deleted_at IS NULL; diff --git a/web/src/app.css b/web/src/app.css index 48e26ed..b11dd96 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -333,3 +333,92 @@ a:hover { border-left: 1px solid var(--border); cursor: col-resize; } + +/* Base markdown rendering — used by every {@html marked.parse(...)} output + (EntityDetailContent, the Knowledge wiki's WikiReader, and as the + foundation ChatThread's fuller "Art Nouveau" chat styling builds on top + of). Global rather than a per-component diff --git a/web/src/lib/components/knowledge/WikiCleanup.svelte b/web/src/lib/components/knowledge/WikiCleanup.svelte new file mode 100644 index 0000000..04909e5 --- /dev/null +++ b/web/src/lib/components/knowledge/WikiCleanup.svelte @@ -0,0 +1,626 @@ + + +
+ + + activate('duplicates')}> + Duplicates + + activate('tags')}> + Tags + + activate('orphans')}> + Orphans + + activate('trash')}> + Trash + + + + +

+ Notes with near-identical titles, grouped for review — not a verdict. Pick a target and the + sources to fold into it; sources are soft-deleted afterward and stay recoverable from Trash. +

+ {#if mergeError} +

+ {mergeError} +

+ {/if} + {#if duplicatesError} +

+ {duplicatesError} + +

+ {/if} + {#if clusters === null} +
+ {#each Array(2) as _, ci (ci)} +
+
+ + +
+
+ {#each Array(ci === 0 ? 3 : 2) as _, ri (ri)} + + {/each} +
+
+ {/each} +
+ {:else if clusters.length === 0} +

No likely duplicates found.

+ {:else} +
+ {#each clusters as c (c.members[0].slug)} + {@const key = c.members[0].slug} + {@const sourceCount = mergeSources[key]?.size ?? 0} +
+
+
+ {c.members.length} similar notes + + + {(c.top_similarity * 100).toFixed(0)}% +
+ +
+ +

+ + keep as target + + + fold into it + +

+
+ {#each c.members as m (m.slug)} + {@const isTarget = mergeTarget[key] === m.slug} + {@const Icon = kindMeta(m.kind).icon} + + {/each} +
+
+ {/each} +
+ {/if} + + + + {#if tagsError} +

+ {tagsError} + +

+ {/if} + {#if tags === null} + + + + + + + + + + + {#each Array(10) as _, i (i)} + + + + + + + + {/each} + +
TagUsesVariants
+ +
+ {:else} + {@const maxUses = Math.max(1, ...tags.map((t) => t.uses))} + + + + + + + + + + + {#each tags as t (t.tag)} + + + + + + + + + {/each} + +
TagUsesVariants
+ {#if renaming === t.tag} +
+ + +
+ {:else} + + {/if} + 		{t.uses} + + + + + + {#if t.split} + {t.variants.join(', ')} + {:else} + — + {/if} + + {#if t.split} + + {/if} +
+ {/if} + + + +

+ Notes untagged, unlinked to any entity, or untouched for 90+ days — invisible to most + navigation paths and easy to lose track of. + {#if orphanCounts.untagged || orphanCounts.unlinked || orphanCounts.stale} + ({orphanCounts.untagged ?? 0} untagged · {orphanCounts.unlinked ?? 0} unlinked · {orphanCounts.stale ?? + 0} stale) + {/if} +

+ {#if orphansError} +

+ {orphansError} + +

+ {/if} + {#if orphans === null} +
+ {#each Array(7) as _, i (i)} +
+ + + +
+ {/each} +
+ {:else if orphans.length === 0} +

Nothing orphaned.

+ {:else} +
+ {#each orphans as o (o.slug)} + + {/each} +
+ {/if} + + + + {#if trashError} +

+ {trashError} + +

+ {/if} + {#if trash === null} +
+ {#each Array(4) as _, i (i)} +
+ + + +
+ {/each} +
+ {:else if trash.length === 0} +

Trash is empty.

+ {:else} +
+ {#each trash as t (t.slug)} +
+ {t.title} + + deleted {relativeTime(t.deleted_at)} by {t.deleted_by || 'unknown'} + + +
+ {/each} +
+ {/if} + + +
diff --git a/web/src/lib/components/knowledge/WikiContextRail.svelte b/web/src/lib/components/knowledge/WikiContextRail.svelte new file mode 100644 index 0000000..26f56b6 --- /dev/null +++ b/web/src/lib/components/knowledge/WikiContextRail.svelte @@ -0,0 +1,124 @@ + + +
+ {#if !item} +

Nothing selected.

+ {:else} + + {#if item.about.length === 0} +

Not linked to any entity.

+ {:else} +
+ {#each item.about as slug (slug)} + + {/each} +
+ {/if} + + + 0} + > + {#if related.length === 0} +

No other notes share a linked entity.

+ {:else} +
+ {#each related as it (it.slug)} + {@const Icon = kindMeta(it.kind).icon} + + {/each} +
+ {/if} + + + + 0 && tagNeighbours.length <= 6} + > + {#if tagNeighbours.length === 0} +

No other notes share a tag.

+ {:else} +
+ {#each tagNeighbours as it (it.slug)} + {@const Icon = kindMeta(it.kind).icon} + + {/each} +
+ {/if} + + {/if} +
diff --git a/web/src/lib/components/knowledge/WikiNewDialog.svelte b/web/src/lib/components/knowledge/WikiNewDialog.svelte new file mode 100644 index 0000000..91e4605 --- /dev/null +++ b/web/src/lib/components/knowledge/WikiNewDialog.svelte @@ -0,0 +1,139 @@ + + + + + + New knowledge note + + +
+ {#if error} +

+ {error} +

+ {/if} + + + +
+ Type +
+ {#each KINDS as k (k)} + + {/each} +
+
+ + + + + +