db as source of truth: wiki→seeds, archive old artifacts, knowledge ingestion
- Migrations 010 (content_hash) + 011 (search tsvector column) - new: internal/knowledge/seed.go — knowledge seed ingest engine - new: internal/httpapi/knowledge.go — SearchKnowledge + GetEntityKnowledge - wire knowledge ingest into oikos seed pipeline - convert all 36 wiki docs + 6 investigations + 12 runbooks → seeds/knowledge.yaml - archive: knowledge/wiki/→archive/, oikos/cards/→archive/, .hermes/plans/→archive/ - delete: 9 superseded Python kernel files, ledger/, mcp/build_host_files.py - remove empty knowledge/ directory tree
This commit is contained in:
142
internal/httpapi/knowledge.go
Normal file
142
internal/httpapi/knowledge.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (s *Server) SearchKnowledge(ctx context.Context, request gen.SearchKnowledgeRequestObject) (gen.SearchKnowledgeResponseObject, error) {
|
||||
q := request.Params.Q
|
||||
limit := clampLimit(request.Params.Limit)
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT e.slug, COALESCE(et.name,''), ke.title, ke.source, ke.tags,
|
||||
ts_rank(ke.search, plainto_tsquery('english', $1)) AS rank,
|
||||
ts_headline('english', ke.content, plainto_tsquery('english', $1),
|
||||
'MaxWords=40, MinWords=15, ShortWord=3, MaxFragments=3,
|
||||
FragmentDelimiter=" ... "') AS snippet
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
JOIN entity_types et ON et.name = e.type
|
||||
WHERE ke.search @@ plainto_tsquery('english', $1)
|
||||
ORDER BY rank DESC
|
||||
LIMIT $2`,
|
||||
q, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := []gen.KnowledgeHit{}
|
||||
|
||||
for rows.Next() {
|
||||
var slug, eType, title, source, tagJSON string
|
||||
var rank float32
|
||||
var snippet *string
|
||||
|
||||
if err := rows.Scan(&slug, &eType, &title, &source, &tagJSON, &rank, &snippet); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tags []string
|
||||
json.Unmarshal([]byte(tagJSON), &tags)
|
||||
|
||||
hitType := gen.Document
|
||||
switch eType {
|
||||
case "investigation":
|
||||
hitType = gen.Investigation
|
||||
case "runbook":
|
||||
hitType = gen.Runbook
|
||||
}
|
||||
|
||||
id, _ := uuid.Parse("")
|
||||
_ = id // not needed for response since we use slug
|
||||
|
||||
items = append(items, gen.KnowledgeHit{
|
||||
Slug: slug,
|
||||
Title: title,
|
||||
Type: hitType,
|
||||
Rank: &rank,
|
||||
Snippet: snippet,
|
||||
SourcePath: &source,
|
||||
})
|
||||
}
|
||||
if rows.Err() != nil {
|
||||
return nil, rows.Err()
|
||||
}
|
||||
|
||||
if items == nil {
|
||||
items = []gen.KnowledgeHit{}
|
||||
}
|
||||
|
||||
return gen.SearchKnowledge200JSONResponse{Items: items}, nil
|
||||
}
|
||||
|
||||
func (s *Server) GetEntityKnowledge(ctx context.Context, request gen.GetEntityKnowledgeRequestObject) (gen.GetEntityKnowledgeResponseObject, error) {
|
||||
entitySlug := request.EntityId
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT e.slug, COALESCE(et.name,''), ke.title, ke.source, ke.tags
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
JOIN entity_types et ON et.name = e.type
|
||||
JOIN relationships r ON r.source_id = ke.entity_id
|
||||
JOIN entities target ON target.id = r.target_id
|
||||
WHERE target.slug = $1
|
||||
AND r.valid_to IS NULL
|
||||
AND r.type IN ('documents', 'about')
|
||||
UNION
|
||||
SELECT e.slug, COALESCE(et.name,''), ke.title, ke.source, ke.tags
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
JOIN entity_types et ON et.name = e.type
|
||||
JOIN relationships r ON r.source_id = ke.entity_id
|
||||
JOIN entity_types target_type ON target_type.name = r.target_id::text
|
||||
JOIN entities ent ON ent.type = target_type.name AND ent.slug = $1
|
||||
WHERE r.valid_to IS NULL
|
||||
AND r.type = 'procedure-for'
|
||||
ORDER BY 1`,
|
||||
entitySlug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := []gen.KnowledgeHit{}
|
||||
for rows.Next() {
|
||||
var slug, eType, title, source, tagJSON string
|
||||
var tags []string
|
||||
|
||||
if err := rows.Scan(&slug, &eType, &title, &source, &tagJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
json.Unmarshal([]byte(tagJSON), &tags)
|
||||
|
||||
hitType := gen.Document
|
||||
switch eType {
|
||||
case "investigation":
|
||||
hitType = gen.Investigation
|
||||
case "runbook":
|
||||
hitType = gen.Runbook
|
||||
}
|
||||
|
||||
items = append(items, gen.KnowledgeHit{
|
||||
Slug: slug,
|
||||
Title: title,
|
||||
Type: hitType,
|
||||
SourcePath: &source,
|
||||
})
|
||||
}
|
||||
if rows.Err() != nil {
|
||||
return nil, rows.Err()
|
||||
}
|
||||
|
||||
if items == nil {
|
||||
items = []gen.KnowledgeHit{}
|
||||
}
|
||||
|
||||
return gen.GetEntityKnowledge200JSONResponse{Items: items}, nil
|
||||
}
|
||||
@@ -1738,16 +1738,6 @@ func float32Ptr(f float32) *float32 {
|
||||
return &f
|
||||
}
|
||||
|
||||
// ─── Knowledge (stubs — tables don't exist yet) ────────────────────────
|
||||
|
||||
func (s *Server) SearchKnowledge(ctx context.Context, request gen.SearchKnowledgeRequestObject) (gen.SearchKnowledgeResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) GetEntityKnowledge(ctx context.Context, request gen.GetEntityKnowledgeRequestObject) (gen.GetEntityKnowledgeResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
// ─── Agent Activity (stub) ─────────────────────────────────────────────
|
||||
|
||||
func (s *Server) QueryAgentActivity(ctx context.Context, request gen.QueryAgentActivityRequestObject) (gen.QueryAgentActivityResponseObject, error) {
|
||||
|
||||
276
internal/knowledge/seed.go
Normal file
276
internal/knowledge/seed.go
Normal file
@@ -0,0 +1,276 @@
|
||||
package knowledge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type SeedResult struct {
|
||||
Documents int
|
||||
Investigations int
|
||||
Runbooks int
|
||||
}
|
||||
|
||||
func contentHash(s string) string {
|
||||
h := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func Ingest(ctx context.Context, tx pgx.Tx, data map[string]any) (*SeedResult, error) {
|
||||
r := &SeedResult{}
|
||||
|
||||
docs, _ := data["documents"].([]any)
|
||||
for _, raw := range docs {
|
||||
d, _ := raw.(map[string]any)
|
||||
if err := ingestDocument(ctx, tx, d); err != nil {
|
||||
return nil, fmt.Errorf("document %v: %w", str(d, "slug"), err)
|
||||
}
|
||||
r.Documents++
|
||||
}
|
||||
|
||||
invs, _ := data["investigations"].([]any)
|
||||
for _, raw := range invs {
|
||||
m, _ := raw.(map[string]any)
|
||||
if err := ingestInvestigation(ctx, tx, m); err != nil {
|
||||
return nil, fmt.Errorf("investigation %v: %w", str(m, "slug"), err)
|
||||
}
|
||||
r.Investigations++
|
||||
}
|
||||
|
||||
rbs, _ := data["runbooks"].([]any)
|
||||
for _, raw := range rbs {
|
||||
m, _ := raw.(map[string]any)
|
||||
if err := ingestRunbook(ctx, tx, m); err != nil {
|
||||
return nil, fmt.Errorf("runbook %v: %w", str(m, "slug"), err)
|
||||
}
|
||||
r.Runbooks++
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func str(m map[string]any, key string) string {
|
||||
s, _ := m[key].(string)
|
||||
return s
|
||||
}
|
||||
|
||||
func strSlice(m map[string]any, key string) []string {
|
||||
raw, _ := m[key].([]any)
|
||||
var out []string
|
||||
for _, v := range raw {
|
||||
if s, ok := v.(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mapVal(m map[string]any, key string) map[string]any {
|
||||
v, _ := m[key].(map[string]any)
|
||||
return v
|
||||
}
|
||||
|
||||
func ingestDocument(ctx context.Context, tx pgx.Tx, m map[string]any) error {
|
||||
slug := str(m, "slug")
|
||||
title := str(m, "title")
|
||||
content := str(m, "content")
|
||||
entitySlug := str(m, "entity_slug")
|
||||
tags := strSlice(m, "tags")
|
||||
atGlance, _ := m["at_glance"].(map[string]any)
|
||||
clRaw, _ := m["changelog"].([]any)
|
||||
|
||||
entityDocSlug := "document:" + slug
|
||||
|
||||
if err := upsertKnowledgeEntity(ctx, tx, entityDocSlug, "document", title, content, slug, tags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attrs := map[string]any{}
|
||||
if len(atGlance) > 0 {
|
||||
attrs["at_glance"] = atGlance
|
||||
}
|
||||
if len(clRaw) > 0 {
|
||||
attrs["changelog"] = clRaw
|
||||
}
|
||||
if len(attrs) > 0 {
|
||||
attrsBytes, _ := json.Marshal(attrs)
|
||||
_, err := tx.Exec(ctx,
|
||||
`UPDATE entities SET attributes = attributes || $1, updated_at = now()
|
||||
WHERE slug = $2`, string(attrsBytes), entityDocSlug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update document attrs: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if entitySlug != "" {
|
||||
if err := createEdge(ctx, tx, entityDocSlug, entitySlug, "documents", nil); err != nil {
|
||||
return fmt.Errorf("link document: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(atGlance) > 0 && entitySlug != "" {
|
||||
backfillAttrs, _ := json.Marshal(atGlance)
|
||||
_, err := tx.Exec(ctx,
|
||||
`UPDATE entities SET attributes = $1 || attributes, updated_at = now()
|
||||
WHERE slug = $2`, string(backfillAttrs), entitySlug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backfill entity attrs: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ingestInvestigation(ctx context.Context, tx pgx.Tx, m map[string]any) error {
|
||||
slug := str(m, "slug")
|
||||
title := str(m, "title")
|
||||
content := str(m, "content")
|
||||
date := str(m, "date")
|
||||
status := str(m, "status")
|
||||
duration := str(m, "duration")
|
||||
aboutSlugs := strSlice(m, "about_slugs")
|
||||
tags := strSlice(m, "tags")
|
||||
|
||||
entitySlug := "investigation:" + slug
|
||||
|
||||
if err := upsertKnowledgeEntity(ctx, tx, entitySlug, "investigation", title, content, slug, tags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attrs := map[string]any{}
|
||||
if date != "" {
|
||||
attrs["date"] = date
|
||||
}
|
||||
if status != "" {
|
||||
attrs["status"] = status
|
||||
}
|
||||
if duration != "" {
|
||||
attrs["duration"] = duration
|
||||
}
|
||||
if len(attrs) > 0 {
|
||||
attrsBytes, _ := json.Marshal(attrs)
|
||||
_, err := tx.Exec(ctx,
|
||||
`UPDATE entities SET attributes = attributes || $1, updated_at = now()
|
||||
WHERE slug = $2`, string(attrsBytes), entitySlug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update investigation attrs: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, aboutSlug := range aboutSlugs {
|
||||
if err := createEdge(ctx, tx, entitySlug, aboutSlug, "about", nil); err != nil {
|
||||
return fmt.Errorf("link investigation about %s: %w", aboutSlug, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ingestRunbook(ctx context.Context, tx pgx.Tx, m map[string]any) error {
|
||||
slug := str(m, "slug")
|
||||
name := str(m, "name")
|
||||
riskClass := str(m, "risk_class")
|
||||
entityType := str(m, "entity_type")
|
||||
content := str(m, "content")
|
||||
tags := strSlice(m, "tags")
|
||||
procedure, _ := m["procedure"].(map[string]any)
|
||||
|
||||
entitySlug := "runbook:" + slug
|
||||
|
||||
if err := upsertKnowledgeEntity(ctx, tx, entitySlug, "runbook", name, content, slug, tags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attrs := map[string]any{}
|
||||
if riskClass != "" {
|
||||
attrs["risk_class"] = riskClass
|
||||
}
|
||||
if len(procedure) > 0 {
|
||||
attrs["procedure"] = procedure
|
||||
}
|
||||
if len(attrs) > 0 {
|
||||
attrsBytes, _ := json.Marshal(attrs)
|
||||
_, err := tx.Exec(ctx,
|
||||
`UPDATE entities SET attributes = attributes || $1, updated_at = now()
|
||||
WHERE slug = $2`, string(attrsBytes), entitySlug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update runbook attrs: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if entityType != "" {
|
||||
if err := createEdge(ctx, tx, entitySlug, entityType, "procedure-for", nil); err != nil {
|
||||
return fmt.Errorf("link runbook: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertKnowledgeEntity(ctx context.Context, tx pgx.Tx, slug, entityType, title, content, source string, tags []string) error {
|
||||
id, err := getOrCreateEntity(ctx, tx, slug, entityType, title)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hash := contentHash(content)
|
||||
tagsJSON, _ := json.Marshal(tags)
|
||||
|
||||
_, err = tx.Exec(ctx,
|
||||
`INSERT INTO knowledge_entities (entity_id, title, content, source, tags, content_hash, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, now(), now())
|
||||
ON CONFLICT (entity_id) DO UPDATE SET
|
||||
title = $2, content = $3, source = $4, tags = $5,
|
||||
content_hash = $6, updated_at = now()`,
|
||||
id, title, content, source, string(tagsJSON), hash)
|
||||
return err
|
||||
}
|
||||
|
||||
func getOrCreateEntity(ctx context.Context, tx pgx.Tx, slug, entityType, name string) (uuid.UUID, error) {
|
||||
var id uuid.UUID
|
||||
err := tx.QueryRow(ctx, "SELECT id FROM entities WHERE slug = $1", slug).Scan(&id)
|
||||
if err == nil {
|
||||
return id, nil
|
||||
}
|
||||
if err != pgx.ErrNoRows {
|
||||
return uuid.Nil, fmt.Errorf("lookup entity %s: %w", slug, err)
|
||||
}
|
||||
|
||||
id, err = uuid.NewV7()
|
||||
if err != nil {
|
||||
return uuid.Nil, fmt.Errorf("generate uuid: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx,
|
||||
`INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, NULL, '{}', 1, now(), now())`,
|
||||
id, slug, entityType, name)
|
||||
if err != nil {
|
||||
return uuid.Nil, fmt.Errorf("create entity %s: %w", slug, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func createEdge(ctx context.Context, tx pgx.Tx, sourceSlug, targetSlug, relType string, attrs map[string]any) error {
|
||||
var sourceID, targetID uuid.UUID
|
||||
if err := tx.QueryRow(ctx, "SELECT id FROM entities WHERE slug = $1", sourceSlug).Scan(&sourceID); err != nil {
|
||||
return fmt.Errorf("source %s: %w", sourceSlug, err)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, "SELECT id FROM entities WHERE slug = $1", targetSlug).Scan(&targetID); err != nil {
|
||||
return fmt.Errorf("target %s: %w", targetSlug, err)
|
||||
}
|
||||
|
||||
attrsBytes, _ := json.Marshal(attrs)
|
||||
_, err := tx.Exec(ctx,
|
||||
`INSERT INTO relationships (source_id, target_id, type, attributes, valid_from, valid_to)
|
||||
VALUES ($1, $2, $3, $4, now(), NULL)
|
||||
ON CONFLICT (source_id, target_id, type) WHERE valid_to IS NULL
|
||||
DO UPDATE SET attributes = EXCLUDED.attributes`,
|
||||
sourceID, targetID, relType, string(attrsBytes))
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user