session reliability: reconnect, knowledge loop, retire request_execution
Phase 1 — crash recovery: SSE auto-reconnect + backoff, polling gate during disconnect, connection banner with retry button, empty-response retry 3x, non-terminal resume on empty response, persistent error cards. Phase 2/4 — visibility + continuation: custom ExecutionStatus renderer, approvals extracted on every tool_result (not just done), activity bar with status/goal, SessionDigest live polling, Continue button. Phase 3 — cleanup: complete_task auto-cancels orphaned approvals, deletes assent/destructive window keys, propose_plan marks pending steps as replaced, plan step seq-order enforcement. Phase 5 — knowledge loop: list_lxcs state filter (active/destroyed), SOUL.md unmissable writeback section, propose_plan validation nudge, complete_task writeback check, upsert_knowledge about array support, plan generation grouping in frontend, session approval count badge. Retire request_execution — all mutations now route through run. Updated SOUL.md, AGENTS.md, CLIENTS.md, skills, and agent system notes. Migration 020: plan step generation column, audit_log session_id index, nomos_plan_executions pending-approval index.
This commit is contained in:
@@ -24,7 +24,6 @@ import (
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/observability"
|
||||
"github.com/dtoro/oikos/internal/policy"
|
||||
"github.com/dtoro/oikos/internal/safego"
|
||||
"github.com/google/jsonschema-go/jsonschema"
|
||||
"github.com/google/uuid"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
@@ -212,7 +211,7 @@ func newServer(pool *db.Pool, agentID uuid.UUID) *mcp.Server {
|
||||
InputSchema: objSchema(
|
||||
prop{"title", "string", "Short, specific, searchable title (e.g. 'Dragonfly memlock rlimit in unprivileged LXCs', not 'notes')."},
|
||||
prop{"content", "string", "The knowledge itself, in markdown. Be concrete: symptom, root cause, the exact fix/commands, and any caveats. Written for someone hitting this fresh."},
|
||||
prop{"about", "string", "Optional entity slug this knowledge concerns (e.g. lxc:typetype, host:strong) — links the note to that entity so get_entity_knowledge surfaces it."},
|
||||
prop{"about", "string", "Optional entity slug(s) this knowledge concerns. Pass a single slug (e.g. 'lxc:nfs-export') or a JSON array of slugs (e.g. '[\"lxc:nfs-export\", \"lxc:gitea\"]') to link to multiple entities. get_entity_knowledge surfaces it for each."},
|
||||
prop{"tags", "string", "Optional comma-separated tags (e.g. 'docker,networking,gotcha')."},
|
||||
prop{"kind", "string", "One of: investigation (a finding/incident analysis — default), document (reference), runbook (a repeatable procedure)."},
|
||||
),
|
||||
@@ -358,186 +357,11 @@ func newServer(pool *db.Pool, agentID uuid.UUID) *mcp.Server {
|
||||
nStr(args["status"])), nil
|
||||
})
|
||||
|
||||
register(&mcp.Tool{Name: "request_execution", Description: "Request a gated execution (agent-only mutation path). Supported actions: restart, systemctl, pct_exec, apt_upgrade, pct_create. pct_create provisions a NEW LXC and installs its service in one approved step — you do NOT need follow-up pct_exec calls for package installs.",
|
||||
InputSchema: objSchema(
|
||||
prop{"target", "string", "Target entity slug. For pct_create this MUST be the Proxmox HOST that will run the container (e.g. host:strong) — NOT the new LXC's name. For restart/systemctl/apt_upgrade/pct_exec use the target service/LXC slug (e.g. lxc:caddy)."},
|
||||
prop{"action", "string", "Action: restart, systemctl, pct_exec, apt_upgrade, pct_create"},
|
||||
prop{"params", "string", "For systemctl: 'enable|disable|reload'. For pct_exec: the shell command. For apt_upgrade: 'audit|upgrade'. For pct_create: a JSON object string with keys: vmid (int, required, unused id), hostname (string, required), cores (int), memory (MB int), disk_gb (int), ip (CIDR e.g. 192.168.8.50/24, or omit/\"dhcp\" — DHCP is the safe default, see below), gw (gateway ip, static only), bridge (e.g. vmbr0/vmbr1 — WHICH BRIDGE REACHES WHICH SUBNET IS DIFFERENT PER HOST, never assume vmbr0; see below), storage (default local-lvm), template (optional — omit to auto-pick newest debian on the host), privileged (bool), nesting (bool), mounts ([]string of 'src,mp=/dst'). Example: {\"vmid\":150,\"hostname\":\"typetype\",\"cores\":2,\"memory\":2048,\"disk_gb\":16,\"ip\":\"192.168.8.50/24\",\"gw\":\"192.168.8.2\",\"bridge\":\"vmbr1\",\"nesting\":true}. pct_create is ATOMIC — it ONLY creates and starts the container (no services/post_install params anymore). Once it completes you will be automatically re-invoked with the result; install packages and run setup by issuing your OWN `run` calls against the new lxc:<hostname> target, one step at a time — you'll see each step's real output and can fix exactly the one that fails, instead of one opaque multi-minute install that either fully works or fully doesn't. STATIC IP RULE: before setting ip/gw/bridge to anything other than DHCP, use list_entities/get_entity_knowledge to find an EXISTING lxc on the SAME host whose IP is in the same /28 block, and copy its exact gw+bridge — do not invent a gateway. If no such neighbor exists, prefer ip:\"dhcp\" (proven to work, gets a real routable address) over guessing; a wrong bridge/gateway pair fails a fast pre-flight ping check now (seconds, not minutes) but is still a wasted turn — better to not guess at all."},
|
||||
),
|
||||
}, func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
args := argsMap(req)
|
||||
targetSlug, _ := args["target"].(string)
|
||||
action, _ := args["action"].(string)
|
||||
params, _ := args["params"].(string)
|
||||
sessionID, _ := args["_session_id"].(string)
|
||||
if targetSlug == "" || action == "" {
|
||||
return textResult("error: target and action required"), nil
|
||||
}
|
||||
|
||||
var targetID uuid.UUID
|
||||
if err := pool.QueryRow(ctx, "SELECT id FROM entities WHERE slug = $1", targetSlug).Scan(&targetID); err != nil {
|
||||
return textResult(fmt.Sprintf("target not found: %s", targetSlug)), nil
|
||||
}
|
||||
|
||||
// restart, pct_exec, and systemctl (outside enable/disable) route
|
||||
// through the same classify→gate path as `run` instead of executing
|
||||
// immediately over SSH with a hardcoded risk_class='reversible_low'
|
||||
// that was never actually checked against anything. Found live
|
||||
// 2026-07-10: a chat request to "restart caddy" — the fleet's
|
||||
// reverse proxy — executed instantly with zero approval, because
|
||||
// this action bypassed the classifier entirely. classifyAndGate
|
||||
// applies the same read-only/config-mutation/destructive
|
||||
// classification and approval flow the `run` tool already uses.
|
||||
if action == "restart" || action == "pct_exec" || (action == "systemctl" && params != "enable" && params != "disable") {
|
||||
svc := strings.TrimPrefix(targetSlug, "lxc:")
|
||||
var cmd, purpose string
|
||||
switch action {
|
||||
case "restart":
|
||||
cmd = fmt.Sprintf("systemctl restart %s; sleep 1; systemctl is-active %s", svc, svc)
|
||||
purpose = "restart " + svc
|
||||
case "pct_exec":
|
||||
cmd = params
|
||||
purpose = "pct_exec (legacy) on " + targetSlug
|
||||
case "systemctl":
|
||||
cmd = fmt.Sprintf("systemctl %s %s; sleep 1; systemctl is-active %s", params, svc, svc)
|
||||
purpose = "systemctl " + params + " " + svc
|
||||
}
|
||||
return classifyAndGate(ctx, pool, agentID, targetID, targetSlug, cmd, purpose, "", sessionID), nil
|
||||
}
|
||||
|
||||
// Deduplicate: if a pending execution already exists for the same
|
||||
// target+action, return the existing one instead of creating a
|
||||
// duplicate. Prevents the LLM from re-requesting the same gated
|
||||
// action in a tool-calling loop. Only blocks when a pending
|
||||
// execution exists; completed/failed ones don't block.
|
||||
if action == "systemctl" || action == "apt_upgrade" || action == "pct_create" {
|
||||
execNamePrefix := action + " on " + targetSlug
|
||||
var existingID string
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT e.id::text FROM entities e
|
||||
JOIN executions ex ON ex.entity_id = e.id
|
||||
WHERE e.type = 'execution' AND e.name LIKE $1 AND ex.status = 'pending_approval'
|
||||
ORDER BY e.created_at DESC LIMIT 1`, execNamePrefix+"%").Scan(&existingID)
|
||||
if err == nil && existingID != "" {
|
||||
return textResult(fmt.Sprintf("%s on %s is already queued for approval — execution %s. Wait for operator approval. Do not re-request.",
|
||||
action, targetSlug, existingID)), nil
|
||||
}
|
||||
}
|
||||
|
||||
id, _ := uuid.NewV7()
|
||||
correlationID := uuid.New().String()
|
||||
|
||||
// Full UUID, not a truncated prefix: UUIDv7's leading bytes encode a
|
||||
// millisecond timestamp, so an 8-char prefix collides for real under
|
||||
// back-to-back requests (observed live: two `run` calls seconds
|
||||
// apart hit entities_slug_key). The full string is guaranteed unique.
|
||||
execName := action + " on " + targetSlug + " (" + id.String() + ")"
|
||||
execSlug := "exec:" + targetSlug + ":" + id.String()
|
||||
_, err := pool.Exec(ctx, `INSERT INTO entities (id, slug, type, name, attributes) VALUES ($1, $2, 'execution', $3, '{}')`,
|
||||
id, execSlug, execName)
|
||||
if err != nil {
|
||||
return textResult(fmt.Sprintf("error: failed to create execution: %v", err)), nil
|
||||
}
|
||||
pool.Exec(ctx, `INSERT INTO executions (entity_id, target_entity_id, action, risk_class, status, correlation_id, agent_id) VALUES ($1, $2, $3, 'reversible_low', 'running', $4, $5) ON CONFLICT DO NOTHING`,
|
||||
id, targetID, action+":"+params, correlationID, agentID)
|
||||
pool.Exec(ctx, `
|
||||
INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT $1, $2, 'targets', '{"by":"nomos"}'::jsonb, now()
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM relationships
|
||||
WHERE source_id = $1 AND target_id = $2 AND type = 'targets' AND valid_to IS NULL)`,
|
||||
id, targetID)
|
||||
if sessionID != "" {
|
||||
pool.Exec(ctx, `
|
||||
INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT t.id, $1, 'involves', '{"by":"nomos"}'::jsonb, now()
|
||||
FROM entities t WHERE t.slug = $2
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM relationships
|
||||
WHERE source_id = t.id AND target_id = $1 AND type = 'involves' AND valid_to IS NULL)`,
|
||||
id, "task:"+sessionID)
|
||||
}
|
||||
|
||||
// Execute reversible actions immediately. restart/pct_exec/systemctl
|
||||
// (outside enable/disable) never reach here — they're routed through
|
||||
// classifyAndGate above, before this dedup+insert block.
|
||||
switch action {
|
||||
case "systemctl":
|
||||
// Only enable/disable reach this case now.
|
||||
svc := strings.TrimPrefix(targetSlug, "lxc:")
|
||||
pool.Exec(ctx, `UPDATE executions SET status='pending_approval', risk_class='config_mutation' WHERE entity_id=$1`, id)
|
||||
createApproval(ctx, pool, id, targetID, "systemctl", svc+":"+params, "config_mutation")
|
||||
return textResult(fmt.Sprintf("systemctl %s on %s requires approval — execution %s queued", params, svc, id)), nil
|
||||
|
||||
case "apt_upgrade":
|
||||
if params == "audit" {
|
||||
host, user, err := resolveHost(ctx, pool, targetSlug)
|
||||
if err != nil {
|
||||
return textResult(fmt.Sprintf("resolve: %v", err)), nil
|
||||
}
|
||||
out, err := sshExec(ctx, host, user, "apt update -qq 2>&1 >/dev/null; apt list --upgradable 2>/dev/null | tail -n +2 | wc -l; apt list --upgradable 2>/dev/null | tail -n +2 | head -20")
|
||||
if err != nil {
|
||||
return textResult(fmt.Sprintf("apt audit error: %v", err)), nil
|
||||
}
|
||||
return textResult("apt audit:\n" + out), nil
|
||||
}
|
||||
// During an active assent window, auto-approve.
|
||||
if assentWindowActive(ctx, pool, agentID, sessionID) {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='pending_approval', risk_class='config_mutation' WHERE entity_id=$1`, id)
|
||||
createApproval(ctx, pool, id, targetID, "apt_upgrade", params, "config_mutation")
|
||||
// Do NOT pre-flip approvals/executions status here (that was
|
||||
// the previous, broken "autoApprove" helper). DecideApproval
|
||||
// (invoked below) is the ONE place that transitions
|
||||
// pending_approval -> approved and dispatches the real SSH
|
||||
// work — it specifically looks for status='pending_approval'
|
||||
// to find what to run. Pre-flipping the status past that
|
||||
// state meant DecideApproval's own lookup found nothing,
|
||||
// silently no-opped, and the execution sat at 'approved'
|
||||
// forever with nothing actually running. Found live: every
|
||||
// assent-window auto-approved pct_create/apt_upgrade has
|
||||
// never actually executed, via this exact bug. Calling
|
||||
// executeApprovedViaAPI directly against the untouched
|
||||
// pending_approval row makes this identical to the manual
|
||||
// Approve-button path, just without a human click.
|
||||
//
|
||||
// context.Background(), NOT ctx: ctx is scoped to this MCP
|
||||
// tool call, cancelled the instant the chat turn's HTTP
|
||||
// response completes (every normal turn) — a goroutine
|
||||
// meant to outlive the request must not inherit its context.
|
||||
safego.Go("mcp:executeApprovedViaAPI:apt_upgrade", func() {
|
||||
executeApprovedViaAPI(context.Background(), id, targetSlug, "apt_upgrade:"+params)
|
||||
})
|
||||
slog.Info("mcp: apt_upgrade auto-approved via assent window", "execution_id", id)
|
||||
return textResult(fmt.Sprintf("apt_upgrade on %s auto-approved via assent window — execution %s running.", targetSlug, id)), nil
|
||||
}
|
||||
// upgrade requires approval — queue
|
||||
pool.Exec(ctx, `UPDATE executions SET status='pending_approval', risk_class='config_mutation' WHERE entity_id=$1`, id)
|
||||
createApproval(ctx, pool, id, targetID, "apt_upgrade", params, "config_mutation")
|
||||
return textResult(fmt.Sprintf("apt_upgrade on %s requires approval — execution %s queued", targetSlug, id)), nil
|
||||
|
||||
case "pct_create":
|
||||
// During an active assent window, auto-approve and execute
|
||||
// instead of queuing — the operator already approved the plan.
|
||||
if assentWindowActive(ctx, pool, agentID, sessionID) {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='pending_approval', risk_class='config_mutation' WHERE entity_id=$1`, id)
|
||||
createApproval(ctx, pool, id, targetID, "pct_create", params, "config_mutation")
|
||||
// See the apt_upgrade case above for why there's no
|
||||
// pre-flip-status "autoApprove" step here anymore, and why
|
||||
// this uses context.Background().
|
||||
safego.Go("mcp:executeApprovedViaAPI:pct_create", func() {
|
||||
executeApprovedViaAPI(context.Background(), id, targetSlug, "pct_create:"+params)
|
||||
})
|
||||
slog.Info("mcp: pct_create auto-approved via assent window", "execution_id", id)
|
||||
return textResult(fmt.Sprintf("pct_create on %s auto-approved via assent window — execution %s running. The LXC is being provisioned now.", targetSlug, id)), nil
|
||||
}
|
||||
pool.Exec(ctx, `UPDATE executions SET status='pending_approval', risk_class='config_mutation' WHERE entity_id=$1`, id)
|
||||
createApproval(ctx, pool, id, targetID, "pct_create", params, "config_mutation")
|
||||
return textResult(fmt.Sprintf("pct_create on %s requires approval — execution %s queued. The LXC will be provisioned once approved.", targetSlug, id)), nil
|
||||
|
||||
default:
|
||||
return textResult(fmt.Sprintf("unknown action: %s. Supported: restart, systemctl, pct_exec, apt_upgrade, pct_create", action)), nil
|
||||
}
|
||||
})
|
||||
// ── request_execution (legacy fixed enum) retired 2026-07-14 ──
|
||||
// All mutations now route through `run`. The handler functions
|
||||
// (runRexecRestart, runRexecSystemctl, etc.) are kept as reference
|
||||
// for future runbook extraction — especially pct_create DNS/VMID logic.
|
||||
// DO NOT re-register this tool. See plans/2026-07-10-general-gated-execution.md.
|
||||
|
||||
register(&mcp.Tool{Name: "run", Description: "Run ANY shell command against any host or LXC. This is the general execution primitive — prefer it over asking the operator to run something manually, and don't wait for a matching fixed action to exist. Every command is automatically risk-classified: read-only inspection (cat, systemctl status, docker ps, journalctl, df, git status, ...) runs immediately; anything that changes state requires operator approval (granted by the operator replying \"go ahead\"/\"yes\" in chat, or via the Approve button); commands matching a destructive pattern (rm -rf, dd, mkfs, pct/qm destroy, DROP TABLE, reboot, piping curl into a shell, ...) always require approval regardless of what you declare. You cannot talk your way past the destructive check by declaring a lower risk.",
|
||||
InputSchema: objSchema(
|
||||
@@ -661,17 +485,27 @@ func newServer(pool *db.Pool, agentID uuid.UUID) *mcp.Server {
|
||||
|
||||
// ─── Phase 5: operational MCP tools ──────────────────────────────
|
||||
|
||||
register(&mcp.Tool{Name: "list_lxcs", Description: "List all LXC containers with ID, host, IP, and state",
|
||||
InputSchema: objSchema(),
|
||||
register(&mcp.Tool{Name: "list_lxcs", Description: "List all LXC containers with ID, host, IP, and state. Pass state=\"active\" to exclude destroyed/deprecated containers.",
|
||||
InputSchema: objSchema(
|
||||
prop{"state", "string", "Optional: filter by entity state (active, destroyed, …)"},
|
||||
),
|
||||
}, func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
state, _ := argsMap(req)["state"].(string)
|
||||
var statePtr *string
|
||||
if state != "" {
|
||||
statePtr = &state
|
||||
}
|
||||
return annotateJSONResult(queryRows(ctx, pool, `
|
||||
SELECT e.slug, e.name, e.attributes->>'pve_id' AS pve_id,
|
||||
e.attributes->>'lan_ip' AS lan_ip,
|
||||
e.state,
|
||||
st.health, st.last_check_at
|
||||
FROM entities e
|
||||
LEFT JOIN entity_status st ON st.entity_id = e.id
|
||||
WHERE e.type = 'lxc'
|
||||
ORDER BY (e.attributes->>'pve_id')::int`), "lxc_list"), nil
|
||||
AND ($1::text IS NULL OR e.state = $1)
|
||||
ORDER BY CASE WHEN e.state = 'active' THEN 0 ELSE 1 END,
|
||||
(e.attributes->>'pve_id')::int`, statePtr), "lxc_list"), nil
|
||||
})
|
||||
|
||||
register(&mcp.Tool{Name: "ping_service", Description: "Check if a service is reachable via HTTP",
|
||||
@@ -1657,10 +1491,26 @@ func knowledgeSlug(kind, title string) string {
|
||||
func upsertKnowledge(ctx context.Context, pool *db.Pool, args map[string]any) (*mcp.CallToolResult, error) {
|
||||
title, _ := args["title"].(string)
|
||||
content, _ := args["content"].(string)
|
||||
about, _ := args["about"].(string)
|
||||
tagsRaw, _ := args["tags"].(string)
|
||||
kind, _ := args["kind"].(string)
|
||||
|
||||
// Normalize about: accept a single string slug or an array of slugs.
|
||||
var aboutSlugs []string
|
||||
switch v := args["about"].(type) {
|
||||
case string:
|
||||
if s := strings.TrimSpace(v); s != "" {
|
||||
aboutSlugs = []string{s}
|
||||
}
|
||||
case []interface{}:
|
||||
for _, item := range v {
|
||||
if s, ok := item.(string); ok {
|
||||
if s = strings.TrimSpace(s); s != "" {
|
||||
aboutSlugs = append(aboutSlugs, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
title = strings.TrimSpace(title)
|
||||
content = strings.TrimSpace(content)
|
||||
if title == "" || content == "" {
|
||||
@@ -1707,21 +1557,27 @@ func upsertKnowledge(ctx context.Context, pool *db.Pool, args map[string]any) (*
|
||||
return textResult(fmt.Sprintf("error writing knowledge: %v", err)), nil
|
||||
}
|
||||
|
||||
// Link it to the entity it's about, if given and not already linked.
|
||||
// Link it to the entity(s) it's about, if given and not already linked.
|
||||
linked := ""
|
||||
if about = strings.TrimSpace(about); about != "" {
|
||||
var targetID uuid.UUID
|
||||
if qerr := pool.QueryRow(ctx, "SELECT id FROM entities WHERE slug = $1", about).Scan(&targetID); qerr == nil {
|
||||
pool.Exec(ctx, `
|
||||
INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT $1, $2, 'documents', '{"by":"nomos"}'::jsonb, now()
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM relationships
|
||||
WHERE source_id = $1 AND target_id = $2 AND type = 'documents' AND valid_to IS NULL)`,
|
||||
docID, targetID)
|
||||
linked = " and linked to " + about
|
||||
} else {
|
||||
linked = fmt.Sprintf(" (note: entity %q not found, saved unlinked)", about)
|
||||
if len(aboutSlugs) > 0 {
|
||||
var linkedSlugs []string
|
||||
for _, slug := range aboutSlugs {
|
||||
var targetID uuid.UUID
|
||||
if qerr := pool.QueryRow(ctx, "SELECT id FROM entities WHERE slug = $1", slug).Scan(&targetID); qerr == nil {
|
||||
pool.Exec(ctx, `
|
||||
INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT $1, $2, 'documents', '{"by":"nomos"}'::jsonb, now()
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM relationships
|
||||
WHERE source_id = $1 AND target_id = $2 AND type = 'documents' AND valid_to IS NULL)`,
|
||||
docID, targetID)
|
||||
linkedSlugs = append(linkedSlugs, slug)
|
||||
}
|
||||
}
|
||||
if len(linkedSlugs) == 1 {
|
||||
linked = " and linked to " + linkedSlugs[0]
|
||||
} else if len(linkedSlugs) > 1 {
|
||||
linked = fmt.Sprintf(" and linked to %d entities", len(linkedSlugs))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user