P0 — stop the bleeding: - prevent premature complete_task(success) when goal involves reachability - validate run targets: block host-only commands (qm/pct/pvesh) on LXC/VM - bump MCP client timeout 30s→120s to stop 'context deadline exceeded' P1 — fix the plan system: - add replaced_reason column to session_plan_steps (migration 030) - track WHY steps are replaced (wrong_diagnosis/scope_change/superseded/etc) - force fresh propose_plan on session resume (reopenSession marks old plan) P2 — cognitive guardrails: - SOUL.md scope-gate rule: ask before chasing unrelated subsystems - auto-upsert knowledge entry on every session close P3 — observability (all were empty/NULL): - populate agent_activity.token_count from LLM usage (was always NULL) - populate nomos_plan_executions linking executions to sessions - write plan_completion_rate metric on task close P4 — learning loop (all were empty/NULL): - auto-classify every run call → classifications table (was 0 rows) - auto-feedback on session close (was 0 rows)
957 lines
36 KiB
Go
957 lines
36 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/openai/openai-go"
|
|
"github.com/openai/openai-go/option"
|
|
"github.com/openai/openai-go/shared"
|
|
)
|
|
|
|
// maxIterations bounds one chat turn's tool-calling loop. Provisioning a
|
|
// service is a long chain (research → plan → run → per-step
|
|
// install/verify run calls), so this must be generous; a full deploy with the
|
|
// decomposed pct_create flow can legitimately need many steps. On exhaustion
|
|
// the loop now produces a real summary (finalSummary) rather than a dead end.
|
|
const maxIterations = 40
|
|
const maxLLMRetries = 3
|
|
|
|
// historyWindowSize bounds how many of a session's most recent persisted
|
|
// messages are replayed into the LLM's context on each turn — see
|
|
// store.go's getRecentMessages for why this exists (fix A2 of
|
|
// plans/2026-07-11-nomos-agent-code-review.md: unbounded history replay was
|
|
// a real, observed-in-production cost/latency/eventual-context-limit risk).
|
|
// 30 is a fixed-window choice, not token-budget-aware: simplest option that
|
|
// still keeps roughly the current task's working context, at the cost of
|
|
// occasionally dropping something a very long task still needed — the
|
|
// system note injected when truncation happens tells the model to check
|
|
// upsert_knowledge/search_knowledge rather than assume something didn't
|
|
// happen. A token-aware trim or LLM-summarize-on-drop are documented
|
|
// stretch options if a fixed window proves insufficient in practice.
|
|
const historyWindowSize = 30
|
|
|
|
var refusalDenylist = []string{
|
|
"我没有相关信息",
|
|
"您可以尝试问我其它问题",
|
|
"我无法",
|
|
"抱歉,我无法",
|
|
"关于这个问题,我没有",
|
|
}
|
|
|
|
type agent struct {
|
|
clients *mcpClientPool // one MCP client PER SESSION, not shared — see mcpClientPool's doc comment
|
|
system string
|
|
provider *openai.Client
|
|
model string
|
|
store *store
|
|
agentID uuid.UUID
|
|
reqOpts []option.RequestOption
|
|
apiBase string // oikos HTTP API base, derived from NOMOS_MCP_URL, for chat-assent approvals
|
|
apiToken string // OIKOS_MCP_BEARER_TOKEN — api's combinedAuth requires it (no dev-open bypass)
|
|
httpClient *http.Client
|
|
// gate serializes turns per session (at most one in-flight turn per
|
|
// sessionID). See turngate.go and plan 2026-08-03 F1.
|
|
gate *turnGate
|
|
// queue holds operator messages that arrived while a turn was already
|
|
// running; they are auto-run when the gate frees (plan 2026-08-03 F2).
|
|
// See messagequeue.go.
|
|
queue *messageQueue
|
|
}
|
|
|
|
func newAgent(ctx context.Context, clients *mcpClientPool, st *store, agentSlug string) (*agent, error) {
|
|
system := loadSoul()
|
|
apiKey := os.Getenv("OPENROUTER_API_KEY")
|
|
model := os.Getenv("NOMOS_MODEL")
|
|
if model == "" {
|
|
// v4-pro over v4-flash: the flash tier over-narrates, occasionally
|
|
// emits canned refusals, and is unreliable at multi-step tool use —
|
|
// exactly the agentic provisioning path the operator needs to work.
|
|
model = "deepseek/deepseek-v4-pro"
|
|
}
|
|
|
|
provider := openai.NewClient(
|
|
option.WithBaseURL("https://openrouter.ai/api/v1"),
|
|
option.WithAPIKey(apiKey),
|
|
)
|
|
|
|
agentID := st.resolveAgentID(ctx, agentSlug)
|
|
if agentID == uuid.Nil {
|
|
slog.Warn("nomos: agent entity not found; tool-call activity will not be logged", "slug", agentSlug)
|
|
}
|
|
|
|
// OpenRouter provider routing. data_collection=deny pins to zero-data-
|
|
// retention providers (privacy: conversations + tool results transit
|
|
// OpenRouter); require_parameters ensures the routed provider actually
|
|
// supports tool calling. NOMOS_PROVIDER_SORT (price|throughput|latency)
|
|
// and Exacto tool-accuracy routing are opt-in — the latter via a model
|
|
// suffix in NOMOS_MODEL (e.g. "deepseek/deepseek-v4-flash:exacto"), so an
|
|
// unsupported value never silently breaks the confirmed routing below.
|
|
providerRouting := map[string]any{
|
|
"data_collection": "deny",
|
|
"require_parameters": true,
|
|
}
|
|
if sort := os.Getenv("NOMOS_PROVIDER_SORT"); sort != "" {
|
|
providerRouting["sort"] = sort
|
|
}
|
|
reqOpts := []option.RequestOption{option.WithJSONSet("provider", providerRouting)}
|
|
|
|
// Derive the oikos HTTP API base from the MCP URL (e.g.
|
|
// "http://api:8090/mcp?session_id=..." -> "http://api:8090"). Used for
|
|
// chat-assent approvals, which call the same decision endpoint the UI's
|
|
// Approve button calls.
|
|
mcpURL := os.Getenv("NOMOS_MCP_URL")
|
|
apiBase := ""
|
|
if idx := strings.Index(mcpURL, "/mcp"); idx > 0 {
|
|
apiBase = mcpURL[:idx]
|
|
}
|
|
|
|
return &agent{
|
|
clients: clients,
|
|
system: system,
|
|
provider: &provider,
|
|
model: model,
|
|
store: st,
|
|
agentID: agentID,
|
|
reqOpts: reqOpts,
|
|
apiBase: apiBase,
|
|
apiToken: os.Getenv("OIKOS_MCP_BEARER_TOKEN"),
|
|
httpClient: &http.Client{Timeout: 15 * time.Second},
|
|
gate: newTurnGate(),
|
|
queue: newMessageQueue(),
|
|
}, nil
|
|
}
|
|
|
|
func loadSoul() string {
|
|
paths := []string{"/app/nomos/SOUL.md", "nomos/SOUL.md"}
|
|
for _, p := range paths {
|
|
if data, err := os.ReadFile(p); err == nil {
|
|
return string(data)
|
|
}
|
|
}
|
|
return `You are Nomos, the steward of the oikos — the AI agent for the hubris homelab.
|
|
You have access to MCP tools to query topology, health, knowledge, and request
|
|
gated mutations through run. Be concise. Prefer tools over guessing.`
|
|
}
|
|
|
|
// assentWindowDuration is how long after an operator approves a plan that
|
|
// config_mutation commands auto-run without re-approval. The operator
|
|
// approved the plan; the agent should execute it end-to-end without
|
|
// stopping every step to re-ask. Destructive actions still always need
|
|
// explicit typed confirmation regardless of the window.
|
|
const assentWindowDuration = 30 * time.Minute
|
|
|
|
// openAssentWindow records an active assent window in autonomy_settings so
|
|
// the MCP run tool (separate process) can check it before requiring approval
|
|
// for config_mutation commands. Key is scoped to this agent's UUID AND this
|
|
// session/task — see store.go's assentWindowActive for why: without the
|
|
// session dimension, approving one task's plan would silently auto-run
|
|
// unapproved actions in any other concurrently-running task.
|
|
func (a *agent) openAssentWindow(ctx context.Context, sessionID string) {
|
|
if a.store == nil || a.store.pool == nil || a.agentID == uuid.Nil || sessionID == "" {
|
|
return
|
|
}
|
|
key := assentWindowKey(a.agentID, sessionID)
|
|
expires := time.Now().Add(assentWindowDuration).UTC().Format(time.RFC3339)
|
|
_, err := a.store.pool.Exec(ctx,
|
|
`INSERT INTO autonomy_settings (key, value) VALUES ($1, $2)
|
|
ON CONFLICT (key) DO UPDATE SET value = $2`, key, expires)
|
|
if err != nil {
|
|
slog.Warn("nomos: openAssentWindow", "error", err)
|
|
} else {
|
|
slog.Info("nomos: assent window opened", "agent", a.agentID, "session", sessionID, "expires", expires)
|
|
}
|
|
}
|
|
|
|
type toolDef struct {
|
|
Name string `json:"name"`
|
|
Description string `json:"description"`
|
|
InputSchema map[string]any `json:"inputSchema"`
|
|
}
|
|
|
|
type agentEvent struct {
|
|
Type string `json:"type"`
|
|
Data any `json:"data,omitempty"`
|
|
SessionID string `json:"session_id,omitempty"`
|
|
Iteration int `json:"iteration,omitempty"`
|
|
// IsThinking marks text/text_delta events that carry the model's internal
|
|
// reasoning (text produced before tool calls in the same iteration), as
|
|
// distinct from the final response text. The frontend renders these as
|
|
// collapsible thinking blocks separated from the response.
|
|
IsThinking bool `json:"is_thinking,omitempty"`
|
|
}
|
|
|
|
func (a *agent) chat(ctx context.Context, sessionID, message string, emit func(agentEvent)) {
|
|
a.chatWith(ctx, sessionID, message, "", emit)
|
|
}
|
|
|
|
// chatWith is chat() with an optional system-injected note appended after the
|
|
// replayed history. The auto-continuation worker uses it to resume a session
|
|
// with a finished execution's result ("execution X completed: … — continue the
|
|
// plan") without persisting a fake user turn. message is normally the new user
|
|
// message; for a worker continuation it is empty and systemInject carries the
|
|
// note.
|
|
func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject string, emit func(agentEvent)) {
|
|
correlationID := uuid.New().String()
|
|
|
|
// emitError emits an error event followed by a done event. The done
|
|
// event is CRITICAL on every terminal path: the frontend's
|
|
// onComplete handler (chat.ts) treats a missing `done` as a severed
|
|
// network connection and triggers an auto-reconnect → resumeSession.
|
|
// Before this fix, a model empty-response (the most common case here)
|
|
// returned without `done`, was misclassified as a network drop, and
|
|
// the reconnect logic re-invoked the agent with a generic "report
|
|
// your state" note — which caused the agent to re-propose the plan
|
|
// and duplicate it in the sidebar (operator-reported 2026-07-14).
|
|
// Every error return below must go through emitError so the frontend
|
|
// shows the error inline instead of silently reconnecting.
|
|
emitError := func(data string) {
|
|
emit(agentEvent{Type: "error", Data: data, SessionID: sessionID})
|
|
emit(agentEvent{Type: "done", Data: map[string]any{
|
|
"session_id": sessionID,
|
|
"correlation_id": correlationID,
|
|
"iterations": 0,
|
|
"error": true,
|
|
}, SessionID: sessionID})
|
|
}
|
|
|
|
tools, err := a.buildTools(sessionID)
|
|
if err != nil {
|
|
emitError(fmt.Sprintf("build tools: %v", err))
|
|
return
|
|
}
|
|
|
|
system := a.system
|
|
if snapshot := a.fleetSnapshot(sessionID); snapshot != "" {
|
|
system += "\n\n" + snapshot
|
|
}
|
|
messages := []openai.ChatCompletionMessageParamUnion{openai.SystemMessage(system)}
|
|
history, truncatedHistory, _ := a.store.getRecentMessages(ctx, sessionID, historyWindowSize)
|
|
if truncatedHistory {
|
|
// Tell the model explicitly rather than silently dropping older
|
|
// turns — otherwise it might assume something wasn't done just
|
|
// because it doesn't see the turn that did it.
|
|
messages = append(messages, openai.SystemMessage(fmt.Sprintf(
|
|
"[System: this task has been running long enough that only the most recent %d turns of its history are included above your context — earlier turns happened but aren't shown. If you need to know what was already tried or found, check search_knowledge/get_entity_knowledge (if you recorded it) rather than assuming it didn't happen.]",
|
|
historyWindowSize)))
|
|
}
|
|
// sawSetGoal / sawCompleteTask track whether this session has EVER framed
|
|
// itself as a structured task (set_goal) or already reached a terminal
|
|
// state (complete_task) — across both replayed history and this turn's
|
|
// own tool calls (updated again below as they happen live). Used by the
|
|
// end-of-turn safety net (plans/2026-07-11-task-completion-safety-net.md,
|
|
// fix 1): most sessions are a single trivial Q&A exchange that answers in
|
|
// text and never calls either tool, leaving agent_sessions.status stuck
|
|
// at its creation-time default forever. If a session never framed itself
|
|
// as a task, its first plain-text turn-end IS the task ending.
|
|
var sawSetGoal, sawCompleteTask bool
|
|
var lastAssistantCalls []persistedCall
|
|
for _, m := range history {
|
|
text := extractText(m.Content)
|
|
switch m.Role {
|
|
case "user":
|
|
messages = append(messages, openai.UserMessage(text))
|
|
case "assistant":
|
|
if calls := extractToolCalls(m.Content); len(calls) > 0 {
|
|
messages = append(messages, assistantToolCallMessage(calls))
|
|
for _, c := range calls {
|
|
messages = append(messages, openai.ToolMessage(c.resultText(), c.id))
|
|
switch c.name {
|
|
case "set_goal":
|
|
sawSetGoal = true
|
|
case "complete_task":
|
|
sawCompleteTask = true
|
|
}
|
|
}
|
|
lastAssistantCalls = calls
|
|
}
|
|
if text != "" {
|
|
messages = append(messages, openai.AssistantMessage(text))
|
|
}
|
|
}
|
|
}
|
|
if len(history) == 0 {
|
|
messages = append(messages, openai.UserMessage(message))
|
|
}
|
|
|
|
// Chat-assent approval: if the immediately-preceding assistant turn
|
|
// proposed gated action(s) and the operator's new message reads as
|
|
// authorization ("go ahead", "yes", ...), grant them now — this is the
|
|
// primary approval path; the Approve button in the UI is a fallback for
|
|
// when the operator wants to click instead of type. Destructive-risk
|
|
// actions are never granted by loose assent — they need the stricter
|
|
// isTypedConfirmation ("I confirm ...", per SOUL.md's guidance for what
|
|
// to ask the operator to type).
|
|
pending := extractPendingApprovals(lastAssistantCalls)
|
|
assent := isAssent(message)
|
|
typedConfirm := isTypedConfirmation(message)
|
|
if len(pending) > 0 && (assent || typedConfirm) {
|
|
var granted, blocked []string
|
|
for _, p := range pending {
|
|
if p.destructive && !typedConfirm {
|
|
blocked = append(blocked, p.execID)
|
|
continue
|
|
}
|
|
if !p.destructive && !assent {
|
|
continue // typed-confirm alone doesn't grant a non-destructive item without also reading as assent
|
|
}
|
|
ok, status, aerr := a.approveExecution(ctx, p.execID)
|
|
if aerr != nil {
|
|
slog.Error("nomos: chat-assent approve", "execution", p.execID, "error", aerr)
|
|
continue
|
|
}
|
|
if ok {
|
|
granted = append(granted, p.execID)
|
|
slog.Info("nomos: chat-assent granted", "execution", p.execID, "status", status, "session", sessionID)
|
|
|
|
// An explicit typed confirmation for a destructive action
|
|
// opens a short, target-scoped window so the rest of a
|
|
// destructive recovery sequence on the SAME target (e.g.
|
|
// stop -> destroy) doesn't need a second typed confirmation.
|
|
if p.destructive && typedConfirm {
|
|
if execUUID, perr := uuid.Parse(p.execID); perr == nil {
|
|
if target := a.store.executionTarget(ctx, execUUID); target != "" {
|
|
a.store.openDestructiveWindow(ctx, a.agentID, target, sessionID)
|
|
slog.Info("nomos: destructive window opened", "agent", a.agentID, "target", target, "session", sessionID)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if len(granted) > 0 {
|
|
a.openAssentWindow(ctx, sessionID)
|
|
// Mark approved executions as continued so the continuation
|
|
// worker doesn't call resumeSession while the chat handler is
|
|
// still processing "go ahead" — two concurrent LLM calls for the
|
|
// same session cause empty responses and race conditions.
|
|
for _, execID := range granted {
|
|
if execUUID, perr := uuid.Parse(execID); perr == nil {
|
|
a.store.markContinued(ctx, execUUID)
|
|
}
|
|
}
|
|
// No system note. The model already sees "go ahead" in the
|
|
// replayed history (the user message was saved to the DB before
|
|
// chat() was called). The old note said "they are now running"
|
|
// which made the model think work was being done for it —
|
|
// causing empty responses (finish_reason=stop, content_len=0).
|
|
// The approved executions are dispatched; the model will
|
|
// continue with the remaining plan steps naturally.
|
|
}
|
|
if len(blocked) > 0 {
|
|
note := fmt.Sprintf("[System: execution(s) %s are classified DESTRUCTIVE and were NOT approved by loose assent — you must ask the operator for an explicit typed confirmation before they can run. Once they do confirm, further destructive steps on that SAME target (e.g. finishing a stop-then-destroy sequence) will auto-run for 15 minutes without asking again — but a different target always needs its own confirmation.]", strings.Join(blocked, ", "))
|
|
messages = append(messages, openai.SystemMessage(note))
|
|
}
|
|
} else if assent && len(pending) == 0 {
|
|
// The operator said "proceed"/"go ahead"/"yes" but there are no
|
|
// pending approvals — the agent proposed a plan (via propose_plan)
|
|
// and asked "shall I?" Open the assent window silently. No system
|
|
// note: the model sees "go ahead" in the replayed history and
|
|
// responds naturally.
|
|
a.openAssentWindow(ctx, sessionID)
|
|
}
|
|
|
|
// Worker continuation: append the finished-execution note so the model
|
|
// sees the result and decides the next step (proceed / recover / done).
|
|
if systemInject != "" {
|
|
messages = append(messages, openai.SystemMessage(systemInject))
|
|
}
|
|
|
|
// Retry cap (P0.1 from plans/2026-07-18-session-review-three-sessions.md):
|
|
// track failing `run` calls within this turn so an identical command that
|
|
// keeps failing is refused after maxRunRetries attempts. Without this,
|
|
// session 1e9c7691 retried the same `chown` ~20 times, each retry piling
|
|
// up a zombie process on the target (knfsd was holding a kernel lock).
|
|
// The tracker is per-turn — a fresh turn after the operator responds can
|
|
// retry once more, so this doesn't permanently block recovery.
|
|
retries := newRunRetryTracker()
|
|
|
|
for i := 0; i < maxIterations; i++ {
|
|
params := openai.ChatCompletionNewParams{
|
|
Model: openai.ChatModel(a.model),
|
|
Messages: messages,
|
|
Tools: tools,
|
|
}
|
|
|
|
var msg openai.ChatCompletionMessage
|
|
var acc openai.ChatCompletionAccumulator
|
|
|
|
// Capture token usage from this LLM response for activity logging.
|
|
// Previously always NULL — every agent_activity row had no token
|
|
// count. Now each tool call in this iteration gets the same total.
|
|
totalTokens := 0
|
|
|
|
for attempt := 0; attempt <= maxLLMRetries; attempt++ {
|
|
acc = openai.ChatCompletionAccumulator{}
|
|
stream := a.provider.Chat.Completions.NewStreaming(ctx, params, a.reqOpts...)
|
|
for stream.Next() {
|
|
chunk := stream.Current()
|
|
acc.AddChunk(chunk)
|
|
if len(chunk.Choices) > 0 {
|
|
if delta := chunk.Choices[0].Delta.Content; delta != "" {
|
|
emit(agentEvent{Type: "text_delta", Data: delta, SessionID: sessionID, Iteration: i + 1})
|
|
}
|
|
}
|
|
}
|
|
if err := stream.Err(); err != nil {
|
|
if attempt < maxLLMRetries {
|
|
slog.Warn("nomos: llm stream error, retrying", "error", err, "attempt", attempt+1, "session", sessionID)
|
|
continue
|
|
}
|
|
emitError(fmt.Sprintf("llm: %v", err))
|
|
return
|
|
}
|
|
if len(acc.Choices) == 0 {
|
|
if attempt < maxLLMRetries {
|
|
slog.Warn("nomos: no choices in response, retrying", "attempt", attempt+1, "session", sessionID)
|
|
continue
|
|
}
|
|
emitError("no choices in response (the model returned zero completions — likely a provider or rate-limit issue)")
|
|
return
|
|
}
|
|
|
|
msg = acc.Choices[0].Message
|
|
finishReason := acc.Choices[0].FinishReason
|
|
|
|
// Capture token usage from this iteration.
|
|
if acc.Usage.TotalTokens > 0 {
|
|
totalTokens = int(acc.Usage.TotalTokens)
|
|
}
|
|
|
|
if len(msg.ToolCalls) == 0 {
|
|
if isRefusalOrEmpty(msg.Content) {
|
|
if attempt < maxLLMRetries {
|
|
slog.Warn("nomos: empty or refusal response, retrying",
|
|
"session", sessionID, "iter", i+1, "attempt", attempt+1,
|
|
"content_len", len(msg.Content), "finish_reason", finishReason)
|
|
continue
|
|
}
|
|
// B.4: surface the real error context (finish_reason +
|
|
// refusal text) instead of a generic "empty response" —
|
|
// the operator can tell "content_filter — rephrase" from
|
|
// "length — token limit hit" from "stop — model no-op'd".
|
|
detail := "empty response"
|
|
if msg.Refusal != "" {
|
|
detail = fmt.Sprintf("refusal: %s", msg.Refusal)
|
|
} else if finishReason != "" && finishReason != "stop" {
|
|
detail = fmt.Sprintf("finish_reason=%s", finishReason)
|
|
}
|
|
emitError(fmt.Sprintf("Nomos returned an empty or unusable response (%s). Retry or rephrase.", detail))
|
|
return
|
|
}
|
|
}
|
|
break
|
|
}
|
|
|
|
if len(msg.ToolCalls) == 0 {
|
|
emit(agentEvent{Type: "text", Data: msg.Content, SessionID: sessionID})
|
|
if !sawSetGoal && !sawCompleteTask {
|
|
a.autoCompleteTrivialTask(ctx, sessionID, msg.Content)
|
|
}
|
|
// Safety net: if the agent called set_goal (structured task)
|
|
// but didn't call complete_task, and all plan steps are
|
|
// terminal, auto-complete. The model often does the work but
|
|
// forgets to close the loop (confirmed live: the #1 remaining
|
|
// model reliability gap after D.1).
|
|
if !sawCompleteTask {
|
|
a.autoCompleteIfPlanDone(ctx, sessionID, msg.Content)
|
|
}
|
|
emit(agentEvent{Type: "done", Data: map[string]any{
|
|
"session_id": sessionID,
|
|
"usage": acc.Usage,
|
|
"correlation_id": correlationID,
|
|
"iterations": i + 1,
|
|
}, SessionID: sessionID})
|
|
return
|
|
}
|
|
|
|
// P3: persist intermediate reasoning. When the model produces text
|
|
// AND tool calls in the same iteration, the text is its reasoning
|
|
// before the tool calls — the operator saw it live via text_delta,
|
|
// but without emitting it as a `text` event here, the persist layer
|
|
// (main.go/continue.go) never captures it and a reload shows only
|
|
// the final summary + a flat tool-call list, not the thinking that
|
|
// led to each step. Emitting it lets the persist layer accumulate
|
|
// per-iteration reasoning into the row's text field.
|
|
if strings.TrimSpace(msg.Content) != "" {
|
|
emit(agentEvent{Type: "text", Data: msg.Content, SessionID: sessionID, IsThinking: true})
|
|
}
|
|
|
|
slog.Info("nomos: tool calls", "count", len(msg.ToolCalls), "iter", i+1, "correlation", correlationID)
|
|
|
|
messages = append(messages, msg.ToParam())
|
|
|
|
for _, tc := range msg.ToolCalls {
|
|
var args map[string]any
|
|
if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil {
|
|
args = map[string]any{}
|
|
}
|
|
|
|
switch tc.Function.Name {
|
|
case "set_goal":
|
|
sawSetGoal = true
|
|
case "complete_task":
|
|
sawCompleteTask = true
|
|
}
|
|
|
|
// Retry cap: if this `run` call has already failed
|
|
// maxRunRetries times this turn with the same (target,
|
|
// command), refuse to dispatch it again. Return a synthetic
|
|
// tool result directing the agent to investigate *why* the
|
|
// command hangs instead of retrying. See retrycap.go and
|
|
// plans/2026-07-18-session-review-three-sessions.md P0.1.
|
|
if tc.Function.Name == "run" {
|
|
t, _ := args["target"].(string)
|
|
c, _ := args["command"].(string)
|
|
key := runFailureKey(t, c)
|
|
if n := retries.failures(key); n >= maxRunRetries {
|
|
directive := runRetryDirective(t, c, n)
|
|
slog.Warn("nomos: run retry cap hit — refusing dispatch",
|
|
"target", t, "failures", n, "session", sessionID)
|
|
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args,
|
|
tc.Function.Arguments, directive, 0, false, correlationID, totalTokens)
|
|
emit(agentEvent{
|
|
Type: "tool_result",
|
|
Data: map[string]any{"name": tc.Function.Name, "result": directive, "id": tc.ID, "retry_capped": true},
|
|
SessionID: sessionID,
|
|
Iteration: i + 1,
|
|
})
|
|
messages = append(messages, openai.ToolMessage(directive, tc.ID))
|
|
continue
|
|
}
|
|
}
|
|
|
|
emit(agentEvent{
|
|
Type: "tool_use",
|
|
Data: map[string]any{"name": tc.Function.Name, "args": args, "id": tc.ID},
|
|
SessionID: sessionID,
|
|
Iteration: i + 1,
|
|
})
|
|
|
|
start := time.Now()
|
|
// Session-scoped task tools are handled in-process; everything else
|
|
// is forwarded to the shared MCP server.
|
|
var result any
|
|
var callErr error
|
|
if localRes, handled := a.handleTaskTool(ctx, sessionID, tc.Function.Name, args); handled {
|
|
result = localRes
|
|
} else {
|
|
// _session_id rides along on the wire call only — never in
|
|
// `args` (which is what gets emitted/logged/persisted as the
|
|
// model's own tool call) — so the MCP-side assent/destructive
|
|
// window checks can scope to THIS task instead of bleeding
|
|
// across every concurrently-running one sharing this agent
|
|
// identity. Not part of any tool's declared InputSchema, so
|
|
// the model never sees or supplies it.
|
|
wireArgs := make(map[string]any, len(args)+1)
|
|
for k, v := range args {
|
|
wireArgs[k] = v
|
|
}
|
|
wireArgs["_session_id"] = sessionID
|
|
var client *mcpClient
|
|
client, callErr = a.clients.get(sessionID)
|
|
if callErr == nil {
|
|
result, callErr = client.callTool(tc.Function.Name, wireArgs)
|
|
}
|
|
}
|
|
elapsed := int(time.Since(start).Milliseconds())
|
|
|
|
inputJSON, _ := json.Marshal(args)
|
|
inputStr := string(inputJSON)
|
|
|
|
if callErr != nil {
|
|
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args, inputStr, callErr.Error(), elapsed, false, correlationID, totalTokens)
|
|
|
|
// Retry cap: dispatch errors (e.g. MCP client timeout)
|
|
// count toward the cap too. A command that keeps timing
|
|
// out at the gateway is exactly the pattern we want to
|
|
// break — see session 1e9c7691's 20+ identical
|
|
// `chown` timeouts.
|
|
if tc.Function.Name == "run" {
|
|
t, _ := args["target"].(string)
|
|
c, _ := args["command"].(string)
|
|
key := runFailureKey(t, c)
|
|
n := retries.recordFailure(key)
|
|
if n >= maxRunRetries {
|
|
slog.Warn("nomos: run failure cap reached — next identical call will be refused",
|
|
"target", t, "failures", n, "session", sessionID)
|
|
}
|
|
}
|
|
|
|
emit(agentEvent{
|
|
Type: "tool_result",
|
|
Data: map[string]any{"name": tc.Function.Name, "error": callErr.Error(), "id": tc.ID},
|
|
SessionID: sessionID,
|
|
Iteration: i + 1,
|
|
})
|
|
messages = append(messages, openai.ToolMessage(callErr.Error(), tc.ID))
|
|
slog.Error("nomos: tool error", "tool", tc.Function.Name, "error", callErr, "ms", elapsed)
|
|
continue
|
|
}
|
|
|
|
resultJSON, _ := json.Marshal(result)
|
|
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args, inputStr, string(resultJSON), elapsed, true, correlationID, totalTokens)
|
|
|
|
// Link any execution this tool queued/started back to this
|
|
// session, so the auto-continuation worker can feed its result
|
|
// back here when it finishes (see cmd/nomos/continue.go). Async
|
|
// executions (pct_create, apt_upgrade) are the ones that matter —
|
|
// their result lands after this turn ends.
|
|
for _, execID := range extractExecutionIDs(string(resultJSON)) {
|
|
a.store.linkExecution(ctx, execID, sessionID)
|
|
}
|
|
|
|
// Record which entities this task touched (task —involves→ entity)
|
|
// and pulse them on the live context panel. Args only — never
|
|
// results — so a bulk query doesn't drag the whole fleet in.
|
|
a.store.recordTouched(ctx, sessionID, tc.Function.Name, args)
|
|
|
|
// When the agent records knowledge, link that note to this task so
|
|
// the task's outcome view shows what it learned (and pulse it live).
|
|
if tc.Function.Name == "upsert_knowledge" {
|
|
a.store.linkKnowledgeToTask(ctx, sessionID, string(resultJSON))
|
|
}
|
|
|
|
emit(agentEvent{
|
|
Type: "tool_result",
|
|
Data: map[string]any{"name": tc.Function.Name, "result": result, "id": tc.ID},
|
|
SessionID: sessionID,
|
|
Iteration: i + 1,
|
|
})
|
|
messages = append(messages, openai.ToolMessage(string(resultJSON), tc.ID))
|
|
slog.Info("nomos: tool success", "tool", tc.Function.Name, "ms", elapsed)
|
|
|
|
// Retry cap: record failures of `run` calls so the cap above
|
|
// can refuse a repeated identical failure. A "failure" here
|
|
// means the dispatch errored OR the MCP result text matches
|
|
// the "run on <target>: ERROR …" signature — both indicate
|
|
// the command actually ran and failed, not just that it
|
|
// queued for approval (pending approvals are not failures).
|
|
// Pass the RAW result text (not JSON-encoded) so the helper's
|
|
// HasPrefix check sees "run on …" not "\"run on …\"".
|
|
if isRunFailure(tc.Function.Name, runResultText(result), callErr) {
|
|
t, _ := args["target"].(string)
|
|
c, _ := args["command"].(string)
|
|
key := runFailureKey(t, c)
|
|
n := retries.recordFailure(key)
|
|
if n >= maxRunRetries {
|
|
slog.Warn("nomos: run failure cap reached — next identical call will be refused",
|
|
"target", t, "failures", n, "session", sessionID)
|
|
}
|
|
}
|
|
|
|
// ask_operator pauses the task: the agent has posed a decision only
|
|
// the operator can make. End the turn here so it doesn't barrel past
|
|
// its own question — the answer (panel or chat reply) resumes it.
|
|
// The prompt becomes the assistant's visible message so the question
|
|
// also shows inline in the transcript.
|
|
if tc.Function.Name == "ask_operator" {
|
|
prompt, _ := args["prompt"].(string)
|
|
emit(agentEvent{Type: "text", Data: prompt, SessionID: sessionID})
|
|
emit(agentEvent{Type: "done", Data: map[string]any{
|
|
"session_id": sessionID,
|
|
"correlation_id": correlationID,
|
|
"iteration": i + 1,
|
|
}, SessionID: sessionID})
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// Hitting the step limit used to end the turn with a bare "max iterations
|
|
// reached without final answer" — a dead end that made the operator ask
|
|
// "status?" to find out what actually happened after a long working turn.
|
|
// Instead, spend one final call asking the model to summarize what it did
|
|
// and the current state, so the turn always ends with a real report.
|
|
messages = append(messages, openai.SystemMessage("[System: you've reached the step limit for this turn. STOP calling tools now and write a concise status report: what you accomplished, the current state of the goal, anything that failed, and what remains. This is what the operator sees.]"))
|
|
summary := a.finalSummary(ctx, messages)
|
|
if summary == "" {
|
|
summary = "I hit this turn's step limit while working. I've done a lot but couldn't wrap up cleanly — ask me for a status update and I'll summarize the current state."
|
|
}
|
|
emit(agentEvent{Type: "text", Data: summary, SessionID: sessionID})
|
|
if !sawCompleteTask {
|
|
a.autoCompleteIfPlanDone(ctx, sessionID, summary)
|
|
}
|
|
emit(agentEvent{Type: "done", Data: map[string]any{
|
|
"session_id": sessionID,
|
|
"correlation_id": correlationID,
|
|
"iterations": maxIterations,
|
|
}, SessionID: sessionID})
|
|
}
|
|
|
|
// finalSummary makes one non-tool LLM call to turn an exhausted tool-loop into
|
|
// a real status report instead of a dead-end message. Best-effort: empty on
|
|
// any error, and the caller has a fallback.
|
|
func (a *agent) finalSummary(ctx context.Context, messages []openai.ChatCompletionMessageParamUnion) string {
|
|
params := openai.ChatCompletionNewParams{
|
|
Model: openai.ChatModel(a.model),
|
|
Messages: messages,
|
|
// No Tools: force a text answer.
|
|
}
|
|
resp, err := a.provider.Chat.Completions.New(ctx, params, a.reqOpts...)
|
|
if err != nil || len(resp.Choices) == 0 {
|
|
return ""
|
|
}
|
|
return resp.Choices[0].Message.Content
|
|
}
|
|
|
|
// extractText pulls the "text" field from a persisted message's JSONB content.
|
|
func extractText(content json.RawMessage) string {
|
|
var m struct {
|
|
Text string `json:"text"`
|
|
}
|
|
if err := json.Unmarshal(content, &m); err != nil {
|
|
return ""
|
|
}
|
|
return m.Text
|
|
}
|
|
|
|
// persistedCall is one merged tool_use+tool_result pair from a persisted
|
|
// assistant message's tool_calls array. The store keeps them as two entries
|
|
// sharing the same id (mirroring the SSE event pair); replay needs one
|
|
// entry per id to build a valid tool-calling assistant message.
|
|
type persistedCall struct {
|
|
id string
|
|
name string
|
|
args json.RawMessage
|
|
result json.RawMessage
|
|
errMsg string
|
|
}
|
|
|
|
func (c persistedCall) resultText() string {
|
|
if c.errMsg != "" {
|
|
return c.errMsg
|
|
}
|
|
if len(c.result) > 0 {
|
|
return string(c.result)
|
|
}
|
|
return "null"
|
|
}
|
|
|
|
// extractToolCalls parses and merges a persisted message's tool_calls array,
|
|
// preserving first-seen order across ids.
|
|
func extractToolCalls(content json.RawMessage) []persistedCall {
|
|
var m struct {
|
|
ToolCalls []struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Name string `json:"name"`
|
|
Args json.RawMessage `json:"args"`
|
|
Result json.RawMessage `json:"result"`
|
|
Error string `json:"error"`
|
|
} `json:"tool_calls"`
|
|
}
|
|
if err := json.Unmarshal(content, &m); err != nil || len(m.ToolCalls) == 0 {
|
|
return nil
|
|
}
|
|
|
|
byID := make(map[string]*persistedCall, len(m.ToolCalls))
|
|
var order []string
|
|
for _, tc := range m.ToolCalls {
|
|
if tc.ID == "" {
|
|
continue
|
|
}
|
|
pc, ok := byID[tc.ID]
|
|
if !ok {
|
|
pc = &persistedCall{id: tc.ID}
|
|
byID[tc.ID] = pc
|
|
order = append(order, tc.ID)
|
|
}
|
|
if tc.Name != "" {
|
|
pc.name = tc.Name
|
|
}
|
|
if len(tc.Args) > 0 && string(tc.Args) != "null" {
|
|
pc.args = tc.Args
|
|
}
|
|
if tc.Type == "tool_result" {
|
|
pc.errMsg = tc.Error
|
|
pc.result = tc.Result
|
|
}
|
|
}
|
|
|
|
calls := make([]persistedCall, 0, len(order))
|
|
for _, id := range order {
|
|
calls = append(calls, *byID[id])
|
|
}
|
|
return calls
|
|
}
|
|
|
|
// assistantToolCallMessage builds the tool-calling assistant message that
|
|
// must precede the tool-role results being replayed.
|
|
func assistantToolCallMessage(calls []persistedCall) openai.ChatCompletionMessageParamUnion {
|
|
toolCalls := make([]openai.ChatCompletionMessageToolCallParam, 0, len(calls))
|
|
for _, c := range calls {
|
|
args := string(c.args)
|
|
if args == "" {
|
|
args = "{}"
|
|
}
|
|
toolCalls = append(toolCalls, openai.ChatCompletionMessageToolCallParam{
|
|
ID: c.id,
|
|
Function: openai.ChatCompletionMessageToolCallFunctionParam{
|
|
Name: c.name,
|
|
Arguments: args,
|
|
},
|
|
})
|
|
}
|
|
return openai.ChatCompletionMessageParamUnion{
|
|
OfAssistant: &openai.ChatCompletionAssistantMessageParam{ToolCalls: toolCalls},
|
|
}
|
|
}
|
|
|
|
// fleetSnapshot returns a compact, current-as-of-now fleet health line for
|
|
// the system prompt so the agent starts each turn already oriented instead
|
|
// of spending its first iteration rediscovering topology it already has
|
|
// tools to query. Best-effort: an empty string on any failure just means no
|
|
// snapshot, not an error for the turn.
|
|
func (a *agent) fleetSnapshot(sessionID string) string {
|
|
client, err := a.clients.get(sessionID)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
result, err := client.callTool("get_health_summary", map[string]any{})
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
rows, ok := result.([]any)
|
|
if !ok {
|
|
return ""
|
|
}
|
|
|
|
counts := map[string]int{}
|
|
var attention []string
|
|
for _, r := range rows {
|
|
row, ok := r.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
health, _ := row["health"].(string)
|
|
counts[health]++
|
|
if health != "healthy" && health != "" {
|
|
if slug, ok := row["slug"].(string); ok && len(attention) < 10 {
|
|
attention = append(attention, fmt.Sprintf("%s(%s)", slug, health))
|
|
}
|
|
}
|
|
}
|
|
if len(counts) == 0 {
|
|
return ""
|
|
}
|
|
|
|
summary := fmt.Sprintf("Current fleet snapshot (as of now): healthy=%d degraded=%d down=%d stale=%d unknown=%d.",
|
|
counts["healthy"], counts["degraded"], counts["down"], counts["stale"], counts["unknown"])
|
|
if len(attention) > 0 {
|
|
summary += " Needs attention: " + fmt.Sprintf("%v", attention) + "."
|
|
}
|
|
return summary
|
|
}
|
|
|
|
// isRefusalOrEmpty returns true when the LLM response is blank or looks like a
|
|
// canned non-English refusal to an English-language conversation. Flash-tier
|
|
// models occasionally emit Chinese boilerplate deflection instead of a real
|
|
// answer; this catches it before it reaches the UI.
|
|
func isRefusalOrEmpty(text string) bool {
|
|
if strings.TrimSpace(text) == "" {
|
|
return true
|
|
}
|
|
ascii, nonASCII := 0, 0
|
|
for _, r := range text {
|
|
if r <= 127 {
|
|
ascii++
|
|
} else {
|
|
nonASCII++
|
|
}
|
|
}
|
|
if nonASCII > ascii {
|
|
return true
|
|
}
|
|
for _, pattern := range refusalDenylist {
|
|
if strings.Contains(text, pattern) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (a *agent) buildTools(sessionID string) ([]openai.ChatCompletionToolParam, error) {
|
|
client, err := a.clients.get(sessionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defs, err := client.listToolsFull()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Append nomos-local, session-scoped task tools (complete_task, …) to the
|
|
// MCP tool list. They're routed to handleTaskTool, not the MCP client.
|
|
defs = append(defs, taskToolDefs()...)
|
|
|
|
var tools []openai.ChatCompletionToolParam
|
|
for _, d := range defs {
|
|
params := shared.FunctionParameters(d.InputSchema)
|
|
if params == nil {
|
|
params = shared.FunctionParameters{"type": "object", "properties": map[string]any{}}
|
|
}
|
|
|
|
tools = append(tools, openai.ChatCompletionToolParam{
|
|
Type: "function",
|
|
Function: shared.FunctionDefinitionParam{
|
|
Name: d.Name,
|
|
Description: openai.String(d.Description),
|
|
Parameters: params,
|
|
},
|
|
})
|
|
}
|
|
return tools, nil
|
|
}
|
|
|
|
// listToolsFull returns the MCP server's tool list, cached on this client
|
|
// after the first call (see mcpClient.toolsCache). Fix F1 of
|
|
// plans/2026-07-11-nomos-agent-code-review.md: buildTools calls this at the
|
|
// start of every chat turn, including every auto-continuation resume — the
|
|
// tool list is static for the lifetime of one MCP connection, so re-fetching
|
|
// it every single time was avoidable network+parsing work on the hot path.
|
|
// Cache invalidates on reconnectLocked (an api restart may change what's
|
|
// registered).
|
|
func (c *mcpClient) listToolsFull() ([]toolDef, error) {
|
|
c.toolsMu.Lock()
|
|
if c.toolsCache != nil {
|
|
cached := c.toolsCache
|
|
c.toolsMu.Unlock()
|
|
return cached, nil
|
|
}
|
|
c.toolsMu.Unlock()
|
|
|
|
resp, err := c.doRequest("tools/list", map[string]any{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var tr struct {
|
|
Tools []struct {
|
|
Name string `json:"name"`
|
|
Description string `json:"description"`
|
|
InputSchema map[string]any `json:"inputSchema"`
|
|
} `json:"tools"`
|
|
}
|
|
if err := json.Unmarshal(resp.Result, &tr); err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]toolDef, len(tr.Tools))
|
|
for i, t := range tr.Tools {
|
|
out[i] = toolDef{
|
|
Name: t.Name,
|
|
Description: t.Description,
|
|
InputSchema: t.InputSchema,
|
|
}
|
|
}
|
|
|
|
c.toolsMu.Lock()
|
|
c.toolsCache = out
|
|
c.toolsMu.Unlock()
|
|
return out, nil
|
|
}
|