feat(nomos): retry cap, vm: targets, inspect_path, goal supersession, runbooks
Some checks failed
ci / build-test (push) Has been cancelled
ci / docker-build (push) Has been cancelled
ci / web (push) Has been cancelled
Desktop App / Build Linux (amd64) (push) Has been cancelled
Desktop App / Attach to Release (push) Has been cancelled

Session-review implementation for the three sessions audited in
plans/2026-07-18-session-review-three-sessions.md. v0.7.11 → v0.7.12.

P0.1 — retry cap + investigate-before-retry (cmd/nomos/retrycap.go,
agent.go): after 3 identical failing run calls in a single turn, refuse
to dispatch the call again and return a directive to investigate *why*
(ps/strace/lsof) or surface the blocker. Per-turn scope so a fresh turn
after the operator responds can retry once more. Session 1e9c7691's 20+
identical chown retries (knfsd held a kernel lock on the exported NFS
dir) is the direct motivation.

P0.2 + P1.8 + P2.10 — SOUL.md guidance: hung command is not a failed
command (investigate before retry); ask before proposing a multi-step
migration; multi-goal sessions summarize the arc not just the last goal.

P1.3 — two new runbook entities in seeds/knowledge.yaml:
  - nfs-exported-dir-mutation-hang (the knfsd fchownat lock procedure:
    killall → exportfs -u → mutate → exportfs -a → verify)
  - netbird-mgmt-oidc-race-after-upgrade (docker restart netbird-mgmt
    after ~30s for the traefik/authentik OIDC race)

P1.4 — setGoal emits task.superseded event when prior goal is overwritten
by a different goal (store.go, TestSetGoal_SupersededEvent). Session
55927f0a had two set_goal calls with the first silently abandoned.

P1.5 — inspect_path MCP tool: runs mount/df/ls/stat for one path across
up to 8 targets in one parallel call, replacing the 15+ run-call
fact-gathering fan-out sessions 1 and 2 each spent on cross-target path
tracing (tools.go, server.go: inspectPathAcrossTargets, inspectOneTarget).

P1.6 — vm: target support in run via qm guest exec (no more SSH-hop
with nested quoting). Extracted shared resolveProxmoxHostSlug for
LXC + VM, with hosts-relationship fallback when attributes.host is
absent (server.go, tools.go). Session 55927f0a's SSH-hop workarounds
for vm:zimaos are the direct motivation.

Deferred (documented in plan): P1.7 (approval window auto-extend on
timeout) and P2.9 (long-running command PENDING detection) — both
addressed at lower cost by the retry cap. Session 3's poll-after-timeout
pattern already works; the cap protects against the failure mode.
This commit is contained in:
2026-07-19 00:09:39 +02:00
parent bd44626532
commit 544afae77f
12 changed files with 1265 additions and 19 deletions

View File

@@ -168,10 +168,10 @@ type toolDef struct {
}
type agentEvent struct {
Type string `json:"type"`
Data any `json:"data,omitempty"`
SessionID string `json:"session_id,omitempty"`
Iteration int `json:"iteration,omitempty"`
Type string `json:"type"`
Data any `json:"data,omitempty"`
SessionID string `json:"session_id,omitempty"`
Iteration int `json:"iteration,omitempty"`
}
func (a *agent) chat(ctx context.Context, sessionID, message string, emit func(agentEvent)) {
@@ -349,6 +349,15 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
messages = append(messages, openai.SystemMessage(systemInject))
}
// Retry cap (P0.1 from plans/2026-07-18-session-review-three-sessions.md):
// track failing `run` calls within this turn so an identical command that
// keeps failing is refused after maxRunRetries attempts. Without this,
// session 1e9c7691 retried the same `chown` ~20 times, each retry piling
// up a zombie process on the target (knfsd was holding a kernel lock).
// The tracker is per-turn — a fresh turn after the operator responds can
// retry once more, so this doesn't permanently block recovery.
retries := newRunRetryTracker()
for i := 0; i < maxIterations; i++ {
params := openai.ChatCompletionNewParams{
Model: openai.ChatModel(a.model),
@@ -467,6 +476,33 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
sawCompleteTask = true
}
// Retry cap: if this `run` call has already failed
// maxRunRetries times this turn with the same (target,
// command), refuse to dispatch it again. Return a synthetic
// tool result directing the agent to investigate *why* the
// command hangs instead of retrying. See retrycap.go and
// plans/2026-07-18-session-review-three-sessions.md P0.1.
if tc.Function.Name == "run" {
t, _ := args["target"].(string)
c, _ := args["command"].(string)
key := runFailureKey(t, c)
if n := retries.failures(key); n >= maxRunRetries {
directive := runRetryDirective(t, c, n)
slog.Warn("nomos: run retry cap hit — refusing dispatch",
"target", t, "failures", n, "session", sessionID)
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args,
tc.Function.Arguments, directive, 0, false, correlationID)
emit(agentEvent{
Type: "tool_result",
Data: map[string]any{"name": tc.Function.Name, "result": directive, "id": tc.ID, "retry_capped": true},
SessionID: sessionID,
Iteration: i + 1,
})
messages = append(messages, openai.ToolMessage(directive, tc.ID))
continue
}
}
emit(agentEvent{
Type: "tool_use",
Data: map[string]any{"name": tc.Function.Name, "args": args, "id": tc.ID},
@@ -508,6 +544,22 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
if callErr != nil {
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args, inputStr, callErr.Error(), elapsed, false, correlationID)
// Retry cap: dispatch errors (e.g. MCP client timeout)
// count toward the cap too. A command that keeps timing
// out at the gateway is exactly the pattern we want to
// break — see session 1e9c7691's 20+ identical
// `chown` timeouts.
if tc.Function.Name == "run" {
t, _ := args["target"].(string)
c, _ := args["command"].(string)
key := runFailureKey(t, c)
n := retries.recordFailure(key)
if n >= maxRunRetries {
slog.Warn("nomos: run failure cap reached — next identical call will be refused",
"target", t, "failures", n, "session", sessionID)
}
}
emit(agentEvent{
Type: "tool_result",
Data: map[string]any{"name": tc.Function.Name, "error": callErr.Error(), "id": tc.ID},
@@ -551,6 +603,25 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
messages = append(messages, openai.ToolMessage(string(resultJSON), tc.ID))
slog.Info("nomos: tool success", "tool", tc.Function.Name, "ms", elapsed)
// Retry cap: record failures of `run` calls so the cap above
// can refuse a repeated identical failure. A "failure" here
// means the dispatch errored OR the MCP result text matches
// the "run on <target>: ERROR …" signature — both indicate
// the command actually ran and failed, not just that it
// queued for approval (pending approvals are not failures).
// Pass the RAW result text (not JSON-encoded) so the helper's
// HasPrefix check sees "run on …" not "\"run on …\"".
if isRunFailure(tc.Function.Name, runResultText(result), callErr) {
t, _ := args["target"].(string)
c, _ := args["command"].(string)
key := runFailureKey(t, c)
n := retries.recordFailure(key)
if n >= maxRunRetries {
slog.Warn("nomos: run failure cap reached — next identical call will be refused",
"target", t, "failures", n, "session", sessionID)
}
}
// ask_operator pauses the task: the agent has posed a decision only
// the operator can make. End the turn here so it doesn't barrel past
// its own question — the answer (panel or chat reply) resumes it.

170
cmd/nomos/retrycap.go Normal file
View File

@@ -0,0 +1,170 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"strings"
"sync"
)
// maxRunRetries is the per-turn cap on identical failing `run` tool calls.
// After this many failures with the same (target, command) key, the agent
// loop refuses to dispatch the call again and instead surfaces a directive
// to investigate *why* (ps/strace/lsof) or escalate to the operator.
//
// Background: session 1e9c7691 (2026-07-18) retried the same
// `chown :10000 /mnt/media_local && chmod 2775 …` ~20 times across direct
// runs, SSH-hop-via-hubris, wrapping in a shell script, and bare `echo test`
// sanity checks. Each retry piled up another zombie process on the target
// (knfsd was holding a kernel lock on the exported directory). The agent
// only investigated *why* after the operator explicitly asked
// "the command just keeps running?" — see
// plans/2026-07-18-session-review-three-sessions.md P0.1.
const maxRunRetries = 3
// runRetryTracker deduplicates failing `run` calls within a single chat
// turn (chatWith invocation). It is NOT persisted across turns — the cap
// is per-turn, so a fresh turn after the operator responds can retry once
// more. The intent is to break a tight retry loop within one turn, not to
// permanently block the agent from ever attempting the operation again.
//
// Threading: the agent loop is single-goroutine per turn, but the tracker
// is guarded by a mutex so future callers (e.g. concurrent tool dispatch)
// stay safe. The mutex is uncontended on the current hot path.
type runRetryTracker struct {
mu sync.Mutex
counts map[string]int
}
func newRunRetryTracker() *runRetryTracker {
return &runRetryTracker{counts: make(map[string]int)}
}
// runFailureKey is the dedup key for "this is the same command against the
// same target." Whitespace is collapsed so trivial reformatting
// (newlines vs spaces, trailing whitespace) doesn't escape the cap. The
// purpose field is intentionally NOT part of the key: the agent often
// rephrases purpose between retries while issuing the same command.
func runFailureKey(target, command string) string {
collapsed := strings.Join(strings.Fields(command), " ")
target = strings.TrimSpace(target)
h := sha256.Sum256([]byte(target + "\x00" + collapsed))
return hex.EncodeToString(h[:])
}
// recordFailure increments the failure count for the given key and returns
// the new count. The caller should check `count > maxRunRetries` BEFORE
// dispatching to decide whether to skip the call.
func (r *runRetryTracker) recordFailure(key string) int {
r.mu.Lock()
defer r.mu.Unlock()
r.counts[key]++
return r.counts[key]
}
// failures returns the current failure count for a key (0 if unseen).
func (r *runRetryTracker) failures(key string) int {
r.mu.Lock()
defer r.mu.Unlock()
return r.counts[key]
}
// isRunFailure reports whether a `run` tool call's outcome should count
// as a failure for retry-cap purposes. A call counts as failed when:
// - the dispatch itself errored (callErr != nil), OR
// - the result text starts with "run on <target>: ERROR" — the
// shape classifyAndGate/sshExec produce when SSH or the command fails.
//
// Approvals queued ("requires approval") do NOT count as failures: they
// are pending operator action, not a command execution failure. A read
// of the existing code paths (classifyAndGate in internal/mcp/server.go)
// confirms the "ERROR" prefix is the stable failure signature for `run`.
//
// The resultText parameter is the MCP tool's RAW text result (not JSON-
// re-encoded): when classifyAndGate returns a textResult like
// "run on host:strong: ERROR ...", the MCP client unwraps it back to a
// plain Go string (see mcpClient.callTool). The caller should pass that
// raw string, not json.Marshal's output (which would quote-wrap it).
func isRunFailure(toolName string, resultText string, callErr error) bool {
if callErr != nil {
return true
}
if toolName != "run" {
return false
}
// "run on host:strong: ERROR ..." or "run on lxc:caddy: ERROR ..."
// Both shapes start with "run on ".
if !strings.HasPrefix(resultText, "run on ") {
return false
}
return strings.Contains(resultText, ": ERROR")
}
// runResultText extracts the raw text from a `run` tool's result value as
// returned by mcpClient.callTool — typically a Go string, but may also be
// a []string (multi-content result) or other JSON-decoded shape. Returns
// "" for shapes we don't recognize. Used by the retry-cap path so
// isRunFailure receives the un-quoted text form (see its doc comment).
func runResultText(result any) string {
switch v := result.(type) {
case string:
return v
case []string:
if len(v) > 0 {
return v[0]
}
case []any:
var b strings.Builder
for _, e := range v {
if s, ok := e.(string); ok {
b.WriteString(s)
}
}
return b.String()
}
return ""
}
// runRetryDirective is the synthetic tool result returned to the model
// when the retry cap is hit, in place of dispatching the call again. It
// directs the agent to investigate *why* the command keeps failing before
// retrying, or to surface the blocker to the operator.
func runRetryDirective(target, command string, failures int) string {
return "Refused: this `run` against " + target + " has failed " +
itoa(failures) + " times this turn — retry cap hit. The command:\n " +
command + "\nis almost certainly blocked by something on the target " +
"(a hung process, a kernel lock, an unexported FS, a stuck SSH " +
"session, …) — NOT a transient gateway issue. Do NOT retry with " +
"different routing or quoting. Instead, BEFORE calling `run` again, " +
"investigate *why* the command hangs: e.g. `ps aux | grep <cmd>`, " +
"`lsof <path>`, `strace -f -p <pid>` or `strace -f <cmd>`, " +
"`mount | grep <path>`, `dmesg | tail`. If you find a structural " +
"blocker (e.g. a kernel lock on an exported NFS directory → " +
"unexport → mutate → re-export), say so to the operator and fix it " +
"with a different command. If you genuinely cannot diagnose, " +
"surface the blocker to the operator with what you've tried — do " +
"not just retry the same command."
}
// itoa is a tiny strconv.Itoa to keep this file dependency-free.
func itoa(n int) string {
if n == 0 {
return "0"
}
neg := n < 0
if neg {
n = -n
}
var buf [20]byte
i := len(buf)
for n > 0 {
i--
buf[i] = byte('0' + n%10)
n /= 10
}
if neg {
i--
buf[i] = '-'
}
return string(buf[i:])
}

129
cmd/nomos/retrycap_test.go Normal file
View File

@@ -0,0 +1,129 @@
package main
import (
"strings"
"testing"
)
func TestRunFailureKey_StableAcrossWhitespace(t *testing.T) {
cases := []struct{ a, b string }{
{"chown :10000 /mnt/media_local && chmod 2775 /mnt/media_local",
"chown :10000 /mnt/media_local && chmod 2775 /mnt/media_local"},
{"chown :10000 /mnt/media_local\n&& chmod 2775 /mnt/media_local",
"chown :10000 /mnt/media_local && chmod 2775 /mnt/media_local"},
{"chown :10000 /mnt/media_local && chmod 2775 /mnt/media_local ",
" chown :10000 /mnt/media_local && chmod 2775 /mnt/media_local"},
}
for i, c := range cases {
ka := runFailureKey("host:strong", c.a)
kb := runFailureKey("host:strong", c.b)
if ka != kb {
t.Errorf("case %d: keys differ for whitespace-equivalent commands:\n a=%q\n b=%q", i, c.a, c.b)
}
}
}
func TestRunFailureKey_DiffersByTarget(t *testing.T) {
a := runFailureKey("host:strong", "echo hi")
b := runFailureKey("host:hubris", "echo hi")
if a == b {
t.Error("keys should differ when target differs")
}
}
func TestRunFailureKey_DiffersByCommand(t *testing.T) {
a := runFailureKey("host:strong", "echo hi")
b := runFailureKey("host:strong", "echo bye")
if a == b {
t.Error("keys should differ when command differs")
}
}
func TestRunRetryTracker_CountsAndCaps(t *testing.T) {
r := newRunRetryTracker()
key := runFailureKey("host:strong", "chown :10000 /mnt/media_local")
for i := 1; i <= maxRunRetries; i++ {
if got := r.recordFailure(key); got != i {
t.Errorf("recordFailure #%d = %d, want %d", i, got, i)
}
}
// At the cap, failures() should report maxRunRetries, and the next
// identical call should be refused by the agent loop (failures() >=
// maxRunRetries).
if got := r.failures(key); got != maxRunRetries {
t.Errorf("failures = %d, want %d", got, maxRunRetries)
}
if r.failures(key) < maxRunRetries {
t.Errorf("cap should be enforced at maxRunRetries=%d", maxRunRetries)
}
}
func TestRunRetryTracker_PerTurnIsolation(t *testing.T) {
// Different keys don't interfere.
r := newRunRetryTracker()
k1 := runFailureKey("host:strong", "echo a")
k2 := runFailureKey("host:strong", "echo b")
r.recordFailure(k1)
r.recordFailure(k1)
if got := r.failures(k2); got != 0 {
t.Errorf("k2 failures = %d, want 0 (keys are isolated)", got)
}
}
func TestIsRunFailure(t *testing.T) {
cases := []struct {
desc string
tool string
result string
callErr error
want bool
}{
{"run with ERROR prefix", "run", "run on host:strong: ERROR ssh: signal: killed", nil, true},
{"run with exit error", "run", "run on lxc:caddy: ERROR exit status 1", nil, true},
{"run success (read-only auto)", "run", "run on host:strong (read_only, auto): hello", nil, false},
{"run success (assent window)", "run", "run on host:strong (config_mutation, auto via assent window): done", nil, false},
{"run queued for approval", "run", "run on host:strong requires approval (risk: config_mutation) — execution 019f4930 queued. Present the command and purpose to the operator and wait; do not re-request.", nil, false},
{"non-run tool", "get_entity", "lxc list result", nil, false},
{"callErr set (dispatch failure)", "run", "", errFake{}, true},
{"callErr set on non-run tool", "get_entity", "some result", errFake{}, true}, // callErr trumps name
}
for i, c := range cases {
got := isRunFailure(c.tool, c.result, c.callErr)
if got != c.want {
t.Errorf("case %d (%s): isRunFailure = %v, want %v", i, c.desc, got, c.want)
}
}
}
type errFake struct{}
func (errFake) Error() string { return "fake dispatch error" }
func TestRunRetryDirective_Content(t *testing.T) {
d := runRetryDirective("host:strong", "chown :10000 /mnt/media_local", 3)
for _, want := range []string{
"Refused:",
"host:strong",
"3 times",
"retry cap hit",
"Do NOT retry",
"strace",
"ps aux",
"lsof",
"surface the blocker",
} {
if !strings.Contains(d, want) {
t.Errorf("directive missing %q; got:\n%s", want, d)
}
}
}
func TestItoa(t *testing.T) {
cases := map[int]string{0: "0", 1: "1", 9: "9", 10: "10", 42: "42",
100: "100", -1: "-1", -42: "-42"}
for in, want := range cases {
if got := itoa(in); got != want {
t.Errorf("itoa(%d) = %q, want %q", in, got, want)
}
}
}

View File

@@ -492,10 +492,34 @@ func (s *store) taskEntityPtr(ctx context.Context, sessionID string) *uuid.UUID
// happens — not in reopenSession — because set_goal is the explicit signal
// for "new sub-task." An approval ("go ahead") does NOT call set_goal, so it
// won't destroy the plan the operator just approved.
//
// P1.4 (2026-07-18): when a non-empty prior goal is being overwritten by a
// different goal, emit a `task.superseded` event carrying the prior goal.
// This gives the UI/audit trail a clear signal that the operator pivoted —
// without it, the prior goal just silently disappears from
// agent_sessions.goal and there's no record the session ever had a
// different starting intent. See plans/2026-07-18-session-review-three-
// sessions.md P1.4 (session 55927f0a had two set_goal calls with the first
// implicitly abandoned when the operator said "lets just keep ludo-library
// then").
func (s *store) setGoal(ctx context.Context, sessionID, goal string) error {
if s == nil || sessionID == "" || sessionID == "ephemeral" {
return nil
}
// Capture the prior goal BEFORE the UPDATE overwrites it. If non-empty
// and different from the new goal, emit task.superseded so the audit
// trail records the pivot — the row's goal column won't.
var priorGoal string
s.pool.QueryRow(ctx,
`SELECT COALESCE(goal, '') FROM agent_sessions WHERE id = $1`,
sessionID).Scan(&priorGoal)
if priorGoal != "" && priorGoal != goal {
_ = observability.Event(ctx, sqlcgen.New(s.pool), "task.superseded",
s.taskEntityPtr(ctx, sessionID), "info", "nomos", sessionID,
map[string]any{"prior_goal": priorGoal, "new_goal": goal})
slog.Info("nomos: task goal superseded by a new set_goal",
"session", sessionID, "prior_goal", priorGoal, "new_goal", goal)
}
// Replace any prior plan steps (done/running/pending/...) as `replaced`.
// The rows are kept for the generation counter + audit trail; proposePlan
// excludes `replaced` from its in-flight check, so the next propose_plan
@@ -868,7 +892,7 @@ type staleGoalSession struct {
}
// staleGoalSessions finds sessions that framed themselves as a real task
// (goal != '', so the inline safety net in agent.go intentionally left them
// (goal != , so the inline safety net in agent.go intentionally left them
// alone) but have sat non-terminal past idleThreshold. completion_nudges
// tells the caller whether to nudge (0) or give up and auto-close (>=1) —
// see processIdleSweep in continue.go.

View File

@@ -311,3 +311,69 @@ func TestHadDiscoveryAndWriteback(t *testing.T) {
t.Fatal("hadEntityWriteback = false after run+writeback, want true")
}
}
// TestSetGoal_SupersessionEvent is the store-level proof for P1.4 from
// plans/2026-07-18-session-review-three-sessions.md: when setGoal is called
// and a non-empty prior goal already exists with a DIFFERENT value, a
// task.superseded event must be emitted (so the audit trail records the
// pivot — the row's goal column will be overwritten, losing the prior intent
// without this event). When the goal is identical OR no prior goal exists,
// no supersession event is emitted.
//
// Background: session 55927f0a had two set_goal calls; the first was
// implicitly abandoned when the operator said "lets just keep ludo-library
// then." Without the event, the prior goal silently disappeared.
func TestSetGoal_SupersededEvent(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
sess, err := s.createSession(ctx, "goal pivot test")
if err != nil {
t.Fatalf("createSession: %v", err)
}
// First set_goal — no prior, no supersession event expected.
if err := s.setGoal(ctx, sess.ID, "Fix sabnzbd download folder to use ludo-lvm"); err != nil {
t.Fatalf("setGoal #1: %v", err)
}
if n := countEvents(ctx, s, sess.ID, "task.superseded"); n != 0 {
t.Errorf("after first set_goal: %d task.superseded events, want 0", n)
}
// Second set_goal with a DIFFERENT goal — supersession event expected.
if err := s.setGoal(ctx, sess.ID, "Add NFS export of ludo-lvm to ZimaOS"); err != nil {
t.Fatalf("setGoal #2: %v", err)
}
if n := countEvents(ctx, s, sess.ID, "task.superseded"); n != 1 {
t.Errorf("after second set_goal with a different goal: %d task.superseded events, want 1", n)
}
// Third set_goal with the SAME goal as the second — no new supersession
// event (idempotent: same goal is a no-op, not a pivot).
if err := s.setGoal(ctx, sess.ID, "Add NFS export of ludo-lvm to ZimaOS"); err != nil {
t.Fatalf("setGoal #3: %v", err)
}
if n := countEvents(ctx, s, sess.ID, "task.superseded"); n != 1 {
t.Errorf("after third set_goal with same goal as second: %d task.superseded events, want 1 (no new pivot)", n)
}
// The session's current goal must be the latest one set.
got, err := s.getSession(ctx, sess.ID)
if err != nil {
t.Fatalf("getSession: %v", err)
}
if got.Goal != "Add NFS export of ludo-lvm to ZimaOS" {
t.Errorf("session goal = %q, want the second (latest) goal", got.Goal)
}
}
// countEvents counts observability events of the given type correlated to
// the given session. Used by TestSetGoal_SupersededEvent to assert the
// task.superseded audit-trail signal was emitted.
func countEvents(ctx context.Context, s *store, sessionID, eventType string) int {
var n int
s.pool.QueryRow(ctx,
`SELECT COUNT(*) FROM events WHERE correlation_id = $1 AND type = $2`,
sessionID, eventType).Scan(&n)
return n
}