package main import ( "crypto/sha256" "encoding/hex" "strings" "sync" ) // maxRunRetries is the per-turn cap on identical failing `run` tool calls. // After this many failures with the same (target, command) key, the agent // loop refuses to dispatch the call again and instead surfaces a directive // to investigate *why* (ps/strace/lsof) or escalate to the operator. // // Background: session 1e9c7691 (2026-07-18) retried the same // `chown :10000 /mnt/media_local && chmod 2775 …` ~20 times across direct // runs, SSH-hop-via-hubris, wrapping in a shell script, and bare `echo test` // sanity checks. Each retry piled up another zombie process on the target // (knfsd was holding a kernel lock on the exported directory). The agent // only investigated *why* after the operator explicitly asked // "the command just keeps running?" — see // plans/2026-07-18-session-review-three-sessions.md P0.1. const maxRunRetries = 3 // runRetryTracker deduplicates failing `run` calls within a single chat // turn (chatWith invocation). It is NOT persisted across turns — the cap // is per-turn, so a fresh turn after the operator responds can retry once // more. The intent is to break a tight retry loop within one turn, not to // permanently block the agent from ever attempting the operation again. // // Threading: the agent loop is single-goroutine per turn, but the tracker // is guarded by a mutex so future callers (e.g. concurrent tool dispatch) // stay safe. The mutex is uncontended on the current hot path. type runRetryTracker struct { mu sync.Mutex counts map[string]int } func newRunRetryTracker() *runRetryTracker { return &runRetryTracker{counts: make(map[string]int)} } // runFailureKey is the dedup key for "this is the same command against the // same target." Whitespace is collapsed so trivial reformatting // (newlines vs spaces, trailing whitespace) doesn't escape the cap. The // purpose field is intentionally NOT part of the key: the agent often // rephrases purpose between retries while issuing the same command. func runFailureKey(target, command string) string { collapsed := strings.Join(strings.Fields(command), " ") target = strings.TrimSpace(target) h := sha256.Sum256([]byte(target + "\x00" + collapsed)) return hex.EncodeToString(h[:]) } // recordFailure increments the failure count for the given key and returns // the new count. The caller should check `count > maxRunRetries` BEFORE // dispatching to decide whether to skip the call. func (r *runRetryTracker) recordFailure(key string) int { r.mu.Lock() defer r.mu.Unlock() r.counts[key]++ return r.counts[key] } // failures returns the current failure count for a key (0 if unseen). func (r *runRetryTracker) failures(key string) int { r.mu.Lock() defer r.mu.Unlock() return r.counts[key] } // isRunFailure reports whether a `run` tool call's outcome should count // as a failure for retry-cap purposes. A call counts as failed when: // - the dispatch itself errored (callErr != nil), OR // - the result text starts with "run on : ERROR" — the // shape classifyAndGate/sshExec produce when SSH or the command fails. // // Approvals queued ("requires approval") do NOT count as failures: they // are pending operator action, not a command execution failure. A read // of the existing code paths (classifyAndGate in internal/mcp/server.go) // confirms the "ERROR" prefix is the stable failure signature for `run`. // // The resultText parameter is the MCP tool's RAW text result (not JSON- // re-encoded): when classifyAndGate returns a textResult like // "run on host:strong: ERROR ...", the MCP client unwraps it back to a // plain Go string (see mcpClient.callTool). The caller should pass that // raw string, not json.Marshal's output (which would quote-wrap it). func isRunFailure(toolName string, resultText string, callErr error) bool { if callErr != nil { return true } if toolName != "run" { return false } // "run on host:strong: ERROR ..." or "run on lxc:caddy: ERROR ..." // Both shapes start with "run on ". if !strings.HasPrefix(resultText, "run on ") { return false } return strings.Contains(resultText, ": ERROR") } // runResultText extracts the raw text from a `run` tool's result value as // returned by mcpClient.callTool — typically a Go string, but may also be // a []string (multi-content result) or other JSON-decoded shape. Returns // "" for shapes we don't recognize. Used by the retry-cap path so // isRunFailure receives the un-quoted text form (see its doc comment). func runResultText(result any) string { switch v := result.(type) { case string: return v case []string: if len(v) > 0 { return v[0] } case []any: var b strings.Builder for _, e := range v { if s, ok := e.(string); ok { b.WriteString(s) } } return b.String() } return "" } // runRetryDirective is the synthetic tool result returned to the model // when the retry cap is hit, in place of dispatching the call again. It // directs the agent to investigate *why* the command keeps failing before // retrying, or to surface the blocker to the operator. func runRetryDirective(target, command string, failures int) string { return "Refused: this `run` against " + target + " has failed " + itoa(failures) + " times this turn — retry cap hit. The command:\n " + command + "\nis almost certainly blocked by something on the target " + "(a hung process, a kernel lock, an unexported FS, a stuck SSH " + "session, …) — NOT a transient gateway issue. Do NOT retry with " + "different routing or quoting. Instead, BEFORE calling `run` again, " + "investigate *why* the command hangs: e.g. `ps aux | grep `, " + "`lsof `, `strace -f -p ` or `strace -f `, " + "`mount | grep `, `dmesg | tail`. If you find a structural " + "blocker (e.g. a kernel lock on an exported NFS directory → " + "unexport → mutate → re-export), say so to the operator and fix it " + "with a different command. If you genuinely cannot diagnose, " + "surface the blocker to the operator with what you've tried — do " + "not just retry the same command." } // itoa is a tiny strconv.Itoa to keep this file dependency-free. func itoa(n int) string { if n == 0 { return "0" } neg := n < 0 if neg { n = -n } var buf [20]byte i := len(buf) for n > 0 { i-- buf[i] = byte('0' + n%10) n /= 10 } if neg { i-- buf[i] = '-' } return string(buf[i:]) }