oikos/cmd/nomos/retrycap.go

package main

import (
	"crypto/sha256"
	"encoding/hex"
	"strings"
	"sync"
)

// maxRunRetries is the per-turn cap on identical failing `run` tool calls.
// After this many failures with the same (target, command) key, the agent
// loop refuses to dispatch the call again and instead surfaces a directive
// to investigate *why* (ps/strace/lsof) or escalate to the operator.
//
// Background: session 1e9c7691 (2026-07-18) retried the same
// `chown :10000 /mnt/media_local && chmod 2775 …` ~20 times across direct
// runs, SSH-hop-via-hubris, wrapping in a shell script, and bare `echo test`
// sanity checks. Each retry piled up another zombie process on the target
// (knfsd was holding a kernel lock on the exported directory). The agent
// only investigated *why* after the operator explicitly asked
// "the command just keeps running?" — see
// plans/2026-07-18-session-review-three-sessions.md P0.1.
const maxRunRetries = 3

// runRetryTracker deduplicates failing `run` calls within a single chat
// turn (chatWith invocation). It is NOT persisted across turns — the cap
// is per-turn, so a fresh turn after the operator responds can retry once
// more. The intent is to break a tight retry loop within one turn, not to
// permanently block the agent from ever attempting the operation again.
//
// Threading: the agent loop is single-goroutine per turn, but the tracker
// is guarded by a mutex so future callers (e.g. concurrent tool dispatch)
// stay safe. The mutex is uncontended on the current hot path.
type runRetryTracker struct {
	mu     sync.Mutex
	counts map[string]int
}

func newRunRetryTracker() *runRetryTracker {
	return &runRetryTracker{counts: make(map[string]int)}
}

// runFailureKey is the dedup key for "this is the same command against the
// same target." Whitespace is collapsed so trivial reformatting
// (newlines vs spaces, trailing whitespace) doesn't escape the cap. The
// purpose field is intentionally NOT part of the key: the agent often
// rephrases purpose between retries while issuing the same command.
func runFailureKey(target, command string) string {
	collapsed := strings.Join(strings.Fields(command), " ")
	target = strings.TrimSpace(target)
	h := sha256.Sum256([]byte(target + "\x00" + collapsed))
	return hex.EncodeToString(h[:])
}

// recordFailure increments the failure count for the given key and returns
// the new count. The caller should check `count > maxRunRetries` BEFORE
// dispatching to decide whether to skip the call.
func (r *runRetryTracker) recordFailure(key string) int {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.counts[key]++
	return r.counts[key]
}

// failures returns the current failure count for a key (0 if unseen).
func (r *runRetryTracker) failures(key string) int {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.counts[key]
}

// isRunFailure reports whether a `run` tool call's outcome should count
// as a failure for retry-cap purposes. A call counts as failed when:
//   - the dispatch itself errored (callErr != nil), OR
//   - the result text starts with "run on <target>: ERROR" — the
//     shape classifyAndGate/sshExec produce when SSH or the command fails.
//
// Approvals queued ("requires approval") do NOT count as failures: they
// are pending operator action, not a command execution failure. A read
// of the existing code paths (classifyAndGate in internal/mcp/server.go)
// confirms the "ERROR" prefix is the stable failure signature for `run`.
//
// The resultText parameter is the MCP tool's RAW text result (not JSON-
// re-encoded): when classifyAndGate returns a textResult like
// "run on host:strong: ERROR ...", the MCP client unwraps it back to a
// plain Go string (see mcpClient.callTool). The caller should pass that
// raw string, not json.Marshal's output (which would quote-wrap it).
func isRunFailure(toolName string, resultText string, callErr error) bool {
	if callErr != nil {
		return true
	}
	if toolName != "run" {
		return false
	}
	// "run on host:strong: ERROR ..." or "run on lxc:caddy: ERROR ..."
	// Both shapes start with "run on ".
	if !strings.HasPrefix(resultText, "run on ") {
		return false
	}
	return strings.Contains(resultText, ": ERROR")
}

// runResultText extracts the raw text from a `run` tool's result value as
// returned by mcpClient.callTool — typically a Go string, but may also be
// a []string (multi-content result) or other JSON-decoded shape. Returns
// "" for shapes we don't recognize. Used by the retry-cap path so
// isRunFailure receives the un-quoted text form (see its doc comment).
func runResultText(result any) string {
	switch v := result.(type) {
	case string:
		return v
	case []string:
		if len(v) > 0 {
			return v[0]
		}
	case []any:
		var b strings.Builder
		for _, e := range v {
			if s, ok := e.(string); ok {
				b.WriteString(s)
			}
		}
		return b.String()
	}
	return ""
}

// runRetryDirective is the synthetic tool result returned to the model
// when the retry cap is hit, in place of dispatching the call again. It
// directs the agent to investigate *why* the command keeps failing before
// retrying, or to surface the blocker to the operator.
func runRetryDirective(target, command string, failures int) string {
	return "Refused: this `run` against " + target + " has failed " +
		itoa(failures) + " times this turn — retry cap hit. The command:\n  " +
		command + "\nis almost certainly blocked by something on the target " +
		"(a hung process, a kernel lock, an unexported FS, a stuck SSH " +
		"session, …) — NOT a transient gateway issue. Do NOT retry with " +
		"different routing or quoting. Instead, BEFORE calling `run` again, " +
		"investigate *why* the command hangs: e.g. `ps aux | grep <cmd>`, " +
		"`lsof <path>`, `strace -f -p <pid>` or `strace -f <cmd>`, " +
		"`mount | grep <path>`, `dmesg | tail`. If you find a structural " +
		"blocker (e.g. a kernel lock on an exported NFS directory → " +
		"unexport → mutate → re-export), say so to the operator and fix it " +
		"with a different command. If you genuinely cannot diagnose, " +
		"surface the blocker to the operator with what you've tried — do " +
		"not just retry the same command."
}

// itoa is a tiny strconv.Itoa to keep this file dependency-free.
func itoa(n int) string {
	if n == 0 {
		return "0"
	}
	neg := n < 0
	if neg {
		n = -n
	}
	var buf [20]byte
	i := len(buf)
	for n > 0 {
		i--
		buf[i] = byte('0' + n%10)
		n /= 10
	}
	if neg {
		i--
		buf[i] = '-'
	}
	return string(buf[i:])
}