feat(nomos): retry cap, vm: targets, inspect_path, goal supersession, runbooks
Session-review implementation for the three sessions audited in
plans/2026-07-18-session-review-three-sessions.md. v0.7.11 → v0.7.12.
P0.1 — retry cap + investigate-before-retry (cmd/nomos/retrycap.go,
agent.go): after 3 identical failing run calls in a single turn, refuse
to dispatch the call again and return a directive to investigate *why*
(ps/strace/lsof) or surface the blocker. Per-turn scope so a fresh turn
after the operator responds can retry once more. Session 1e9c7691's 20+
identical chown retries (knfsd held a kernel lock on the exported NFS
dir) is the direct motivation.
P0.2 + P1.8 + P2.10 — SOUL.md guidance: hung command is not a failed
command (investigate before retry); ask before proposing a multi-step
migration; multi-goal sessions summarize the arc not just the last goal.
P1.3 — two new runbook entities in seeds/knowledge.yaml:
- nfs-exported-dir-mutation-hang (the knfsd fchownat lock procedure:
killall → exportfs -u → mutate → exportfs -a → verify)
- netbird-mgmt-oidc-race-after-upgrade (docker restart netbird-mgmt
after ~30s for the traefik/authentik OIDC race)
P1.4 — setGoal emits task.superseded event when prior goal is overwritten
by a different goal (store.go, TestSetGoal_SupersededEvent). Session
55927f0a had two set_goal calls with the first silently abandoned.
P1.5 — inspect_path MCP tool: runs mount/df/ls/stat for one path across
up to 8 targets in one parallel call, replacing the 15+ run-call
fact-gathering fan-out sessions 1 and 2 each spent on cross-target path
tracing (tools.go, server.go: inspectPathAcrossTargets, inspectOneTarget).
P1.6 — vm: target support in run via qm guest exec (no more SSH-hop
with nested quoting). Extracted shared resolveProxmoxHostSlug for
LXC + VM, with hosts-relationship fallback when attributes.host is
absent (server.go, tools.go). Session 55927f0a's SSH-hop workarounds
for vm:zimaos are the direct motivation.
Deferred (documented in plan): P1.7 (approval window auto-extend on
timeout) and P2.9 (long-running command PENDING detection) — both
addressed at lower cost by the retry cap. Session 3's poll-after-timeout
pattern already works; the cap protects against the failure mode.
This commit is contained in:
@@ -511,16 +511,27 @@ func isPrivateHost(host string) bool {
|
||||
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()
|
||||
}
|
||||
|
||||
// resolveExecTarget resolves any target slug (host: or lxc:) to the SSH
|
||||
// resolveExecTarget resolves any target slug (host:, lxc:, or vm:) to the SSH
|
||||
// endpoint that will actually run the command, and a wrap function that turns
|
||||
// a plain shell command into whatever must actually be sent over that SSH
|
||||
// connection: identity for a host, `pct exec <pve_id> -- ...` for an LXC.
|
||||
// connection: identity for a host, `pct exec <pve_id> -- ...` for an LXC,
|
||||
// `qm guest exec <pve_id> -- ...` for a VM.
|
||||
//
|
||||
// The lxc.attributes.host value is stored WITHOUT a "host:" prefix (e.g.
|
||||
// "strong", not "host:strong") — see pct_create's entity registration. The
|
||||
// pre-existing pct_exec handler queried resolveHost with that bare value
|
||||
// directly, which can never match a "host:*" slug and always fails; this
|
||||
// prefixes it correctly.
|
||||
//
|
||||
// vm: support (2026-07-18): VMs in inventory.yaml carry `pve_id` and a `host`
|
||||
// attribute (or a `hosts` relationship) just like LXCs, but they're reached
|
||||
// via `qm guest exec` instead of `pct exec`. Previously the agent had to
|
||||
// SSH-hop via `host:hubris` to reach a VM (e.g. `ssh root@<vm_ip> '...'`),
|
||||
// which broke on nested shell quoting and forced manual escaping workarounds
|
||||
// — see plans/2026-07-18-session-review-three-sessions.md P1.6. A VM's
|
||||
// `host` attribute is optional: if absent, fall back to looking up the
|
||||
// `hosts` relationship on the VM entity, then to hubris (the documented
|
||||
// default Proxmox host) — same fallback chain as LXCs.
|
||||
func resolveExecTarget(ctx context.Context, pool *db.Pool, targetSlug string) (host, user string, wrap func(cmd string) string, err error) {
|
||||
if strings.HasPrefix(targetSlug, "host:") {
|
||||
host, user, err = resolveHost(ctx, pool, targetSlug)
|
||||
@@ -536,13 +547,7 @@ func resolveExecTarget(ctx context.Context, pool *db.Pool, targetSlug string) (h
|
||||
if qerr := pool.QueryRow(ctx, "SELECT attributes->>'pve_id', COALESCE(attributes->>'host', '') FROM entities WHERE slug = $1", targetSlug).Scan(&pveID, &hostAttr); qerr != nil || pveID == "" {
|
||||
return "", "", nil, fmt.Errorf("LXC not found or missing pve_id: %s", targetSlug)
|
||||
}
|
||||
hostSlug := hostAttr
|
||||
if hostSlug == "" {
|
||||
hostSlug = "hubris" // documented default Proxmox host when unset
|
||||
}
|
||||
if !strings.HasPrefix(hostSlug, "host:") {
|
||||
hostSlug = "host:" + hostSlug
|
||||
}
|
||||
hostSlug := resolveProxmoxHostSlug(ctx, pool, targetSlug, hostAttr)
|
||||
host, user, err = resolveHost(ctx, pool, hostSlug)
|
||||
id := pveID
|
||||
return host, user, func(cmd string) string {
|
||||
@@ -550,7 +555,71 @@ func resolveExecTarget(ctx context.Context, pool *db.Pool, targetSlug string) (h
|
||||
return fmt.Sprintf("pct exec %s -- bash -c 'echo %s | base64 -d | bash'", id, b64)
|
||||
}, err
|
||||
}
|
||||
return "", "", nil, fmt.Errorf("unsupported target %q: must be host:<slug> or lxc:<slug>", targetSlug)
|
||||
if strings.HasPrefix(targetSlug, "vm:") {
|
||||
// VMs: same host-resolution chain as LXCs (attributes.host →
|
||||
// `hosts` relationship → hubris default), but reached via
|
||||
// `qm guest exec` instead of `pct exec`. Requires the QEMU
|
||||
// guest agent running inside the VM (the standard Proxmox
|
||||
// setup; ZimaOS/HAOS in this fleet already have it).
|
||||
var pveID, hostAttr string
|
||||
if qerr := pool.QueryRow(ctx, "SELECT attributes->>'pve_id', COALESCE(attributes->>'host', '') FROM entities WHERE slug = $1", targetSlug).Scan(&pveID, &hostAttr); qerr != nil || pveID == "" {
|
||||
return "", "", nil, fmt.Errorf("VM not found or missing pve_id: %s", targetSlug)
|
||||
}
|
||||
hostSlug := resolveProxmoxHostSlug(ctx, pool, targetSlug, hostAttr)
|
||||
host, user, err = resolveHost(ctx, pool, hostSlug)
|
||||
id := pveID
|
||||
return host, user, func(cmd string) string {
|
||||
b64 := base64.StdEncoding.EncodeToString([]byte(cmd))
|
||||
// `qm guest exec <id> -- /bin/bash -c '...'` returns JSON by
|
||||
// default; pipe through `jq -r .out` if available, else cat.
|
||||
// The base64 round-trip mirrors the LXC path so nested quoting
|
||||
// (the original VM-target pain point — session 55927f0a) is
|
||||
// handled identically to LXC dispatch.
|
||||
return fmt.Sprintf(
|
||||
"qm guest exec %s -- /bin/bash -c 'echo %s | base64 -d | bash' | jq -r '.out // .err // empty' 2>/dev/null || qm guest exec %s -- /bin/bash -c 'echo %s | base64 -d | bash'",
|
||||
id, b64, id, b64)
|
||||
}, err
|
||||
}
|
||||
return "", "", nil, fmt.Errorf("unsupported target %q: must be host:<slug>, lxc:<slug>, or vm:<slug>", targetSlug)
|
||||
}
|
||||
|
||||
// resolveProxmoxHostSlug resolves the Proxmox host slug that owns a given
|
||||
// LXC/VM target. Resolution order:
|
||||
// 1. hostAttr if non-empty (the entity's attributes.host — stored without
|
||||
// "host:" prefix in inventory.yaml and pct_create).
|
||||
// 2. the `hosts` relationship on the entity (e.g. host:hubris → vm:zimaos),
|
||||
// looked up in the relationships table — the canonical graph source.
|
||||
// 3. "hubris" as a documented default Proxmox host fallback.
|
||||
//
|
||||
// Returns a slug with the "host:" prefix attached, ready for resolveHost.
|
||||
// Extracted from the inline LXC path (2026-07-18) so the VM path shares the
|
||||
// same chain — see plans/2026-07-18-session-review-three-sessions.md P1.6.
|
||||
func resolveProxmoxHostSlug(ctx context.Context, pool *db.Pool, entitySlug, hostAttr string) string {
|
||||
hostSlug := strings.TrimSpace(hostAttr)
|
||||
if hostSlug == "" {
|
||||
// Fall back to the `hosts` relationship — the graph edge from
|
||||
// the Proxmox host to this LXC/VM. This is the canonical source
|
||||
// for "who owns this VM" in inventory.yaml; the `host` attribute
|
||||
// is a denormalized shortcut that not every entity has.
|
||||
var relHostSlug string
|
||||
// hosts relationship: source=host, target=lxc/vm. Look up the
|
||||
// source slug given the target.
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT e.slug FROM relationships r
|
||||
JOIN entities e ON e.id = r.source_id
|
||||
WHERE r.target_id = (SELECT id FROM entities WHERE slug = $1)
|
||||
AND r.type = 'hosts' AND r.valid_to IS NULL
|
||||
LIMIT 1`, entitySlug).Scan(&relHostSlug); err == nil && relHostSlug != "" {
|
||||
hostSlug = relHostSlug
|
||||
}
|
||||
}
|
||||
if hostSlug == "" {
|
||||
hostSlug = "hubris" // documented default Proxmox host when unset
|
||||
}
|
||||
if !strings.HasPrefix(hostSlug, "host:") {
|
||||
hostSlug = "host:" + hostSlug
|
||||
}
|
||||
return hostSlug
|
||||
}
|
||||
|
||||
// classifyAndGate is the shared classify→execute-or-queue path for every
|
||||
@@ -994,4 +1063,83 @@ func createApproval(ctx context.Context, pool *db.Pool, execID, targetID uuid.UU
|
||||
// gated action is now awaiting a decision.
|
||||
_ = observability.Event(ctx, sqlcgen.New(pool), "approval.created", &execID, "warning", "mcp", "",
|
||||
map[string]any{"action": action, "params": params, "risk_class": riskClass})
|
||||
}
|
||||
}
|
||||
|
||||
// inspectPathAcrossTargets is the bulk fact-gathering helper behind the
|
||||
// inspect_path MCP tool (plans/2026-07-18-session-review-three-sessions.md
|
||||
// P1.5). For each target slug, it runs a single read-only shell command
|
||||
// producing mount/df/ls/stat output for the given path, and returns the
|
||||
// results as a map keyed by target slug.
|
||||
//
|
||||
// Why this exists: sessions 1e9c7691 and 55927f0a each spent ~15 `run`
|
||||
// calls gathering identical facts (`mount | grep`, `df`, `ls -la`, `stat`)
|
||||
// across hosts and LXCs to trace where a path lives, who mounts it, and
|
||||
// what permissions it has. One call here replaces that fan-out. All
|
||||
// commands are read-only — the tool bypasses classifyAndGate and runs
|
||||
// directly via sshExec against resolveExecTarget's host/wrap. Failures
|
||||
// (unresolvable target, SSH error) are reported per-target in the result
|
||||
// map, not as a single tool-level error, so one bad target doesn't lose
|
||||
// the others.
|
||||
//
|
||||
// The per-target command is intentionally compact: one combined shell
|
||||
// invocation that prints mount source/dest, df, ls -la of the path's
|
||||
// parent + the path itself, and stat. Output is truncated to 4KB per
|
||||
// target to keep the total result reasonable for an 8-target call.
|
||||
func inspectPathAcrossTargets(ctx context.Context, pool *db.Pool, path string, targets []string) map[string]any {
|
||||
results := make(map[string]any, len(targets))
|
||||
path = strings.TrimSpace(path)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
wg.Add(len(targets))
|
||||
|
||||
for _, tgt := range targets {
|
||||
go func(target string) {
|
||||
defer wg.Done()
|
||||
entry := inspectOneTarget(ctx, pool, path, target)
|
||||
mu.Lock()
|
||||
results[target] = entry
|
||||
mu.Unlock()
|
||||
}(tgt)
|
||||
}
|
||||
wg.Wait()
|
||||
return results
|
||||
}
|
||||
|
||||
// inspectOneTarget runs the read-only inspection for one target. Returns a
|
||||
// map with keys: "ok" (bool), "output" (string, on success), "error"
|
||||
// (string, on failure). Kept small so the JSON shape is stable across the
|
||||
// parallel-call path.
|
||||
func inspectOneTarget(ctx context.Context, pool *db.Pool, path, target string) map[string]any {
|
||||
host, user, wrap, rerr := resolveExecTarget(ctx, pool, target)
|
||||
if rerr != nil {
|
||||
return map[string]any{"ok": false, "error": fmt.Sprintf("resolve target: %v", rerr)}
|
||||
}
|
||||
// One shell invocation, four sections, each guarded by `2>&1 || true`
|
||||
// so a missing path doesn't kill the rest. Stat with -c gives a
|
||||
// stable machine-readable line for ownership/perms; ls -la gives the
|
||||
// human-readable listing of the path and its parent (so we can see
|
||||
// both "what's in here" and "how the parent is laid out" — useful for
|
||||
// NFS-root-vs-subdir permission mismatches, the exact issue in
|
||||
// session 1e9c7691).
|
||||
cmd := fmt.Sprintf(
|
||||
`echo "=== mount ==="; mount 2>/dev/null | grep -- "%[1]s" || echo "(not a mount point)";
|
||||
echo "=== df ==="; df -h "%[1]s" 2>&1 || true;
|
||||
echo "=== stat ==="; stat -c '%%a %%U:%%G (size=%%s, type=%%F)' "%[1]s" 2>&1 || true;
|
||||
echo "=== ls -la path ==="; ls -la "%[1]s" 2>&1 | head -40 || true;
|
||||
echo "=== ls -la parent ==="; ls -la "$(dirname "%[1]s")" 2>&1 | head -20 || true`,
|
||||
path)
|
||||
out, xerr := sshExec(ctx, host, user, wrap(cmd))
|
||||
if xerr != nil {
|
||||
return map[string]any{"ok": false, "error": fmt.Sprintf("ssh: %v: %s", xerr, out)}
|
||||
}
|
||||
// Truncate per-target output to keep an 8-target call's total under
|
||||
// ~32KB. 4KB per target is enough for the head -40/head -20 listings
|
||||
// above; if a directory is enormous, the truncation keeps the result
|
||||
// usable without flooding the model's context.
|
||||
const maxPerTarget = 4096
|
||||
if len(out) > maxPerTarget {
|
||||
out = out[:maxPerTarget] + fmt.Sprintf("\n...truncated (%d bytes total)", len(out))
|
||||
}
|
||||
return map[string]any{"ok": true, "output": out}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user