feat: event-driven auto-continuation — agent runs an approved plan to completion
The root cause behind "the agent stops at the first error and doesn't recover":
provisioning executions run ASYNCHRONOUSLY (pct_create fires the SSH work in a
goroutine and returns "running" immediately), so the agent's turn ENDS before
the result exists. The agent literally isn't running when the step fails — it
can't react to a failure it never observes. The only thing that fed results
back was the operator typing "continue" after every async step: the human was
the event loop. (In the flagged 18-message session the operator typed
continue/proceed/?? eight times while the agent correctly diagnosed each failure
but couldn't advance a step on its own.)
This makes the system the event loop instead:
- migrations/017: nomos_plan_executions links each gated execution to the chat
session that started it.
- cmd/nomos: after a tool result, any "execution <uuid>" it started is linked
to the session. A background worker (continue.go) polls for those executions
reaching a terminal state and — while the agent has an open assent window (an
approved plan is in flight) — re-invokes the agent with the result
("execution X completed/failed: <result>"), so it proceeds to the next step
or diagnoses+fixes the failure, with no operator tick. Guarded against loops
(mark-continued before running) and bounded by the 30-min window.
- chatWith(): chat() variant that injects the finished-execution note after
replayed history without persisting a fake user turn.
- DecideApproval: approving a step by ANY route (button or chat-assent) now
opens the assent window, so auto-continuation works regardless of how the
operator approved — previously only typing "go ahead" opened it.
- SOUL: the agent is told it will be auto-re-invoked when async steps finish —
don't poll get_execution_status, don't wait for "continue"; end the turn and
keep going step by step until the goal is verified or a genuine blocker.
This is the root fix, not another per-command patch: you can't enumerate every
failure of an unbounded action space, but you can give the agent a loop that
observes each result and adapts — because "do anything" always includes "the
first attempt failed."
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
135
cmd/nomos/continue.go
Normal file
135
cmd/nomos/continue.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// execIDRe matches "execution <uuid>" in a tool result — the phrasing shared
|
||||
// by request_execution / run when they queue or start a gated execution.
|
||||
// Only these async executions need continuation; the synchronous auto-run
|
||||
// path returns its output inline and is already observed in-turn.
|
||||
var execIDRe = regexp.MustCompile(`(?i)execution\s+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})`)
|
||||
|
||||
func extractExecutionIDs(toolResult string) []uuid.UUID {
|
||||
matches := execIDRe.FindAllStringSubmatch(toolResult, -1)
|
||||
seen := map[uuid.UUID]bool{}
|
||||
var out []uuid.UUID
|
||||
for _, m := range matches {
|
||||
if id, err := uuid.Parse(m[1]); err == nil && !seen[id] {
|
||||
seen[id] = true
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runContinuationWorker is the event loop that replaces the human typing
|
||||
// "continue". It polls for gated executions that (a) were initiated by a chat
|
||||
// session and (b) have just finished, and — while that agent has an open assent
|
||||
// window (an approved plan is in flight) — feeds each result back into the
|
||||
// agent so it proceeds to the next step or recovers from the failure, all
|
||||
// without an operator tick. Blocks until ctx is cancelled.
|
||||
func (a *agent) runContinuationWorker(ctx context.Context) {
|
||||
if a.store == nil {
|
||||
slog.Warn("nomos: continuation worker disabled (no store)")
|
||||
return
|
||||
}
|
||||
slog.Info("nomos: continuation worker started")
|
||||
ticker := time.NewTicker(4 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
a.processContinuations(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *agent) processContinuations(ctx context.Context) {
|
||||
pending := a.store.pendingContinuations(ctx, 5)
|
||||
windowOpen := a.store.assentWindowActive(ctx, a.agentID)
|
||||
for _, p := range pending {
|
||||
// Scope gate: only auto-continue while an approved plan is active.
|
||||
// A finished one-off execution with no window is left as-is (marked
|
||||
// continued so we don't re-check it forever) — the operator decides
|
||||
// what happens next, as today.
|
||||
if !windowOpen {
|
||||
a.store.markContinued(ctx, p.ExecID)
|
||||
continue
|
||||
}
|
||||
a.store.markContinued(ctx, p.ExecID) // stamp first: a failure here must not cause a re-continue loop
|
||||
a.continueSession(ctx, p)
|
||||
}
|
||||
}
|
||||
|
||||
// continueSession re-invokes the agent for one finished execution, persisting
|
||||
// the resulting assistant turn just like handleChat does. The operator sees it
|
||||
// on their next load of the session (live push is a follow-up).
|
||||
func (a *agent) continueSession(ctx context.Context, p pendingContinuation) {
|
||||
note := buildContinuationNote(p)
|
||||
slog.Info("nomos: auto-continuing session", "session", p.SessionID, "execution", p.ExecID, "status", p.Status)
|
||||
|
||||
var toolCalls []map[string]any
|
||||
var finalText string
|
||||
emit := func(ev agentEvent) {
|
||||
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
||||
if m, ok := ev.Data.(map[string]any); ok {
|
||||
m["type"] = ev.Type
|
||||
toolCalls = append(toolCalls, m)
|
||||
}
|
||||
}
|
||||
if ev.Type == "text" {
|
||||
finalText, _ = ev.Data.(string)
|
||||
}
|
||||
}
|
||||
|
||||
// Use a generous timeout: a continuation may itself launch further steps.
|
||||
cctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
a.chatWith(cctx, p.SessionID, "", note, emit)
|
||||
|
||||
assistantMsg, _ := json.Marshal(map[string]any{
|
||||
"role": "assistant",
|
||||
"text": finalText,
|
||||
"tool_calls": toolCalls,
|
||||
"auto": true, // marks this as an autonomous continuation, not an operator turn
|
||||
})
|
||||
a.store.saveMessage(ctx, p.SessionID, "assistant", assistantMsg)
|
||||
}
|
||||
|
||||
// buildContinuationNote frames the finished execution for the model: what
|
||||
// happened, and what to do about it. The persist-through-errors instruction
|
||||
// lives here (and in SOUL) so the agent recovers instead of stopping.
|
||||
func buildContinuationNote(p pendingContinuation) string {
|
||||
action := p.Action
|
||||
if i := strings.IndexByte(action, ':'); i > 0 && len(action) > 40 {
|
||||
action = action[:i] // keep just the action verb for brevity; params are in the DB
|
||||
}
|
||||
result := p.Result
|
||||
if len(result) > 3000 {
|
||||
result = result[:3000] + "…[truncated]"
|
||||
}
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "[System: execution %s (%s) finished with status=%s.\nResult: %s\n\n",
|
||||
p.ExecID, action, p.Status, result)
|
||||
switch p.Status {
|
||||
case "completed":
|
||||
b.WriteString("It SUCCEEDED. Continue the approved plan: run the next step. If this was the final step, verify the end goal actually works (e.g. curl the service) and then report success to the operator. Do NOT stop and wait for the operator to say 'continue'.")
|
||||
case "failed", "cancelled":
|
||||
b.WriteString("It FAILED. Do NOT give up or hand back to the operator. Diagnose the cause from the result above (and by running read-only inspection commands if needed), form a hypothesis, fix it, and retry or take an alternative approach. You have an active assent window, so config_mutation steps run without re-approval. Only stop and ask the operator if you are genuinely blocked (need information only they have) or the fix would require a destructive action they haven't approved.")
|
||||
default: // denied / revoked
|
||||
b.WriteString("The operator denied or revoked this step. Stop executing this plan and briefly acknowledge.")
|
||||
}
|
||||
b.WriteString("]")
|
||||
return b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user