fix: auto-continuation silently dropped LLM errors + added outer retry
Verified live: the new auto-continuation worker (previous commit) worked end
to end for the happy path (provision -> auto-verify -> report success, zero
operator ticks). But testing a failure-recovery case (a destroy that failed
because the container was still running) surfaced a real bug: continueSession's
emit closure only captured "text" events, so when chatWith ended the turn on
an "error" event (LLM returned an empty/refusal response, internal retry also
empty), the worker persisted a completely blank, uninformative "auto" message
— no sign anything had gone wrong, undermining observability of the very
mechanism just built.
- Capture "error" events and, if the turn produced no text/tool_calls at all,
persist an explanatory placeholder instead of blank.
- Add one outer retry of the whole chatWith call when the first attempt
produces nothing — the principle behind this whole feature ("don't give up
on the first error") should apply to the continuation mechanism itself, not
just the homelab commands it's continuing.
Also verified live: recovery-from-failure works via the normal chat path once
prompted, and cleaned up the test container.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -80,27 +80,50 @@ func (a *agent) continueSession(ctx context.Context, p pendingContinuation) {
|
|||||||
slog.Info("nomos: auto-continuing session", "session", p.SessionID, "execution", p.ExecID, "status", p.Status)
|
slog.Info("nomos: auto-continuing session", "session", p.SessionID, "execution", p.ExecID, "status", p.Status)
|
||||||
|
|
||||||
var toolCalls []map[string]any
|
var toolCalls []map[string]any
|
||||||
var finalText string
|
var finalText, errText string
|
||||||
emit := func(ev agentEvent) {
|
|
||||||
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
// One retry if the LLM call itself produced nothing (transient flake /
|
||||||
if m, ok := ev.Data.(map[string]any); ok {
|
// empty-response) — the whole point of this mechanism is "don't give up
|
||||||
m["type"] = ev.Type
|
// on the first error," which should apply to the continuation call
|
||||||
toolCalls = append(toolCalls, m)
|
// itself, not just the homelab commands it's continuing. Found live: a
|
||||||
|
// destructive-recovery continuation hit an empty LLM response, its
|
||||||
|
// internal retry (chatWith's own maxLLMRetries=1) also came up empty, and
|
||||||
|
// without this outer retry the operator would see nothing at all.
|
||||||
|
cctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
for attempt := 0; attempt < 2; attempt++ {
|
||||||
|
toolCalls, finalText, errText = nil, "", ""
|
||||||
|
emit := func(ev agentEvent) {
|
||||||
|
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
||||||
|
if m, ok := ev.Data.(map[string]any); ok {
|
||||||
|
m["type"] = ev.Type
|
||||||
|
toolCalls = append(toolCalls, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ev.Type == "text" {
|
||||||
|
finalText, _ = ev.Data.(string)
|
||||||
|
}
|
||||||
|
if ev.Type == "error" {
|
||||||
|
errText, _ = ev.Data.(string)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ev.Type == "text" {
|
a.chatWith(cctx, p.SessionID, "", note, emit)
|
||||||
finalText, _ = ev.Data.(string)
|
if finalText != "" || len(toolCalls) > 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if attempt == 0 {
|
||||||
|
slog.Warn("nomos: auto-continuation produced nothing, retrying once", "session", p.SessionID, "execution", p.ExecID, "error", errText)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use a generous timeout: a continuation may itself launch further steps.
|
text := finalText
|
||||||
cctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
if text == "" && errText != "" {
|
||||||
defer cancel()
|
text = fmt.Sprintf("(auto-continuation hit an internal error and did not respond: %s — the execution's own result is above; you may need to prompt the agent again)", errText)
|
||||||
a.chatWith(cctx, p.SessionID, "", note, emit)
|
slog.Error("nomos: auto-continuation produced no response after retry", "session", p.SessionID, "execution", p.ExecID, "error", errText)
|
||||||
|
}
|
||||||
assistantMsg, _ := json.Marshal(map[string]any{
|
assistantMsg, _ := json.Marshal(map[string]any{
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"text": finalText,
|
"text": text,
|
||||||
"tool_calls": toolCalls,
|
"tool_calls": toolCalls,
|
||||||
"auto": true, // marks this as an autonomous continuation, not an operator turn
|
"auto": true, // marks this as an autonomous continuation, not an operator turn
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user