fix: auto-continuation silently dropped LLM errors + added outer retry
Verified live: the new auto-continuation worker (previous commit) worked end
to end for the happy path (provision -> auto-verify -> report success, zero
operator ticks). But testing a failure-recovery case (a destroy that failed
because the container was still running) surfaced a real bug: continueSession's
emit closure only captured "text" events, so when chatWith ended the turn on
an "error" event (LLM returned an empty/refusal response, internal retry also
empty), the worker persisted a completely blank, uninformative "auto" message
— no sign anything had gone wrong, undermining observability of the very
mechanism just built.
- Capture "error" events and, if the turn produced no text/tool_calls at all,
persist an explanatory placeholder instead of blank.
- Add one outer retry of the whole chatWith call when the first attempt
produces nothing — the principle behind this whole feature ("don't give up
on the first error") should apply to the continuation mechanism itself, not
just the homelab commands it's continuing.
Also verified live: recovery-from-failure works via the normal chat path once
prompted, and cleaned up the test container.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -80,27 +80,50 @@ func (a *agent) continueSession(ctx context.Context, p pendingContinuation) {
|
||||
slog.Info("nomos: auto-continuing session", "session", p.SessionID, "execution", p.ExecID, "status", p.Status)
|
||||
|
||||
var toolCalls []map[string]any
|
||||
var finalText string
|
||||
emit := func(ev agentEvent) {
|
||||
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
||||
if m, ok := ev.Data.(map[string]any); ok {
|
||||
m["type"] = ev.Type
|
||||
toolCalls = append(toolCalls, m)
|
||||
var finalText, errText string
|
||||
|
||||
// One retry if the LLM call itself produced nothing (transient flake /
|
||||
// empty-response) — the whole point of this mechanism is "don't give up
|
||||
// on the first error," which should apply to the continuation call
|
||||
// itself, not just the homelab commands it's continuing. Found live: a
|
||||
// destructive-recovery continuation hit an empty LLM response, its
|
||||
// internal retry (chatWith's own maxLLMRetries=1) also came up empty, and
|
||||
// without this outer retry the operator would see nothing at all.
|
||||
cctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
toolCalls, finalText, errText = nil, "", ""
|
||||
emit := func(ev agentEvent) {
|
||||
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
||||
if m, ok := ev.Data.(map[string]any); ok {
|
||||
m["type"] = ev.Type
|
||||
toolCalls = append(toolCalls, m)
|
||||
}
|
||||
}
|
||||
if ev.Type == "text" {
|
||||
finalText, _ = ev.Data.(string)
|
||||
}
|
||||
if ev.Type == "error" {
|
||||
errText, _ = ev.Data.(string)
|
||||
}
|
||||
}
|
||||
if ev.Type == "text" {
|
||||
finalText, _ = ev.Data.(string)
|
||||
a.chatWith(cctx, p.SessionID, "", note, emit)
|
||||
if finalText != "" || len(toolCalls) > 0 {
|
||||
break
|
||||
}
|
||||
if attempt == 0 {
|
||||
slog.Warn("nomos: auto-continuation produced nothing, retrying once", "session", p.SessionID, "execution", p.ExecID, "error", errText)
|
||||
}
|
||||
}
|
||||
|
||||
// Use a generous timeout: a continuation may itself launch further steps.
|
||||
cctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
a.chatWith(cctx, p.SessionID, "", note, emit)
|
||||
|
||||
text := finalText
|
||||
if text == "" && errText != "" {
|
||||
text = fmt.Sprintf("(auto-continuation hit an internal error and did not respond: %s — the execution's own result is above; you may need to prompt the agent again)", errText)
|
||||
slog.Error("nomos: auto-continuation produced no response after retry", "session", p.SessionID, "execution", p.ExecID, "error", errText)
|
||||
}
|
||||
assistantMsg, _ := json.Marshal(map[string]any{
|
||||
"role": "assistant",
|
||||
"text": finalText,
|
||||
"text": text,
|
||||
"tool_calls": toolCalls,
|
||||
"auto": true, // marks this as an autonomous continuation, not an operator turn
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user