package main import ( "context" "encoding/json" "fmt" "log/slog" "regexp" "strings" "time" "github.com/dtoro/oikos/internal/safego" "github.com/google/uuid" ) // execIDRe matches "execution " in a tool result — the phrasing shared // by request_execution / run when they queue or start a gated execution. // Only these async executions need continuation; the synchronous auto-run // path returns its output inline and is already observed in-turn. var execIDRe = regexp.MustCompile(`(?i)execution\s+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})`) func extractExecutionIDs(toolResult string) []uuid.UUID { matches := execIDRe.FindAllStringSubmatch(toolResult, -1) seen := map[uuid.UUID]bool{} var out []uuid.UUID for _, m := range matches { if id, err := uuid.Parse(m[1]); err == nil && !seen[id] { seen[id] = true out = append(out, id) } } return out } // idleTaskThreshold is how long a goal-bearing session can sit non-terminal // with no activity before the idle sweep nudges it, per // plans/2026-07-11-task-completion-safety-net.md. Arbitrary starting point, // not measured against real task durations — long enough that it won't fire // mid-turn, short enough the board doesn't lie for hours. const idleTaskThreshold = 15 * time.Minute // runIdleSweepWorker is the safety net for case 2 of // plans/2026-07-11-task-completion-safety-net.md: sessions that called // set_goal (so the inline safety net in agent.go correctly left them alone, // since they framed themselves as a real task) but then stalled without // ever calling complete_task. Coarser than runContinuationWorker's 4s tick // since "gone idle" is a much slower signal than "an execution just // finished." Blocks until ctx is cancelled. func (a *agent) runIdleSweepWorker(ctx context.Context) { if a.store == nil { slog.Warn("nomos: idle sweep worker disabled (no store)") return } slog.Info("nomos: idle sweep worker started") ticker := time.NewTicker(2 * time.Minute) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: a.processIdleSweep(ctx) } } } // processIdleSweep nudges a stalled goal-bearing session once; if it's still // non-terminal on the NEXT sweep (meaning the nudge itself went unanswered, // not just that the model is still working), auto-closes it with a // visible "auto-closed" outcome instead of leaving it stuck forever — same // reasoning resumeSession already applies below for a different failure // mode (a resume that produces no response at all). func (a *agent) processIdleSweep(ctx context.Context) { stale := a.store.staleGoalSessions(ctx, idleTaskThreshold, 5) for _, s := range stale { s := s if s.CompletionNudges == 0 { safego.Go("nomos:idle-nudge:"+s.ID, func() { note := fmt.Sprintf("[System: this task ('%s') has been idle for %s with no complete_task call. "+ "If the goal is done (or can't be completed), call complete_task now with the outcome and a "+ "one-line summary. If you're still genuinely working through the plan, ignore this and continue.]", s.Goal, idleTaskThreshold) note = a.store.enrichResumeNote(ctx, s.ID, note) // P1: only count the nudge if it actually delivered. resumeSession // skips (returns false) when a turn is already active; bumping the // counter anyway would make the next sweep auto-close a merely-busy // session as "unanswered." if a.resumeSession(ctx, s.ID, note) { if err := a.store.bumpCompletionNudge(ctx, s.ID); err != nil { slog.Error("nomos: idle nudge bump failed", "session", s.ID, "error", err) } } }) continue } safego.Go("nomos:idle-autoclose:"+s.ID, func() { summary := fmt.Sprintf("Auto-closed after %s idle with no response to a completion nudge.", idleTaskThreshold) if err := a.store.completeTask(ctx, s.ID, "partial", summary); err != nil { slog.Error("nomos: idle auto-close failed", "session", s.ID, "error", err) } }) } } // runContinuationWorker is the event loop that replaces the human typing // "continue". It polls for gated executions that (a) were initiated by a chat // session and (b) have just finished, and — while that agent has an open assent // window (an approved plan is in flight) — feeds each result back into the // agent so it proceeds to the next step or recovers from the failure, all // without an operator tick. Blocks until ctx is cancelled. func (a *agent) runContinuationWorker(ctx context.Context) { if a.store == nil { slog.Warn("nomos: continuation worker disabled (no store)") return } slog.Info("nomos: continuation worker started") ticker := time.NewTicker(4 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: a.processContinuations(ctx) } } } // processContinuations dispatches each pending item as its OWN goroutine // (safego.Go, so a panic deep in one task's resumed turn — JSON parsing of // model output, an unexpected nil in a tool result — is recovered and logged // instead of taking down this whole function, which used to run every // item sequentially in the SAME goroutine as the ticker loop. Two problems // that fixed: (1) throughput — task B's continuation no longer waits for // task A's full (up to 10-minute) resumed turn to finish first, the exact // per-task blocking this session's earlier concurrency work removed from the // live-chat path but had left in place here; (2) survivability — since Go // panics unwind the goroutine they occur in, an unrecovered one here used to // mean this call (and every future tick, since the whole ticker loop runs in // one goroutine) would simply stop — auto-continuation for every task would // silently die until nomos restarted. Now a single bad item can only ever // take down its own goroutine. func (a *agent) processContinuations(ctx context.Context) { pending := a.store.pendingContinuations(ctx, 5) for _, p := range pending { // Scope gate: only auto-continue while an approved plan is active FOR // THIS SESSION. Checked per-item, not once for the whole batch — with // multiple tasks in flight, one task's open window must never cover a // pending continuation belonging to a different task. if !a.store.assentWindowActive(ctx, a.agentID, p.SessionID) { // Re-open the assent window if this session is genuinely // executing (plan was approved, work is in progress) — the // window may have expired while the execution ran. Don't // penalize timing: the plan was approved, the work happened, // the result should flow back. sesh, seshErr := a.store.getSession(ctx, p.SessionID) if seshErr == nil && sesh.Goal != "" && (sesh.Status == "executing" || sesh.Status == "planning") { a.openAssentWindow(ctx, p.SessionID) slog.Info("nomos: re-opened assent window for continuing session", "session", p.SessionID, "execution", p.ExecID) } else { // Genuinely no plan — inject a visible note so the // operator knows WHY the agent didn't auto-continue. note := fmt.Sprintf("[System: execution %s finished with status=%s, but the assent window for this session is not active. The agent will not auto-continue. Reply 'continue' or re-approve the plan to resume.]", p.ExecID, p.Status) body, _ := json.Marshal(map[string]any{"role": "assistant", "text": note, "auto": true}) a.store.saveMessage(context.Background(), p.SessionID, "assistant", body) a.store.markContinued(ctx, p.ExecID) continue } } // markContinued now happens inside continueSession, AFTER resumeSession // actually runs (P0). Pre-marking here consumed the item even when // resumeSession skipped on a busy session, losing the result. safego.Go("nomos:continue-session:"+p.SessionID, func() { a.continueSession(ctx, p) }) } } // continueSession re-invokes the agent for one finished execution. Persists // progress LIVE — a placeholder row immediately, updated in place as each // tool call completes — instead of only saving once the whole continuation // finishes. The frontend polls (see chat.ts startPolling); without // incremental persistence here, a continuation that runs several tool calls // before concluding would look like total silence in the UI for however long // that takes, which is exactly the "I just wait while nothing happens" // complaint this exists to fix — polling alone only helps if there's // something new to poll for. func (a *agent) continueSession(ctx context.Context, p pendingContinuation) { slog.Info("nomos: auto-continuing session", "session", p.SessionID, "execution", p.ExecID, "status", p.Status) // P0 (plans/2026-08-03-nomos-chat-changes-review.md): mark the execution // continued ONLY after the turn actually ran. resumeSession skips (returns // false) when another turn is already active for this session; marking // before that — as the old code did — consumed the item (continued_at set, // never re-queued by pendingContinuations) and silently lost the result. // On a skip, leave it pending so the next worker tick retries once the // active turn frees the permit. if !a.resumeSession(ctx, p.SessionID, buildContinuationNote(p)) { slog.Info("nomos: continuation deferred — a turn is active; will retry next tick", "session", p.SessionID, "execution", p.ExecID) return } a.store.markContinued(ctx, p.ExecID) } // resumeSession re-invokes the agent for a session with a system-injected note — // a finished execution (continueSession) or an operator's answer to a question // (handleAnswerQuestion) — persisting progress LIVE (a placeholder row updated // in place as each tool call lands) so the frontend poller sees each step, // instead of total silence until the whole resume concludes. // // F1 (plan 2026-08-03): this is the single entry point for EVERY background // turn — the continuation worker, idle sweep, answer-question, /resume, and the // empty-message reconnect all funnel through here. It acquires the session's // turn permit non-blocking and SKIPS if a turn is already running. A duplicate // resume while a turn (live or background) is active is exactly the // interleaving that corrupted the activity panel and made tasks feel stuck. // // Returns whether the turn actually ran. Callers that mutate state before // resuming (the continuation worker's markContinued, the idle sweep's nudge // bump) MUST gate that mutation on a true return — otherwise a busy-skip leaves // the state changed but the work undone (lost continuation / false auto-close). // See plans/2026-08-03-nomos-chat-changes-review.md P0/P1. func (a *agent) resumeSession(ctx context.Context, sessionID, note string) bool { if !a.gate.acquire(sessionID, 0) { slog.Info("nomos: turn already active, skipping background resume", "session", sessionID) return false } // Release the gate, then drain any operator message that was queued while // this background turn ran (plan 2026-08-03 F2). Queued messages are run as // real user turns server-side; resumeSession itself never enqueues. defer func() { a.gate.release(sessionID) safego.Go("nomos:drain:"+sessionID, func() { a.drainQueued(context.Background(), sessionID) }) }() placeholder, _ := json.Marshal(map[string]any{ "role": "assistant", "text": "", "auto": true, }) msgID, err := a.store.insertMessageReturningID(ctx, sessionID, "assistant", placeholder) if err != nil { slog.Error("nomos: resume placeholder insert failed", "session", sessionID, "error", err) } var toolCalls []map[string]any var finalText, errText string var finalThinking string persist := func() { if msgID == uuid.Nil { return } text := finalText if text == "" && errText != "" { text = fmt.Sprintf("(auto-continuation hit an internal error and did not respond: %s — the execution's own result is above; you may need to prompt the agent again)", errText) } body, _ := json.Marshal(map[string]any{ "role": "assistant", "text": text, "thinking": finalThinking, "tool_calls": toolCalls, "auto": true, // marks this as an autonomous continuation, not an operator turn }) a.store.updateMessage(ctx, msgID, body) } // One retry if the LLM call itself produced nothing (transient flake / // empty-response) — the whole point of this mechanism is "don't give up // on the first error," which should apply to the continuation call // itself, not just the homelab commands it's continuing. Found live: a // destructive-recovery continuation hit an empty LLM response, its // internal retry (chatWith's own maxLLMRetries=1) also came up empty, and // without this outer retry the operator would see nothing at all. cctx, cancel := context.WithTimeout(ctx, 10*time.Minute) defer cancel() // B.3: escalate the recovery note across attempts — a transient flake // needs a different prompt than a model that's stuck no-op'ing. The // final attempt is maximally directive ("do this specific thing now"). // B.5: back off between retries (4s, 8s) so a transient provider issue // has time to clear — 3 identical calls in 3 seconds just get 3 // identical empties. notes := []string{ note, // attempt 0: the original (already enriched per B.2) note fmt.Sprintf("[System: your previous turn produced no response. %s. Produce a response now — call the next tool or report progress in one sentence.]", note), fmt.Sprintf("[System: two consecutive empty responses. Stop trying to be clever. The next action is: pick the lowest-pending plan step, mark it running with update_plan_step, and call run for its target. Do that now.]"), } for attempt := 0; attempt < 3; attempt++ { if attempt > 0 { select { case <-cctx.Done(): return true // a turn ran on an earlier attempt; consume, don't re-loop case <-time.After(time.Duration(2< 0 { break } if attempt < 2 { slog.Warn("nomos: resume produced nothing, retrying", "session", sessionID, "error", errText, "attempt", attempt+1) } } if errText != "" && finalText == "" { slog.Error("nomos: resume produced no response after retry", "session", sessionID, "error", errText) // Persist a visible system note in the transcript so the // operator sees what happened, but do NOT auto-complete the // task — leave it in 'executing' so a follow-up chat message // can resume it. Before this fix, the task was marked 'failed' // here, which ended it permanently and required starting over. resumeFailedNote := fmt.Sprintf("[System: auto-resume failed after retrying: %s. The task is paused — send another message to continue.]", errText) body, _ := json.Marshal(map[string]any{ "role": "assistant", "text": resumeFailedNote, "auto": true, }) if msgID != uuid.Nil { a.store.updateMessage(context.Background(), msgID, body) } else { // No placeholder was inserted (rare), save directly. a.store.saveMessage(context.Background(), sessionID, "assistant", body) } return true // do not call persist() again — already persisted above } persist() // final state — same row, updated one last time with the concluding text return true } // buildContinuationNote frames the finished execution for the model: what // happened, and what to do about it. The persist-through-errors instruction // lives here (and in SOUL) so the agent recovers instead of stopping. func buildContinuationNote(p pendingContinuation) string { action := p.Action if i := strings.IndexByte(action, ':'); i > 0 && len(action) > 40 { action = action[:i] // keep just the action verb for brevity; params are in the DB } result := p.Result if len(result) > 3000 { result = result[:3000] + "…[truncated]" } var b strings.Builder fmt.Fprintf(&b, "[System: execution %s (%s) finished with status=%s.\nResult: %s\n\n", p.ExecID, action, p.Status, result) switch p.Status { case "completed": b.WriteString("It SUCCEEDED. Continue the approved plan: run the next step. If this was the final step, verify the end goal actually works (e.g. curl the service) and then report success to the operator. Do NOT stop and wait for the operator to say 'continue'.") case "failed", "cancelled": b.WriteString("It FAILED. Do NOT give up or hand back to the operator. Diagnose the cause from the result above (and by running read-only inspection commands if needed), form a hypothesis, fix it, and retry or take an alternative approach. You have an active assent window, so config_mutation steps run without re-approval. Only stop and ask the operator if you are genuinely blocked (need information only they have) or the fix would require a destructive action they haven't approved.") default: // denied / revoked b.WriteString("The operator denied or revoked this step. Stop executing this plan and briefly acknowledge.") } b.WriteString("]") return b.String() }