session reliability: reconnect, knowledge loop, retire request_execution
Phase 1 — crash recovery: SSE auto-reconnect + backoff, polling gate during disconnect, connection banner with retry button, empty-response retry 3x, non-terminal resume on empty response, persistent error cards. Phase 2/4 — visibility + continuation: custom ExecutionStatus renderer, approvals extracted on every tool_result (not just done), activity bar with status/goal, SessionDigest live polling, Continue button. Phase 3 — cleanup: complete_task auto-cancels orphaned approvals, deletes assent/destructive window keys, propose_plan marks pending steps as replaced, plan step seq-order enforcement. Phase 5 — knowledge loop: list_lxcs state filter (active/destroyed), SOUL.md unmissable writeback section, propose_plan validation nudge, complete_task writeback check, upsert_knowledge about array support, plan generation grouping in frontend, session approval count badge. Retire request_execution — all mutations now route through run. Updated SOUL.md, AGENTS.md, CLIENTS.md, skills, and agent system notes. Migration 020: plan step generation column, audit_log session_id index, nomos_plan_executions pending-approval index.
This commit is contained in:
@@ -40,7 +40,7 @@ function extractApprovals(tools: ToolCallResult[]): PendingApproval[] {
|
||||
if (m) {
|
||||
out.push({
|
||||
executionId: m[1],
|
||||
action: t.args?.action ?? t.args?.purpose ?? t.name ?? 'unknown',
|
||||
action: t.args?.purpose?.slice(0, 60) ?? t.args?.action ?? t.name ?? 'unknown',
|
||||
target: t.args?.target ?? 'unknown',
|
||||
destructive: /\bDESTRUCTIVE\b/.test(text),
|
||||
command: t.args?.command,
|
||||
@@ -66,10 +66,20 @@ function mid(): string {
|
||||
|
||||
export const messages = writable<ChatMessage[]>([])
|
||||
export const streaming = writable(false)
|
||||
export const connectionState = writable<'connected' | 'disconnected' | 'reconnecting'>('connected')
|
||||
export const currentSession = writable<string | null>(null)
|
||||
export const sessions = writable<Session[]>([])
|
||||
export const sessionMessages = writable<Message[]>([])
|
||||
export const error = writable<string | null>(null)
|
||||
export const chatErrors = writable<{ id: string; message: string; action?: string }[]>([])
|
||||
|
||||
export function dismissError(id: string) {
|
||||
chatErrors.update((e) => e.filter((x) => x.id !== id))
|
||||
}
|
||||
|
||||
export function addChatError(message: string, action?: string) {
|
||||
chatErrors.update((e) => [...e, { id: crypto.randomUUID(), message, action }])
|
||||
}
|
||||
|
||||
// Per-session controller tracking. Multiple tasks can stream concurrently
|
||||
// (see sendMessage's session guard above this used to be a single global
|
||||
@@ -153,7 +163,9 @@ function startPolling(sessionId: string) {
|
||||
stopPolling()
|
||||
pollingSessionId = sessionId
|
||||
pollTimer = setInterval(async () => {
|
||||
if (get(streaming)) return
|
||||
// Allow polling while disconnected — the agent is still working
|
||||
// server-side and the poller is the only way to see it.
|
||||
if (get(streaming) && get(connectionState) === 'connected') return
|
||||
if (pollingSessionId !== sessionId || get(currentSession) !== sessionId) return
|
||||
const msgs = await fetchMessages(sessionId)
|
||||
if (get(streaming) || pollingSessionId !== sessionId) return // re-check: the fetch itself takes time
|
||||
@@ -217,6 +229,7 @@ export function sendMessage(text: string) {
|
||||
// auto-continuation.
|
||||
const openedFor = get(currentSession)
|
||||
let streamSessionID = openedFor
|
||||
let receivedDone = false
|
||||
|
||||
const controller = streamChat(
|
||||
text,
|
||||
@@ -271,6 +284,7 @@ export function sendMessage(text: string) {
|
||||
last.tools = last.tools.map((t) =>
|
||||
t.id === ev.data.id ? updated : t
|
||||
)
|
||||
last.pendingApprovals = extractApprovals(last.tools)
|
||||
}
|
||||
return [...ms]
|
||||
})
|
||||
@@ -293,6 +307,8 @@ export function sendMessage(text: string) {
|
||||
return [...ms]
|
||||
})
|
||||
} else if (ev.type === 'done') {
|
||||
receivedDone = true
|
||||
connectionState.set('connected')
|
||||
messages.update((ms) => {
|
||||
const last = ms[ms.length - 1]
|
||||
if (last && last.role === 'assistant') {
|
||||
@@ -314,14 +330,29 @@ export function sendMessage(text: string) {
|
||||
}
|
||||
},
|
||||
(err: string) => {
|
||||
if (get(currentSession) === streamSessionID) error.set(err)
|
||||
// Distinguish user abort from network drop.
|
||||
if (err === 'AbortError' || err.includes('aborted')) {
|
||||
if (get(currentSession) === streamSessionID) streaming.set(false)
|
||||
return
|
||||
}
|
||||
// Network blip / server restart — initiate reconnect.
|
||||
if (get(currentSession) === streamSessionID) {
|
||||
error.set(err)
|
||||
if (!receivedDone && streamSessionID) {
|
||||
handleDisconnect(streamSessionID)
|
||||
} else {
|
||||
streaming.set(false)
|
||||
}
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (get(currentSession) === streamSessionID) streaming.set(false)
|
||||
// Clean up whichever slot this controller ended up in — normally
|
||||
// activeControllers[streamSessionID] once the 'session' event has
|
||||
// fired, but fall back to pendingController for the (rare) case where
|
||||
// the stream errored/completed before ever getting one.
|
||||
// SSE stream completed without error. If we never received 'done',
|
||||
// the connection was severed mid-turn — treat as disconnect.
|
||||
if (!receivedDone && streamSessionID && get(currentSession) === streamSessionID) {
|
||||
handleDisconnect(streamSessionID)
|
||||
} else if (get(currentSession) === streamSessionID) {
|
||||
streaming.set(false)
|
||||
}
|
||||
if (streamSessionID && activeControllers.get(streamSessionID) === controller) {
|
||||
activeControllers.delete(streamSessionID)
|
||||
}
|
||||
@@ -340,12 +371,87 @@ export function sendMessage(text: string) {
|
||||
}
|
||||
}
|
||||
|
||||
// handleDisconnect is called when the SSE stream drops mid-turn without
|
||||
// receiving a 'done' event. Falls back to polling and attempts reconnection.
|
||||
function handleDisconnect(sessionId: string) {
|
||||
const MAX_RECONNECT = 3
|
||||
connectionState.set('disconnected')
|
||||
startPolling(sessionId)
|
||||
addChatError('Agent connection lost. The task is still running — retrying…', 'Dismiss')
|
||||
|
||||
let attempts = 0
|
||||
let delay = 1000
|
||||
|
||||
const attemptReconnect = () => {
|
||||
if (get(currentSession) !== sessionId || attempts >= MAX_RECONNECT) {
|
||||
connectionState.set('disconnected')
|
||||
streaming.set(false)
|
||||
return
|
||||
}
|
||||
if (attempts > 0) {
|
||||
connectionState.set('reconnecting')
|
||||
addChatError(`Reconnecting to agent (attempt ${attempts + 1}/${MAX_RECONNECT})…`, 'Dismiss')
|
||||
}
|
||||
attempts++
|
||||
const controller = streamChat(
|
||||
'',
|
||||
sessionId,
|
||||
(_ev: ChatEvent) => {},
|
||||
(_err: string) => {
|
||||
delay = Math.min(delay * 2, 8000)
|
||||
setTimeout(attemptReconnect, delay)
|
||||
},
|
||||
() => {
|
||||
if (get(currentSession) === sessionId) {
|
||||
connectionState.set('connected')
|
||||
streaming.set(false)
|
||||
loadSessionMessages(sessionId)
|
||||
}
|
||||
}
|
||||
)
|
||||
if (activeControllers.get(sessionId)) {
|
||||
activeControllers.get(sessionId)?.abort()
|
||||
}
|
||||
activeControllers.set(sessionId, controller)
|
||||
}
|
||||
|
||||
setTimeout(attemptReconnect, delay)
|
||||
}
|
||||
|
||||
export function reconnect() {
|
||||
const sid = get(currentSession)
|
||||
if (!sid) return
|
||||
connectionState.set('reconnecting')
|
||||
const controller = streamChat(
|
||||
'',
|
||||
sid,
|
||||
(_ev: ChatEvent) => {},
|
||||
(_err: string) => {
|
||||
connectionState.set('disconnected')
|
||||
addChatError('Reconnect failed. The task may still be running — try sending a message to wake the agent.', 'Dismiss')
|
||||
},
|
||||
() => {
|
||||
if (get(currentSession) === sid) {
|
||||
connectionState.set('connected')
|
||||
streaming.set(false)
|
||||
loadSessionMessages(sid)
|
||||
}
|
||||
}
|
||||
)
|
||||
if (activeControllers.get(sid)) {
|
||||
activeControllers.get(sid)?.abort()
|
||||
}
|
||||
activeControllers.set(sid, controller)
|
||||
}
|
||||
|
||||
export function newChat() {
|
||||
cancelStream()
|
||||
stopPolling()
|
||||
connectionState.set('connected')
|
||||
currentSession.set(null)
|
||||
messages.set([])
|
||||
error.set(null)
|
||||
chatErrors.set([])
|
||||
streaming.set(false) // fresh view — see loadSessionMessages for why this must not depend on cancelStream's own reset
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user