Files
oikos/web/src/lib/stores/chat.ts
dtoro 8950bada44
Some checks failed
ci / build-test (push) Has been cancelled
ci / docker-build (push) Has been cancelled
fix: sshExec had no timeout — a hung remote command blocked forever
Root cause of "running for 10+ minutes without stopping": a real production
execution (TypeType pct_create) was found genuinely stuck 17+ minutes into a
single blocking SSH call. The container's post_install script was looping on
`getent hosts deb.debian.org`, waiting on a network that could never come up
— the operator's static IP config used gw:192.168.8.1, but the actual gateway
on that subnet is 192.168.8.2, so every network call hung instead of failing
fast (packets dropped, not rejected).

Two compounding bugs made this unrecoverable without manual intervention:

1. sshExec (both internal/httpapi/phase3.go and internal/mcp/server.go) had
   NO execution timeout — `session.CombinedOutput()` blocks until the remote
   command exits, with no deadline. A hung remote process blocks the Go
   goroutine forever; the execution can never leave 'running', and the
   operator has no way to make it stop. Fixed: both now race the SSH call
   against a 10-minute hard timeout, closing the session/client and
   returning a clear "timed out after 10m0s" error if exceeded. (The
   mcp/server.go copy also still had the original "swallowed non-zero exit"
   bug from before that fix was applied to httpapi's copy only — fixed here
   too.)

2. provisionScript's DNS-wait loop assumed `getent hosts` fails fast on no
   connectivity — it doesn't; a black-holed network can make each call hang
   far past the resolver's nominal timeout, so the documented "~90s" budget
   was never real. Wrapped every attempt in `timeout 3` so the wall-clock
   budget is now actually enforced (~2min worst case), and the failure
   message now suggests checking the net0 gateway.

Also fixes the matching UI-side gap (operator's literal question: "is there
a way to get more details? it has been running for 10+ minutes without
stopping"):

- InlineApproval's track() polling loop had its own ~6min ceiling and simply
  STOPPED polling after that — silently going stale before the backend (now
  correctly capped at 10min) could ever resolve. Raised to a 14min ceiling
  with margin, and added a distinct 'stalled' state if that's ever exceeded
  (explicitly says something's wrong, rather than freezing silently).
- The running-card now shows live elapsed time (ticking, from the
  execution's created_at), the actual command being run, and the execution
  ID — previously just a static "this can take a minute" with zero
  information. Also added command display to the destructive pending-
  approval card for full transparency before confirming.

Verified live end-to-end in a real browser (dev server proxying to
production): queued a real command via chat, approved via the button,
watched the elapsed-time counter tick in real time, and saw it transition to
a completed card with real output once the command finished.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-10 10:45:47 +02:00

244 lines
7.3 KiB
TypeScript

import { writable, get } from 'svelte/store'
import { streamChat, fetchSessions, fetchMessages, deleteSession as apiDeleteSession } from '$lib/api'
import type { ChatEvent, Session, Message } from '$lib/api'
export interface PendingApproval {
executionId: string
action: string
target: string
destructive: boolean
command?: string
purpose?: string
}
export interface ChatMessage {
id: string
role: 'user' | 'assistant'
text: string
tools: ToolCallResult[]
pendingApprovals: PendingApproval[]
}
const APPROVAL_RE = /execution\s+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i
// Deliberately NOT filtered by tool name. There is no fixed set of gated
// tools — `run` can execute anything, and any future tool that queues an
// approval should surface a card the same way. A prior version hardcoded
// `t.name === 'request_execution'`, so approvals raised by the newer `run`
// tool were silently invisible in chat: no card, no feedback, nothing to
// self-heal, forcing the operator to the Ops page with zero acknowledgement
// back in the conversation. Matching on the response shape (not the tool
// name) is what makes this robust to new gated tools without another
// silent breakage.
function extractApprovals(tools: ToolCallResult[]): PendingApproval[] {
const out: PendingApproval[] = []
for (const t of tools) {
if (t.type !== 'tool_result') continue
const text = typeof t.result === 'string' ? t.result : JSON.stringify(t.result ?? '')
if (!text.includes('requires approval')) continue
const m = text.match(APPROVAL_RE)
if (m) {
out.push({
executionId: m[1],
action: t.args?.action ?? t.args?.purpose ?? t.name ?? 'unknown',
target: t.args?.target ?? 'unknown',
destructive: /\bDESTRUCTIVE\b/.test(text),
command: t.args?.command,
purpose: t.args?.purpose
})
}
}
return out
}
export interface ToolCallResult {
type: 'tool_use' | 'tool_result'
name: string
id?: string
args?: any
result?: any
error?: string
}
function mid(): string {
return crypto.randomUUID()
}
export const messages = writable<ChatMessage[]>([])
export const streaming = writable(false)
export const currentSession = writable<string | null>(null)
export const sessions = writable<Session[]>([])
export const sessionMessages = writable<Message[]>([])
export const error = writable<string | null>(null)
let activeController: AbortController | null = null
export async function loadSessions() {
const list = await fetchSessions()
sessions.set(list)
}
// mergeToolCalls collapses a persisted tool_calls array into one entry per
// call id. Nomos persists the tool_use and tool_result as two separate
// entries sharing the same id (matching the SSE event pair); Chat.svelte
// renders tools in a keyed {#each ... (tool.id)}, which throws on duplicate
// keys and silently aborts the whole message list. Live-streamed messages
// never hit this because sendMessage() merges tool_result into the existing
// tool_use entry in place rather than appending a second one.
function mergeToolCalls(raw: ToolCallResult[] | undefined): ToolCallResult[] {
const byId = new Map<string, ToolCallResult>()
for (const tc of raw ?? []) {
const key = tc.id ?? crypto.randomUUID()
const existing = byId.get(key)
byId.set(key, existing ? { ...existing, ...tc, id: key } : { ...tc, id: key })
}
return Array.from(byId.values())
}
export async function loadSessionMessages(sessionId: string) {
currentSession.set(sessionId)
const msgs = await fetchMessages(sessionId)
sessionMessages.set(msgs)
const chatMsgs: ChatMessage[] = msgs.map((m) => {
const tools = mergeToolCalls(m.content?.tool_calls)
return {
id: m.id,
role: m.role as 'user' | 'assistant',
text: m.content?.text ?? (typeof m.content === 'string' ? m.content : ''),
tools,
pendingApprovals: extractApprovals(tools)
}
})
messages.set(chatMsgs)
}
export function sendMessage(text: string) {
error.set(null)
streaming.set(true)
const userMsg: ChatMessage = {
id: mid(),
role: 'user',
text,
tools: [],
pendingApprovals: []
}
messages.update((ms) => [...ms, userMsg])
const assistantMsg: ChatMessage = {
id: mid(),
role: 'assistant',
text: '',
tools: [],
pendingApprovals: []
}
messages.update((ms) => [...ms, assistantMsg])
let activeTools: Map<string, ToolCallResult> = new Map()
activeController = streamChat(
text,
get(currentSession), // continue the active session so the agent keeps context
(ev: ChatEvent) => {
if (ev.type === 'session') {
currentSession.set(ev.data)
} else if (ev.type === 'tool_use') {
const tr: ToolCallResult = {
type: 'tool_use',
name: ev.data.name,
id: ev.data.id,
args: ev.data.args
}
activeTools.set(ev.data.id, tr)
messages.update((ms) => {
const last = ms[ms.length - 1]
if (last && last.role === 'assistant') {
last.tools = [...last.tools, tr]
}
return [...ms]
})
} else if (ev.type === 'tool_result') {
const existing = activeTools.get(ev.data.id)
if (existing) {
const updated: ToolCallResult = {
...existing,
type: 'tool_result',
result: ev.data.result,
error: ev.data.error
}
activeTools.set(ev.data.id, updated)
messages.update((ms) => {
const last = ms[ms.length - 1]
if (last && last.role === 'assistant') {
last.tools = last.tools.map((t) =>
t.id === ev.data.id ? updated : t
)
}
return [...ms]
})
}
} else if (ev.type === 'text_delta') {
messages.update((ms) => {
const last = ms[ms.length - 1]
if (last && last.role === 'assistant') {
last.text += ev.data
}
return [...ms]
})
} else if (ev.type === 'text') {
// Final authoritative content for the turn; replaces accumulated deltas.
messages.update((ms) => {
const last = ms[ms.length - 1]
if (last && last.role === 'assistant') {
last.text = ev.data
}
return [...ms]
})
} else if (ev.type === 'done') {
messages.update((ms) => {
const last = ms[ms.length - 1]
if (last && last.role === 'assistant') {
last.pendingApprovals = extractApprovals(last.tools)
}
return [...ms]
})
currentSession.set(ev.data?.session_id ?? ev.session_id)
} else if (ev.type === 'error') {
error.set(ev.data)
}
},
(err: string) => {
error.set(err)
},
() => {
streaming.set(false)
activeController = null
loadSessions()
}
)
}
export function newChat() {
cancelStream()
currentSession.set(null)
messages.set([])
error.set(null)
}
export function cancelStream() {
if (activeController) {
activeController.abort()
activeController = null
streaming.set(false)
}
}
export async function deleteSession(sessionId: string) {
const ok = await apiDeleteSession(sessionId)
if (!ok) return
if (get(currentSession) === sessionId) {
newChat()
}
loadSessions()
}