Make background/long/desynced turns visible and queueable, fixing the four
symptoms that survived the v0.15.0 chat reliability pass.
F1 - status-driven working signal (workspace.ts taskWorking/currentWorking =
streaming OR status in {planning,executing}). Drives the chat trace, indicator,
and activity spinner so a turn with no live stream (background resume, a dropped
SSE, an idle-close mid long turn) still looks alive.
F2 - operator messages sent during an in-flight turn are now QUEUED and
auto-run when the gate frees, replacing the "still finishing a previous step...
send it again" rejection. Per-session in-memory FIFO (messagequeue.go, capped at
20) drained one-at-a-time under the turn gate; a `queued` SSE event drives a
"Queued" hint. drainQueued releases via a per-iteration deferred closure so a
runChatTurn panic can't deadlock the session's gate.
F3 - SSE keepalive (12s `:keepalive` comment) in handleChat so 20-40s
inter-iteration gaps no longer trip a proxy/browser idle close (the desync root
cause). All SSE writes serialized through one mutex.
F4 - generation-aware activity timeline (only the last propose_plan renders;
superseded ones collapse to one "Earlier plan revised" marker; step-attribution
follows only the current generation) + debounced plan refetch on lifecycle
events so a missed plan.proposed self-heals.
Verified against the last session (23da10db: 6m33s turn, operator "status"
deferred at 19:48:05). go test ./cmd/nomos/ green (new messagequeue tests);
web vitest 72/72 (new F4 generation tests); vite build clean.
VERSION: 0.16.0 -> 0.17.0
409 lines
15 KiB
TypeScript
409 lines
15 KiB
TypeScript
import { writable, derived, get, type Writable, type Readable } from 'svelte/store'
|
|
import { liveEvents, subscribeEvents } from './events'
|
|
import { currentSession, sessions, loadSessions, chatFor, streaming } from './chat'
|
|
import {
|
|
fetchPlan,
|
|
fetchQuestions,
|
|
type PlanStep,
|
|
type SessionQuestion,
|
|
type Session
|
|
} from '$lib/api'
|
|
import type {
|
|
PlanProposedData,
|
|
PlanStepEventData,
|
|
QuestionRaisedData,
|
|
QuestionAnsweredData,
|
|
EntityTouchedData,
|
|
HealthChangedData
|
|
} from '$lib/types'
|
|
|
|
// workspace.ts is the live "what is this task doing right now" surface for the
|
|
// TaskContextPanel: plan progress, the pinned operator question, and entities
|
|
// the agent is touching or whose health just changed. It is deliberately driven
|
|
// by the ALWAYS-ON global events stream (subscribeEvents), not the per-turn
|
|
// chat SSE — the auto-continuation worker and resumeSession run entirely
|
|
// server-side with no chat turn open, so a chat-bound panel would go stale
|
|
// exactly when the agent is working autonomously. This also means the panel
|
|
// keeps updating across a tab reload: hydrate() re-fetches REST state, then
|
|
// live events carry deltas from there.
|
|
|
|
export interface TouchedEntity {
|
|
slug: string
|
|
tool: string
|
|
ts: number
|
|
}
|
|
const TOUCHED_MAX = 12
|
|
const TOUCHED_PULSE_MS = 6000
|
|
|
|
export interface HealthDiff {
|
|
slug: string
|
|
from: string
|
|
to: string
|
|
ts: number
|
|
}
|
|
const HEALTH_DIFF_MS = 8000
|
|
|
|
export interface WorkspaceState {
|
|
planSteps: Writable<PlanStep[]>
|
|
questions: Writable<SessionQuestion[]>
|
|
openQuestion: Readable<SessionQuestion | null>
|
|
touched: Writable<TouchedEntity[]>
|
|
healthDiffs: Writable<HealthDiff[]>
|
|
}
|
|
|
|
function createWorkspaceState(): WorkspaceState {
|
|
const questions = writable<SessionQuestion[]>([])
|
|
return {
|
|
planSteps: writable<PlanStep[]>([]),
|
|
questions,
|
|
openQuestion: derived(questions, (qs) => qs.find((q) => q.status === 'open') ?? null),
|
|
touched: writable<TouchedEntity[]>([]),
|
|
healthDiffs: writable<HealthDiff[]>([])
|
|
}
|
|
}
|
|
|
|
// ─── global "current session" workspace — used by the main Chat page's rail ─
|
|
const globalWorkspace = createWorkspaceState()
|
|
export const planSteps = globalWorkspace.planSteps
|
|
export const questions = globalWorkspace.questions
|
|
export const openQuestion = globalWorkspace.openQuestion
|
|
export const touched = globalWorkspace.touched
|
|
export const healthDiffs = globalWorkspace.healthDiffs
|
|
|
|
// The task's own fields (goal/status/outcome/summary) live on the session row.
|
|
// Rather than a dedicated endpoint, derive from the sessions list (already
|
|
// fetched for the task board) and keep it fresh here on task-lifecycle events.
|
|
export const currentTask = derived(
|
|
[sessions, currentSession],
|
|
([$sessions, $id]) => $sessions.find((s) => s.id === $id) ?? null
|
|
)
|
|
|
|
// Events that can change agent_sessions.status/goal/outcome — see applyEventTo.
|
|
const STATUS_AFFECTING = new Set([
|
|
'goal.set',
|
|
'task.status',
|
|
'plan.proposed',
|
|
'question.raised',
|
|
'question.answered'
|
|
])
|
|
|
|
let refreshTimer: ReturnType<typeof setTimeout> | null = null
|
|
function scheduleSessionsRefresh() {
|
|
if (refreshTimer) clearTimeout(refreshTimer)
|
|
refreshTimer = setTimeout(() => loadSessions(), 300)
|
|
}
|
|
|
|
// A dropped plan-step event is one that matched no step on screen —
|
|
// historically a silent return, which made a disagreeing backend look like a
|
|
// dead UI (the panel froze showing pending steps while work happened
|
|
// elsewhere). Surface it so the next divergence is visible. Exported so a
|
|
// debug surface (or a test) can read the count.
|
|
export let droppedPlanStepEvents = 0
|
|
export function resetDroppedPlanStepEvents(): void {
|
|
droppedPlanStepEvents = 0
|
|
}
|
|
|
|
function applyPlanStepEventTo(ws: WorkspaceState, data: PlanStepEventData) {
|
|
const stepID = data?.step_id
|
|
const seq = data?.seq
|
|
ws.planSteps.update((steps) => {
|
|
const i = steps.findIndex((s) => (stepID && s.id === stepID) || (seq != null && s.seq === seq))
|
|
if (i === -1) {
|
|
droppedPlanStepEvents++
|
|
console.warn('plan step event matched no step on screen', {
|
|
stepID,
|
|
seq,
|
|
status: data?.status
|
|
})
|
|
return steps
|
|
}
|
|
const next = [...steps]
|
|
next[i] = {
|
|
...next[i],
|
|
status: data.status ?? next[i].status,
|
|
execution_id: data.execution_id ?? next[i].execution_id
|
|
}
|
|
return next
|
|
})
|
|
}
|
|
|
|
// Applies a live event to `ws` if it belongs to session `sid` — shared by the
|
|
// global "current session" watcher and every per-session floating-window
|
|
// watcher, each passing its own target state and session id.
|
|
function applyEventTo(
|
|
ws: WorkspaceState,
|
|
sid: string,
|
|
ev: { type: string; correlation_id?: string | null; data?: unknown }
|
|
) {
|
|
if (ev.correlation_id !== sid) return
|
|
const data = (ev.data ?? {}) as Record<string, unknown>
|
|
|
|
// Task fields (status/goal/outcome) live on the session row — refetch the
|
|
// (cheap) session list so the UI picks up the change without a
|
|
// dedicated endpoint. Every event that can change agent_sessions.status
|
|
// (goal.set → planning, propose_plan → executing, ask_operator →
|
|
// awaiting_input, answerQuestion → executing, complete_task → done/failed)
|
|
// must trigger this, not just goal.set/task.status — otherwise the status
|
|
// pill goes stale exactly when resumeSession runs the next turn entirely
|
|
// server-side, with no client-streaming 'done' event to piggyback a refresh
|
|
// on (found live: answering a question via the panel left the header stuck
|
|
// on "Needs your input" after the agent had already resumed). Debounced
|
|
// since several of these can land in one burst, and shared across
|
|
// sessions since it just refreshes the one global session list.
|
|
if (STATUS_AFFECTING.has(ev.type)) scheduleSessionsRefresh()
|
|
|
|
switch (ev.type) {
|
|
case 'plan.proposed': {
|
|
const d = data as unknown as PlanProposedData
|
|
if (Array.isArray(d.steps)) {
|
|
const incoming = d.steps.map((s) => ({
|
|
id: s.id,
|
|
seq: s.seq,
|
|
title: s.title,
|
|
detail: s.detail ?? '',
|
|
status: 'pending' as const,
|
|
target_slug: s.target_slug || undefined
|
|
}))
|
|
ws.planSteps.update((existing) => (d.appended ? [...existing, ...incoming] : incoming))
|
|
}
|
|
break
|
|
}
|
|
case 'plan.step.started':
|
|
case 'plan.step.finished':
|
|
applyPlanStepEventTo(ws, data as unknown as PlanStepEventData)
|
|
break
|
|
case 'question.raised': {
|
|
const d = data as unknown as QuestionRaisedData
|
|
ws.questions.update((qs) => [
|
|
{
|
|
id: d.question_id,
|
|
prompt: d.prompt ?? '',
|
|
context: { why: d.why, options: d.options, entities: d.entities },
|
|
status: 'open',
|
|
created_at: new Date().toISOString()
|
|
},
|
|
...qs.filter((q) => q.id !== d.question_id)
|
|
])
|
|
break
|
|
}
|
|
case 'question.answered': {
|
|
const d = data as unknown as QuestionAnsweredData
|
|
ws.questions.update((qs) =>
|
|
qs.map((q) => (q.id === d.question_id ? { ...q, status: 'answered', answer: d.answer } : q))
|
|
)
|
|
break
|
|
}
|
|
case 'entity.touched': {
|
|
const d = data as unknown as EntityTouchedData
|
|
if (d.slug) {
|
|
const now = Date.now()
|
|
ws.touched.update((t) =>
|
|
[{ slug: d.slug, tool: d.tool ?? '', ts: now }, ...t].slice(0, TOUCHED_MAX)
|
|
)
|
|
}
|
|
break
|
|
}
|
|
case 'knowledge.recorded':
|
|
break
|
|
}
|
|
}
|
|
|
|
// health.changed is task-agnostic (fleet-wide), so it's matched separately:
|
|
// show the diff whenever the changed entity is one this task has touched, not
|
|
// by correlation_id (health events don't carry one).
|
|
function applyHealthChangedTo(ws: WorkspaceState, ev: { type: string; data?: unknown }) {
|
|
if (ev.type !== 'health.changed') return
|
|
const data = (ev.data ?? {}) as HealthChangedData
|
|
if (!data.slug) return
|
|
const isRelevant = get(ws.touched).some((t) => t.slug === data.slug)
|
|
if (!isRelevant) return
|
|
ws.healthDiffs.update((d) =>
|
|
[{ slug: data.slug, from: data.from ?? '', to: data.to ?? '', ts: Date.now() }, ...d].slice(
|
|
0,
|
|
8
|
|
)
|
|
)
|
|
}
|
|
|
|
let hydratedFor: string | null = null
|
|
let unsubStream: (() => void) | null = null
|
|
let unsubLive: (() => void) | null = null
|
|
let lastSeenId = 0
|
|
|
|
async function hydrate(sessionId: string) {
|
|
hydratedFor = sessionId
|
|
globalWorkspace.planSteps.set([])
|
|
globalWorkspace.questions.set([])
|
|
globalWorkspace.touched.set([])
|
|
globalWorkspace.healthDiffs.set([])
|
|
const [steps, qs] = await Promise.all([fetchPlan(sessionId), fetchQuestions(sessionId)])
|
|
if (get(currentSession) !== sessionId) return // switched away while loading
|
|
globalWorkspace.planSteps.set(steps)
|
|
globalWorkspace.questions.set(qs)
|
|
}
|
|
|
|
// startWorkspace opens the global event subscription and begins tracking the
|
|
// active session. Call once from the panel's onMount; call the returned
|
|
// cleanup on unmount. Safe to call multiple times (ref-counted underneath).
|
|
export function startWorkspace(): () => void {
|
|
unsubStream = subscribeEvents()
|
|
|
|
const unsubSession = currentSession.subscribe((sid) => {
|
|
if (sid && sid !== hydratedFor) hydrate(sid)
|
|
if (!sid) {
|
|
hydratedFor = null
|
|
globalWorkspace.planSteps.set([])
|
|
globalWorkspace.questions.set([])
|
|
globalWorkspace.touched.set([])
|
|
globalWorkspace.healthDiffs.set([])
|
|
}
|
|
})
|
|
|
|
unsubLive = liveEvents.subscribe((evs) => {
|
|
if (evs.length === 0) return
|
|
const maxId = evs[0].id
|
|
if (maxId <= lastSeenId) {
|
|
return
|
|
}
|
|
const fresh = evs.filter((e) => e.id > lastSeenId)
|
|
lastSeenId = maxId
|
|
const sid = get(currentSession)
|
|
if (!sid) return
|
|
// Oldest-first application so ordering (e.g. plan.step.started before
|
|
// .finished) is preserved.
|
|
let planAffecting = false
|
|
for (const e of fresh.slice().reverse()) {
|
|
applyEventTo(globalWorkspace, sid, e)
|
|
applyHealthChangedTo(globalWorkspace, e)
|
|
if (e.correlation_id === sid && STATUS_AFFECTING.has(e.type)) planAffecting = true
|
|
}
|
|
if (planAffecting) schedulePlanRefetch(globalWorkspace, sid)
|
|
})
|
|
|
|
return () => {
|
|
unsubSession()
|
|
unsubLive?.()
|
|
unsubStream?.()
|
|
}
|
|
}
|
|
|
|
// ─── per-session workspace, for floating task windows ───────────────────────
|
|
//
|
|
// Same shape as the global workspace above, but keyed by session id instead
|
|
// of "whatever's on screen" — mirrors chat.ts's chatFor(). A window's
|
|
// TaskContextPanel calls startSessionWorkspace(sessionId) instead of
|
|
// startWorkspace(), and reads workspaceFor(sessionId)'s stores instead of the
|
|
// global ones, so several sessions' panels can be open and live at once.
|
|
const workspaces = new Map<string, WorkspaceState>()
|
|
|
|
export function workspaceFor(sessionId: string): WorkspaceState {
|
|
let w = workspaces.get(sessionId)
|
|
if (!w) {
|
|
w = createWorkspaceState()
|
|
workspaces.set(sessionId, w)
|
|
}
|
|
return w
|
|
}
|
|
|
|
export function taskFor(sessionId: string): Readable<Session | null> {
|
|
return derived(sessions, ($sessions) => $sessions.find((s) => s.id === sessionId) ?? null)
|
|
}
|
|
|
|
// Session statuses where the server is actively running a turn for this task.
|
|
// Deliberately EXCLUDES `awaiting_input` (paused for the operator) and the
|
|
// terminal states (done/failed/abandoned). This is the reliable "the agent is
|
|
// working" truth that survives a dropped SSE stream or an autonomous/background
|
|
// turn (which has no chat stream at all) — see plan 2026-08-03 F1.
|
|
const ACTIVE_TURN_STATUS = new Set(['planning', 'executing'])
|
|
|
|
function isWorking($streaming: boolean, $task: Session | null): boolean {
|
|
return $streaming || (!!$task && !!$task.status && ACTIVE_TURN_STATUS.has($task.status))
|
|
}
|
|
|
|
// taskWorking(sessionId): true while this session has a live stream OR its
|
|
// server-side status says a turn is running. Used by the chat window's
|
|
// "working" indicator, trace running state, and the activity spinner so a
|
|
// background/long/desynced turn still looks alive (the symptom: "can't tell
|
|
// the agent is working").
|
|
export function taskWorking(sessionId: string): Readable<boolean> {
|
|
const chat = chatFor(sessionId)
|
|
return derived([chat.streaming, taskFor(sessionId)], ([$s, $t]) => isWorking($s, $t))
|
|
}
|
|
|
|
// Global "current session" working signal for the main view's panel.
|
|
export const currentWorking = derived(
|
|
[streaming, currentTask],
|
|
([$s, $t]) => isWorking($s, $t)
|
|
)
|
|
|
|
async function hydrateSession(ws: WorkspaceState, sessionId: string) {
|
|
const [steps, qs] = await Promise.all([fetchPlan(sessionId), fetchQuestions(sessionId)])
|
|
ws.planSteps.set(steps)
|
|
ws.questions.set(qs)
|
|
}
|
|
|
|
// Self-heal for the plan panel (plan 2026-08-03 F4): plan steps are otherwise
|
|
// driven ONLY by live plan.proposed/plan.step.* events plus a one-time hydrate
|
|
// on mount. If an event is missed (window opened mid-turn, a brief events-
|
|
// stream gap), the panel freezes on a stale generation. Refetching the plan
|
|
// (current generation) on any task-lifecycle event makes it converge back to
|
|
// truth. Debounced per session since several of these land in one burst.
|
|
const planRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
|
function schedulePlanRefetch(ws: WorkspaceState, sessionId: string) {
|
|
const existing = planRefreshTimers.get(sessionId)
|
|
if (existing) clearTimeout(existing)
|
|
planRefreshTimers.set(
|
|
sessionId,
|
|
setTimeout(async () => {
|
|
planRefreshTimers.delete(sessionId)
|
|
try {
|
|
ws.planSteps.set(await fetchPlan(sessionId))
|
|
} catch {
|
|
// network blip — the next lifecycle event retries
|
|
}
|
|
}, 400)
|
|
)
|
|
}
|
|
|
|
export function startSessionWorkspace(sessionId: string): () => void {
|
|
const ws = workspaceFor(sessionId)
|
|
const unsub = subscribeEvents()
|
|
hydrateSession(ws, sessionId)
|
|
|
|
// Own "seen" watermark rather than the global lastSeenId — several
|
|
// windows, each watching a different session, can be reading off the same
|
|
// liveEvents feed at once.
|
|
let lastSeen = 0
|
|
const unsubLive = liveEvents.subscribe((evs) => {
|
|
if (evs.length === 0) return
|
|
const maxId = evs[0].id
|
|
if (maxId <= lastSeen) return
|
|
const fresh = evs.filter((e) => e.id > lastSeen)
|
|
lastSeen = maxId
|
|
let planAffecting = false
|
|
for (const e of fresh.slice().reverse()) {
|
|
applyEventTo(ws, sessionId, e)
|
|
applyHealthChangedTo(ws, e)
|
|
if (e.correlation_id === sessionId && STATUS_AFFECTING.has(e.type)) planAffecting = true
|
|
}
|
|
if (planAffecting) schedulePlanRefetch(ws, sessionId)
|
|
})
|
|
|
|
return () => {
|
|
unsubLive()
|
|
unsub()
|
|
}
|
|
}
|
|
|
|
// Sweep expired pulses/diffs on an interval so old touches stop glowing —
|
|
// across the global workspace and every per-session one currently in use.
|
|
setInterval(() => {
|
|
const now = Date.now()
|
|
const sweep = (ws: WorkspaceState) => {
|
|
ws.touched.update((t) => t.filter((e) => now - e.ts < TOUCHED_PULSE_MS))
|
|
ws.healthDiffs.update((d) => d.filter((e) => now - e.ts < HEALTH_DIFF_MS))
|
|
}
|
|
sweep(globalWorkspace)
|
|
for (const ws of workspaces.values()) sweep(ws)
|
|
}, 1000)
|