feat(observability): restore monitoring coverage, make gaps visible, stream executions
Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by discarded errors in checkdefaults: - writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT (slug) DO NOTHING, then wrote a check_defs row referencing it. On any re-seed the slug already existed, the entity insert no-oped, and the FK violated — aborting the ingest transaction and surfacing as an unrelated failure several entities later. Re-seeding has been broken since; prod's coverage was frozen at its first successful seed. This is what TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting. - shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed to ".network" and overwrote each other; service:jellyfin collided with lxc:jellyfin. - The ssh-script checker never read the `args` config checkdefaults wrote, so process_check.sh always ran without its unit name and returned "unknown". Coverage is now 75/89. Monitoring is declared per entity type in seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap. coverageSweep raises an `unmonitored` signal only where a type declares monitoring it lacks — 8 real gaps, no false positives. Also: - entity_types.attribute_schema was never ingested: the seed loader read "attribute_schema" but the YAML says "attributes", so all 60 types stored JSON null. - ListExecutions ignored its declared target/action/correlation_id filters and paginated on a non-unique target slug, dropping and repeating rows. - started_at was captured but only written at terminal state, so a running execution reported NULL for its whole life. The three MCP auto-run copies wrote no timing at all; they are now one autoRun helper. - SSH output was buffered to completion and discarded entirely on timeout. Both sshExec copies now stream through a shared execlog sink into execution_logs, and keep partial output when a command is cancelled. - executions.correlation_id was a random per-execution uuid that correlated nothing; it is now the chat session id, which is what lets the chat tail live output. - reversible_low had no auto-run branch despite policy declaring it unattended. Since computeCommandRisk never returns it, the class only arises when an agent declares it over a read_only command — so gating it penalised candor without adding safety. - backup-target gains a backup-freshness checker (portable find -mmin, since the first target is on macOS), resolving its host by walking backs-up-to backwards. The pre-deploy pg_dump is now a tracked backup target. UI: an Executions section on entity detail with live output tailing, and streamed output under a running `run` call in the chat timeline. Migrations 022-024. Ops.svelte and context.ts exclude execution.output from their refetch triggers, which would otherwise fire once a second per command. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -706,6 +706,25 @@ export async function fetchEntitySignals(entityId: string): Promise<Signal[]> {
|
||||
return data.items ?? []
|
||||
}
|
||||
|
||||
export interface ExecutionLogChunk {
|
||||
seq: number
|
||||
stream: string
|
||||
chunk: string
|
||||
ts: string
|
||||
}
|
||||
|
||||
// Streamed command output. Chunks land in execution_logs as the command runs,
|
||||
// so this returns output for an execution that is still going — unlike
|
||||
// `result.output`, which is only written once at the terminal state.
|
||||
export async function fetchExecutionLogs(
|
||||
executionId: string
|
||||
): Promise<{ items: ExecutionLogChunk[]; combined: string }> {
|
||||
const res = await fetchWithAuth(`${API}/executions/${executionId}/logs?limit=2000`)
|
||||
if (!res.ok) return { items: [], combined: '' }
|
||||
const data = await res.json()
|
||||
return { items: data.items ?? [], combined: data.combined ?? '' }
|
||||
}
|
||||
|
||||
export async function fetchEntityExecutions(entityId: string): Promise<Execution[]> {
|
||||
const params = new URLSearchParams({ target: entityId, limit: '50' })
|
||||
const res = await fetchWithAuth(`${API}/executions?${params}`)
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
fetchMetrics,
|
||||
fetchEntityEvents,
|
||||
fetchEntitySignals,
|
||||
fetchEntityExecutions,
|
||||
fetchExecutionLogs,
|
||||
fetchEntityTasks,
|
||||
fetchEntityKnowledge,
|
||||
fetchKnowledgeContent,
|
||||
@@ -24,6 +26,7 @@
|
||||
type Relationship,
|
||||
type MetricSeries,
|
||||
type Signal,
|
||||
type Execution,
|
||||
type EntityTask,
|
||||
type KnowledgeHit,
|
||||
type KnowledgeContent,
|
||||
@@ -32,7 +35,7 @@
|
||||
type AuditEntry
|
||||
} from '$lib/api'
|
||||
import { relativeTime, truncateMiddle } from '$lib/utils'
|
||||
import type { OikosEvent } from '$lib/stores/events'
|
||||
import { liveEvents, subscribeEvents, type OikosEvent } from '$lib/stores/events'
|
||||
import DetailSection from '$lib/components/DetailSection.svelte'
|
||||
import { Badge } from '$lib/components/ui/badge'
|
||||
import { Button } from '$lib/components/ui/button'
|
||||
@@ -48,6 +51,15 @@
|
||||
let metrics = $state<MetricSeries[]>([])
|
||||
let events = $state<OikosEvent[]>([])
|
||||
let signals = $state<Signal[]>([])
|
||||
let executions = $state<Execution[]>([])
|
||||
let expandedExecution = $state<string | null>(null)
|
||||
// Streamed output for the expanded execution, kept separate from
|
||||
// result.output: result is only written at the terminal state, so a running
|
||||
// command has nothing there and these chunks are the only thing to show.
|
||||
let streamedOutput = $state('')
|
||||
let streamEl = $state<HTMLPreElement | null>(null)
|
||||
// Follow the tail unless the operator has scrolled up to read something.
|
||||
let followTail = $state(true)
|
||||
let tasks = $state<EntityTask[]>([])
|
||||
let knowledge = $state<KnowledgeHit[]>([])
|
||||
let ownContent = $state<KnowledgeContent | null>(null)
|
||||
@@ -76,11 +88,12 @@
|
||||
loading = false
|
||||
return
|
||||
}
|
||||
const [rel, m, ev, sig, tk, kh, oc, ch, aa, au] = await Promise.all([
|
||||
const [rel, m, ev, sig, ex, tk, kh, oc, ch, aa, au] = await Promise.all([
|
||||
fetchEntityRelations(entity.id),
|
||||
fetchMetrics(entity.id),
|
||||
fetchEntityEvents(entity.id),
|
||||
fetchEntitySignals(entity.id),
|
||||
fetchEntityExecutions(entity.id),
|
||||
fetchEntityTasks(entity),
|
||||
fetchEntityKnowledge(entity.id),
|
||||
KNOWLEDGE_TYPES.has(entity.type) ? fetchKnowledgeContent(entity.id) : Promise.resolve(null),
|
||||
@@ -92,6 +105,7 @@
|
||||
metrics = m
|
||||
events = ev
|
||||
signals = sig
|
||||
executions = ex
|
||||
tasks = tk
|
||||
knowledge = kh
|
||||
ownContent = oc
|
||||
@@ -141,8 +155,80 @@
|
||||
}
|
||||
}
|
||||
|
||||
const RUNNING_STATUSES = new Set(['running', 'approved', 'executing', 'pending_approval'])
|
||||
|
||||
function isRunning(execution: Execution): boolean {
|
||||
return RUNNING_STATUSES.has(execution.status)
|
||||
}
|
||||
|
||||
async function loadExecutionLogs(executionId: string) {
|
||||
const logs = await fetchExecutionLogs(executionId)
|
||||
// Ignore a response that arrives after the operator collapsed the row or
|
||||
// opened a different one.
|
||||
if (expandedExecution !== executionId) return
|
||||
streamedOutput = logs.combined
|
||||
if (followTail) {
|
||||
await tick()
|
||||
if (streamEl) streamEl.scrollTop = streamEl.scrollHeight
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleExecution(execution: Execution) {
|
||||
if (expandedExecution === execution.id) {
|
||||
expandedExecution = null
|
||||
streamedOutput = ''
|
||||
return
|
||||
}
|
||||
expandedExecution = execution.id
|
||||
streamedOutput = ''
|
||||
followTail = true
|
||||
await loadExecutionLogs(execution.id)
|
||||
}
|
||||
|
||||
function onStreamScroll() {
|
||||
if (!streamEl) return
|
||||
// Re-engage following once the operator scrolls back to the bottom.
|
||||
followTail = streamEl.scrollHeight - streamEl.scrollTop - streamEl.clientHeight < 24
|
||||
}
|
||||
|
||||
// Live tail. The backend throttles execution.output to one event per second
|
||||
// per execution and the NOTIFY payload deliberately omits the data, so the
|
||||
// event is only a "there is more" ping — the chunks are re-read here.
|
||||
//
|
||||
// Lifecycle events (execution.completed/failed) refresh the list instead:
|
||||
// without that the row keeps its `running` badge and empty duration forever,
|
||||
// which only became visible once running executions were shown at all.
|
||||
$effect(() => {
|
||||
const ev = $liveEvents[0]
|
||||
if (!ev || !ev.type.startsWith('execution.')) return
|
||||
|
||||
if (ev.type === 'execution.output') {
|
||||
const target = expandedExecution
|
||||
if (target && ev.entity_id === target) loadExecutionLogs(target)
|
||||
return
|
||||
}
|
||||
|
||||
if (!entity) return
|
||||
// Only refetch for an execution this panel is actually showing, so an
|
||||
// unrelated command elsewhere in the fleet doesn't cause a request here.
|
||||
if (executions.some((e) => e.id === ev.entity_id)) {
|
||||
refreshExecutions()
|
||||
}
|
||||
})
|
||||
|
||||
async function refreshExecutions() {
|
||||
if (!entity) return
|
||||
const id = entity.id
|
||||
const next = await fetchEntityExecutions(id)
|
||||
// Guard against the panel having switched entity mid-flight.
|
||||
if (entity?.id === id) executions = next
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
load(slug)
|
||||
// One shared, reference-counted SSE connection; this just registers
|
||||
// interest so the tail receives events while the window is open.
|
||||
return subscribeEvents()
|
||||
})
|
||||
|
||||
$effect(() => {
|
||||
@@ -171,6 +257,53 @@
|
||||
}
|
||||
}
|
||||
|
||||
// The `run` tool encodes its action as `run:{"command":…,"purpose":…}`, so
|
||||
// the raw string is unreadable. Show the command when there is one, the bare
|
||||
// verb otherwise. Mirrors splitAction() in internal/httpapi/activity.go.
|
||||
function executionSummary(execution: Execution): string {
|
||||
const idx = execution.action.indexOf(':')
|
||||
if (idx < 0) return execution.action
|
||||
const verb = execution.action.slice(0, idx)
|
||||
const rest = execution.action.slice(idx + 1)
|
||||
try {
|
||||
const params = JSON.parse(rest)
|
||||
if (typeof params?.command === 'string') return params.command
|
||||
if (typeof params?.purpose === 'string') return `${verb} — ${params.purpose}`
|
||||
} catch {
|
||||
// Not JSON — older actions use `verb:plain-params`.
|
||||
return `${verb} ${rest}`
|
||||
}
|
||||
return verb
|
||||
}
|
||||
|
||||
// result is {"output": …} on success and {"output": …, "error": …} on
|
||||
// failure. Until now nothing in the UI rendered either.
|
||||
function executionOutput(execution: Execution): string {
|
||||
const result = execution.result
|
||||
if (!result) return ''
|
||||
const parts: string[] = []
|
||||
if (typeof result.error === 'string' && result.error) parts.push(result.error)
|
||||
if (typeof result.output === 'string' && result.output) parts.push(result.output)
|
||||
return parts.join('\n\n').trim()
|
||||
}
|
||||
|
||||
function executionStatusVariant(
|
||||
status: string
|
||||
): 'default' | 'secondary' | 'destructive' | 'outline' {
|
||||
if (status === 'failed' || status === 'denied' || status === 'revoked') return 'destructive'
|
||||
if (status === 'completed') return 'default'
|
||||
if (status === 'running' || status === 'pending_approval') return 'secondary'
|
||||
return 'outline'
|
||||
}
|
||||
|
||||
function formatDuration(ms: number | null): string {
|
||||
if (ms == null) return '—'
|
||||
if (ms < 1000) return `${ms}ms`
|
||||
const s = Math.round(ms / 1000)
|
||||
if (s < 60) return `${s}s`
|
||||
return `${Math.floor(s / 60)}m ${s % 60}s`
|
||||
}
|
||||
|
||||
function severityVariant(sev: string): 'default' | 'secondary' | 'destructive' {
|
||||
if (sev === 'critical') return 'destructive'
|
||||
if (sev === 'warning') return 'secondary'
|
||||
@@ -550,6 +683,64 @@
|
||||
</div>
|
||||
{/snippet}
|
||||
|
||||
{#snippet executionsContent()}
|
||||
<div class="flex flex-col gap-1.5">
|
||||
{#each executions as execution (execution.id)}
|
||||
{@const output = executionOutput(execution)}
|
||||
{@const running = isRunning(execution)}
|
||||
{@const expanded = expandedExecution === execution.id}
|
||||
<div class="flex flex-col gap-1 border-b pb-1.5 text-xs last:border-0 last:pb-0">
|
||||
<div class="flex items-center justify-between gap-2">
|
||||
<button
|
||||
type="button"
|
||||
class="flex-1 truncate text-left font-mono hover:underline disabled:cursor-default disabled:no-underline"
|
||||
disabled={!output && !running}
|
||||
title={output || running ? 'Show output' : undefined}
|
||||
onclick={() => toggleExecution(execution)}
|
||||
>
|
||||
{executionSummary(execution)}
|
||||
</button>
|
||||
<div class="flex shrink-0 items-center gap-1">
|
||||
{#if execution.duration_ms != null}
|
||||
<span class="text-muted-foreground">{formatDuration(execution.duration_ms)}</span>
|
||||
{:else if running && execution.started_at}
|
||||
<!-- started_at is now written when the status flips to
|
||||
running, so an in-flight command can show how long it
|
||||
has been going instead of nothing at all. -->
|
||||
<span class="text-muted-foreground">{relativeTime(execution.started_at)}</span>
|
||||
{/if}
|
||||
<Badge variant={executionStatusVariant(execution.status)}>{execution.status}</Badge>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex items-center gap-2 text-muted-foreground">
|
||||
<span>{relativeTime(execution.started_at ?? execution.created_at)}</span>
|
||||
<span>·</span>
|
||||
<span>{execution.risk_class}</span>
|
||||
</div>
|
||||
{#if expanded}
|
||||
{@const shown = running ? streamedOutput : streamedOutput || output}
|
||||
{#if shown}
|
||||
<pre
|
||||
bind:this={streamEl}
|
||||
onscroll={onStreamScroll}
|
||||
class="mt-1 max-h-64 overflow-auto rounded bg-muted p-2 font-mono text-[11px] leading-snug whitespace-pre-wrap">{shown}</pre>
|
||||
{:else if running}
|
||||
<p class="mt-1 text-xs text-muted-foreground italic">Waiting for output…</p>
|
||||
{/if}
|
||||
{#if running}
|
||||
<div class="flex items-center gap-1.5 text-[11px] text-muted-foreground">
|
||||
<span class="size-1.5 animate-pulse rounded-full bg-warning"></span>
|
||||
<span>{followTail ? 'following output' : 'scrolled up — paused'}</span>
|
||||
</div>
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
{:else}
|
||||
<p class="text-xs text-muted-foreground">None.</p>
|
||||
{/each}
|
||||
</div>
|
||||
{/snippet}
|
||||
|
||||
{#snippet tasksContent()}
|
||||
<div class="flex flex-col gap-1">
|
||||
{#each tasks as { task, executionCount } (task.id)}
|
||||
@@ -674,6 +865,12 @@
|
||||
},
|
||||
{ key: 'metrics', title: 'Metrics', count: metrics.length, content: metricsContent },
|
||||
{ key: 'signals', title: 'Signals', count: signals.length, content: signalsContent },
|
||||
{
|
||||
key: 'executions',
|
||||
title: 'Executions',
|
||||
count: executions.length,
|
||||
content: executionsContent
|
||||
},
|
||||
{ key: 'tasks', title: 'Tasks', count: tasks.length, content: tasksContent },
|
||||
{ key: 'knowledge', title: 'Knowledge', count: knowledge.length, content: knowledgeContent },
|
||||
{ key: 'events', title: 'Recent events', count: events.length, content: eventsContent },
|
||||
|
||||
@@ -53,6 +53,18 @@
|
||||
stepToggles.set(step.id, !stepOpen(step))
|
||||
stepToggles = new Map(stepToggles)
|
||||
}
|
||||
// Pin each streaming output pane to its tail as chunks arrive. Keyed by
|
||||
// tool id because several run entries can be on screen, though only the
|
||||
// newest one is ever actually streaming.
|
||||
let liveOutputEls = $state<Record<string, HTMLPreElement | null>>({})
|
||||
$effect(() => {
|
||||
for (const e of entries) {
|
||||
if (!e.liveOutput) continue
|
||||
const el = liveOutputEls[e.id]
|
||||
if (el) el.scrollTop = el.scrollHeight
|
||||
}
|
||||
})
|
||||
|
||||
function toggleTool(id: string) {
|
||||
if (expandedTools.has(id)) expandedTools.delete(id)
|
||||
else expandedTools.add(id)
|
||||
@@ -366,7 +378,7 @@
|
||||
{#if expandedWithTools}
|
||||
<div transition:slide={{ duration: 150 }} class="flex flex-col">
|
||||
{#each item.tools as tool (tool.id)}
|
||||
{@const tOpen = expandedTools.has(tool.id)}
|
||||
{@const tOpen = expandedTools.has(tool.id) || !!tool.liveOutput}
|
||||
<div class="relative" data-tl-id={tool.id}>
|
||||
<!-- Branch stub: backbone → tool -->
|
||||
<span
|
||||
@@ -379,10 +391,12 @@
|
||||
<button
|
||||
type="button"
|
||||
class="flex w-full items-center gap-1.5 py-1 pl-9 pr-3 text-left text-[11px] {tool.args ||
|
||||
tool.detail
|
||||
tool.detail ||
|
||||
tool.liveOutput
|
||||
? 'cursor-pointer hover:bg-muted/20'
|
||||
: 'cursor-default'}"
|
||||
onclick={() => (tool.args || tool.detail) && toggleTool(tool.id)}
|
||||
onclick={() =>
|
||||
(tool.args || tool.detail || tool.liveOutput) && toggleTool(tool.id)}
|
||||
>
|
||||
<span class="flex size-3 shrink-0 items-center justify-center">
|
||||
{#if tool.status === 'running'}
|
||||
@@ -428,6 +442,13 @@
|
||||
tool.args
|
||||
)}</pre>
|
||||
{/if}
|
||||
{#if tool.liveOutput}
|
||||
<!-- Streaming while the command runs. Bound so it
|
||||
can be pinned to the tail as chunks arrive. -->
|
||||
<pre
|
||||
bind:this={liveOutputEls[tool.id]}
|
||||
class="max-h-36 overflow-auto whitespace-pre-wrap break-words rounded bg-muted/50 p-1.5 font-mono text-[9px] leading-relaxed text-muted-foreground">{tool.liveOutput}</pre>
|
||||
{/if}
|
||||
{#if tool.detail}
|
||||
<pre
|
||||
class="max-h-36 overflow-auto whitespace-pre-wrap break-words rounded bg-muted/50 p-1.5 font-mono text-[9px] leading-relaxed {tool.status ===
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { derived, type Readable } from 'svelte/store'
|
||||
import { messages, chatFor, type ChatMessage, type ToolCallResult } from './chat'
|
||||
import { planSteps, currentTask, workspaceFor, taskFor } from './workspace'
|
||||
import { liveExecutionOutputFor, type LiveExecutionOutput } from './execstream'
|
||||
import type { PlanStep, Session } from '$lib/api'
|
||||
|
||||
export { type ToolCallResult }
|
||||
@@ -28,6 +29,10 @@ export interface ActivityEntry {
|
||||
stepSeq?: number
|
||||
indent?: boolean
|
||||
status: 'running' | 'done' | 'failed'
|
||||
// Command output streaming in while a `run` tool call is still executing.
|
||||
// Distinct from `detail`, which is only populated once the tool_result
|
||||
// arrives — for an auto-run that is the moment the command finishes.
|
||||
liveOutput?: string
|
||||
}
|
||||
|
||||
// Detail text is kept full-length (not hard-truncated to a preview snippet)
|
||||
@@ -199,12 +204,31 @@ export const activityLog = derived([messages, planSteps, currentTask], ([$msgs,
|
||||
computeActivityLog($msgs, $steps, $task)
|
||||
)
|
||||
|
||||
// Attach streaming output to the `run` entry that is currently executing.
|
||||
// Nomos runs tools sequentially, so the last still-running run entry is the
|
||||
// one the output belongs to.
|
||||
function withLiveOutput(
|
||||
entries: ActivityEntry[],
|
||||
live: LiveExecutionOutput | null
|
||||
): ActivityEntry[] {
|
||||
if (!live?.output) return entries
|
||||
for (let i = entries.length - 1; i >= 0; i--) {
|
||||
const e = entries[i]
|
||||
if (e.type === 'tool_running' && e.status === 'running' && e.toolName === 'run') {
|
||||
entries[i] = { ...e, liveOutput: live.output }
|
||||
break
|
||||
}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
export function activityLogFor(sessionId: string): Readable<ActivityEntry[]> {
|
||||
const chat = chatFor(sessionId)
|
||||
const ws = workspaceFor(sessionId)
|
||||
const task = taskFor(sessionId)
|
||||
return derived([chat.messages, ws.planSteps, task], ([$msgs, $steps, $task]) =>
|
||||
computeActivityLog($msgs, $steps, $task)
|
||||
const live = liveExecutionOutputFor(sessionId)
|
||||
return derived([chat.messages, ws.planSteps, task, live], ([$msgs, $steps, $task, $live]) =>
|
||||
withLiveOutput(computeActivityLog($msgs, $steps, $task), $live)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -22,10 +22,13 @@ export async function refreshContext() {
|
||||
function onEvent(ev: OikosEvent) {
|
||||
if (ev.id <= lastSeenEventId) return
|
||||
lastSeenEventId = ev.id
|
||||
// execution.output carries no summary-level change — it just signals that a
|
||||
// running command printed more. Excluded so a single noisy command doesn't
|
||||
// refresh the dashboard summary once a second.
|
||||
if (
|
||||
ev.type.startsWith('approval.') ||
|
||||
ev.type.startsWith('signal.') ||
|
||||
ev.type.startsWith('execution.') ||
|
||||
(ev.type.startsWith('execution.') && ev.type !== 'execution.output') ||
|
||||
ev.type === 'health.changed'
|
||||
) {
|
||||
refreshContext()
|
||||
|
||||
137
web/src/lib/stores/execstream.test.ts
Normal file
137
web/src/lib/stores/execstream.test.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { get, writable } from 'svelte/store'
|
||||
import type { OikosEvent } from './events'
|
||||
|
||||
// The store is driven entirely by SSE events plus a log fetch, so both are
|
||||
// mocked. What matters is the correlation logic: an execution.output event is
|
||||
// matched to a chat session by correlation_id, which MCP-initiated executions
|
||||
// now carry (it used to be a random per-execution UUID that correlated
|
||||
// nothing).
|
||||
|
||||
const liveEvents = writable<OikosEvent[]>([])
|
||||
const subscribeEvents = vi.fn(() => () => {})
|
||||
const fetchExecutionLogs = vi.fn(async (id: string) => ({
|
||||
items: [],
|
||||
combined: `output-for-${id}`
|
||||
}))
|
||||
|
||||
vi.mock('./events', () => ({
|
||||
liveEvents,
|
||||
subscribeEvents
|
||||
}))
|
||||
vi.mock('$lib/api', () => ({
|
||||
fetchExecutionLogs: (id: string) => fetchExecutionLogs(id)
|
||||
}))
|
||||
|
||||
let mod: typeof import('./execstream')
|
||||
|
||||
function event(partial: Partial<OikosEvent>): OikosEvent {
|
||||
return {
|
||||
id: Math.floor(Math.random() * 1e9),
|
||||
ts: new Date().toISOString(),
|
||||
type: 'execution.output',
|
||||
entity_id: 'exec-1',
|
||||
severity: 'info',
|
||||
source: 'actuator',
|
||||
data: {},
|
||||
correlation_id: 'session-1',
|
||||
...partial
|
||||
} as OikosEvent
|
||||
}
|
||||
|
||||
// The store fetches asynchronously; let the microtask queue drain.
|
||||
const settle = () => new Promise((r) => setTimeout(r, 0))
|
||||
|
||||
beforeEach(async () => {
|
||||
liveEvents.set([])
|
||||
fetchExecutionLogs.mockClear()
|
||||
vi.resetModules()
|
||||
mod = await import('./execstream')
|
||||
})
|
||||
|
||||
describe('liveExecutionOutputFor', () => {
|
||||
it('picks up output for its own session', async () => {
|
||||
const store = mod.liveExecutionOutputFor('session-1')
|
||||
const stop = store.subscribe(() => {})
|
||||
|
||||
liveEvents.set([event({ entity_id: 'exec-1', correlation_id: 'session-1' })])
|
||||
await settle()
|
||||
|
||||
expect(fetchExecutionLogs).toHaveBeenCalledWith('exec-1')
|
||||
expect(get(store)).toEqual({ executionId: 'exec-1', output: 'output-for-exec-1' })
|
||||
stop()
|
||||
})
|
||||
|
||||
// Without this every open chat window would tail every other session's
|
||||
// commands.
|
||||
it('ignores output belonging to a different session', async () => {
|
||||
const store = mod.liveExecutionOutputFor('session-1')
|
||||
const stop = store.subscribe(() => {})
|
||||
|
||||
liveEvents.set([event({ entity_id: 'exec-9', correlation_id: 'session-2' })])
|
||||
await settle()
|
||||
|
||||
expect(fetchExecutionLogs).not.toHaveBeenCalled()
|
||||
expect(get(store)).toBeNull()
|
||||
stop()
|
||||
})
|
||||
|
||||
it('ignores unrelated event types', async () => {
|
||||
const store = mod.liveExecutionOutputFor('session-1')
|
||||
const stop = store.subscribe(() => {})
|
||||
|
||||
liveEvents.set([event({ type: 'signal.raised' })])
|
||||
await settle()
|
||||
|
||||
expect(fetchExecutionLogs).not.toHaveBeenCalled()
|
||||
stop()
|
||||
})
|
||||
|
||||
// A session runs commands one after another; the second must not inherit
|
||||
// the first one's output.
|
||||
it('resets when a new execution starts in the same session', async () => {
|
||||
const store = mod.liveExecutionOutputFor('session-1')
|
||||
const stop = store.subscribe(() => {})
|
||||
|
||||
liveEvents.set([event({ entity_id: 'exec-1' })])
|
||||
await settle()
|
||||
expect(get(store)?.executionId).toBe('exec-1')
|
||||
|
||||
liveEvents.set([event({ entity_id: 'exec-2' })])
|
||||
await settle()
|
||||
expect(get(store)).toEqual({ executionId: 'exec-2', output: 'output-for-exec-2' })
|
||||
stop()
|
||||
})
|
||||
|
||||
// Once the command finishes its output belongs to the tool_result, not to a
|
||||
// still-"running" entry — leaving it set would show stale output against
|
||||
// the next command.
|
||||
it('clears on a terminal execution event', async () => {
|
||||
const store = mod.liveExecutionOutputFor('session-1')
|
||||
const stop = store.subscribe(() => {})
|
||||
|
||||
liveEvents.set([event({ entity_id: 'exec-1' })])
|
||||
await settle()
|
||||
expect(get(store)).not.toBeNull()
|
||||
|
||||
liveEvents.set([event({ type: 'execution.completed', entity_id: 'exec-1' })])
|
||||
await settle()
|
||||
expect(get(store)).toBeNull()
|
||||
stop()
|
||||
})
|
||||
|
||||
it('does not clear on another session completing', async () => {
|
||||
const store = mod.liveExecutionOutputFor('session-1')
|
||||
const stop = store.subscribe(() => {})
|
||||
|
||||
liveEvents.set([event({ entity_id: 'exec-1' })])
|
||||
await settle()
|
||||
|
||||
liveEvents.set([
|
||||
event({ type: 'execution.completed', entity_id: 'exec-5', correlation_id: 'session-2' })
|
||||
])
|
||||
await settle()
|
||||
expect(get(store)).not.toBeNull()
|
||||
stop()
|
||||
})
|
||||
})
|
||||
102
web/src/lib/stores/execstream.ts
Normal file
102
web/src/lib/stores/execstream.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
// Live command output for a chat session's currently-running execution.
|
||||
//
|
||||
// The chat renders a `run` tool call as "running" from the moment the tool_use
|
||||
// arrives until its tool_result comes back — and for an auto-run that gap IS
|
||||
// the command's runtime. Until now that window showed only the arguments; the
|
||||
// output appeared all at once at the end.
|
||||
//
|
||||
// Correlation works because MCP-initiated executions now carry the chat
|
||||
// session id as their correlation_id, and every execution.output event carries
|
||||
// that through. So an event can be matched to the session on screen without a
|
||||
// lookup. Nomos runs tools sequentially, so at most one execution is in flight
|
||||
// per session — no ambiguity about which entry the output belongs to.
|
||||
//
|
||||
// Only auto-runs stream. A gated run returns "execution queued" immediately,
|
||||
// so its tool entry is already `done` and the command executes minutes later
|
||||
// after approval — the entity detail window is where that one is watched.
|
||||
|
||||
import { readable, type Readable } from 'svelte/store'
|
||||
import { liveEvents, subscribeEvents } from './events'
|
||||
import { fetchExecutionLogs } from '$lib/api'
|
||||
|
||||
export interface LiveExecutionOutput {
|
||||
executionId: string
|
||||
output: string
|
||||
}
|
||||
|
||||
const cache = new Map<string, Readable<LiveExecutionOutput | null>>()
|
||||
|
||||
/**
|
||||
* Live output for whichever execution this session is currently running.
|
||||
* Resets when a different execution starts, so output from a previous command
|
||||
* never bleeds into the next one's entry.
|
||||
*/
|
||||
export function liveExecutionOutputFor(sessionId: string): Readable<LiveExecutionOutput | null> {
|
||||
const existing = cache.get(sessionId)
|
||||
if (existing) return existing
|
||||
|
||||
const store = readable<LiveExecutionOutput | null>(null, (set) => {
|
||||
let currentId: string | null = null
|
||||
let inFlight = false
|
||||
// Coalesce: a refetch already running means the next event's data will be
|
||||
// covered by a single follow-up, rather than queueing a request per event.
|
||||
let queued = false
|
||||
|
||||
async function refresh(executionId: string) {
|
||||
if (inFlight) {
|
||||
queued = true
|
||||
return
|
||||
}
|
||||
inFlight = true
|
||||
try {
|
||||
const logs = await fetchExecutionLogs(executionId)
|
||||
// Drop a response for an execution that is no longer current.
|
||||
if (currentId === executionId) set({ executionId, output: logs.combined })
|
||||
} finally {
|
||||
inFlight = false
|
||||
if (queued) {
|
||||
queued = false
|
||||
if (currentId) refresh(currentId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const unsubscribeSSE = subscribeEvents()
|
||||
const unsubscribeEvents = liveEvents.subscribe((events) => {
|
||||
const ev = events[0]
|
||||
if (!ev) return
|
||||
|
||||
// Lifecycle end: clear so the finished command's output stops being
|
||||
// shown against a new "running" entry.
|
||||
if (
|
||||
ev.correlation_id === sessionId &&
|
||||
(ev.type === 'execution.completed' ||
|
||||
ev.type === 'execution.failed' ||
|
||||
ev.type === 'execution.cancelled')
|
||||
) {
|
||||
currentId = null
|
||||
set(null)
|
||||
return
|
||||
}
|
||||
|
||||
if (ev.type !== 'execution.output') return
|
||||
if (ev.correlation_id !== sessionId) return
|
||||
if (!ev.entity_id) return
|
||||
|
||||
if (ev.entity_id !== currentId) {
|
||||
currentId = ev.entity_id
|
||||
set({ executionId: currentId, output: '' })
|
||||
}
|
||||
refresh(ev.entity_id)
|
||||
})
|
||||
|
||||
return () => {
|
||||
unsubscribeEvents()
|
||||
unsubscribeSSE()
|
||||
cache.delete(sessionId)
|
||||
}
|
||||
})
|
||||
|
||||
cache.set(sessionId, store)
|
||||
return store
|
||||
}
|
||||
@@ -45,7 +45,12 @@
|
||||
const ev = $liveEvents[0]
|
||||
if (!ev) return
|
||||
if (ev.type.startsWith('approval.')) loadApprovals()
|
||||
if (ev.type.startsWith('execution.')) loadActivity()
|
||||
// execution.output is a "more command output arrived" ping for one
|
||||
// execution, not a lifecycle change — it fires up to once a second per
|
||||
// running command and changes nothing this table shows. Refetching the
|
||||
// whole activity list on it would turn a chatty apt upgrade into a
|
||||
// refetch storm.
|
||||
if (ev.type.startsWith('execution.') && ev.type !== 'execution.output') loadActivity()
|
||||
})
|
||||
|
||||
async function decide(id: string, decision: 'approve' | 'deny') {
|
||||
|
||||
Reference in New Issue
Block a user