feat(observability): restore monitoring coverage, make gaps visible, stream executions
Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by discarded errors in checkdefaults: - writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT (slug) DO NOTHING, then wrote a check_defs row referencing it. On any re-seed the slug already existed, the entity insert no-oped, and the FK violated — aborting the ingest transaction and surfacing as an unrelated failure several entities later. Re-seeding has been broken since; prod's coverage was frozen at its first successful seed. This is what TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting. - shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed to ".network" and overwrote each other; service:jellyfin collided with lxc:jellyfin. - The ssh-script checker never read the `args` config checkdefaults wrote, so process_check.sh always ran without its unit name and returned "unknown". Coverage is now 75/89. Monitoring is declared per entity type in seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap. coverageSweep raises an `unmonitored` signal only where a type declares monitoring it lacks — 8 real gaps, no false positives. Also: - entity_types.attribute_schema was never ingested: the seed loader read "attribute_schema" but the YAML says "attributes", so all 60 types stored JSON null. - ListExecutions ignored its declared target/action/correlation_id filters and paginated on a non-unique target slug, dropping and repeating rows. - started_at was captured but only written at terminal state, so a running execution reported NULL for its whole life. The three MCP auto-run copies wrote no timing at all; they are now one autoRun helper. - SSH output was buffered to completion and discarded entirely on timeout. Both sshExec copies now stream through a shared execlog sink into execution_logs, and keep partial output when a command is cancelled. - executions.correlation_id was a random per-execution uuid that correlated nothing; it is now the chat session id, which is what lets the chat tail live output. - reversible_low had no auto-run branch despite policy declaring it unattended. Since computeCommandRisk never returns it, the class only arises when an agent declares it over a read_only command — so gating it penalised candor without adding safety. - backup-target gains a backup-freshness checker (portable find -mmin, since the first target is on macOS), resolving its host by walking backs-up-to backwards. The pre-deploy pg_dump is now a tracked backup target. UI: an Executions section on entity detail with live output tailing, and streamed output under a running `run` call in the chat timeline. Migrations 022-024. Ops.svelte and context.ts exclude execution.output from their refetch triggers, which would otherwise fire once a second per command. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,8 @@
|
||||
fetchMetrics,
|
||||
fetchEntityEvents,
|
||||
fetchEntitySignals,
|
||||
fetchEntityExecutions,
|
||||
fetchExecutionLogs,
|
||||
fetchEntityTasks,
|
||||
fetchEntityKnowledge,
|
||||
fetchKnowledgeContent,
|
||||
@@ -24,6 +26,7 @@
|
||||
type Relationship,
|
||||
type MetricSeries,
|
||||
type Signal,
|
||||
type Execution,
|
||||
type EntityTask,
|
||||
type KnowledgeHit,
|
||||
type KnowledgeContent,
|
||||
@@ -32,7 +35,7 @@
|
||||
type AuditEntry
|
||||
} from '$lib/api'
|
||||
import { relativeTime, truncateMiddle } from '$lib/utils'
|
||||
import type { OikosEvent } from '$lib/stores/events'
|
||||
import { liveEvents, subscribeEvents, type OikosEvent } from '$lib/stores/events'
|
||||
import DetailSection from '$lib/components/DetailSection.svelte'
|
||||
import { Badge } from '$lib/components/ui/badge'
|
||||
import { Button } from '$lib/components/ui/button'
|
||||
@@ -48,6 +51,15 @@
|
||||
let metrics = $state<MetricSeries[]>([])
|
||||
let events = $state<OikosEvent[]>([])
|
||||
let signals = $state<Signal[]>([])
|
||||
let executions = $state<Execution[]>([])
|
||||
let expandedExecution = $state<string | null>(null)
|
||||
// Streamed output for the expanded execution, kept separate from
|
||||
// result.output: result is only written at the terminal state, so a running
|
||||
// command has nothing there and these chunks are the only thing to show.
|
||||
let streamedOutput = $state('')
|
||||
let streamEl = $state<HTMLPreElement | null>(null)
|
||||
// Follow the tail unless the operator has scrolled up to read something.
|
||||
let followTail = $state(true)
|
||||
let tasks = $state<EntityTask[]>([])
|
||||
let knowledge = $state<KnowledgeHit[]>([])
|
||||
let ownContent = $state<KnowledgeContent | null>(null)
|
||||
@@ -76,11 +88,12 @@
|
||||
loading = false
|
||||
return
|
||||
}
|
||||
const [rel, m, ev, sig, tk, kh, oc, ch, aa, au] = await Promise.all([
|
||||
const [rel, m, ev, sig, ex, tk, kh, oc, ch, aa, au] = await Promise.all([
|
||||
fetchEntityRelations(entity.id),
|
||||
fetchMetrics(entity.id),
|
||||
fetchEntityEvents(entity.id),
|
||||
fetchEntitySignals(entity.id),
|
||||
fetchEntityExecutions(entity.id),
|
||||
fetchEntityTasks(entity),
|
||||
fetchEntityKnowledge(entity.id),
|
||||
KNOWLEDGE_TYPES.has(entity.type) ? fetchKnowledgeContent(entity.id) : Promise.resolve(null),
|
||||
@@ -92,6 +105,7 @@
|
||||
metrics = m
|
||||
events = ev
|
||||
signals = sig
|
||||
executions = ex
|
||||
tasks = tk
|
||||
knowledge = kh
|
||||
ownContent = oc
|
||||
@@ -141,8 +155,80 @@
|
||||
}
|
||||
}
|
||||
|
||||
const RUNNING_STATUSES = new Set(['running', 'approved', 'executing', 'pending_approval'])
|
||||
|
||||
function isRunning(execution: Execution): boolean {
|
||||
return RUNNING_STATUSES.has(execution.status)
|
||||
}
|
||||
|
||||
async function loadExecutionLogs(executionId: string) {
|
||||
const logs = await fetchExecutionLogs(executionId)
|
||||
// Ignore a response that arrives after the operator collapsed the row or
|
||||
// opened a different one.
|
||||
if (expandedExecution !== executionId) return
|
||||
streamedOutput = logs.combined
|
||||
if (followTail) {
|
||||
await tick()
|
||||
if (streamEl) streamEl.scrollTop = streamEl.scrollHeight
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleExecution(execution: Execution) {
|
||||
if (expandedExecution === execution.id) {
|
||||
expandedExecution = null
|
||||
streamedOutput = ''
|
||||
return
|
||||
}
|
||||
expandedExecution = execution.id
|
||||
streamedOutput = ''
|
||||
followTail = true
|
||||
await loadExecutionLogs(execution.id)
|
||||
}
|
||||
|
||||
function onStreamScroll() {
|
||||
if (!streamEl) return
|
||||
// Re-engage following once the operator scrolls back to the bottom.
|
||||
followTail = streamEl.scrollHeight - streamEl.scrollTop - streamEl.clientHeight < 24
|
||||
}
|
||||
|
||||
// Live tail. The backend throttles execution.output to one event per second
|
||||
// per execution and the NOTIFY payload deliberately omits the data, so the
|
||||
// event is only a "there is more" ping — the chunks are re-read here.
|
||||
//
|
||||
// Lifecycle events (execution.completed/failed) refresh the list instead:
|
||||
// without that the row keeps its `running` badge and empty duration forever,
|
||||
// which only became visible once running executions were shown at all.
|
||||
$effect(() => {
|
||||
const ev = $liveEvents[0]
|
||||
if (!ev || !ev.type.startsWith('execution.')) return
|
||||
|
||||
if (ev.type === 'execution.output') {
|
||||
const target = expandedExecution
|
||||
if (target && ev.entity_id === target) loadExecutionLogs(target)
|
||||
return
|
||||
}
|
||||
|
||||
if (!entity) return
|
||||
// Only refetch for an execution this panel is actually showing, so an
|
||||
// unrelated command elsewhere in the fleet doesn't cause a request here.
|
||||
if (executions.some((e) => e.id === ev.entity_id)) {
|
||||
refreshExecutions()
|
||||
}
|
||||
})
|
||||
|
||||
async function refreshExecutions() {
|
||||
if (!entity) return
|
||||
const id = entity.id
|
||||
const next = await fetchEntityExecutions(id)
|
||||
// Guard against the panel having switched entity mid-flight.
|
||||
if (entity?.id === id) executions = next
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
load(slug)
|
||||
// One shared, reference-counted SSE connection; this just registers
|
||||
// interest so the tail receives events while the window is open.
|
||||
return subscribeEvents()
|
||||
})
|
||||
|
||||
$effect(() => {
|
||||
@@ -171,6 +257,53 @@
|
||||
}
|
||||
}
|
||||
|
||||
// The `run` tool encodes its action as `run:{"command":…,"purpose":…}`, so
|
||||
// the raw string is unreadable. Show the command when there is one, the bare
|
||||
// verb otherwise. Mirrors splitAction() in internal/httpapi/activity.go.
|
||||
function executionSummary(execution: Execution): string {
|
||||
const idx = execution.action.indexOf(':')
|
||||
if (idx < 0) return execution.action
|
||||
const verb = execution.action.slice(0, idx)
|
||||
const rest = execution.action.slice(idx + 1)
|
||||
try {
|
||||
const params = JSON.parse(rest)
|
||||
if (typeof params?.command === 'string') return params.command
|
||||
if (typeof params?.purpose === 'string') return `${verb} — ${params.purpose}`
|
||||
} catch {
|
||||
// Not JSON — older actions use `verb:plain-params`.
|
||||
return `${verb} ${rest}`
|
||||
}
|
||||
return verb
|
||||
}
|
||||
|
||||
// result is {"output": …} on success and {"output": …, "error": …} on
|
||||
// failure. Until now nothing in the UI rendered either.
|
||||
function executionOutput(execution: Execution): string {
|
||||
const result = execution.result
|
||||
if (!result) return ''
|
||||
const parts: string[] = []
|
||||
if (typeof result.error === 'string' && result.error) parts.push(result.error)
|
||||
if (typeof result.output === 'string' && result.output) parts.push(result.output)
|
||||
return parts.join('\n\n').trim()
|
||||
}
|
||||
|
||||
function executionStatusVariant(
|
||||
status: string
|
||||
): 'default' | 'secondary' | 'destructive' | 'outline' {
|
||||
if (status === 'failed' || status === 'denied' || status === 'revoked') return 'destructive'
|
||||
if (status === 'completed') return 'default'
|
||||
if (status === 'running' || status === 'pending_approval') return 'secondary'
|
||||
return 'outline'
|
||||
}
|
||||
|
||||
function formatDuration(ms: number | null): string {
|
||||
if (ms == null) return '—'
|
||||
if (ms < 1000) return `${ms}ms`
|
||||
const s = Math.round(ms / 1000)
|
||||
if (s < 60) return `${s}s`
|
||||
return `${Math.floor(s / 60)}m ${s % 60}s`
|
||||
}
|
||||
|
||||
function severityVariant(sev: string): 'default' | 'secondary' | 'destructive' {
|
||||
if (sev === 'critical') return 'destructive'
|
||||
if (sev === 'warning') return 'secondary'
|
||||
@@ -550,6 +683,64 @@
|
||||
</div>
|
||||
{/snippet}
|
||||
|
||||
{#snippet executionsContent()}
|
||||
<div class="flex flex-col gap-1.5">
|
||||
{#each executions as execution (execution.id)}
|
||||
{@const output = executionOutput(execution)}
|
||||
{@const running = isRunning(execution)}
|
||||
{@const expanded = expandedExecution === execution.id}
|
||||
<div class="flex flex-col gap-1 border-b pb-1.5 text-xs last:border-0 last:pb-0">
|
||||
<div class="flex items-center justify-between gap-2">
|
||||
<button
|
||||
type="button"
|
||||
class="flex-1 truncate text-left font-mono hover:underline disabled:cursor-default disabled:no-underline"
|
||||
disabled={!output && !running}
|
||||
title={output || running ? 'Show output' : undefined}
|
||||
onclick={() => toggleExecution(execution)}
|
||||
>
|
||||
{executionSummary(execution)}
|
||||
</button>
|
||||
<div class="flex shrink-0 items-center gap-1">
|
||||
{#if execution.duration_ms != null}
|
||||
<span class="text-muted-foreground">{formatDuration(execution.duration_ms)}</span>
|
||||
{:else if running && execution.started_at}
|
||||
<!-- started_at is now written when the status flips to
|
||||
running, so an in-flight command can show how long it
|
||||
has been going instead of nothing at all. -->
|
||||
<span class="text-muted-foreground">{relativeTime(execution.started_at)}</span>
|
||||
{/if}
|
||||
<Badge variant={executionStatusVariant(execution.status)}>{execution.status}</Badge>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex items-center gap-2 text-muted-foreground">
|
||||
<span>{relativeTime(execution.started_at ?? execution.created_at)}</span>
|
||||
<span>·</span>
|
||||
<span>{execution.risk_class}</span>
|
||||
</div>
|
||||
{#if expanded}
|
||||
{@const shown = running ? streamedOutput : streamedOutput || output}
|
||||
{#if shown}
|
||||
<pre
|
||||
bind:this={streamEl}
|
||||
onscroll={onStreamScroll}
|
||||
class="mt-1 max-h-64 overflow-auto rounded bg-muted p-2 font-mono text-[11px] leading-snug whitespace-pre-wrap">{shown}</pre>
|
||||
{:else if running}
|
||||
<p class="mt-1 text-xs text-muted-foreground italic">Waiting for output…</p>
|
||||
{/if}
|
||||
{#if running}
|
||||
<div class="flex items-center gap-1.5 text-[11px] text-muted-foreground">
|
||||
<span class="size-1.5 animate-pulse rounded-full bg-warning"></span>
|
||||
<span>{followTail ? 'following output' : 'scrolled up — paused'}</span>
|
||||
</div>
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
{:else}
|
||||
<p class="text-xs text-muted-foreground">None.</p>
|
||||
{/each}
|
||||
</div>
|
||||
{/snippet}
|
||||
|
||||
{#snippet tasksContent()}
|
||||
<div class="flex flex-col gap-1">
|
||||
{#each tasks as { task, executionCount } (task.id)}
|
||||
@@ -674,6 +865,12 @@
|
||||
},
|
||||
{ key: 'metrics', title: 'Metrics', count: metrics.length, content: metricsContent },
|
||||
{ key: 'signals', title: 'Signals', count: signals.length, content: signalsContent },
|
||||
{
|
||||
key: 'executions',
|
||||
title: 'Executions',
|
||||
count: executions.length,
|
||||
content: executionsContent
|
||||
},
|
||||
{ key: 'tasks', title: 'Tasks', count: tasks.length, content: tasksContent },
|
||||
{ key: 'knowledge', title: 'Knowledge', count: knowledge.length, content: knowledgeContent },
|
||||
{ key: 'events', title: 'Recent events', count: events.length, content: eventsContent },
|
||||
|
||||
@@ -53,6 +53,18 @@
|
||||
stepToggles.set(step.id, !stepOpen(step))
|
||||
stepToggles = new Map(stepToggles)
|
||||
}
|
||||
// Pin each streaming output pane to its tail as chunks arrive. Keyed by
|
||||
// tool id because several run entries can be on screen, though only the
|
||||
// newest one is ever actually streaming.
|
||||
let liveOutputEls = $state<Record<string, HTMLPreElement | null>>({})
|
||||
$effect(() => {
|
||||
for (const e of entries) {
|
||||
if (!e.liveOutput) continue
|
||||
const el = liveOutputEls[e.id]
|
||||
if (el) el.scrollTop = el.scrollHeight
|
||||
}
|
||||
})
|
||||
|
||||
function toggleTool(id: string) {
|
||||
if (expandedTools.has(id)) expandedTools.delete(id)
|
||||
else expandedTools.add(id)
|
||||
@@ -366,7 +378,7 @@
|
||||
{#if expandedWithTools}
|
||||
<div transition:slide={{ duration: 150 }} class="flex flex-col">
|
||||
{#each item.tools as tool (tool.id)}
|
||||
{@const tOpen = expandedTools.has(tool.id)}
|
||||
{@const tOpen = expandedTools.has(tool.id) || !!tool.liveOutput}
|
||||
<div class="relative" data-tl-id={tool.id}>
|
||||
<!-- Branch stub: backbone → tool -->
|
||||
<span
|
||||
@@ -379,10 +391,12 @@
|
||||
<button
|
||||
type="button"
|
||||
class="flex w-full items-center gap-1.5 py-1 pl-9 pr-3 text-left text-[11px] {tool.args ||
|
||||
tool.detail
|
||||
tool.detail ||
|
||||
tool.liveOutput
|
||||
? 'cursor-pointer hover:bg-muted/20'
|
||||
: 'cursor-default'}"
|
||||
onclick={() => (tool.args || tool.detail) && toggleTool(tool.id)}
|
||||
onclick={() =>
|
||||
(tool.args || tool.detail || tool.liveOutput) && toggleTool(tool.id)}
|
||||
>
|
||||
<span class="flex size-3 shrink-0 items-center justify-center">
|
||||
{#if tool.status === 'running'}
|
||||
@@ -428,6 +442,13 @@
|
||||
tool.args
|
||||
)}</pre>
|
||||
{/if}
|
||||
{#if tool.liveOutput}
|
||||
<!-- Streaming while the command runs. Bound so it
|
||||
can be pinned to the tail as chunks arrive. -->
|
||||
<pre
|
||||
bind:this={liveOutputEls[tool.id]}
|
||||
class="max-h-36 overflow-auto whitespace-pre-wrap break-words rounded bg-muted/50 p-1.5 font-mono text-[9px] leading-relaxed text-muted-foreground">{tool.liveOutput}</pre>
|
||||
{/if}
|
||||
{#if tool.detail}
|
||||
<pre
|
||||
class="max-h-36 overflow-auto whitespace-pre-wrap break-words rounded bg-muted/50 p-1.5 font-mono text-[9px] leading-relaxed {tool.status ===
|
||||
|
||||
Reference in New Issue
Block a user