Files
oikos/web/src/lib/stores/activity.ts
dtoro 1dca2cfd7a feat(observability): restore monitoring coverage, make gaps visible, stream executions
Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by
discarded errors in checkdefaults:

- writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT
  (slug) DO NOTHING, then wrote a check_defs row referencing it. On any
  re-seed the slug already existed, the entity insert no-oped, and the FK
  violated — aborting the ingest transaction and surfacing as an unrelated
  failure several entities later. Re-seeding has been broken since; prod's
  coverage was frozen at its first successful seed. This is what
  TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting.
- shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed
  to ".network" and overwrote each other; service:jellyfin collided with
  lxc:jellyfin.
- The ssh-script checker never read the `args` config checkdefaults wrote, so
  process_check.sh always ran without its unit name and returned "unknown".

Coverage is now 75/89. Monitoring is declared per entity type in
seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say
it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap.
coverageSweep raises an `unmonitored` signal only where a type declares
monitoring it lacks — 8 real gaps, no false positives.

Also:
- entity_types.attribute_schema was never ingested: the seed loader read
  "attribute_schema" but the YAML says "attributes", so all 60 types stored
  JSON null.
- ListExecutions ignored its declared target/action/correlation_id filters and
  paginated on a non-unique target slug, dropping and repeating rows.
- started_at was captured but only written at terminal state, so a running
  execution reported NULL for its whole life. The three MCP auto-run copies
  wrote no timing at all; they are now one autoRun helper.
- SSH output was buffered to completion and discarded entirely on timeout.
  Both sshExec copies now stream through a shared execlog sink into
  execution_logs, and keep partial output when a command is cancelled.
- executions.correlation_id was a random per-execution uuid that correlated
  nothing; it is now the chat session id, which is what lets the chat tail
  live output.
- reversible_low had no auto-run branch despite policy declaring it
  unattended. Since computeCommandRisk never returns it, the class only arises
  when an agent declares it over a read_only command — so gating it penalised
  candor without adding safety.
- backup-target gains a backup-freshness checker (portable find -mmin, since
  the first target is on macOS), resolving its host by walking backs-up-to
  backwards. The pre-deploy pg_dump is now a tracked backup target.

UI: an Executions section on entity detail with live output tailing, and
streamed output under a running `run` call in the chat timeline.

Migrations 022-024. Ops.svelte and context.ts exclude execution.output from
their refetch triggers, which would otherwise fire once a second per command.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-28 13:51:14 +02:00

287 lines
9.9 KiB
TypeScript

import { derived, type Readable } from 'svelte/store'
import { messages, chatFor, type ChatMessage, type ToolCallResult } from './chat'
import { planSteps, currentTask, workspaceFor, taskFor } from './workspace'
import { liveExecutionOutputFor, type LiveExecutionOutput } from './execstream'
import type { PlanStep, Session } from '$lib/api'
export { type ToolCallResult }
export interface ActivityEntry {
id: string
type:
| 'goal'
| 'plan'
| 'step_running'
| 'step_done'
| 'step_failed'
| 'tool_running'
| 'tool_done'
| 'tool_error'
| 'knowledge'
| 'complete'
| 'question'
| 'error'
description: string
detail?: string
args?: string
timestamp: number
toolName?: string
stepSeq?: number
indent?: boolean
status: 'running' | 'done' | 'failed'
// Command output streaming in while a `run` tool call is still executing.
// Distinct from `detail`, which is only populated once the tool_result
// arrives — for an auto-run that is the moment the command finishes.
liveOutput?: string
}
// Detail text is kept full-length (not hard-truncated to a preview snippet)
// so the expanded view has something worth pretty-printing — capped only as
// a safety net against pathological payloads (a full fleet dump, etc).
const DETAIL_MAX = 8000
function summarizeArgs(args: unknown): string | undefined {
if (!args || typeof args !== 'object' || Array.isArray(args)) return undefined
if (Object.keys(args).length === 0) return undefined
try {
return JSON.stringify(args)
} catch {
return undefined
}
}
function stringifyResult(result: unknown): string {
const s = typeof result === 'string' ? result : JSON.stringify(result ?? '')
return s.length > DETAIL_MAX ? `${s.slice(0, DETAIL_MAX)}\n… truncated` : s
}
// Pure derivation, parameterized so it can back both the global "current
// session" activityLog below and a per-session activityLogFor(sessionId) for
// a floating task window.
function computeActivityLog(
$msgs: ChatMessage[],
$steps: PlanStep[],
$task: Session | null
): ActivityEntry[] {
const entries: ActivityEntry[] = []
const now = Date.now()
// Goal
if ($task?.goal) {
entries.push({
id: 'goal',
type: 'goal',
description: $task.goal,
timestamp: 0,
status: 'done'
})
}
// Plan steps
for (const s of $steps) {
if (s.status === 'pending') continue
const stepLabel = s.title || `Step ${s.seq}`
entries.push({
id: s.id,
type:
s.status === 'running' ? 'step_running' : s.status === 'done' ? 'step_done' : 'step_failed',
description: `Step ${s.seq}: ${stepLabel}`,
detail: s.detail || undefined,
timestamp: s.started_at ? new Date(s.started_at).getTime() : now,
status: s.status === 'running' ? 'running' : s.status === 'done' ? 'done' : 'failed'
})
}
// Tool calls (from messages). Tag each tool with the plan step that's
// currently running when it fires.
let currentStepSeq = 0
let entryIdx = 0
for (let mi = 0; mi < $msgs.length; mi++) {
for (const t of $msgs[mi].tools) {
// Track current step from update_plan_step calls
if (t.type === 'tool_use' && t.name === 'update_plan_step') {
const s = typeof t.args?.seq === 'number' ? t.args.seq : undefined
const status = typeof t.args?.status === 'string' ? t.args.status : undefined
if (s && status === 'running') currentStepSeq = s
} else if (t.name === 'set_goal' || t.name === 'propose_plan' || t.name === 'complete_task') {
currentStepSeq = 0
}
const label = toolActivityLabel(t)
const stepTag = currentStepSeq > 0 ? currentStepSeq : undefined
if (t.type === 'tool_use') {
entries.push({
id: t.id ?? `tool_${mi}_${entryIdx++}`,
type: 'tool_running',
description: label,
args: summarizeArgs(t.args),
timestamp: now - ($msgs.length - mi) * 1000,
toolName: t.name,
stepSeq: stepTag,
indent: stepTag != null,
status: 'running'
})
} else if (t.type === 'tool_result') {
const running = entries.find(
(e) => e.type === 'tool_running' && e.id === t.id && e.status === 'running'
)
if (running && t.error) {
running.type = 'tool_error'
running.status = 'failed'
running.description = `${label}: ${t.error.slice(0, 80)}`
running.detail = t.error
} else if (running) {
running.type = 'tool_done'
running.status = 'done'
running.detail = stringifyResult(t.result)
} else {
// Historical/persisted tool calls arrive as one merged record (args
// + result on the same object, see mergeToolCalls in chat.ts) rather
// than a separate tool_use/tool_result pair — there's never a
// "running" entry to attach to, so this branch has to build the
// full entry itself. It used to fall back to the raw tool name
// (e.g. "get_entity") instead of the humanized label here.
entries.push({
id: t.id ?? `tool_${mi}_${entryIdx++}`,
type: t.error ? 'tool_error' : 'tool_done',
description: t.error ? `${label}: ${t.error.slice(0, 80)}` : label,
detail: t.error ? t.error : stringifyResult(t.result),
args: summarizeArgs(t.args),
timestamp: now - ($msgs.length - mi) * 1000,
toolName: t.name,
stepSeq: stepTag,
indent: stepTag != null,
status: t.error ? 'failed' : 'done'
})
}
}
}
}
// Knowledge recorded — detect from upsert_knowledge tool results
for (let mi = 0; mi < $msgs.length; mi++) {
for (const t of $msgs[mi].tools) {
if (t.type === 'tool_result' && t.name === 'upsert_knowledge' && !t.error) {
const title = typeof t.args?.title === 'string' ? t.args.title : ''
entries.push({
id: `knowledge_${mi}`,
type: 'knowledge',
description: title ? `Recorded: ${title.slice(0, 60)}` : 'Recorded knowledge',
timestamp: now - ($msgs.length - mi) * 1000,
status: 'done'
})
}
}
}
// Task completion
if ($task?.outcome) {
entries.push({
id: 'complete',
type: 'complete',
description: $task.summary || `Task ${$task.outcome}`,
timestamp: now,
status: $task.outcome === 'failure' ? 'failed' : 'done'
})
}
// Note: approval entries were removed from activityLog (2026-07-15).
// They were always `status: 'running'` and never transitioned to 'done'
// (the derived store builds from tool-call text, not execution status),
// which caused the AgentIndicator to latch onto a stale "Approval: ..."
// entry and never clear — even after the session completed. Approvals
// are tracked via the REST /approvals endpoint (context.ts, Ops.svelte)
// and rendered as inline approval cards in the chat (or Ops page), not
// in the activity log.
// Sort oldest first
entries.sort((a, b) => a.timestamp - b.timestamp)
return entries
}
export const activityLog = derived([messages, planSteps, currentTask], ([$msgs, $steps, $task]) =>
computeActivityLog($msgs, $steps, $task)
)
// Attach streaming output to the `run` entry that is currently executing.
// Nomos runs tools sequentially, so the last still-running run entry is the
// one the output belongs to.
function withLiveOutput(
entries: ActivityEntry[],
live: LiveExecutionOutput | null
): ActivityEntry[] {
if (!live?.output) return entries
for (let i = entries.length - 1; i >= 0; i--) {
const e = entries[i]
if (e.type === 'tool_running' && e.status === 'running' && e.toolName === 'run') {
entries[i] = { ...e, liveOutput: live.output }
break
}
}
return entries
}
export function activityLogFor(sessionId: string): Readable<ActivityEntry[]> {
const chat = chatFor(sessionId)
const ws = workspaceFor(sessionId)
const task = taskFor(sessionId)
const live = liveExecutionOutputFor(sessionId)
return derived([chat.messages, ws.planSteps, task, live], ([$msgs, $steps, $task, $live]) =>
withLiveOutput(computeActivityLog($msgs, $steps, $task), $live)
)
}
// Humanized, past/present-tense description of what a tool call is doing
// ("Check execution", "Research: …") rather than its raw wire name. Exported
// so the chat's agent trace can read as a thinking log instead of an API log.
export function toolActivityLabel(t: ToolCallResult): string {
const args = t.args ?? {}
const str = (v: unknown): string => (typeof v === 'string' ? v : '')
switch (t.name) {
case 'set_goal':
return 'Set goal'
case 'propose_plan':
return 'Proposed plan'
case 'search_knowledge':
return `Research: ${str(args.query)}`
case 'get_entity':
return `Lookup: ${str(args.slug_or_id)}`
case 'get_entity_knowledge':
return 'Check prior knowledge'
case 'get_relations':
return 'Check relationships'
case 'list_lxcs':
return 'List containers'
case 'list_entities':
return 'List entities'
case 'get_health_summary':
return 'Fleet health'
case 'get_state_snapshot':
return 'State snapshot'
case 'run': {
const purpose = str(args.purpose)
const target = str(args.target)
if (purpose) return purpose
if (target) return `Run on ${target}`
return 'Run command'
}
case 'get_execution_status':
return 'Check execution'
case 'update_plan_step':
return 'Update plan'
case 'upsert_knowledge':
return 'Record knowledge'
case 'complete_task':
return 'Complete task'
case 'ping_service':
return 'Check service'
case 'ask_operator':
return 'Ask operator'
// Unmapped tool (new/uncommon) — humanize the raw name rather than
// showing it verbatim, e.g. "revoke_execution" -> "Revoke execution".
default:
return t.name.replace(/_/g, ' ').replace(/^./, (c) => c.toUpperCase())
}
}