diff --git a/web/src/lib/components/EntityDetailContent.svelte b/web/src/lib/components/EntityDetailContent.svelte index 15ec043..309561e 100644 --- a/web/src/lib/components/EntityDetailContent.svelte +++ b/web/src/lib/components/EntityDetailContent.svelte @@ -36,6 +36,7 @@ } from '$lib/api' import { relativeTime, truncateMiddle } from '$lib/utils' import { liveEvents, subscribeEvents, type OikosEvent } from '$lib/stores/events' + import { isHealthEvent, applyHealthEventTo } from '$lib/health' import DetailSection from '$lib/components/DetailSection.svelte' import { Badge } from '$lib/components/ui/badge' import { Button } from '$lib/components/ui/button' @@ -217,36 +218,31 @@ // Health and signals for THIS entity. The panel loaded these once on // open, so a window left on screen kept showing the health it had at - // mount — the exact staleness the fleet-wide work was about, reproduced - // one window at a time. Scoped by entity_id so an unrelated entity going - // down elsewhere costs nothing here. + // mount. Scoped by entity_id so an unrelated entity going down elsewhere + // costs nothing here. if (!entity || ev.entity_id !== entity.id) return - if ( - ev.type === 'health.changed' || - ev.type === 'health.stale' || - ev.type.startsWith('signal.') || - ev.type.startsWith('coverage.') - ) { - refreshStatus() + + // Health is patched from the event itself — it carries the new value, so + // there is nothing to go and ask for. + if (isHealthEvent(ev)) { + entity = applyHealthEventTo(entity, ev) + return + } + + // Signals genuinely need a read: the event says one was raised or + // resolved, not what the full signal list now looks like. Just the + // signals, though — not the entity, checks, and everything else. + if (ev.type.startsWith('signal.') || ev.type.startsWith('coverage.')) { + refreshSignals() } }) - // Re-reads only what a health or signal event can change, rather than - // re-running the full 11-request load(). - async function refreshStatus() { + async function refreshSignals() { if (!entity) return const id = entity.id - const slugAtStart = entity.slug - const [fresh, sig, ch] = await Promise.all([ - fetchEntity(slugAtStart), - fetchEntitySignals(id), - fetchChecksForTarget(slugAtStart) - ]) - // The panel may have switched entity while these were in flight. - if (!entity || entity.id !== id) return - if (fresh) entity = fresh - signals = sig - checks = ch + const next = await fetchEntitySignals(id) + // The panel may have switched entity while this was in flight. + if (entity?.id === id) signals = next } async function refreshExecutions() { diff --git a/web/src/lib/components/FleetMap.svelte b/web/src/lib/components/FleetMap.svelte index c48f52d..f2e1d12 100644 --- a/web/src/lib/components/FleetMap.svelte +++ b/web/src/lib/components/FleetMap.svelte @@ -14,6 +14,7 @@ import { onMount, onDestroy } from 'svelte' import { fetchGraph, type GraphView, type Entity } from '$lib/api' import { liveEvents, subscribeEvents } from '$lib/stores/events' + import { isHealthEvent, applyHealthEvent, healthFromEvent } from '$lib/health' import { Skeleton } from '$lib/components/ui/skeleton' import { Button } from '$lib/components/ui/button' import GlobeIcon from '@lucide/svelte/icons/globe' @@ -56,16 +57,29 @@ return subscribeEvents() }) - // Refetch on structural or health changes, same triggers as the old graph. + // Structural changes need a refetch — they change which nodes and edges + // exist. Health does not: the event carries the new value, so the node is + // patched in place instead of pulling the entire fleet graph (and its + // layout) down again for one colour change. $effect(() => { const ev = $liveEvents[0] if (!ev) return - if ( - ev.type.startsWith('entity.') || - ev.type.startsWith('relationship.') || - ev.type === 'health.changed' - ) { + if (ev.type.startsWith('entity.') || ev.type.startsWith('relationship.')) { load() + return + } + if (isHealthEvent(ev) && graph) { + const health = healthFromEvent(ev) + if (!health || !ev.entity_id) return + // healthOf() reads graph.health[id] in preference to the node's own + // field (include=status attaches it as a side map), so patching only + // the nodes would leave the rendered colour unchanged. Patch both. + const nodes = applyHealthEvent(graph.nodes, ev) + graph = { + ...graph, + nodes, + health: { ...(graph.health ?? {}), [ev.entity_id]: health as Health } + } } }) diff --git a/web/src/lib/health.test.ts b/web/src/lib/health.test.ts new file mode 100644 index 0000000..7b7e642 --- /dev/null +++ b/web/src/lib/health.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect } from 'vitest' +import { isHealthEvent, healthFromEvent, applyHealthEvent, applyHealthEventTo } from './health' +import type { Entity } from './api' +import type { OikosEvent } from './stores/events' + +function ev(partial: Partial): OikosEvent { + return { + id: 1, + ts: '2026-07-28T12:00:00Z', + type: 'health.changed', + entity_id: 'e1', + severity: 'info', + source: 'scheduler', + data: { to: 'degraded' }, + correlation_id: null, + ...partial + } as OikosEvent +} + +function entity(partial: Partial = {}): Entity { + return { + id: 'e1', + slug: 'lxc:apps', + type: 'lxc', + name: 'apps', + attributes: {}, + version: 1, + created_at: '', + updated_at: '', + health: 'healthy', + ...partial + } as Entity +} + +describe('health event application', () => { + it('reads the new health out of the payload', () => { + expect(healthFromEvent(ev({}))).toBe('degraded') + expect(healthFromEvent(ev({ type: 'health.stale', data: { to: 'stale' } }))).toBe('stale') + }) + + it('ignores events that are not health events', () => { + expect(isHealthEvent(ev({ type: 'signal.raised' }))).toBe(false) + expect(healthFromEvent(ev({ type: 'execution.output' }))).toBeNull() + }) + + it('ignores a health event with no usable payload', () => { + expect(healthFromEvent(ev({ data: {} }))).toBeNull() + expect(healthFromEvent(ev({ entity_id: null }))).toBeNull() + }) + + it('patches the matching entity in a list and leaves the rest alone', () => { + const list = [entity(), entity({ id: 'e2', slug: 'lxc:dns', health: 'healthy' })] + const next = applyHealthEvent(list, ev({})) + expect(next[0].health).toBe('degraded') + expect(next[0].last_check_at).toBe('2026-07-28T12:00:00Z') + // untouched entities keep their identity, so rows that did not change do + // not re-render + expect(next[1]).toBe(list[1]) + }) + + // Returning the same array reference matters: assigning a fresh array on + // every unrelated event would churn the whole table. + it('returns the original array when the event does not apply', () => { + const list = [entity()] + expect(applyHealthEvent(list, ev({ entity_id: 'nobody' }))).toBe(list) + expect(applyHealthEvent(list, ev({ type: 'signal.raised' }))).toBe(list) + expect(applyHealthEvent(list, ev({ data: { to: 'healthy' } }))).toBe(list) // already healthy + }) + + it('patches a single entity for a detail view', () => { + const e = entity() + const next = applyHealthEventTo(e, ev({})) + expect(next?.health).toBe('degraded') + expect(applyHealthEventTo(e, ev({ entity_id: 'other' }))).toBe(e) + expect(applyHealthEventTo(null, ev({}))).toBeNull() + }) + + // The scheduler emits health.changed with entity_id = the observed entity + // but data.slug = the *check's* slug, so slug must never be used to match. + it('matches on entity_id, never on the payload slug', () => { + const list = [entity()] + const next = applyHealthEvent(list, ev({ data: { to: 'down', slug: 'check:ping:whatever:0' } })) + expect(next[0].health).toBe('down') + }) +}) diff --git a/web/src/lib/health.ts b/web/src/lib/health.ts new file mode 100644 index 0000000..4393f27 --- /dev/null +++ b/web/src/lib/health.ts @@ -0,0 +1,54 @@ +// Applying health events in place. +// +// health.changed / health.stale already carry the new value in their payload, +// so a view that renders health does not need to refetch anything: it can +// patch the one entity it already holds. Refetching the whole fleet because a +// single container went degraded costs a full round trip, re-runs the parent +// grouping, and churns the table on every transition — for information the +// event had already delivered. +// +// NOTE: match on entity_id, never on data.slug. The scheduler emits +// health.changed with entity_id = the observed entity but data.slug = the +// *check's* slug (e.g. "check:ping:host:hubris:0"), so the slug in the +// payload does not identify the thing whose health changed. + +import type { Entity, EntityHealth } from '$lib/api' +import type { OikosEvent } from '$lib/stores/events' + +const HEALTH_EVENTS = new Set(['health.changed', 'health.stale']) + +export function isHealthEvent(ev: OikosEvent): boolean { + return HEALTH_EVENTS.has(ev.type) +} + +/** The new health an event reports, or null if it isn't a usable health event. */ +export function healthFromEvent(ev: OikosEvent): EntityHealth | null { + if (!isHealthEvent(ev) || !ev.entity_id) return null + const to = (ev.data as Record | undefined)?.to + return typeof to === 'string' ? (to as EntityHealth) : null +} + +/** + * Returns a copy of `list` with the event's entity patched, or the original + * array when the event doesn't apply — so callers can assign unconditionally + * without forcing a re-render for an entity they aren't showing. + */ +export function applyHealthEvent(list: Entity[], ev: OikosEvent): Entity[] { + const health = healthFromEvent(ev) + if (!health) return list + const i = list.findIndex((e) => e.id === ev.entity_id) + if (i < 0) return list + if (list[i].health === health) return list + const next = list.slice() + // ev.ts is when the check ran, which is exactly what "checked N ago" means. + next[i] = { ...next[i], health, last_check_at: ev.ts } + return next +} + +/** Single-entity form, for a detail view holding one entity. */ +export function applyHealthEventTo(entity: Entity | null, ev: OikosEvent): Entity | null { + if (!entity || ev.entity_id !== entity.id) return entity + const health = healthFromEvent(ev) + if (!health || entity.health === health) return entity + return { ...entity, health, last_check_at: ev.ts } +} diff --git a/web/src/lib/stores/events.ts b/web/src/lib/stores/events.ts index b030650..9f3143a 100644 --- a/web/src/lib/stores/events.ts +++ b/web/src/lib/stores/events.ts @@ -18,14 +18,29 @@ export const liveEvents = writable([]) let source: EventSource | null = null let subscriberCount = 0 +let retry: ReturnType | null = null +let backoff = 0 + +/** True while the stream is live. Every live surface is only as fresh as this. */ +export const eventsConnected = writable(true) + +const RETRY_BASE_MS = 1000 +const RETRY_MAX_MS = 30000 async function connect() { if (source) return - // The browser's EventSource sends Last-event-ID automatically on reconnect. + // The browser's EventSource sends Last-event-ID automatically on reconnect, + // so a reconnect replays whatever was missed rather than leaving a hole. // sseUrl is async so the OIDC access token is refreshed if expired. - source = new EventSource(await sseUrl('/api/v1/events/stream')) + const es = new EventSource(await sseUrl('/api/v1/events/stream')) + source = es - source.onmessage = (ev) => { + es.onopen = () => { + backoff = 0 + eventsConnected.set(true) + } + + es.onmessage = (ev) => { try { const parsed: OikosEvent = JSON.parse(ev.data) liveEvents.update((events) => [parsed, ...events].slice(0, MAX_BUFFERED)) @@ -34,12 +49,33 @@ async function connect() { } } - source.onerror = () => { - // browser will auto-reconnect; nothing to surface here + // EventSource only auto-reconnects from a *transient* failure. Once it + // reaches CLOSED — which is what an HTTP error on (re)connect produces, e.g. + // the API restarting during a deploy — it stays closed and never retries. + // Leaving that to the browser meant a single blip silently froze every live + // surface in the app: health, signals and executions all just stopped + // updating, with nothing on screen to say so. That is precisely the + // stale-UI failure this whole change set exists to remove. + es.onerror = () => { + if (es.readyState !== EventSource.CLOSED) return // transient; browser retries + eventsConnected.set(false) + if (source === es) source = null + es.close() + if (subscriberCount === 0 || retry) return + backoff = backoff ? Math.min(backoff * 2, RETRY_MAX_MS) : RETRY_BASE_MS + retry = setTimeout(() => { + retry = null + if (subscriberCount > 0) connect() + }, backoff) } } function disconnect() { + if (retry) { + clearTimeout(retry) + retry = null + } + backoff = 0 source?.close() source = null } diff --git a/web/src/pages/KnowledgeBase.svelte b/web/src/pages/KnowledgeBase.svelte index 86b766e..e7989fa 100644 --- a/web/src/pages/KnowledgeBase.svelte +++ b/web/src/pages/KnowledgeBase.svelte @@ -9,6 +9,7 @@ type Ontology } from '$lib/api' import { liveEvents, subscribeEvents } from '$lib/stores/events' + import { isHealthEvent, applyHealthEvent } from '$lib/health' import EntityTable from '$lib/components/EntityTable.svelte' import FleetMap from '$lib/components/FleetMap.svelte' import { openEntityWindow, wmState } from '$lib/stores/windows' @@ -179,18 +180,13 @@ // Health arrives on its own events, not entity.*, so the table's Health // column used to sit at whatever it was when the page mounted while the - // graph view beside it updated live. Coalesced because health.stale fires - // once per entity during a sweep, and refetching the whole fleet for each - // would mean a burst of identical requests. - let fleetRefresh: ReturnType | null = null - function refreshFleetSoon() { - if (fleetRefresh) return - fleetRefresh = setTimeout(() => { - fleetRefresh = null - loadEntities() - }, 400) - } - + // graph view beside it updated live. + // + // Patched in place rather than refetched: the event already carries the new + // health, so a fleet-wide reload (plus its parent-grouping pass) would be a + // round trip to learn something we were just told — and would churn the + // whole table on every transition. Only entity.* changes the SET of + // entities, so only that needs a fetch. $effect(() => { const ev = $liveEvents[0] if (!ev) return @@ -198,7 +194,7 @@ loadEntities() return } - if (ev.type === 'health.changed' || ev.type === 'health.stale') refreshFleetSoon() + if (isHealthEvent(ev)) allEntities = applyHealthEvent(allEntities, ev) }) const filteredEntities = $derived.by(() => {