perf(web): patch health in place instead of refetching, and reconnect the SSE stream

Refetching everything on a health event was wasteful and churned the UI: one
container going degraded pulled down the entire fleet entity list (plus its
parent-grouping pass), or the whole fleet graph, to learn something the event
had already delivered.

health.changed / health.stale carry the new value in their payload, so the
views that hold the entity just patch it:

- Fleet table: patch the row. Only entity.* changes which entities exist, so
  only that still refetches.
- Fleet map: patch the node AND graph.health[id] — healthOf() reads the side
  map in preference to the node's own field, so patching only the nodes would
  have left the rendered colour unchanged.
- Entity detail: patch the open entity. Signals still need a read (the event
  says one was raised, not what the list now contains) but only the signals,
  not the entity and checks alongside them.

Shared in $lib/health.ts, which returns the original array when an event does
not apply so unrelated rows keep their identity and do not re-render. Note it
matches on entity_id, never data.slug: the scheduler emits health.changed with
entity_id = the observed entity but slug = the *check's* slug.

Separately, events.ts had no reconnect. onerror was empty on the assumption
the browser retries, but EventSource only does that for a transient failure --
once it reaches CLOSED (an HTTP error on connect, e.g. the API restarting
during a deploy) it stays closed forever. A single blip silently froze every
live surface in the app with nothing on screen to say so. Now reconnects with
capped exponential backoff, and exports eventsConnected so a future indicator
can show when the stream is down.

Verified against live prod: flipping lxc:apps health recoloured the map node
and moved its counts (30 healthy -> 29, 9 down -> 10) with ZERO network
requests.

Co-Authored-By: Claude <noreply@anthropic.com>

web/src/lib/components/EntityDetailContent.svelte

@@ -36,6 +36,7 @@
   } from '$lib/api'
   import { relativeTime, truncateMiddle } from '$lib/utils'
   import { liveEvents, subscribeEvents, type OikosEvent } from '$lib/stores/events'
+  import { isHealthEvent, applyHealthEventTo } from '$lib/health'
   import DetailSection from '$lib/components/DetailSection.svelte'
   import { Badge } from '$lib/components/ui/badge'
   import { Button } from '$lib/components/ui/button'
@@ -217,36 +218,31 @@
 
     // Health and signals for THIS entity. The panel loaded these once on
     // open, so a window left on screen kept showing the health it had at
-    // mount — the exact staleness the fleet-wide work was about, reproduced
-    // one window at a time. Scoped by entity_id so an unrelated entity going
-    // down elsewhere costs nothing here.
+    // mount. Scoped by entity_id so an unrelated entity going down elsewhere
+    // costs nothing here.
     if (!entity || ev.entity_id !== entity.id) return
-    if (
-      ev.type === 'health.changed' ||
-      ev.type === 'health.stale' ||
-      ev.type.startsWith('signal.') ||
-      ev.type.startsWith('coverage.')
-    ) {
-      refreshStatus()
+
+    // Health is patched from the event itself — it carries the new value, so
+    // there is nothing to go and ask for.
+    if (isHealthEvent(ev)) {
+      entity = applyHealthEventTo(entity, ev)
+      return
+    }
+
+    // Signals genuinely need a read: the event says one was raised or
+    // resolved, not what the full signal list now looks like. Just the
+    // signals, though — not the entity, checks, and everything else.
+    if (ev.type.startsWith('signal.') || ev.type.startsWith('coverage.')) {
+      refreshSignals()
     }
   })
 
-  // Re-reads only what a health or signal event can change, rather than
-  // re-running the full 11-request load().
-  async function refreshStatus() {
+  async function refreshSignals() {
     if (!entity) return
     const id = entity.id
-    const slugAtStart = entity.slug
-    const [fresh, sig, ch] = await Promise.all([
-      fetchEntity(slugAtStart),
-      fetchEntitySignals(id),
-      fetchChecksForTarget(slugAtStart)
-    ])
-    // The panel may have switched entity while these were in flight.
-    if (!entity || entity.id !== id) return
-    if (fresh) entity = fresh
-    signals = sig
-    checks = ch
+    const next = await fetchEntitySignals(id)
+    // The panel may have switched entity while this was in flight.
+    if (entity?.id === id) signals = next
   }
 
   async function refreshExecutions() {

web/src/lib/components/FleetMap.svelte

@@ -14,6 +14,7 @@
   import { onMount, onDestroy } from 'svelte'
   import { fetchGraph, type GraphView, type Entity } from '$lib/api'
   import { liveEvents, subscribeEvents } from '$lib/stores/events'
+  import { isHealthEvent, applyHealthEvent, healthFromEvent } from '$lib/health'
   import { Skeleton } from '$lib/components/ui/skeleton'
   import { Button } from '$lib/components/ui/button'
   import GlobeIcon from '@lucide/svelte/icons/globe'
@@ -56,16 +57,29 @@
     return subscribeEvents()
   })
 
-  // Refetch on structural or health changes, same triggers as the old graph.
+  // Structural changes need a refetch — they change which nodes and edges
+  // exist. Health does not: the event carries the new value, so the node is
+  // patched in place instead of pulling the entire fleet graph (and its
+  // layout) down again for one colour change.
   $effect(() => {
     const ev = $liveEvents[0]
     if (!ev) return
-    if (
-      ev.type.startsWith('entity.') ||
-      ev.type.startsWith('relationship.') ||
-      ev.type === 'health.changed'
-    ) {
+    if (ev.type.startsWith('entity.') || ev.type.startsWith('relationship.')) {
       load()
+      return
+    }
+    if (isHealthEvent(ev) && graph) {
+      const health = healthFromEvent(ev)
+      if (!health || !ev.entity_id) return
+      // healthOf() reads graph.health[id] in preference to the node's own
+      // field (include=status attaches it as a side map), so patching only
+      // the nodes would leave the rendered colour unchanged. Patch both.
+      const nodes = applyHealthEvent(graph.nodes, ev)
+      graph = {
+        ...graph,
+        nodes,
+        health: { ...(graph.health ?? {}), [ev.entity_id]: health as Health }
+      }
     }
   })
 

web/src/lib/health.test.ts

@@ -0,0 +1,85 @@
+import { describe, it, expect } from 'vitest'
+import { isHealthEvent, healthFromEvent, applyHealthEvent, applyHealthEventTo } from './health'
+import type { Entity } from './api'
+import type { OikosEvent } from './stores/events'
+
+function ev(partial: Partial<OikosEvent>): OikosEvent {
+  return {
+    id: 1,
+    ts: '2026-07-28T12:00:00Z',
+    type: 'health.changed',
+    entity_id: 'e1',
+    severity: 'info',
+    source: 'scheduler',
+    data: { to: 'degraded' },
+    correlation_id: null,
+    ...partial
+  } as OikosEvent
+}
+
+function entity(partial: Partial<Entity> = {}): Entity {
+  return {
+    id: 'e1',
+    slug: 'lxc:apps',
+    type: 'lxc',
+    name: 'apps',
+    attributes: {},
+    version: 1,
+    created_at: '',
+    updated_at: '',
+    health: 'healthy',
+    ...partial
+  } as Entity
+}
+
+describe('health event application', () => {
+  it('reads the new health out of the payload', () => {
+    expect(healthFromEvent(ev({}))).toBe('degraded')
+    expect(healthFromEvent(ev({ type: 'health.stale', data: { to: 'stale' } }))).toBe('stale')
+  })
+
+  it('ignores events that are not health events', () => {
+    expect(isHealthEvent(ev({ type: 'signal.raised' }))).toBe(false)
+    expect(healthFromEvent(ev({ type: 'execution.output' }))).toBeNull()
+  })
+
+  it('ignores a health event with no usable payload', () => {
+    expect(healthFromEvent(ev({ data: {} }))).toBeNull()
+    expect(healthFromEvent(ev({ entity_id: null }))).toBeNull()
+  })
+
+  it('patches the matching entity in a list and leaves the rest alone', () => {
+    const list = [entity(), entity({ id: 'e2', slug: 'lxc:dns', health: 'healthy' })]
+    const next = applyHealthEvent(list, ev({}))
+    expect(next[0].health).toBe('degraded')
+    expect(next[0].last_check_at).toBe('2026-07-28T12:00:00Z')
+    // untouched entities keep their identity, so rows that did not change do
+    // not re-render
+    expect(next[1]).toBe(list[1])
+  })
+
+  // Returning the same array reference matters: assigning a fresh array on
+  // every unrelated event would churn the whole table.
+  it('returns the original array when the event does not apply', () => {
+    const list = [entity()]
+    expect(applyHealthEvent(list, ev({ entity_id: 'nobody' }))).toBe(list)
+    expect(applyHealthEvent(list, ev({ type: 'signal.raised' }))).toBe(list)
+    expect(applyHealthEvent(list, ev({ data: { to: 'healthy' } }))).toBe(list) // already healthy
+  })
+
+  it('patches a single entity for a detail view', () => {
+    const e = entity()
+    const next = applyHealthEventTo(e, ev({}))
+    expect(next?.health).toBe('degraded')
+    expect(applyHealthEventTo(e, ev({ entity_id: 'other' }))).toBe(e)
+    expect(applyHealthEventTo(null, ev({}))).toBeNull()
+  })
+
+  // The scheduler emits health.changed with entity_id = the observed entity
+  // but data.slug = the *check's* slug, so slug must never be used to match.
+  it('matches on entity_id, never on the payload slug', () => {
+    const list = [entity()]
+    const next = applyHealthEvent(list, ev({ data: { to: 'down', slug: 'check:ping:whatever:0' } }))
+    expect(next[0].health).toBe('down')
+  })
+})

web/src/lib/health.ts

@@ -0,0 +1,54 @@
+// Applying health events in place.
+//
+// health.changed / health.stale already carry the new value in their payload,
+// so a view that renders health does not need to refetch anything: it can
+// patch the one entity it already holds. Refetching the whole fleet because a
+// single container went degraded costs a full round trip, re-runs the parent
+// grouping, and churns the table on every transition — for information the
+// event had already delivered.
+//
+// NOTE: match on entity_id, never on data.slug. The scheduler emits
+// health.changed with entity_id = the observed entity but data.slug = the
+// *check's* slug (e.g. "check:ping:host:hubris:0"), so the slug in the
+// payload does not identify the thing whose health changed.
+
+import type { Entity, EntityHealth } from '$lib/api'
+import type { OikosEvent } from '$lib/stores/events'
+
+const HEALTH_EVENTS = new Set(['health.changed', 'health.stale'])
+
+export function isHealthEvent(ev: OikosEvent): boolean {
+  return HEALTH_EVENTS.has(ev.type)
+}
+
+/** The new health an event reports, or null if it isn't a usable health event. */
+export function healthFromEvent(ev: OikosEvent): EntityHealth | null {
+  if (!isHealthEvent(ev) || !ev.entity_id) return null
+  const to = (ev.data as Record<string, unknown> | undefined)?.to
+  return typeof to === 'string' ? (to as EntityHealth) : null
+}
+
+/**
+ * Returns a copy of `list` with the event's entity patched, or the original
+ * array when the event doesn't apply — so callers can assign unconditionally
+ * without forcing a re-render for an entity they aren't showing.
+ */
+export function applyHealthEvent(list: Entity[], ev: OikosEvent): Entity[] {
+  const health = healthFromEvent(ev)
+  if (!health) return list
+  const i = list.findIndex((e) => e.id === ev.entity_id)
+  if (i < 0) return list
+  if (list[i].health === health) return list
+  const next = list.slice()
+  // ev.ts is when the check ran, which is exactly what "checked N ago" means.
+  next[i] = { ...next[i], health, last_check_at: ev.ts }
+  return next
+}
+
+/** Single-entity form, for a detail view holding one entity. */
+export function applyHealthEventTo(entity: Entity | null, ev: OikosEvent): Entity | null {
+  if (!entity || ev.entity_id !== entity.id) return entity
+  const health = healthFromEvent(ev)
+  if (!health || entity.health === health) return entity
+  return { ...entity, health, last_check_at: ev.ts }
+}

web/src/lib/stores/events.ts

@@ -18,14 +18,29 @@ export const liveEvents = writable<OikosEvent[]>([])
 
 let source: EventSource | null = null
 let subscriberCount = 0
+let retry: ReturnType<typeof setTimeout> | null = null
+let backoff = 0
+
+/** True while the stream is live. Every live surface is only as fresh as this. */
+export const eventsConnected = writable(true)
+
+const RETRY_BASE_MS = 1000
+const RETRY_MAX_MS = 30000
 
 async function connect() {
   if (source) return
-  // The browser's EventSource sends Last-event-ID automatically on reconnect.
+  // The browser's EventSource sends Last-event-ID automatically on reconnect,
+  // so a reconnect replays whatever was missed rather than leaving a hole.
   // sseUrl is async so the OIDC access token is refreshed if expired.
-  source = new EventSource(await sseUrl('/api/v1/events/stream'))
+  const es = new EventSource(await sseUrl('/api/v1/events/stream'))
+  source = es
 
-  source.onmessage = (ev) => {
+  es.onopen = () => {
+    backoff = 0
+    eventsConnected.set(true)
+  }
+
+  es.onmessage = (ev) => {
     try {
       const parsed: OikosEvent = JSON.parse(ev.data)
       liveEvents.update((events) => [parsed, ...events].slice(0, MAX_BUFFERED))
@@ -34,12 +49,33 @@ async function connect() {
     }
   }
 
-  source.onerror = () => {
-    // browser will auto-reconnect; nothing to surface here
+  // EventSource only auto-reconnects from a *transient* failure. Once it
+  // reaches CLOSED — which is what an HTTP error on (re)connect produces, e.g.
+  // the API restarting during a deploy — it stays closed and never retries.
+  // Leaving that to the browser meant a single blip silently froze every live
+  // surface in the app: health, signals and executions all just stopped
+  // updating, with nothing on screen to say so. That is precisely the
+  // stale-UI failure this whole change set exists to remove.
+  es.onerror = () => {
+    if (es.readyState !== EventSource.CLOSED) return // transient; browser retries
+    eventsConnected.set(false)
+    if (source === es) source = null
+    es.close()
+    if (subscriberCount === 0 || retry) return
+    backoff = backoff ? Math.min(backoff * 2, RETRY_MAX_MS) : RETRY_BASE_MS
+    retry = setTimeout(() => {
+      retry = null
+      if (subscriberCount > 0) connect()
+    }, backoff)
   }
 }
 
 function disconnect() {
+  if (retry) {
+    clearTimeout(retry)
+    retry = null
+  }
+  backoff = 0
   source?.close()
   source = null
 }

web/src/pages/KnowledgeBase.svelte

@@ -9,6 +9,7 @@
     type Ontology
   } from '$lib/api'
   import { liveEvents, subscribeEvents } from '$lib/stores/events'
+  import { isHealthEvent, applyHealthEvent } from '$lib/health'
   import EntityTable from '$lib/components/EntityTable.svelte'
   import FleetMap from '$lib/components/FleetMap.svelte'
   import { openEntityWindow, wmState } from '$lib/stores/windows'
@@ -179,18 +180,13 @@
 
   // Health arrives on its own events, not entity.*, so the table's Health
   // column used to sit at whatever it was when the page mounted while the
-  // graph view beside it updated live. Coalesced because health.stale fires
-  // once per entity during a sweep, and refetching the whole fleet for each
-  // would mean a burst of identical requests.
-  let fleetRefresh: ReturnType<typeof setTimeout> | null = null
-  function refreshFleetSoon() {
-    if (fleetRefresh) return
-    fleetRefresh = setTimeout(() => {
-      fleetRefresh = null
-      loadEntities()
-    }, 400)
-  }
-
+  // graph view beside it updated live.
+  //
+  // Patched in place rather than refetched: the event already carries the new
+  // health, so a fleet-wide reload (plus its parent-grouping pass) would be a
+  // round trip to learn something we were just told — and would churn the
+  // whole table on every transition. Only entity.* changes the SET of
+  // entities, so only that needs a fetch.
   $effect(() => {
     const ev = $liveEvents[0]
     if (!ev) return
@@ -198,7 +194,7 @@
       loadEntities()
       return
     }
-    if (ev.type === 'health.changed' || ev.type === 'health.stale') refreshFleetSoon()
+    if (isHealthEvent(ev)) allEntities = applyHealthEvent(allEntities, ev)
   })
 
   const filteredEntities = $derived.by(() => {