dtoro
cc8eae4979
Refetching everything on a health event was wasteful and churned the UI: one container going degraded pulled down the entire fleet entity list (plus its parent-grouping pass), or the whole fleet graph, to learn something the event had already delivered. health.changed / health.stale carry the new value in their payload, so the views that hold the entity just patch it: - Fleet table: patch the row. Only entity.* changes which entities exist, so only that still refetches. - Fleet map: patch the node AND graph.health[id] — healthOf() reads the side map in preference to the node's own field, so patching only the nodes would have left the rendered colour unchanged. - Entity detail: patch the open entity. Signals still need a read (the event says one was raised, not what the list now contains) but only the signals, not the entity and checks alongside them. Shared in $lib/health.ts, which returns the original array when an event does not apply so unrelated rows keep their identity and do not re-render. Note it matches on entity_id, never data.slug: the scheduler emits health.changed with entity_id = the observed entity but slug = the *check's* slug. Separately, events.ts had no reconnect. onerror was empty on the assumption the browser retries, but EventSource only does that for a transient failure -- once it reaches CLOSED (an HTTP error on connect, e.g. the API restarting during a deploy) it stays closed forever. A single blip silently froze every live surface in the app with nothing on screen to say so. Now reconnects with capped exponential backoff, and exports eventsConnected so a future indicator can show when the stream is down. Verified against live prod: flipping lxc:apps health recoloured the map node and moved its counts (30 healthy -> 29, 9 down -> 10) with ZERO network requests. Co-Authored-By: Claude <noreply@anthropic.com>
92 lines
2.6 KiB
TypeScript
92 lines
2.6 KiB
TypeScript
import { writable } from 'svelte/store'
|
|
import { sseUrl } from '$lib/config'
|
|
|
|
export interface OikosEvent {
|
|
id: number
|
|
ts: string
|
|
type: string
|
|
entity_id?: string | null
|
|
severity: 'info' | 'warning' | 'critical'
|
|
source: string
|
|
data?: unknown
|
|
correlation_id?: string | null
|
|
}
|
|
|
|
const MAX_BUFFERED = 200
|
|
|
|
export const liveEvents = writable<OikosEvent[]>([])
|
|
|
|
let source: EventSource | null = null
|
|
let subscriberCount = 0
|
|
let retry: ReturnType<typeof setTimeout> | null = null
|
|
let backoff = 0
|
|
|
|
/** True while the stream is live. Every live surface is only as fresh as this. */
|
|
export const eventsConnected = writable(true)
|
|
|
|
const RETRY_BASE_MS = 1000
|
|
const RETRY_MAX_MS = 30000
|
|
|
|
async function connect() {
|
|
if (source) return
|
|
// The browser's EventSource sends Last-event-ID automatically on reconnect,
|
|
// so a reconnect replays whatever was missed rather than leaving a hole.
|
|
// sseUrl is async so the OIDC access token is refreshed if expired.
|
|
const es = new EventSource(await sseUrl('/api/v1/events/stream'))
|
|
source = es
|
|
|
|
es.onopen = () => {
|
|
backoff = 0
|
|
eventsConnected.set(true)
|
|
}
|
|
|
|
es.onmessage = (ev) => {
|
|
try {
|
|
const parsed: OikosEvent = JSON.parse(ev.data)
|
|
liveEvents.update((events) => [parsed, ...events].slice(0, MAX_BUFFERED))
|
|
} catch {
|
|
// skip malformed
|
|
}
|
|
}
|
|
|
|
// EventSource only auto-reconnects from a *transient* failure. Once it
|
|
// reaches CLOSED — which is what an HTTP error on (re)connect produces, e.g.
|
|
// the API restarting during a deploy — it stays closed and never retries.
|
|
// Leaving that to the browser meant a single blip silently froze every live
|
|
// surface in the app: health, signals and executions all just stopped
|
|
// updating, with nothing on screen to say so. That is precisely the
|
|
// stale-UI failure this whole change set exists to remove.
|
|
es.onerror = () => {
|
|
if (es.readyState !== EventSource.CLOSED) return // transient; browser retries
|
|
eventsConnected.set(false)
|
|
if (source === es) source = null
|
|
es.close()
|
|
if (subscriberCount === 0 || retry) return
|
|
backoff = backoff ? Math.min(backoff * 2, RETRY_MAX_MS) : RETRY_BASE_MS
|
|
retry = setTimeout(() => {
|
|
retry = null
|
|
if (subscriberCount > 0) connect()
|
|
}, backoff)
|
|
}
|
|
}
|
|
|
|
function disconnect() {
|
|
if (retry) {
|
|
clearTimeout(retry)
|
|
retry = null
|
|
}
|
|
backoff = 0
|
|
source?.close()
|
|
source = null
|
|
}
|
|
|
|
// Reference-counted: the stream stays open as long as at least one page subscribes.
|
|
export function subscribeEvents(): () => void {
|
|
subscriberCount++
|
|
if (subscriberCount === 1) connect()
|
|
return () => {
|
|
subscriberCount--
|
|
if (subscriberCount === 0) disconnect()
|
|
}
|
|
}
|