Problem: every host/service/lxc/etc. entity_status row was permanently stuck at 'unknown' since creation. Verified against the live DB: metric_samples had 17,559 rows, 100% attached to type='check' probe entities and 0% to any real monitored entity; only 25 check entities ever had real health written. check_defs.entity_id (the probe's own bookkeeping entity) and check_defs.target_id (the host/service actually being observed) were both real fields, but the scheduler wrote UpsertEntityStatus/InsertMetricSample/emitSchedulerEvent keyed by entity_id instead of target_id — so every check ran and every result was real, it just landed on the wrong row. This is the mechanism behind observed drift: the agent's dashboard/health tools reported the internal probes' state, never the actual fleet. Change: - scheduler.go: runCheck/resolveSignal now resolve targetID from cd.TargetID (falling back to the check's own id if unset) and write status/metrics/events there. Signals stay keyed by the check entity, unchanged, matching their existing resolution logic. - Added a staleness sweep to housekeeping(): an entity whose last observation is older than 3x its fastest enabled check's interval (floor 5m) is marked 'stale' and emits health.stale, so a stalled scheduler or disabled check_def can no longer look like current data forever. - migrations/016: deletes the now-orphaned check-entity entity_status rows so dashboard/fleet-health rollups stop double-counting probes as monitored entities. Historical metric_samples on check entities are left as-is (time-series data, not safe to reattribute). - openapi.yaml + regenerated gen code: Entity gains health/last_check_at; 'stale' added to the health enum everywhere it's used. - dashboard.go / GetFleetHealth / nomos's get_health_summary MCP tool: exclude type='check' entities from rollups. - nomos/agent.go: replay prior turns' tool_use/tool_result pairs into the conversation instead of dropping them (previously only final text was replayed, forcing the agent to re-derive fleet state every turn), and inject a compact live fleet-health snapshot into the system prompt each turn so it starts oriented instead of spending an iteration on discovery. Risk: config_mutation (schema-adjacent — new migration, no destructive DDL, additive DELETE only on orphaned rows). No behavior change until oikos-api/oikos-scheduler/nomos are rebuilt and redeployed. Verification: go build/vet clean across the repo. Ran this worktree's own API binary against the live dev Postgres on an alternate port (read-only from the live containers' perspective) and confirmed /api/v1/entities now returns health/last_check_at, and the dashboard health rollup dropped from double-counting to an honest 168 unmonitored entities (matches reality pre-deploy — the live scheduler hasn't run the fixed code yet). Confirmed check_defs.target_id correctly maps multiple checks to host:hubris via direct psql query. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
173 lines
3.9 KiB
Go
173 lines
3.9 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/dtoro/oikos/internal/httpapi/gen"
|
|
)
|
|
|
|
// GetDashboardSummary returns one round-trip overview for the control room
|
|
// home page: entity counts, health rollup, open signals, pending approvals,
|
|
// executions in the last 24h, and an event-rate sparkline.
|
|
func (s *Server) GetDashboardSummary(ctx context.Context, req gen.GetDashboardSummaryRequestObject) (gen.GetDashboardSummaryResponseObject, error) {
|
|
resp := gen.DashboardSummary{
|
|
EntitiesByType: map[string]int{},
|
|
EntitiesByState: map[string]int{},
|
|
SignalsBySeverity: map[string]int{},
|
|
ExecutionsByState: map[string]int{},
|
|
}
|
|
|
|
rows, err := s.pool.Query(ctx, `SELECT type, count(*) FROM entities GROUP BY type`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for rows.Next() {
|
|
var typ string
|
|
var n int
|
|
if err := rows.Scan(&typ, &n); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
resp.EntitiesByType[typ] = n
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rows, err = s.pool.Query(ctx, `SELECT coalesce(state, 'unknown'), count(*) FROM entities GROUP BY state`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for rows.Next() {
|
|
var state string
|
|
var n int
|
|
if err := rows.Scan(&state, &n); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
resp.EntitiesByState[state] = n
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Exclude 'check' entities (internal probes) — only entities actually
|
|
// being monitored should count toward the fleet health rollup.
|
|
rows, err = s.pool.Query(ctx, `
|
|
SELECT st.health, count(*)
|
|
FROM entity_status st JOIN entities e ON e.id = st.entity_id
|
|
WHERE e.type <> 'check'
|
|
GROUP BY st.health`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stale := 0
|
|
for rows.Next() {
|
|
var health string
|
|
var n int
|
|
if err := rows.Scan(&health, &n); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
switch health {
|
|
case "healthy":
|
|
resp.Health.Healthy = n
|
|
case "degraded":
|
|
resp.Health.Degraded = n
|
|
case "down":
|
|
resp.Health.Down = n
|
|
case "stale":
|
|
stale = n
|
|
default:
|
|
resp.Health.Unknown = n
|
|
}
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if stale > 0 {
|
|
resp.Health.Stale = &stale
|
|
}
|
|
|
|
rows, err = s.pool.Query(ctx, `
|
|
SELECT severity, count(*) FROM signals
|
|
WHERE state NOT IN ('resolved', 'failed')
|
|
GROUP BY severity`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for rows.Next() {
|
|
var severity string
|
|
var n int
|
|
if err := rows.Scan(&severity, &n); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
resp.SignalsBySeverity[severity] = n
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := s.pool.QueryRow(ctx,
|
|
`SELECT count(*) FROM approvals WHERE status = 'pending'`,
|
|
).Scan(&resp.ApprovalsPending); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rows, err = s.pool.Query(ctx, `
|
|
SELECT status, count(*) FROM executions
|
|
WHERE created_at > now() - interval '24 hours'
|
|
GROUP BY status`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for rows.Next() {
|
|
var status string
|
|
var n int
|
|
if err := rows.Scan(&status, &n); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
resp.ExecutionsByState[status] = n
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rows, err = s.pool.Query(ctx, `
|
|
SELECT date_trunc('hour', ts) + (extract(minute FROM ts)::int / 5) * interval '5 minutes' AS bucket,
|
|
count(*)
|
|
FROM events
|
|
WHERE ts > now() - interval '6 hours'
|
|
GROUP BY bucket
|
|
ORDER BY bucket`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for rows.Next() {
|
|
var bucket time.Time
|
|
var n int
|
|
if err := rows.Scan(&bucket, &n); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
resp.EventRate = append(resp.EventRate, struct {
|
|
Bucket time.Time `json:"bucket"`
|
|
Count int `json:"count"`
|
|
}{Bucket: bucket, Count: n})
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return gen.GetDashboardSummary200JSONResponse(resp), nil
|
|
}
|