dtoro/oikos
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: scheduler wrote health/metrics/events to probe entities, not targets

Browse Source 
Problem: every host/service/lxc/etc. entity_status row was permanently
stuck at 'unknown' since creation. Verified against the live DB:
metric_samples had 17,559 rows, 100% attached to type='check' probe
entities and 0% to any real monitored entity; only 25 check entities
ever had real health written. check_defs.entity_id (the probe's own
bookkeeping entity) and check_defs.target_id (the host/service actually
being observed) were both real fields, but the scheduler wrote
UpsertEntityStatus/InsertMetricSample/emitSchedulerEvent keyed by
entity_id instead of target_id — so every check ran and every result
was real, it just landed on the wrong row. This is the mechanism behind
observed drift: the agent's dashboard/health tools reported the
internal probes' state, never the actual fleet.

Change:
- scheduler.go: runCheck/resolveSignal now resolve targetID from
  cd.TargetID (falling back to the check's own id if unset) and write
  status/metrics/events there. Signals stay keyed by the check entity,
  unchanged, matching their existing resolution logic.
- Added a staleness sweep to housekeeping(): an entity whose last
  observation is older than 3x its fastest enabled check's interval
  (floor 5m) is marked 'stale' and emits health.stale, so a stalled
  scheduler or disabled check_def can no longer look like current data
  forever.
- migrations/016: deletes the now-orphaned check-entity entity_status
  rows so dashboard/fleet-health rollups stop double-counting probes as
  monitored entities. Historical metric_samples on check entities are
  left as-is (time-series data, not safe to reattribute).
- openapi.yaml + regenerated gen code: Entity gains health/last_check_at;
  'stale' added to the health enum everywhere it's used.
- dashboard.go / GetFleetHealth / nomos's get_health_summary MCP tool:
  exclude type='check' entities from rollups.
- nomos/agent.go: replay prior turns' tool_use/tool_result pairs into
  the conversation instead of dropping them (previously only final text
  was replayed, forcing the agent to re-derive fleet state every turn),
  and inject a compact live fleet-health snapshot into the system prompt
  each turn so it starts oriented instead of spending an iteration on
  discovery.

Risk: config_mutation (schema-adjacent — new migration, no destructive
DDL, additive DELETE only on orphaned rows). No behavior change until
oikos-api/oikos-scheduler/nomos are rebuilt and redeployed.

Verification: go build/vet clean across the repo. Ran this worktree's
own API binary against the live dev Postgres on an alternate port
(read-only from the live containers' perspective) and confirmed
/api/v1/entities now returns health/last_check_at, and the dashboard
health rollup dropped from double-counting to an honest 168 unmonitored
entities (matches reality pre-deploy — the live scheduler hasn't run
the fixed code yet). Confirmed check_defs.target_id correctly maps
multiple checks to host:hubris via direct psql query.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
authentik Default Admin
2026-07-09 00:26:04 +02:00
parent a39e67b6e9
commit 279549c8c9
10 changed files with 798 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
internal/httpapi/impl.go
View File

@@ -75,22 +75,31 @@ func (s *Server) resolveEntityID(ctx context.Context, idOrSlug string) (uuid.UUI
return id, err
}
// entityCols requires the entities table to be aliased as `e`.
// entityCols requires the entities table to be aliased as `e`, with
// entity_status left-joined and aliased as `st` (see withEntityStatus).
const entityCols = `e.id, e.slug, e.type, e.name, e.state, e.attributes,
e.maintenance_until, e.version, e.created_at, e.updated_at`
e.maintenance_until, e.version, e.created_at, e.updated_at,
st.health, st.last_check_at`
func scanEntity(row pgx.Row) (gen.Entity, error) {
var e gen.Entity
var state *string
var attrsJSON []byte
var maint *time.Time
var health *string
var lastCheckAt *time.Time
err := row.Scan(&e.Id, &e.Slug, &e.Type, &e.Name, &state, &attrsJSON,
&maint, &e.Version, &e.CreatedAt, &e.UpdatedAt)
&maint, &e.Version, &e.CreatedAt, &e.UpdatedAt, &health, &lastCheckAt)
if err != nil {
return e, err
}
e.State = state
e.MaintenanceUntil = maint
if health != nil {
h := gen.EntityHealth(*health)
e.Health = &h
}
e.LastCheckAt = lastCheckAt
var attrs map[string]any
if len(attrsJSON) > 0 && json.Unmarshal(attrsJSON, &attrs) == nil && len(attrs) > 0 {
e.Attributes = &attrs
@@ -113,6 +122,7 @@ func (s *Server) ListEntities(ctx context.Context, req gen.ListEntitiesRequestOb
)
SELECT ` + entityCols + ` FROM entities e
JOIN entity_types et ON et.name = e.type
LEFT JOIN entity_status st ON st.entity_id = e.id
WHERE e.type IN (SELECT name FROM tt)
AND ($2::text IS NULL OR e.state = $2)
AND ($3::text IS NULL OR et.domain = $3)
@@ -159,7 +169,7 @@ func (s *Server) GetEntity(ctx context.Context, req gen.GetEntityRequestObject)
return nil, err
}
e, err := scanEntity(s.pool.QueryRow(ctx,
"SELECT "+entityCols+" FROM entities e WHERE e.id = $1", id))
"SELECT "+entityCols+" FROM entities e LEFT JOIN entity_status st ON st.entity_id = e.id WHERE e.id = $1", id))
if err != nil {
return nil, err
}
@@ -231,6 +241,7 @@ func (s *Server) GetBlastRadius(ctx context.Context, req gen.GetBlastRadiusReque
SELECT `+entityCols+`, b.depth
FROM blast_radius($1, $2) b
JOIN entities e ON e.id = b.entity_id
LEFT JOIN entity_status st ON st.entity_id = e.id
ORDER BY b.depth, e.slug`, id, depth)
if err != nil {
return nil, err
@@ -246,13 +257,20 @@ func (s *Server) GetBlastRadius(ctx context.Context, req gen.GetBlastRadiusReque
var state *string
var attrsJSON []byte
var maint *time.Time
var health *string
var lastCheckAt *time.Time
var d int
if err := rows.Scan(&e.Id, &e.Slug, &e.Type, &e.Name, &state, &attrsJSON,
&maint, &e.Version, &e.CreatedAt, &e.UpdatedAt, &d); err != nil {
&maint, &e.Version, &e.CreatedAt, &e.UpdatedAt, &health, &lastCheckAt, &d); err != nil {
return nil, err
}
e.State = state
e.MaintenanceUntil = maint
if health != nil {
h := gen.EntityHealth(*health)
e.Health = &h
}
e.LastCheckAt = lastCheckAt
var attrs map[string]any
if len(attrsJSON) > 0 && json.Unmarshal(attrsJSON, &attrs) == nil && len(attrs) > 0 {
e.Attributes = &attrs
@@ -283,10 +301,13 @@ func (s *Server) GetGraph(ctx context.Context, req gen.GetGraphRequestObject) (g
nodes, err = s.queryEntities(ctx, `
SELECT `+entityCols+`
FROM blast_radius($1, $2, $3) b JOIN entities e ON e.id = b.entity_id
LEFT JOIN entity_status st ON st.entity_id = e.id
ORDER BY e.slug`, rootID, depth, req.Params.RelType)
} else {
nodes, err = s.queryEntities(ctx, `
SELECT `+entityCols+` FROM entities e ORDER BY e.slug LIMIT $1`,
SELECT `+entityCols+` FROM entities e
LEFT JOIN entity_status st ON st.entity_id = e.id
ORDER BY e.slug LIMIT $1`,
graphNodeCap+1)
if err == nil && len(nodes) > graphNodeCap {
nodes = nodes[:graphNodeCap]
@@ -523,14 +544,18 @@ func (s *Server) GetFleetHealth(ctx context.Context, req gen.GetFleetHealthReque
Type string `json:"type"`
}{}
// Exclude 'check' entities (internal probes) — only entities actually
// being monitored should count toward fleet health.
rows, err := s.pool.Query(ctx, `
SELECT e.slug, e.type, st.health, st.last_check_at
FROM entity_status st JOIN entities e ON e.id = st.entity_id
WHERE e.type <> 'check'
ORDER BY e.slug`)
if err != nil {
return nil, err
}
defer rows.Close()
stale := 0
for rows.Next() {
var slug, typ, health string
var lastCheck *time.Time
@@ -544,6 +569,8 @@ func (s *Server) GetFleetHealth(ctx context.Context, req gen.GetFleetHealthReque
resp.Summary.Degraded++
case "down":
resp.Summary.Down++
case "stale":
stale++
default:
resp.Summary.Unknown++
}
@@ -560,6 +587,9 @@ func (s *Server) GetFleetHealth(ctx context.Context, req gen.GetFleetHealthReque
Type: typ,
})
}
if stale > 0 {
resp.Summary.Stale = &stale
}
return resp, rows.Err()
}