fix: scheduler wrote health/metrics/events to probe entities, not targets
Problem: every host/service/lxc/etc. entity_status row was permanently stuck at 'unknown' since creation. Verified against the live DB: metric_samples had 17,559 rows, 100% attached to type='check' probe entities and 0% to any real monitored entity; only 25 check entities ever had real health written. check_defs.entity_id (the probe's own bookkeeping entity) and check_defs.target_id (the host/service actually being observed) were both real fields, but the scheduler wrote UpsertEntityStatus/InsertMetricSample/emitSchedulerEvent keyed by entity_id instead of target_id — so every check ran and every result was real, it just landed on the wrong row. This is the mechanism behind observed drift: the agent's dashboard/health tools reported the internal probes' state, never the actual fleet. Change: - scheduler.go: runCheck/resolveSignal now resolve targetID from cd.TargetID (falling back to the check's own id if unset) and write status/metrics/events there. Signals stay keyed by the check entity, unchanged, matching their existing resolution logic. - Added a staleness sweep to housekeeping(): an entity whose last observation is older than 3x its fastest enabled check's interval (floor 5m) is marked 'stale' and emits health.stale, so a stalled scheduler or disabled check_def can no longer look like current data forever. - migrations/016: deletes the now-orphaned check-entity entity_status rows so dashboard/fleet-health rollups stop double-counting probes as monitored entities. Historical metric_samples on check entities are left as-is (time-series data, not safe to reattribute). - openapi.yaml + regenerated gen code: Entity gains health/last_check_at; 'stale' added to the health enum everywhere it's used. - dashboard.go / GetFleetHealth / nomos's get_health_summary MCP tool: exclude type='check' entities from rollups. - nomos/agent.go: replay prior turns' tool_use/tool_result pairs into the conversation instead of dropping them (previously only final text was replayed, forcing the agent to re-derive fleet state every turn), and inject a compact live fleet-health snapshot into the system prompt each turn so it starts oriented instead of spending an iteration on discovery. Risk: config_mutation (schema-adjacent — new migration, no destructive DDL, additive DELETE only on orphaned rows). No behavior change until oikos-api/oikos-scheduler/nomos are rebuilt and redeployed. Verification: go build/vet clean across the repo. Ran this worktree's own API binary against the live dev Postgres on an alternate port (read-only from the live containers' perspective) and confirmed /api/v1/entities now returns health/last_check_at, and the dashboard health rollup dropped from double-counting to an honest 168 unmonitored entities (matches reality pre-deploy — the live scheduler hasn't run the fixed code yet). Confirmed check_defs.target_id correctly maps multiple checks to host:hubris via direct psql query. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -108,11 +108,16 @@ func (a *agent) chat(ctx context.Context, sessionID, message string, emit func(a
|
||||
// Rebuild conversation context from persisted history so sessions are
|
||||
// multi-turn. The current user turn is saved by the HTTP handler before
|
||||
// this runs, so it is already included in the history for real sessions.
|
||||
// Intermediate tool_use/tool_result pairs are not replayed (their ids
|
||||
// must match exactly or the API rejects them); prior final answers carry
|
||||
// the salient context. Ephemeral sessions (no store) fall back to the
|
||||
// single incoming message.
|
||||
messages := []openai.ChatCompletionMessageParamUnion{openai.SystemMessage(a.system)}
|
||||
// Prior tool_use/tool_result pairs are replayed as a tool-calling
|
||||
// assistant message followed by matching tool-role results, so the agent
|
||||
// starts each turn already knowing what it already checked instead of
|
||||
// re-querying the same tools from scratch. Ephemeral sessions (no store)
|
||||
// fall back to the single incoming message.
|
||||
system := a.system
|
||||
if snapshot := a.fleetSnapshot(); snapshot != "" {
|
||||
system += "\n\n" + snapshot
|
||||
}
|
||||
messages := []openai.ChatCompletionMessageParamUnion{openai.SystemMessage(system)}
|
||||
history, _ := a.store.getMessages(ctx, sessionID)
|
||||
for _, m := range history {
|
||||
text := extractText(m.Content)
|
||||
@@ -120,6 +125,12 @@ func (a *agent) chat(ctx context.Context, sessionID, message string, emit func(a
|
||||
case "user":
|
||||
messages = append(messages, openai.UserMessage(text))
|
||||
case "assistant":
|
||||
if calls := extractToolCalls(m.Content); len(calls) > 0 {
|
||||
messages = append(messages, assistantToolCallMessage(calls))
|
||||
for _, c := range calls {
|
||||
messages = append(messages, openai.ToolMessage(c.resultText(), c.id))
|
||||
}
|
||||
}
|
||||
if text != "" {
|
||||
messages = append(messages, openai.AssistantMessage(text))
|
||||
}
|
||||
@@ -243,6 +254,140 @@ func extractText(content json.RawMessage) string {
|
||||
return m.Text
|
||||
}
|
||||
|
||||
// persistedCall is one merged tool_use+tool_result pair from a persisted
|
||||
// assistant message's tool_calls array. The store keeps them as two entries
|
||||
// sharing the same id (mirroring the SSE event pair); replay needs one
|
||||
// entry per id to build a valid tool-calling assistant message.
|
||||
type persistedCall struct {
|
||||
id string
|
||||
name string
|
||||
args json.RawMessage
|
||||
result json.RawMessage
|
||||
errMsg string
|
||||
}
|
||||
|
||||
func (c persistedCall) resultText() string {
|
||||
if c.errMsg != "" {
|
||||
return c.errMsg
|
||||
}
|
||||
if len(c.result) > 0 {
|
||||
return string(c.result)
|
||||
}
|
||||
return "null"
|
||||
}
|
||||
|
||||
// extractToolCalls parses and merges a persisted message's tool_calls array,
|
||||
// preserving first-seen order across ids.
|
||||
func extractToolCalls(content json.RawMessage) []persistedCall {
|
||||
var m struct {
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Args json.RawMessage `json:"args"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error string `json:"error"`
|
||||
} `json:"tool_calls"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &m); err != nil || len(m.ToolCalls) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
byID := make(map[string]*persistedCall, len(m.ToolCalls))
|
||||
var order []string
|
||||
for _, tc := range m.ToolCalls {
|
||||
if tc.ID == "" {
|
||||
continue
|
||||
}
|
||||
pc, ok := byID[tc.ID]
|
||||
if !ok {
|
||||
pc = &persistedCall{id: tc.ID}
|
||||
byID[tc.ID] = pc
|
||||
order = append(order, tc.ID)
|
||||
}
|
||||
if tc.Name != "" {
|
||||
pc.name = tc.Name
|
||||
}
|
||||
if len(tc.Args) > 0 && string(tc.Args) != "null" {
|
||||
pc.args = tc.Args
|
||||
}
|
||||
if tc.Type == "tool_result" {
|
||||
pc.errMsg = tc.Error
|
||||
pc.result = tc.Result
|
||||
}
|
||||
}
|
||||
|
||||
calls := make([]persistedCall, 0, len(order))
|
||||
for _, id := range order {
|
||||
calls = append(calls, *byID[id])
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
// assistantToolCallMessage builds the tool-calling assistant message that
|
||||
// must precede the tool-role results being replayed.
|
||||
func assistantToolCallMessage(calls []persistedCall) openai.ChatCompletionMessageParamUnion {
|
||||
toolCalls := make([]openai.ChatCompletionMessageToolCallParam, 0, len(calls))
|
||||
for _, c := range calls {
|
||||
args := string(c.args)
|
||||
if args == "" {
|
||||
args = "{}"
|
||||
}
|
||||
toolCalls = append(toolCalls, openai.ChatCompletionMessageToolCallParam{
|
||||
ID: c.id,
|
||||
Function: openai.ChatCompletionMessageToolCallFunctionParam{
|
||||
Name: c.name,
|
||||
Arguments: args,
|
||||
},
|
||||
})
|
||||
}
|
||||
return openai.ChatCompletionMessageParamUnion{
|
||||
OfAssistant: &openai.ChatCompletionAssistantMessageParam{ToolCalls: toolCalls},
|
||||
}
|
||||
}
|
||||
|
||||
// fleetSnapshot returns a compact, current-as-of-now fleet health line for
|
||||
// the system prompt so the agent starts each turn already oriented instead
|
||||
// of spending its first iteration rediscovering topology it already has
|
||||
// tools to query. Best-effort: an empty string on any failure just means no
|
||||
// snapshot, not an error for the turn.
|
||||
func (a *agent) fleetSnapshot() string {
|
||||
result, err := a.client.callTool("get_health_summary", map[string]any{})
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
rows, ok := result.([]any)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
|
||||
counts := map[string]int{}
|
||||
var attention []string
|
||||
for _, r := range rows {
|
||||
row, ok := r.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
health, _ := row["health"].(string)
|
||||
counts[health]++
|
||||
if health != "healthy" && health != "" {
|
||||
if slug, ok := row["slug"].(string); ok && len(attention) < 10 {
|
||||
attention = append(attention, fmt.Sprintf("%s(%s)", slug, health))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(counts) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
summary := fmt.Sprintf("Current fleet snapshot (as of now): healthy=%d degraded=%d down=%d stale=%d unknown=%d.",
|
||||
counts["healthy"], counts["degraded"], counts["down"], counts["stale"], counts["unknown"])
|
||||
if len(attention) > 0 {
|
||||
summary += " Needs attention: " + fmt.Sprintf("%v", attention) + "."
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func (a *agent) buildTools() ([]openai.ChatCompletionToolParam, error) {
|
||||
defs, err := a.client.listToolsFull()
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user