fix(scheduler): derive entity health from all its checks, not the last one
Some checks failed
ci / build-test (push) Has been cancelled
ci / docker-build (push) Has been cancelled
ci / web (push) Has been cancelled
Desktop App / Build Linux (amd64) (push) Has been cancelled
Desktop App / Attach to Release (push) Has been cancelled

host:strong logged 226 health.changed events in one hour, oscillating
down/healthy while the host was fine throughout. host:hubris did it 126 times.

runCheck wrote entity_status.health on every check completion, so an entity's
health was simply whichever of its checks finished most recently. A host with
six checks reported whichever facet happened to be sampled last, and one
failing probe alternating with five passing ones flapped forever. resolveSignal
forced "healthy" too, a second path by which one passing probe erased another
probe's genuine failure.

On this fleet the trigger is a known false positive: the scheduler's network
vantage point cannot ICMP host:strong, so its ping check fails while every
ssh-script check succeeds. Under last-writer-wins that single probe declared
the whole host down, twice a minute.

Each check now records its own verdict (check_defs.last_health, migration 027)
and the entity's health is the worst across its enabled checks. A failing probe
now degrades the entity honestly and *stably*, without erasing what the other
five report, and health.changed fires only when that aggregate actually moves.
Checks that have never run are ignored rather than counted as unknown, so
adding a check cannot drag a known-good entity down before it has a verdict.

Also declares service:oikos in the seed. The previous commit re-pointed the mcp
ingress at it, but the entity only ever existed in the production database — so
a fresh seed (a new install, or a DR restore) failed on an unresolvable edge.
Caught by seeding an empty database rather than a copy of prod, which is the
only way that class of bug shows up.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-07-28 21:33:48 +02:00
parent af450dac2a
commit 6ca6d5b352
6 changed files with 161 additions and 34 deletions

View File

@@ -68,7 +68,21 @@ WHERE cd.enabled = true
OR cd.last_run_at <= now() - make_interval(secs => cd.interval_s));
-- name: MarkCheckRun :exec
UPDATE check_defs SET last_run_at = now() WHERE entity_id = $1;
UPDATE check_defs SET last_run_at = now(), last_health = $2 WHERE entity_id = $1;
-- name: WorstHealthForTarget :one
-- An entity is as healthy as its unhealthiest check. Checks that have not run
-- yet (last_health IS NULL) are ignored rather than counted as unknown, so a
-- newly added check does not drag a known-good entity down before it has
-- produced a verdict.
SELECT COALESCE(
(SELECT last_health FROM check_defs
WHERE enabled AND target_id = $1 AND last_health IS NOT NULL
ORDER BY CASE last_health
WHEN 'down' THEN 0 WHEN 'degraded' THEN 1 WHEN 'stale' THEN 2
WHEN 'unknown' THEN 3 ELSE 4 END
LIMIT 1),
'unknown')::text AS health;
-- name: GetCheckDef :one
SELECT * FROM check_defs WHERE entity_id = $1;

View File

@@ -114,6 +114,8 @@ type CheckDef struct {
UpdatedAt time.Time
// When this check last executed. NULL = never, due immediately. Compared against interval_s to decide due-ness.
LastRunAt *time.Time
// This check's own most recent verdict (healthy/degraded/down/unknown). entity_status.health is the worst of these across the target's enabled checks.
LastHealth *string
}
type Classification struct {
@@ -255,6 +257,20 @@ type KnowledgeEntity struct {
UpdatedAt time.Time
ContentHash *string
Search interface{}
EditedBy string
DeletedAt *time.Time
}
type KnowledgeRevision struct {
ID int64
EntityID uuid.UUID
Title string
Content string
Source *string
Tags []string
EditedBy string
VersionAt time.Time
RevisedAt time.Time
}
type Ledger struct {

View File

@@ -51,7 +51,7 @@ func (q *Queries) GetAutonomySetting(ctx context.Context, key string) (string, e
}
const getCheckDef = `-- name: GetCheckDef :one
SELECT entity_id, target_id, target_type, kind, config, interval_s, timeout_s, zone, enabled, updated_at, last_run_at FROM check_defs WHERE entity_id = $1
SELECT entity_id, target_id, target_type, kind, config, interval_s, timeout_s, zone, enabled, updated_at, last_run_at, last_health FROM check_defs WHERE entity_id = $1
`
func (q *Queries) GetCheckDef(ctx context.Context, entityID uuid.UUID) (CheckDef, error) {
@@ -69,6 +69,7 @@ func (q *Queries) GetCheckDef(ctx context.Context, entityID uuid.UUID) (CheckDef
&i.Enabled,
&i.UpdatedAt,
&i.LastRunAt,
&i.LastHealth,
)
return i, err
}
@@ -1133,11 +1134,16 @@ func (q *Queries) ListSkills(ctx context.Context, status *string) ([]Skill, erro
}
const markCheckRun = `-- name: MarkCheckRun :exec
UPDATE check_defs SET last_run_at = now() WHERE entity_id = $1
UPDATE check_defs SET last_run_at = now(), last_health = $2 WHERE entity_id = $1
`
func (q *Queries) MarkCheckRun(ctx context.Context, entityID uuid.UUID) error {
_, err := q.db.Exec(ctx, markCheckRun, entityID)
type MarkCheckRunParams struct {
EntityID uuid.UUID
LastHealth *string
}
func (q *Queries) MarkCheckRun(ctx context.Context, arg MarkCheckRunParams) error {
_, err := q.db.Exec(ctx, markCheckRun, arg.EntityID, arg.LastHealth)
return err
}
@@ -1462,3 +1468,25 @@ func (q *Queries) UpsertSignal(ctx context.Context, arg UpsertSignalParams) (Sig
)
return i, err
}
const worstHealthForTarget = `-- name: WorstHealthForTarget :one
SELECT COALESCE(
(SELECT last_health FROM check_defs
WHERE enabled AND target_id = $1 AND last_health IS NOT NULL
ORDER BY CASE last_health
WHEN 'down' THEN 0 WHEN 'degraded' THEN 1 WHEN 'stale' THEN 2
WHEN 'unknown' THEN 3 ELSE 4 END
LIMIT 1),
'unknown')::text AS health
`
// An entity is as healthy as its unhealthiest check. Checks that have not run
// yet (last_health IS NULL) are ignored rather than counted as unknown, so a
// newly added check does not drag a known-good entity down before it has
// produced a verdict.
func (q *Queries) WorstHealthForTarget(ctx context.Context, targetID *uuid.UUID) (string, error) {
row := q.db.QueryRow(ctx, worstHealthForTarget, targetID)
var health string
err := row.Scan(&health)
return health, err
}