fix(scheduler): derive entity health from all its checks, not the last one

host:strong logged 226 health.changed events in one hour, oscillating
down/healthy while the host was fine throughout. host:hubris did it 126 times.

runCheck wrote entity_status.health on every check completion, so an entity's
health was simply whichever of its checks finished most recently. A host with
six checks reported whichever facet happened to be sampled last, and one
failing probe alternating with five passing ones flapped forever. resolveSignal
forced "healthy" too, a second path by which one passing probe erased another
probe's genuine failure.

On this fleet the trigger is a known false positive: the scheduler's network
vantage point cannot ICMP host:strong, so its ping check fails while every
ssh-script check succeeds. Under last-writer-wins that single probe declared
the whole host down, twice a minute.

Each check now records its own verdict (check_defs.last_health, migration 027)
and the entity's health is the worst across its enabled checks. A failing probe
now degrades the entity honestly and *stably*, without erasing what the other
five report, and health.changed fires only when that aggregate actually moves.
Checks that have never run are ignored rather than counted as unknown, so
adding a check cannot drag a known-good entity down before it has a verdict.

Also declares service:oikos in the seed. The previous commit re-pointed the mcp
ingress at it, but the entity only ever existed in the production database — so
a fresh seed (a new install, or a DR restore) failed on an unresolvable edge.
Caught by seeding an empty database rather than a copy of prod, which is the
only way that class of bug shows up.

Co-Authored-By: Claude <noreply@anthropic.com>

internal/scheduler/scheduler.go

@@ -115,8 +115,16 @@ func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDef
 
 	// Stamp the run before processing the result: due-ness must advance even
 	// when a check fails, or a permanently failing check would be re-run on
-	// every pass instead of at its declared interval.
-	if err := q.MarkCheckRun(ctx, cd.EntityID); err != nil {
+	// every pass instead of at its declared interval. last_health records THIS
+	// check's own verdict, which is what makes the aggregation below possible.
+	checkHealth := result.health
+	if checkHealth == "" {
+		checkHealth = "healthy"
+	}
+	if err := q.MarkCheckRun(ctx, sqlcgen.MarkCheckRunParams{
+		EntityID:   cd.EntityID,
+		LastHealth: &checkHealth,
+	}); err != nil {
 		slog.Error("scheduler: mark check run", "entity", cd.EntitySlug, "error", err)
 	}
 
@@ -150,16 +158,12 @@ func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDef
 
 	if result.signalKind == "" || result.health == "healthy" {
 		resolveSignal(ctx, pool, cd.EntityID, targetID, cd.EntitySlug)
-		_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
-			EntityID:    targetID,
-			Health:      "healthy",
-			LastCheckAt: &[]time.Time{time.Now()}[0],
-			Details:     []byte(`{}`),
-		})
-		if prevHealth != "" && prevHealth != "healthy" {
-			emitSchedulerEvent(ctx, pool, "health.changed", targetID, "info",
-				map[string]any{"slug": cd.EntitySlug, "from": prevHealth, "to": "healthy"})
-		}
+		// NOT unconditionally "healthy": this check passing says nothing about
+		// the entity's other checks. Writing healthy here is what let one
+		// passing probe erase a genuine failure reported by another — and,
+		// alternating with a failing probe, produced 226 health flips an hour
+		// on a host that was fine throughout.
+		applyAggregateHealth(ctx, pool, q, targetID, cd.EntitySlug, prevHealth)
 		return
 	}
 
@@ -180,22 +184,52 @@ func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDef
 		return
 	}
 
-	_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
-		EntityID:    targetID,
-		Health:      result.health,
-		LastCheckAt: &[]time.Time{time.Now()}[0],
-		Details:     []byte(`{}`),
-	})
 	_ = sig
 
 	if prevHealth == "" || prevHealth == "healthy" {
 		emitSchedulerEvent(ctx, pool, "signal.raised", targetID, severity,
 			map[string]any{"slug": cd.EntitySlug, "kind": result.signalKind, "evidence": result.evidence})
 	}
-	if prevHealth != result.health {
-		emitSchedulerEvent(ctx, pool, "health.changed", targetID, severity,
-			map[string]any{"slug": cd.EntitySlug, "from": prevHealth, "to": result.health})
+	applyAggregateHealth(ctx, pool, q, targetID, cd.EntitySlug, prevHealth)
+}
+
+// applyAggregateHealth sets the target's health to the worst verdict across
+// all of its enabled checks, and emits health.changed only when that aggregate
+// actually moves.
+//
+// Health is a property of the entity, but each check only ever observes one
+// facet of it — reachability, disk, a systemd unit. Letting whichever check
+// finished last overwrite the entity's health meant a host with six checks
+// reported whichever facet was sampled most recently, so one failing probe and
+// five passing ones oscillated forever instead of settling on "degraded".
+func applyAggregateHealth(ctx context.Context, pool *db.Pool, q *sqlcgen.Queries,
+	targetID uuid.UUID, checkSlug, prevHealth string) {
+
+	health, err := q.WorstHealthForTarget(ctx, &targetID)
+	if err != nil {
+		slog.Error("scheduler: aggregate health", "entity", checkSlug, "error", err)
+		return
 	}
+
+	_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
+		EntityID:    targetID,
+		Health:      health,
+		LastCheckAt: &[]time.Time{time.Now()}[0],
+		Details:     []byte(`{}`),
+	})
+
+	if prevHealth == health {
+		return
+	}
+	severity := "info"
+	switch health {
+	case "down":
+		severity = "critical"
+	case "degraded", "stale":
+		severity = "warning"
+	}
+	emitSchedulerEvent(ctx, pool, "health.changed", targetID, severity,
+		map[string]any{"slug": checkSlug, "from": prevHealth, "to": health})
 }
 
 // currentHealth reads the last recorded health for an entity, or "" if none.
@@ -217,7 +251,6 @@ func emitSchedulerEvent(ctx context.Context, pool *db.Pool, eventType string, en
 // checkID matches how signals are keyed (UpsertSignal uses the check's own
 // entity id); targetID is the observed entity whose status this affects.
 func resolveSignal(ctx context.Context, pool *db.Pool, checkID, targetID uuid.UUID, slug string) {
-	q := sqlcgen.New(pool)
 	// Check if there's an open signal on this entity
 	tag, err := pool.Exec(ctx, `UPDATE signals SET state = 'resolved', updated_at = now()
 		WHERE entity_id = $1 AND state = 'raised'`, checkID)
@@ -227,14 +260,12 @@ func resolveSignal(ctx context.Context, pool *db.Pool, checkID, targetID uuid.UU
 	if tag.RowsAffected() > 0 {
 		emitSchedulerEvent(ctx, pool, "signal.resolved", targetID, "info",
 			map[string]any{"slug": slug})
+		slog.Info("scheduler: signal resolved", "entity", slug)
 	}
-	_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
-		EntityID:    targetID,
-		Health:      "healthy",
-		LastCheckAt: &[]time.Time{time.Now()}[0],
-		Details:     []byte(`{}`),
-	})
-	slog.Info("scheduler: signal resolved", "entity", slug)
+	// Deliberately does NOT write health. Resolving THIS check's signal says
+	// nothing about the target's other checks; the caller re-derives health
+	// from all of them. Forcing "healthy" here was a second path by which one
+	// passing probe erased another probe's genuine failure.
 }
 
 // checkResult bundles the outcome of a single check execution.