feat(observability): restore monitoring coverage, make gaps visible, stream executions

Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by
discarded errors in checkdefaults:

- writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT
  (slug) DO NOTHING, then wrote a check_defs row referencing it. On any
  re-seed the slug already existed, the entity insert no-oped, and the FK
  violated — aborting the ingest transaction and surfacing as an unrelated
  failure several entities later. Re-seeding has been broken since; prod's
  coverage was frozen at its first successful seed. This is what
  TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting.
- shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed
  to ".network" and overwrote each other; service:jellyfin collided with
  lxc:jellyfin.
- The ssh-script checker never read the `args` config checkdefaults wrote, so
  process_check.sh always ran without its unit name and returned "unknown".

Coverage is now 75/89. Monitoring is declared per entity type in
seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say
it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap.
coverageSweep raises an `unmonitored` signal only where a type declares
monitoring it lacks — 8 real gaps, no false positives.

Also:
- entity_types.attribute_schema was never ingested: the seed loader read
  "attribute_schema" but the YAML says "attributes", so all 60 types stored
  JSON null.
- ListExecutions ignored its declared target/action/correlation_id filters and
  paginated on a non-unique target slug, dropping and repeating rows.
- started_at was captured but only written at terminal state, so a running
  execution reported NULL for its whole life. The three MCP auto-run copies
  wrote no timing at all; they are now one autoRun helper.
- SSH output was buffered to completion and discarded entirely on timeout.
  Both sshExec copies now stream through a shared execlog sink into
  execution_logs, and keep partial output when a command is cancelled.
- executions.correlation_id was a random per-execution uuid that correlated
  nothing; it is now the chat session id, which is what lets the chat tail
  live output.
- reversible_low had no auto-run branch despite policy declaring it
  unattended. Since computeCommandRisk never returns it, the class only arises
  when an agent declares it over a read_only command — so gating it penalised
  candor without adding safety.
- backup-target gains a backup-freshness checker (portable find -mmin, since
  the first target is on macOS), resolving its host by walking backs-up-to
  backwards. The pre-deploy pg_dump is now a tracked backup target.

UI: an Executions section on entity detail with live output tailing, and
streamed output under a running `run` call in the chat timeline.

Migrations 022-024. Ops.svelte and context.ts exclude execution.output from
their refetch triggers, which would otherwise fire once a second per command.

Co-Authored-By: Claude <noreply@anthropic.com>

internal/scheduler/backup.go

@@ -0,0 +1,116 @@
+package scheduler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dtoro/oikos/internal/db/sqlcgen"
+)
+
+// checkBackupFreshness reports whether a backup target has a recent artifact.
+//
+// The ontology has carried a `backup-target` type and a `backs-up-to` edge
+// since the first seed, but nothing ever verified that a backup actually
+// happened — a silent backup failure looked exactly like a working one. This
+// makes staleness a Signal like any other, so it flows through the existing
+// dedup, auto-resolve and notifier path rather than needing its own machinery.
+//
+// Config: {"path": "/opt/oikos/backups", "max_age_s": 86400, "host": …}
+//
+// Deliberately uses `find -mmin` rather than `-printf '%T@'` or `stat`:
+// -printf is GNU-only and stat's format flag differs between GNU (-c) and BSD
+// (-f). The first real target for this check is the pre-deploy pg_dump on the
+// mac-mini, which is macOS — so a GNU-only probe would have silently reported
+// "unknown" on the one target that motivated the check.
+func checkBackupFreshness(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
+	cfg := struct {
+		Path    string `json:"path"`
+		MaxAgeS int    `json:"max_age_s"`
+		Host    string `json:"host"`
+		User    string `json:"user"`
+		Port    int    `json:"port"`
+	}{
+		MaxAgeS: 86400, // a daily backup that has not run in 24h is stale
+	}
+	if len(cd.Config) > 0 {
+		_ = json.Unmarshal(cd.Config, &cfg)
+	}
+	if cfg.Path == "" || cfg.Host == "" {
+		return checkResult{health: "unknown", signalKind: "backup-misconfigured",
+			evidence: "backup check needs both a path and a host"}
+	}
+	if cfg.Port == 0 {
+		cfg.Port = 22
+	}
+	if cfg.User == "" {
+		cfg.User = sshUser
+	}
+	if cfg.MaxAgeS <= 0 {
+		cfg.MaxAgeS = 86400
+	}
+
+	timeout := time.Duration(cd.TimeoutS) * time.Second
+	if timeout <= 0 {
+		timeout = 30 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	minutes := cfg.MaxAgeS / 60
+	if minutes < 1 {
+		minutes = 1
+	}
+	quoted := shellSingleQuote(cfg.Path)
+
+	// Two questions in one round trip: is there anything at all, and is any of
+	// it recent? "no backups ever" and "backups stopped" are different
+	// failures and deserve different severities.
+	cmd := fmt.Sprintf(
+		`if [ ! -d %s ]; then echo missing; else `+
+			`f=$(find %s -type f -mmin -%d 2>/dev/null | head -1); `+
+			`a=$(find %s -type f 2>/dev/null | head -1); `+
+			`if [ -n "$f" ]; then echo fresh; elif [ -n "$a" ]; then echo stale; else echo empty; fi; fi`,
+		quoted, quoted, minutes, quoted)
+
+	out, err := sshExec(ctx, cfg.Host, strconv.Itoa(cfg.Port), cfg.User, cmd, timeout)
+	if err != nil {
+		return checkResult{
+			health: "unknown", signalKind: "backup-unreachable",
+			evidence: fmt.Sprintf("ssh %s: %v", cfg.Host, err),
+			err:      err,
+		}
+	}
+
+	age := time.Duration(cfg.MaxAgeS) * time.Second
+	switch strings.TrimSpace(string(out)) {
+	case "fresh":
+		return checkResult{health: "healthy"}
+	case "stale":
+		return checkResult{
+			health: "degraded", signalKind: "backup-stale",
+			evidence: fmt.Sprintf("no backup in %s under %s on %s", age, cfg.Path, cfg.Host),
+		}
+	case "empty":
+		return checkResult{
+			health: "down", signalKind: "backup-missing",
+			evidence: fmt.Sprintf("%s on %s exists but contains no files", cfg.Path, cfg.Host),
+		}
+	case "missing":
+		return checkResult{
+			health: "down", signalKind: "backup-missing",
+			evidence: fmt.Sprintf("backup directory %s does not exist on %s", cfg.Path, cfg.Host),
+		}
+	}
+	return checkResult{health: "unknown", signalKind: "backup-unreachable",
+		evidence: fmt.Sprintf("unexpected probe output: %q", strings.TrimSpace(string(out)))}
+}
+
+// shellSingleQuote makes a path safe to embed in the remote sh command. Paths
+// come from check_defs config, which an operator or the agent can write.
+func shellSingleQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}

internal/scheduler/backup_test.go

@@ -0,0 +1,101 @@
+package scheduler
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/dtoro/oikos/internal/db/sqlcgen"
+)
+
+func backupCheckDef(t *testing.T, config string) sqlcgen.ListEnabledCheckDefsRow {
+	t.Helper()
+	return sqlcgen.ListEnabledCheckDefsRow{
+		Kind:     "backup-freshness",
+		Config:   []byte(config),
+		TimeoutS: 15,
+	}
+}
+
+// A misconfigured check must say so rather than quietly reporting healthy —
+// "no path configured" and "backup ran fine" must never look the same.
+func TestBackupFreshnessRejectsIncompleteConfig(t *testing.T) {
+	for _, c := range []struct{ desc, config string }{
+		{"no path", `{"host":"localhost"}`},
+		{"no host", `{"path":"/tmp"}`},
+		{"empty", `{}`},
+	} {
+		got := checkBackupFreshness(context.Background(), backupCheckDef(t, c.config))
+		if got.health != "unknown" || got.signalKind != "backup-misconfigured" {
+			t.Errorf("%s: got health=%q kind=%q, want unknown/backup-misconfigured",
+				c.desc, got.health, got.signalKind)
+		}
+	}
+}
+
+// Live probe against a real SSH endpoint. Guarded by OIKOS_SSH_TEST_HOST:
+//
+//	OIKOS_SSH_TEST_HOST=localhost OIKOS_SSH_USER=$USER \
+//	OIKOS_SSH_KEY_PATH=~/.ssh/id_ed25519 go test ./internal/scheduler/ -run TestBackupFreshnessLive
+//
+// The probe shell has to work on both GNU and BSD find — the first real target
+// is the pre-deploy pg_dump on the macOS mac-mini, so a GNU-only construct
+// would fail exactly where it matters.
+func TestBackupFreshnessLiveDistinguishesTheFourStates(t *testing.T) {
+	host := os.Getenv("OIKOS_SSH_TEST_HOST")
+	if host == "" {
+		t.Skip("OIKOS_SSH_TEST_HOST not set — skipping live backup probe")
+	}
+	sshKeyPath = os.Getenv("OIKOS_SSH_KEY_PATH")
+	sshUser = os.Getenv("OIKOS_SSH_USER")
+
+	dir := t.TempDir()
+	fresh := filepath.Join(dir, "fresh")
+	stale := filepath.Join(dir, "stale")
+	empty := filepath.Join(dir, "empty")
+	for _, d := range []string{fresh, stale, empty} {
+		if err := os.Mkdir(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(fresh, "dump.sql"), []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	oldFile := filepath.Join(stale, "dump.sql")
+	if err := os.WriteFile(oldFile, []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	old := time.Now().Add(-72 * time.Hour)
+	if err := os.Chtimes(oldFile, old, old); err != nil {
+		t.Fatal(err)
+	}
+
+	cases := []struct {
+		desc, path, wantHealth, wantKind string
+	}{
+		{"recent artifact", fresh, "healthy", ""},
+		{"artifact older than max_age", stale, "degraded", "backup-stale"},
+		{"directory exists but is empty", empty, "down", "backup-missing"},
+		{"directory does not exist", filepath.Join(dir, "nope"), "down", "backup-missing"},
+	}
+
+	for _, c := range cases {
+		cfg := `{"host":"` + host + `","path":"` + c.path + `","max_age_s":86400}`
+		got := checkBackupFreshness(context.Background(), backupCheckDef(t, cfg))
+		if got.health != c.wantHealth || got.signalKind != c.wantKind {
+			t.Errorf("%s: got health=%q kind=%q evidence=%q, want %q/%q",
+				c.desc, got.health, got.signalKind, got.evidence, c.wantHealth, c.wantKind)
+		}
+	}
+}
+
+// A path with a quote in it must not break out of the remote sh command.
+func TestShellSingleQuoteEscapes(t *testing.T) {
+	got := shellSingleQuote(`/tmp/it's; rm -rf /`)
+	want := `'/tmp/it'\''s; rm -rf /'`
+	if got != want {
+		t.Errorf("shellSingleQuote = %s, want %s", got, want)
+	}
+}

internal/scheduler/coverage.go

@@ -0,0 +1,191 @@
+package scheduler
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/dtoro/oikos/internal/db"
+	"github.com/google/uuid"
+)
+
+// UnmonitoredKind is the signal kind raised for an entity whose type declares
+// monitoring it does not have.
+const UnmonitoredKind = "unmonitored"
+
+// coverageSweep reports entities that should be monitored and are not.
+//
+// staleSweep can only protect an entity that already has a check — it INNER
+// JOINs check_defs, so an entity with none is structurally invisible to it and
+// keeps reporting its last-known health forever. This sweep covers the other
+// half: it notices the absence itself.
+//
+// It fires only where the entity type *declares* monitoring. Types that
+// declare `monitoring: none` (site, cluster, lan, mesh — topological groupings
+// with nothing to probe) are working as intended and must never raise a
+// signal; a permanent unresolvable warning against six healthy entities would
+// discredit the whole thing. Types that declare nothing at all are a modelling
+// gap, reported once per pass at debug level rather than as a fleet problem.
+func coverageSweep(ctx context.Context, pool *db.Pool) {
+	// Resolution walks parent_type — inheriting types (lxc, proxmox-host, lan)
+	// carry NULL in their own monitoring_spec column, so reading it directly
+	// would flag every one of them. Reuse the Go resolver instead of
+	// duplicating the hierarchy walk in SQL.
+	tx, err := pool.Begin(ctx)
+	if err != nil {
+		slog.Error("scheduler: coverage sweep begin", "error", err)
+		return
+	}
+	tree, err := db.LoadTypeTree(ctx, tx)
+	if err != nil {
+		_ = tx.Rollback(ctx)
+		slog.Error("scheduler: coverage sweep load type tree", "error", err)
+		return
+	}
+	_ = tx.Rollback(ctx) // read-only
+
+	rows, err := pool.Query(ctx, `
+		SELECT e.id, e.slug, e.type, (cd.target_id IS NOT NULL) AS has_check
+		FROM entities e
+		LEFT JOIN (
+			SELECT DISTINCT target_id FROM check_defs
+			WHERE enabled AND target_id IS NOT NULL
+		) cd ON cd.target_id = e.id
+		WHERE e.state = 'active' AND e.type <> 'check'`)
+	if err != nil {
+		slog.Error("scheduler: coverage sweep query", "error", err)
+		return
+	}
+
+	type entity struct {
+		id       uuid.UUID
+		slug     string
+		typ      string
+		hasCheck bool
+	}
+	var all []entity
+	for rows.Next() {
+		var e entity
+		if err := rows.Scan(&e.id, &e.slug, &e.typ, &e.hasCheck); err != nil {
+			continue
+		}
+		all = append(all, e)
+	}
+	rows.Close()
+	if rows.Err() != nil {
+		slog.Error("scheduler: coverage sweep scan", "error", rows.Err())
+		return
+	}
+
+	var raised, resolved, undeclared int
+	for _, e := range all {
+		mon := tree.Monitoring(e.typ)
+
+		switch {
+		case !mon.Declared:
+			undeclared++
+		case mon.None():
+			// Explicitly unmonitorable. Nothing to say.
+		case e.hasCheck:
+			if resolveCoverageSignal(ctx, pool, e.id) {
+				resolved++
+				slog.Info("scheduler: entity is monitored again", "entity", e.slug)
+			}
+		default:
+			if raiseCoverageSignal(ctx, pool, e.id, e.slug, e.typ, mon.Kinds) {
+				raised++
+			}
+		}
+	}
+
+	if raised > 0 || resolved > 0 {
+		slog.Warn("scheduler: coverage sweep",
+			"unmonitored_raised", raised, "resolved", resolved, "scanned", len(all))
+	}
+	if undeclared > 0 {
+		slog.Debug("scheduler: entity types declare no monitoring", "entities", undeclared)
+	}
+}
+
+// raiseCoverageSignal raises (or refreshes) the unmonitored signal for one
+// entity. Reports whether this was a new raise.
+func raiseCoverageSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID, slug, typ string, want []string) bool {
+	// A signal is a dual entity: signals.entity_id is a PK referencing
+	// entities(id), so the row has to exist first. The scheduler's other
+	// signals borrow the check entity's id — there is no check here, which is
+	// the whole point, so this sweep owns a signal entity per target.
+	//
+	// The slug is stable per target, which makes the signal row stable too and
+	// lets a resolved signal be re-raised by primary key rather than colliding
+	// with it.
+	signalSlug := fmt.Sprintf("signal:%s:%s", UnmonitoredKind, slug)
+
+	newID, err := uuid.NewV7()
+	if err != nil {
+		newID = uuid.New()
+	}
+
+	var signalID uuid.UUID
+	// Upsert RETURNING id, never insert-and-assume: assuming is what made
+	// checkdefaults write foreign keys to rows it had not created.
+	if err := pool.QueryRow(ctx,
+		`INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
+		 VALUES ($1, $2, 'signal', $2, 'active', '{}', 1, now(), now())
+		 ON CONFLICT (slug) DO UPDATE SET updated_at = now()
+		 RETURNING id`,
+		newID, signalSlug).Scan(&signalID); err != nil {
+		slog.Error("scheduler: upsert signal entity", "entity", slug, "error", err)
+		return false
+	}
+
+	evidence := fmt.Sprintf("type %s declares monitoring %v but the entity has no enabled check_def", typ, want)
+
+	// Conflict on the primary key rather than on the (target, kind) partial
+	// index: that index only covers OPEN signals, so a previously resolved
+	// signal would not conflict there and would collide on the PK instead.
+	tag, err := pool.Exec(ctx,
+		`INSERT INTO signals (entity_id, kind, severity, target_entity_id, evidence, state)
+		 VALUES ($1, $2, 'warning', $3, $4, 'raised')
+		 ON CONFLICT (entity_id) DO UPDATE
+		   SET state = CASE WHEN signals.state IN ('resolved','failed') THEN 'raised' ELSE signals.state END,
+		       occurrence_count = signals.occurrence_count + 1,
+		       evidence = EXCLUDED.evidence,
+		       last_seen_at = now(), updated_at = now()`,
+		signalID, UnmonitoredKind, entityID, evidence)
+	if err != nil {
+		slog.Error("scheduler: raise unmonitored signal", "entity", slug, "error", err)
+		return false
+	}
+
+	// RowsAffected is 1 for both insert and update, so ask the signal itself
+	// whether this was the first occurrence.
+	var occurrences int
+	if err := pool.QueryRow(ctx,
+		`SELECT occurrence_count FROM signals WHERE entity_id = $1`, signalID).Scan(&occurrences); err != nil {
+		return tag.RowsAffected() > 0
+	}
+	if occurrences <= 1 {
+		slog.Warn("scheduler: entity is unmonitored",
+			"entity", slug, "type", typ, "declared", want)
+		emitSchedulerEvent(ctx, pool, "coverage.unmonitored", entityID, "warning",
+			map[string]any{"slug": slug, "type": typ, "declared": want})
+		return true
+	}
+	return false
+}
+
+// resolveCoverageSignal closes the unmonitored signal once the entity has a
+// check. The scheduler's normal auto-resolve keys on the *check* entity id and
+// only from state 'raised', so it can never clear one of these.
+func resolveCoverageSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID) bool {
+	tag, err := pool.Exec(ctx,
+		`UPDATE signals SET state = 'resolved', updated_at = now()
+		 WHERE target_entity_id = $1 AND kind = $2
+		   AND state NOT IN ('resolved', 'failed')`,
+		entityID, UnmonitoredKind)
+	if err != nil {
+		slog.Error("scheduler: resolve unmonitored signal", "error", err)
+		return false
+	}
+	return tag.RowsAffected() > 0
+}

internal/scheduler/coverage_test.go

@@ -0,0 +1,309 @@
+package scheduler
+
+// Integration tests for coverageSweep against a real TimescaleDB. Guarded by
+// OIKOS_TEST_DATABASE_URL — skipped when unset, same convention as
+// internal/db/integration_test.go. Each run creates a throwaway database.
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/dtoro/oikos/internal/db"
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5"
+)
+
+func newCoveragePool(t *testing.T) *db.Pool {
+	t.Helper()
+	baseURL := os.Getenv("OIKOS_TEST_DATABASE_URL")
+	if baseURL == "" {
+		t.Skip("OIKOS_TEST_DATABASE_URL not set — skipping integration test")
+	}
+	ctx := context.Background()
+
+	admin, err := pgx.Connect(ctx, baseURL)
+	if err != nil {
+		t.Fatalf("connect admin: %v", err)
+	}
+	dbName := fmt.Sprintf("oikos_cov_%08x", rand.Int63())
+	if _, err := admin.Exec(ctx, "CREATE DATABASE "+dbName); err != nil {
+		admin.Close(ctx)
+		t.Fatalf("create test db: %v", err)
+	}
+	admin.Close(ctx)
+
+	at := strings.LastIndex(baseURL, "/")
+	testURL := baseURL[:at+1] + dbName
+	if q := strings.Index(baseURL[at:], "?"); q >= 0 {
+		testURL += baseURL[at+q:]
+	}
+
+	pool, err := db.New(ctx, testURL)
+	if err != nil {
+		t.Fatalf("connect test db: %v", err)
+	}
+	if err := pool.Migrate(ctx); err != nil {
+		t.Fatalf("migrate: %v", err)
+	}
+	t.Cleanup(func() {
+		pool.Close()
+		if admin, err := pgx.Connect(ctx, baseURL); err == nil {
+			admin.Exec(ctx, "DROP DATABASE IF EXISTS "+dbName+" WITH (FORCE)")
+			admin.Close(ctx)
+		}
+	})
+	return pool
+}
+
+// fixture builds the four cases that matter, without the full seed:
+// a declared+monitored type, a declared+unmonitored one, an explicitly
+// unmonitorable one, and one that inherits its declaration from a parent.
+func coverageFixture(t *testing.T, pool *db.Pool) map[string]uuid.UUID {
+	t.Helper()
+	ctx := context.Background()
+
+	exec := func(sql string, args ...any) {
+		t.Helper()
+		if _, err := pool.Exec(ctx, sql, args...); err != nil {
+			t.Fatalf("fixture %q: %v", sql, err)
+		}
+	}
+
+	exec(`INSERT INTO entity_types (name, parent_type, is_abstract, domain, layer, monitoring_spec, status)
+	      VALUES ('t-container', NULL, true, 'compute', 'infrastructure', '["resource"]', 'active')
+	      ON CONFLICT (name) DO UPDATE SET monitoring_spec = EXCLUDED.monitoring_spec`)
+	exec(`INSERT INTO entity_types (name, parent_type, is_abstract, domain, layer, monitoring_spec, status)
+	      VALUES ('t-lxc', 't-container', false, 'compute', 'infrastructure', NULL, 'active')
+	      ON CONFLICT (name) DO NOTHING`)
+	exec(`INSERT INTO entity_types (name, parent_type, is_abstract, domain, layer, monitoring_spec, status)
+	      VALUES ('t-service', NULL, false, 'software', 'infrastructure', '["http"]', 'active')
+	      ON CONFLICT (name) DO NOTHING`)
+	exec(`INSERT INTO entity_types (name, parent_type, is_abstract, domain, layer, monitoring_spec, status)
+	      VALUES ('t-site', NULL, false, 'physical', 'infrastructure', '[]', 'active')
+	      ON CONFLICT (name) DO NOTHING`)
+	// `check` and `signal` normally arrive with the ontology seed, which this
+	// fixture skips; the sweep creates signal entities and needs both.
+	exec(`INSERT INTO entity_types (name, parent_type, is_abstract, domain, layer, monitoring_spec, status)
+	      VALUES ('check', NULL, false, 'operations', 'cognition', '[]', 'active')
+	      ON CONFLICT (name) DO NOTHING`)
+	exec(`INSERT INTO entity_types (name, parent_type, is_abstract, domain, layer, monitoring_spec, status)
+	      VALUES ('signal', NULL, false, 'operations', 'cognition', '[]', 'active')
+	      ON CONFLICT (name) DO NOTHING`)
+
+	ids := map[string]uuid.UUID{}
+	for _, e := range []struct{ slug, typ string }{
+		{"monitored-svc", "t-service"},   // declared + has a check
+		{"unmonitored-svc", "t-service"}, // declared + no check  → signal
+		{"inheriting-lxc", "t-lxc"},      // inherits [resource], no check → signal
+		{"a-site", "t-site"},             // explicitly none → never a signal
+	} {
+		id := uuid.New()
+		exec(`INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
+		      VALUES ($1,$2,$3,$2,'active','{}',1,now(),now())`, id, e.slug, e.typ)
+		ids[e.slug] = id
+	}
+
+	// Only monitored-svc gets a check.
+	checkID := uuid.New()
+	exec(`INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
+	      VALUES ($1,'check:t:monitored-svc:0','check','c-monitored-svc','active','{}',1,now(),now())`, checkID)
+	exec(`INSERT INTO check_defs (entity_id, target_id, kind, config, interval_s, timeout_s, enabled)
+	      VALUES ($1,$2,'http','{}',60,30,true)`, checkID, ids["monitored-svc"])
+
+	return ids
+}
+
+func openSignals(t *testing.T, pool *db.Pool, targetID uuid.UUID) (int, int) {
+	t.Helper()
+	var count, occurrences int
+	err := pool.QueryRow(context.Background(),
+		`SELECT count(*), COALESCE(max(occurrence_count),0) FROM signals
+		 WHERE target_entity_id = $1 AND kind = $2 AND state NOT IN ('resolved','failed')`,
+		targetID, UnmonitoredKind).Scan(&count, &occurrences)
+	if err != nil {
+		t.Fatalf("count signals: %v", err)
+	}
+	return count, occurrences
+}
+
+func TestCoverageSweepOnlyFlagsDeclaredButUnmonitored(t *testing.T) {
+	pool := newCoveragePool(t)
+	ids := coverageFixture(t, pool)
+	ctx := context.Background()
+
+	coverageSweep(ctx, pool)
+
+	for _, c := range []struct {
+		slug string
+		want int
+		why  string
+	}{
+		{"unmonitored-svc", 1, "declares http, has no check"},
+		{"inheriting-lxc", 1, "inherits [resource] from its parent, has no check"},
+		{"monitored-svc", 0, "has a check"},
+		{"a-site", 0, "declares monitoring: none — flagging it would be a permanent false positive"},
+	} {
+		got, _ := openSignals(t, pool, ids[c.slug])
+		if got != c.want {
+			t.Errorf("%s (%s): %d open unmonitored signals, want %d", c.slug, c.why, got, c.want)
+		}
+	}
+}
+
+func TestCoverageSweepDedupsRatherThanDuplicating(t *testing.T) {
+	pool := newCoveragePool(t)
+	ids := coverageFixture(t, pool)
+	ctx := context.Background()
+
+	coverageSweep(ctx, pool)
+	coverageSweep(ctx, pool)
+	coverageSweep(ctx, pool)
+
+	count, occurrences := openSignals(t, pool, ids["unmonitored-svc"])
+	if count != 1 {
+		t.Errorf("three sweeps produced %d signals, want 1", count)
+	}
+	if occurrences != 3 {
+		t.Errorf("occurrence_count = %d after three sweeps, want 3", occurrences)
+	}
+}
+
+func TestCoverageSweepResolvesWhenACheckAppears(t *testing.T) {
+	pool := newCoveragePool(t)
+	ids := coverageFixture(t, pool)
+	ctx := context.Background()
+
+	coverageSweep(ctx, pool)
+	if count, _ := openSignals(t, pool, ids["unmonitored-svc"]); count != 1 {
+		t.Fatalf("expected an open signal before the check is added, got %d", count)
+	}
+
+	// The entity acquires a check. The scheduler's normal auto-resolve keys on
+	// the check entity id and only from 'raised', so it could never clear this.
+	checkID := uuid.New()
+	if _, err := pool.Exec(ctx,
+		// entities has a unique (type, name), so this cannot reuse the
+		// fixture check's name.
+		`INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
+		 VALUES ($1,'check:t:unmonitored-svc:0','check','c-unmonitored-svc','active','{}',1,now(),now())`, checkID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := pool.Exec(ctx,
+		`INSERT INTO check_defs (entity_id, target_id, kind, config, interval_s, timeout_s, enabled)
+		 VALUES ($1,$2,'http','{}',60,30,true)`, checkID, ids["unmonitored-svc"]); err != nil {
+		t.Fatal(err)
+	}
+
+	coverageSweep(ctx, pool)
+
+	if count, _ := openSignals(t, pool, ids["unmonitored-svc"]); count != 0 {
+		t.Errorf("signal should have resolved once the entity had a check, %d still open", count)
+	}
+}
+
+// Against the real ontology and inventory rather than a fixture: the sweep
+// must stay silent for types that opted out and speak up for the genuine gaps.
+// Asserted as properties, not exact counts, so it does not break every time
+// the fleet changes.
+func TestCoverageSweepAgainstTheRealSeed(t *testing.T) {
+	pool := newCoveragePool(t)
+	ctx := context.Background()
+
+	for _, f := range []string{"ontology.yaml", "inventory.yaml", "policy.yaml"} {
+		content, err := os.ReadFile("../../seeds/" + f)
+		if err != nil {
+			t.Fatalf("read %s: %v", f, err)
+		}
+		err = pool.SeedIngest(ctx, f, content,
+			func(ctx context.Context, tx pgx.Tx, data map[string]any) error {
+				switch f {
+				case "ontology.yaml":
+					_, err := db.IngestOntologySeed(ctx, tx, data)
+					return err
+				case "inventory.yaml":
+					_, err := db.IngestInventorySeed(ctx, tx, data)
+					return err
+				default:
+					_, err := db.IngestPolicySeed(ctx, tx, data)
+					return err
+				}
+			})
+		if err != nil {
+			t.Fatalf("ingest %s: %v", f, err)
+		}
+	}
+
+	coverageSweep(ctx, pool)
+
+	flagged := map[string]int{}
+	rows, err := pool.Query(ctx, `
+		SELECT e.type, count(*)
+		FROM signals s JOIN entities e ON e.id = s.target_entity_id
+		WHERE s.kind = $1 AND s.state NOT IN ('resolved','failed')
+		GROUP BY e.type`, UnmonitoredKind)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for rows.Next() {
+		var typ string
+		var n int
+		if err := rows.Scan(&typ, &n); err != nil {
+			t.Fatal(err)
+		}
+		flagged[typ] = n
+	}
+	rows.Close()
+
+	t.Logf("unmonitored signals by entity type: %v", flagged)
+
+	// Declared `monitoring: none` — flagging these would be a permanent,
+	// unresolvable false positive, which is the failure mode that would make
+	// the signal worthless.
+	for _, typ := range []string{"site", "lan", "mesh", "cluster"} {
+		if n := flagged[typ]; n != 0 {
+			t.Errorf("%s declares monitoring: none but %d were flagged unmonitored", typ, n)
+		}
+	}
+
+	// Types that now get real checks must not be flagged either.
+	for _, typ := range []string{"service", "lxc", "ingress-route", "proxmox-host"} {
+		if n := flagged[typ]; n != 0 {
+			t.Errorf("%s should be covered by checkdefaults, but %d were flagged", typ, n)
+		}
+	}
+
+	// Genuine gaps: no `dns` checker, no edge from a pool to its machine.
+	for _, typ := range []string{"dns-zone", "storage-pool"} {
+		if flagged[typ] == 0 {
+			t.Errorf("%s is a known gap and should have been flagged", typ)
+		}
+	}
+}
+
+// A resolved signal must be re-raisable. The (target, kind) partial unique
+// index only covers open signals, so a resolved row does not conflict there —
+// it collides on the primary key instead, which is why the upsert targets the
+// PK.
+func TestCoverageSweepReRaisesAfterResolution(t *testing.T) {
+	pool := newCoveragePool(t)
+	ids := coverageFixture(t, pool)
+	ctx := context.Background()
+
+	coverageSweep(ctx, pool)
+	if _, err := pool.Exec(ctx,
+		`UPDATE signals SET state='resolved' WHERE target_entity_id=$1 AND kind=$2`,
+		ids["unmonitored-svc"], UnmonitoredKind); err != nil {
+		t.Fatal(err)
+	}
+
+	coverageSweep(ctx, pool)
+
+	count, _ := openSignals(t, pool, ids["unmonitored-svc"])
+	if count != 1 {
+		t.Errorf("a resolved signal should be re-raisable, got %d open", count)
+	}
+}

internal/scheduler/scheduler.go

@@ -16,6 +16,7 @@ import (
 	"regexp"
 	"runtime"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/dtoro/oikos/internal/config"
@@ -73,7 +74,12 @@ func runCheckPass(ctx context.Context, pool *db.Pool) {
 		return
 	}
 	if len(defs) == 0 {
+		// Still run housekeeping: a fleet with no enabled check_defs is
+		// precisely the case coverageSweep exists to report, and returning
+		// here would mean the one situation that most needs reporting is the
+		// one situation that stays silent.
 		slog.Debug("scheduler: no enabled check_defs")
+		housekeeping(ctx, pool)
 		return
 	}
 
@@ -248,6 +254,8 @@ func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) check
 		return checkPing(ctx, cd)
 	case "ssh-script":
 		return checkSSHScript(ctx, cd)
+	case "backup-freshness":
+		return checkBackupFreshness(ctx, cd)
 	default:
 		return checkResult{health: "unknown"}
 	}
@@ -265,6 +273,7 @@ func housekeeping(ctx context.Context, pool *db.Pool) {
 	}
 
 	staleSweep(ctx, pool)
+	coverageSweep(ctx, pool)
 
 	// Log housekeeping completion
 	slog.Debug("scheduler: housekeeping done", "pruned_idempotency_before", cutoff.Format(time.RFC3339))
@@ -337,9 +346,14 @@ func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkRes
 	cfg := struct {
 		URL            string `json:"url"`
 		ExpectedStatus int    `json:"expected_status"`
-		Insecure       bool   `json:"insecure"`
+		// MaxStatus accepts a range instead of one exact code. Most services
+		// sit behind Authentik and answer 302 or 401 — a working service, but
+		// an exact-match on 200 reports it degraded and raises a signal.
+		// Unset expected_status means "any response below MaxStatus is fine".
+		MaxStatus int  `json:"max_status"`
+		Insecure  bool `json:"insecure"`
 	}{
-		ExpectedStatus: 200,
+		MaxStatus: 500,
 	}
 	if len(cd.Config) > 0 {
 		_ = json.Unmarshal(cd.Config, &cfg)
@@ -379,10 +393,17 @@ func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkRes
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode != cfg.ExpectedStatus {
+	if cfg.ExpectedStatus != 0 {
+		if resp.StatusCode != cfg.ExpectedStatus {
+			return checkResult{
+				health: "degraded", signalKind: "http",
+				evidence: fmt.Sprintf("GET %s returned %d (expected %d)", cfg.URL, resp.StatusCode, cfg.ExpectedStatus),
+			}
+		}
+	} else if resp.StatusCode >= cfg.MaxStatus {
 		return checkResult{
 			health: "degraded", signalKind: "http",
-			evidence: fmt.Sprintf("GET %s returned %d (expected %d)", cfg.URL, resp.StatusCode, cfg.ExpectedStatus),
+			evidence: fmt.Sprintf("GET %s returned %d (expected below %d)", cfg.URL, resp.StatusCode, cfg.MaxStatus),
 		}
 	}
 
@@ -616,6 +637,11 @@ func checkSSHScript(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) che
 		Port   int    `json:"port"`
 		User   string `json:"user"`
 		Script string `json:"script"`
+		// Args is a single positional argument for the script. checkdefaults
+		// has always written it for process_check.sh, but nothing read it —
+		// so every process check ran argument-less and process_check.sh
+		// answered "no service name provided" with health unknown.
+		Args string `json:"args"`
 	}{}
 	if len(cd.Config) > 0 {
 		_ = json.Unmarshal(cd.Config, &cfg)
@@ -646,6 +672,11 @@ func checkSSHScript(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) che
 	defer cancel()
 
 	scriptPath := "/opt/oikos/checks/" + cfg.Script
+	if cfg.Args != "" {
+		// Single-quote the argument so an entity name can never break out of
+		// the remote command. The script name itself is allowlisted above.
+		scriptPath += " '" + strings.ReplaceAll(cfg.Args, "'", `'\''`) + "'"
+	}
 	port := strconv.Itoa(cfg.Port)
 
 	output, err := sshExec(ctx, cfg.Host, port, cfg.User, scriptPath, timeout)