oikos/internal/scheduler/coverage.go

package scheduler

import (
	"context"
	"fmt"
	"log/slog"

	"github.com/dtoro/oikos/internal/db"
	"github.com/google/uuid"
)

// UnmonitoredKind is the signal kind raised for an entity whose type declares
// monitoring it does not have.
const UnmonitoredKind = "unmonitored"

// coverageSweep reports entities that should be monitored and are not.
//
// staleSweep can only protect an entity that already has a check — it INNER
// JOINs check_defs, so an entity with none is structurally invisible to it and
// keeps reporting its last-known health forever. This sweep covers the other
// half: it notices the absence itself.
//
// It fires only where the entity type *declares* monitoring. Types that
// declare `monitoring: none` (site, cluster, lan, mesh — topological groupings
// with nothing to probe) are working as intended and must never raise a
// signal; a permanent unresolvable warning against six healthy entities would
// discredit the whole thing. Types that declare nothing at all are a modelling
// gap, reported once per pass at debug level rather than as a fleet problem.
func coverageSweep(ctx context.Context, pool *db.Pool) {
	// Resolution walks parent_type — inheriting types (lxc, proxmox-host, lan)
	// carry NULL in their own monitoring_spec column, so reading it directly
	// would flag every one of them. Reuse the Go resolver instead of
	// duplicating the hierarchy walk in SQL.
	tx, err := pool.Begin(ctx)
	if err != nil {
		slog.Error("scheduler: coverage sweep begin", "error", err)
		return
	}
	tree, err := db.LoadTypeTree(ctx, tx)
	if err != nil {
		_ = tx.Rollback(ctx)
		slog.Error("scheduler: coverage sweep load type tree", "error", err)
		return
	}
	_ = tx.Rollback(ctx) // read-only

	rows, err := pool.Query(ctx, `
		SELECT e.id, e.slug, e.type, (cd.target_id IS NOT NULL) AS has_check
		FROM entities e
		LEFT JOIN (
			SELECT DISTINCT target_id FROM check_defs
			WHERE enabled AND target_id IS NOT NULL
		) cd ON cd.target_id = e.id
		WHERE e.state = 'active' AND e.type <> 'check'`)
	if err != nil {
		slog.Error("scheduler: coverage sweep query", "error", err)
		return
	}

	type entity struct {
		id       uuid.UUID
		slug     string
		typ      string
		hasCheck bool
	}
	var all []entity
	for rows.Next() {
		var e entity
		if err := rows.Scan(&e.id, &e.slug, &e.typ, &e.hasCheck); err != nil {
			continue
		}
		all = append(all, e)
	}
	rows.Close()
	if rows.Err() != nil {
		slog.Error("scheduler: coverage sweep scan", "error", rows.Err())
		return
	}

	var raised, resolved, undeclared int
	for _, e := range all {
		mon := tree.Monitoring(e.typ)

		switch {
		case !mon.Declared:
			undeclared++
		case mon.None():
			// Explicitly unmonitorable. Nothing to say.
		case e.hasCheck:
			if resolveCoverageSignal(ctx, pool, e.id) {
				resolved++
				slog.Info("scheduler: entity is monitored again", "entity", e.slug)
			}
		default:
			if raiseCoverageSignal(ctx, pool, e.id, e.slug, e.typ, mon.Kinds) {
				raised++
			}
		}
	}

	if raised > 0 || resolved > 0 {
		slog.Warn("scheduler: coverage sweep",
			"unmonitored_raised", raised, "resolved", resolved, "scanned", len(all))
	}
	if undeclared > 0 {
		slog.Debug("scheduler: entity types declare no monitoring", "entities", undeclared)
	}
}

// raiseCoverageSignal raises (or refreshes) the unmonitored signal for one
// entity. Reports whether this was a new raise.
func raiseCoverageSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID, slug, typ string, want []string) bool {
	// A signal is a dual entity: signals.entity_id is a PK referencing
	// entities(id), so the row has to exist first. The scheduler's other
	// signals borrow the check entity's id — there is no check here, which is
	// the whole point, so this sweep owns a signal entity per target.
	//
	// The slug is stable per target, which makes the signal row stable too and
	// lets a resolved signal be re-raised by primary key rather than colliding
	// with it.
	signalSlug := fmt.Sprintf("signal:%s:%s", UnmonitoredKind, slug)

	newID, err := uuid.NewV7()
	if err != nil {
		newID = uuid.New()
	}

	var signalID uuid.UUID
	// Upsert RETURNING id, never insert-and-assume: assuming is what made
	// checkdefaults write foreign keys to rows it had not created.
	if err := pool.QueryRow(ctx,
		`INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
		 VALUES ($1, $2, 'signal', $2, 'active', '{}', 1, now(), now())
		 ON CONFLICT (slug) DO UPDATE SET updated_at = now()
		 RETURNING id`,
		newID, signalSlug).Scan(&signalID); err != nil {
		slog.Error("scheduler: upsert signal entity", "entity", slug, "error", err)
		return false
	}

	evidence := fmt.Sprintf("type %s declares monitoring %v but the entity has no enabled check_def", typ, want)

	// Conflict on the primary key rather than on the (target, kind) partial
	// index: that index only covers OPEN signals, so a previously resolved
	// signal would not conflict there and would collide on the PK instead.
	tag, err := pool.Exec(ctx,
		`INSERT INTO signals (entity_id, kind, severity, target_entity_id, evidence, state)
		 VALUES ($1, $2, 'warning', $3, $4, 'raised')
		 ON CONFLICT (entity_id) DO UPDATE
		   SET state = CASE WHEN signals.state IN ('resolved','failed') THEN 'raised' ELSE signals.state END,
		       occurrence_count = signals.occurrence_count + 1,
		       evidence = EXCLUDED.evidence,
		       last_seen_at = now(), updated_at = now()`,
		signalID, UnmonitoredKind, entityID, evidence)
	if err != nil {
		slog.Error("scheduler: raise unmonitored signal", "entity", slug, "error", err)
		return false
	}

	// RowsAffected is 1 for both insert and update, so ask the signal itself
	// whether this was the first occurrence.
	var occurrences int
	if err := pool.QueryRow(ctx,
		`SELECT occurrence_count FROM signals WHERE entity_id = $1`, signalID).Scan(&occurrences); err != nil {
		return tag.RowsAffected() > 0
	}
	if occurrences <= 1 {
		slog.Warn("scheduler: entity is unmonitored",
			"entity", slug, "type", typ, "declared", want)
		emitSchedulerEvent(ctx, pool, "coverage.unmonitored", entityID, "warning",
			map[string]any{"slug": slug, "type": typ, "declared": want})
		return true
	}
	return false
}

// resolveCoverageSignal closes the unmonitored signal once the entity has a
// check. The scheduler's normal auto-resolve keys on the *check* entity id and
// only from state 'raised', so it can never clear one of these.
func resolveCoverageSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID) bool {
	tag, err := pool.Exec(ctx,
		`UPDATE signals SET state = 'resolved', updated_at = now()
		 WHERE target_entity_id = $1 AND kind = $2
		   AND state NOT IN ('resolved', 'failed')`,
		entityID, UnmonitoredKind)
	if err != nil {
		slog.Error("scheduler: resolve unmonitored signal", "error", err)
		return false
	}
	return tag.RowsAffected() > 0
}