package scheduler import ( "context" "fmt" "log/slog" "github.com/dtoro/oikos/internal/db" "github.com/google/uuid" ) // UnmonitoredKind is the signal kind raised for an entity whose type declares // monitoring it does not have. const UnmonitoredKind = "unmonitored" // coverageSweep reports entities that should be monitored and are not. // // staleSweep can only protect an entity that already has a check — it INNER // JOINs check_defs, so an entity with none is structurally invisible to it and // keeps reporting its last-known health forever. This sweep covers the other // half: it notices the absence itself. // // It fires only where the entity type *declares* monitoring. Types that // declare `monitoring: none` (site, cluster, lan, mesh — topological groupings // with nothing to probe) are working as intended and must never raise a // signal; a permanent unresolvable warning against six healthy entities would // discredit the whole thing. Types that declare nothing at all are a modelling // gap, reported once per pass at debug level rather than as a fleet problem. func coverageSweep(ctx context.Context, pool *db.Pool) { // Resolution walks parent_type — inheriting types (lxc, proxmox-host, lan) // carry NULL in their own monitoring_spec column, so reading it directly // would flag every one of them. Reuse the Go resolver instead of // duplicating the hierarchy walk in SQL. tx, err := pool.Begin(ctx) if err != nil { slog.Error("scheduler: coverage sweep begin", "error", err) return } tree, err := db.LoadTypeTree(ctx, tx) if err != nil { _ = tx.Rollback(ctx) slog.Error("scheduler: coverage sweep load type tree", "error", err) return } _ = tx.Rollback(ctx) // read-only rows, err := pool.Query(ctx, ` SELECT e.id, e.slug, e.type, (cd.target_id IS NOT NULL) AS has_check FROM entities e LEFT JOIN ( SELECT DISTINCT target_id FROM check_defs WHERE enabled AND target_id IS NOT NULL ) cd ON cd.target_id = e.id WHERE e.state = 'active' AND e.type <> 'check'`) if err != nil { slog.Error("scheduler: coverage sweep query", "error", err) return } type entity struct { id uuid.UUID slug string typ string hasCheck bool } var all []entity for rows.Next() { var e entity if err := rows.Scan(&e.id, &e.slug, &e.typ, &e.hasCheck); err != nil { continue } all = append(all, e) } rows.Close() if rows.Err() != nil { slog.Error("scheduler: coverage sweep scan", "error", rows.Err()) return } var raised, resolved, undeclared int for _, e := range all { mon := tree.Monitoring(e.typ) switch { case !mon.Declared: undeclared++ case mon.None(): // Explicitly unmonitorable. Nothing to raise — but a type that // USED to declare monitoring (e.g. dns-zone, [dns]→none) may have // open `unmonitored` signals from before the change. They are no // longer a gap, so close them; otherwise they linger forever, // because resolveCoverageSignal only runs from the hasCheck path // and a None() entity never gains a check. if resolveCoverageSignal(ctx, pool, e.id) { resolved++ slog.Info("scheduler: type now unmonitorable, resolving stale signal", "entity", e.slug) } case e.hasCheck: if resolveCoverageSignal(ctx, pool, e.id) { resolved++ slog.Info("scheduler: entity is monitored again", "entity", e.slug) } default: if raiseCoverageSignal(ctx, pool, e.id, e.slug, e.typ, mon.Kinds) { raised++ } } } if raised > 0 || resolved > 0 { slog.Warn("scheduler: coverage sweep", "unmonitored_raised", raised, "resolved", resolved, "scanned", len(all)) } if undeclared > 0 { slog.Debug("scheduler: entity types declare no monitoring", "entities", undeclared) } } // raiseCoverageSignal raises (or refreshes) the unmonitored signal for one // entity. Reports whether this was a new raise. func raiseCoverageSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID, slug, typ string, want []string) bool { // A signal is a dual entity: signals.entity_id is a PK referencing // entities(id), so the row has to exist first. The scheduler's other // signals borrow the check entity's id — there is no check here, which is // the whole point, so this sweep owns a signal entity per target. // // The slug is stable per target, which makes the signal row stable too and // lets a resolved signal be re-raised by primary key rather than colliding // with it. signalSlug := fmt.Sprintf("signal:%s:%s", UnmonitoredKind, slug) newID, err := uuid.NewV7() if err != nil { newID = uuid.New() } var signalID uuid.UUID // Upsert RETURNING id, never insert-and-assume: assuming is what made // checkdefaults write foreign keys to rows it had not created. if err := pool.QueryRow(ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at) VALUES ($1, $2, 'signal', $2, 'active', '{}', 1, now(), now()) ON CONFLICT (slug) DO UPDATE SET updated_at = now() RETURNING id`, newID, signalSlug).Scan(&signalID); err != nil { slog.Error("scheduler: upsert signal entity", "entity", slug, "error", err) return false } evidence := fmt.Sprintf("type %s declares monitoring %v but the entity has no enabled check_def", typ, want) // Conflict on the primary key rather than on the (target, kind) partial // index: that index only covers OPEN signals, so a previously resolved // signal would not conflict there and would collide on the PK instead. tag, err := pool.Exec(ctx, `INSERT INTO signals (entity_id, kind, severity, target_entity_id, evidence, state) VALUES ($1, $2, 'warning', $3, $4, 'raised') ON CONFLICT (entity_id) DO UPDATE SET state = CASE WHEN signals.state IN ('resolved','failed') THEN 'raised' ELSE signals.state END, occurrence_count = signals.occurrence_count + 1, evidence = EXCLUDED.evidence, last_seen_at = now(), updated_at = now()`, signalID, UnmonitoredKind, entityID, evidence) if err != nil { slog.Error("scheduler: raise unmonitored signal", "entity", slug, "error", err) return false } // RowsAffected is 1 for both insert and update, so ask the signal itself // whether this was the first occurrence. var occurrences int if err := pool.QueryRow(ctx, `SELECT occurrence_count FROM signals WHERE entity_id = $1`, signalID).Scan(&occurrences); err != nil { return tag.RowsAffected() > 0 } if occurrences <= 1 { slog.Warn("scheduler: entity is unmonitored", "entity", slug, "type", typ, "declared", want) emitSchedulerEvent(ctx, pool, "coverage.unmonitored", entityID, "warning", map[string]any{"slug": slug, "type": typ, "declared": want}) return true } return false } // resolveCoverageSignal closes the unmonitored signal once the entity has a // check. The scheduler's normal auto-resolve keys on the *check* entity id and // only from state 'raised', so it can never clear one of these. func resolveCoverageSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID) bool { tag, err := pool.Exec(ctx, `UPDATE signals SET state = 'resolved', updated_at = now() WHERE target_entity_id = $1 AND kind = $2 AND state NOT IN ('resolved', 'failed')`, entityID, UnmonitoredKind) if err != nil { slog.Error("scheduler: resolve unmonitored signal", "error", err) return false } return tag.RowsAffected() > 0 }