// Package scheduler implements the Oikos observe + decide loop (Phase 3). // It loads enabled check_defs, runs checks on schedule, manages signal // lifecycle (dedup, flap suppression, maintenance mode), and writes metrics. package scheduler import ( "context" "log/slog" "time" "github.com/dtoro/oikos/internal/config" "github.com/dtoro/oikos/internal/db" "github.com/dtoro/oikos/internal/db/sqlcgen" "github.com/google/uuid" "golang.org/x/sync/errgroup" ) // Run starts the scheduler loop. Blocks until ctx is cancelled. func Run(ctx context.Context, pool *db.Pool, cfg config.Config) { slog.Info("scheduler: starting", "interval", cfg.SchedulerInterval) interval := cfg.SchedulerInterval if interval <= 0 { interval = 30 * time.Second } ticker := time.NewTicker(interval) defer ticker.Stop() // Immediate first pass runCheckPass(ctx, pool) for { select { case <-ctx.Done(): slog.Info("scheduler: shutting down") return case <-ticker.C: runCheckPass(ctx, pool) } } } // runCheckPass executes one full cycle of check evaluation. func runCheckPass(ctx context.Context, pool *db.Pool) { q := sqlcgen.New(pool) defs, err := q.ListEnabledCheckDefs(ctx) if err != nil { slog.Error("scheduler: list check defs", "error", err) return } if len(defs) == 0 { slog.Debug("scheduler: no enabled check_defs") return } g, gctx := errgroup.WithContext(ctx) g.SetLimit(10) // bounded worker pool for _, def := range defs { cd := def g.Go(func() error { runCheck(gctx, pool, cd) return nil }) } g.Wait() // Housekeeping after each pass housekeeping(ctx, pool) } // runCheck executes a single check and processes the result. func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDefsRow) { q := sqlcgen.New(pool) start := time.Now() health, signalKind, evidence, checkErr := executeCheck(ctx, cd) latency := time.Since(start).Milliseconds() // Write metric _ = q.InsertMetricSample(ctx, sqlcgen.InsertMetricSampleParams{ EntityID: cd.EntityID, Metric: "probe_latency_ms", Value: float64(latency), Tags: []byte(`{}`), }) if checkErr != nil { slog.Warn("scheduler: check failed", "entity", cd.EntitySlug, "kind", cd.Kind, "error", checkErr) } if signalKind == "" || health == "healthy" { // Recovery: resolve any open signal for this check resolveSignal(ctx, pool, cd.EntityID, cd.EntitySlug) // Update entity_status to healthy _ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{ EntityID: cd.EntityID, Health: "healthy", LastCheckAt: &[]time.Time{time.Now()}[0], Details: []byte(`{}`), }) return } // Failure: upsert signal (dedup via partial unique index) slog.Warn("scheduler: raising signal", "entity", cd.EntitySlug, "kind", signalKind, "evidence", evidence) severity := "warning" if signalKind == "down" { severity = "critical" } sig, err := q.UpsertSignal(ctx, sqlcgen.UpsertSignalParams{ EntityID: cd.EntityID, Kind: signalKind, Severity: severity, TargetEntityID: cd.TargetID, Evidence: &evidence, }) if err != nil { slog.Error("scheduler: upsert signal", "error", err) return } // Update entity_status _ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{ EntityID: cd.EntityID, Health: health, LastCheckAt: &[]time.Time{time.Now()}[0], Details: []byte(`{}`), }) _ = sig // used for flap detection below } // resolveSignal resolves any open signal for the given check entity. func resolveSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID, slug string) { q := sqlcgen.New(pool) // Check if there's an open signal on this entity _, err := pool.Exec(ctx, `UPDATE signals SET state = 'resolved', updated_at = now() WHERE entity_id = $1 AND state = 'raised'`, entityID) if err != nil { return } _ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{ EntityID: entityID, Health: "healthy", LastCheckAt: &[]time.Time{time.Now()}[0], Details: []byte(`{}`), }) slog.Info("scheduler: signal resolved", "entity", slug) } // executeCheck dispatches to the appropriate checker by kind. func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (health string, signalKind string, evidence string, err error) { switch cd.Kind { case "http": return checkHTTP(ctx, cd) case "tcp": return checkTCP(ctx, cd) case "disk": return checkDisk(ctx, cd) case "cert-expiry": return checkCertExpiry(ctx, cd) default: return "unknown", "", "", nil } } // housekeeping runs background maintenance tasks. func housekeeping(ctx context.Context, pool *db.Pool) { // Prune expired idempotency keys (older than 24h) cutoff := time.Now().Add(-24 * time.Hour) _, err := pool.Exec(ctx, "DELETE FROM idempotency_keys WHERE created_at < $1", cutoff) if err != nil { slog.Error("scheduler: prune idempotency keys", "error", err) } // Log housekeeping completion slog.Debug("scheduler: housekeeping done", "pruned_idempotency_before", cutoff.Format(time.RFC3339)) } // checkHTTP performs an HTTP health check. func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) { // Stub: always returns healthy return "healthy", "", "", nil } // checkTCP performs a TCP dial check. func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) { // Stub: always returns healthy return "healthy", "", "", nil } // checkDisk performs a disk usage check via SSH. func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) { // Stub: always returns healthy return "healthy", "", "", nil } // checkCertExpiry checks TLS certificate expiry. func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) { // Stub: always returns healthy return "healthy", "", "", nil }