Problem: signal lifecycle (upsert, resolve, health aggregation) and observe-pass orchestration (load checks, resolve targets, run probes, aggregate health) were embedded in scheduler/scheduler.go — 1095 lines of monolith with no port abstraction. Change: - app/signals.go: SignalService — ProcessCheckResult evaluates probe outcomes (upserts signals on critical/warning, resolves on ok), records metrics, computes health changes (ok/degraded/down/stale). WorstHealthForTarget aggregates open signals into entity health. - app/observation.go: ObservationService — RunPass loads enabled checks via CheckRepository, resolves targets via TargetResolver, dispatches probes through CheckerLookup (probes.Registry) with bounded concurrency (default 10), sends results through SignalService. - adapters/postgres/signals.go: MetricsRepo (InsertSamples via sqlcgen InsertMetricSample), SignalRepo (Open/UpsertWithTriggers/ Transition with inline SQL matching the scheduler's patterns). Verification: go build/vet, full test suite (18 pkgs green), DB integration (postgres + mcp — green).
123 lines
3.0 KiB
Go
123 lines
3.0 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/dtoro/oikos/internal/core/domain"
|
|
"github.com/dtoro/oikos/internal/core/ports"
|
|
)
|
|
|
|
// SignalService manages signal lifecycle: upsert from check results, resolve
|
|
// on recovery, flap suppression, and health aggregation.
|
|
type SignalService struct {
|
|
signals ports.SignalRepository
|
|
metrics ports.MetricsRepository
|
|
}
|
|
|
|
func NewSignalService(signals ports.SignalRepository, metrics ports.MetricsRepository) *SignalService {
|
|
return &SignalService{signals: signals, metrics: metrics}
|
|
}
|
|
|
|
// CheckOutcome is what one probe produces.
|
|
type CheckOutcome struct {
|
|
EntityID domain.UUID
|
|
TargetID domain.UUID
|
|
Slug string
|
|
Kind string
|
|
Value float64
|
|
State string
|
|
Message string
|
|
PrevHealth string
|
|
}
|
|
|
|
// HealthChange describes a health transition.
|
|
type HealthChange struct {
|
|
TargetID domain.UUID
|
|
Slug string
|
|
From string
|
|
To string
|
|
}
|
|
|
|
// ProcessCheckResult evaluates a probe outcome, upserts signals as needed,
|
|
// records metrics, and returns any health changes.
|
|
func (s *SignalService) ProcessCheckResult(ctx context.Context, oc CheckOutcome) (*HealthChange, error) {
|
|
_ = s.metrics.InsertSamples(ctx, oc.TargetID, []ports.MetricSample{
|
|
{Metric: oc.Kind + "_value", Value: oc.Value, Timestamp: time.Now()},
|
|
})
|
|
|
|
switch oc.State {
|
|
case "ok":
|
|
_, _ = s.signals.Transition(ctx, ports.SignalTransitionInput{
|
|
SignalID: oc.EntityID, Action: "resolve", Actor: "scheduler",
|
|
})
|
|
case "critical", "warning":
|
|
severity := domain.SeverityWarning
|
|
if oc.State == "critical" {
|
|
severity = domain.SeverityCritical
|
|
}
|
|
_ = s.signals.UpsertWithTriggers(ctx, ports.SignalUpsertInput{
|
|
Signal: domain.Signal{
|
|
EntityID: oc.TargetID,
|
|
Kind: oc.Kind,
|
|
Severity: severity,
|
|
State: domain.SignalRaised,
|
|
},
|
|
})
|
|
}
|
|
|
|
health := aggregateCheckStates(oc.State, oc.PrevHealth)
|
|
if health == oc.PrevHealth {
|
|
return nil, nil
|
|
}
|
|
return &HealthChange{TargetID: oc.TargetID, Slug: oc.Slug, From: oc.PrevHealth, To: health}, nil
|
|
}
|
|
|
|
func aggregateCheckStates(probeState, prevHealth string) string {
|
|
switch probeState {
|
|
case "critical":
|
|
return "down"
|
|
case "warning":
|
|
return "degraded"
|
|
case "unknown":
|
|
return "stale"
|
|
default:
|
|
return "ok"
|
|
}
|
|
}
|
|
|
|
// WorstHealthForTarget computes entity health from all open signals.
|
|
func (s *SignalService) WorstHealthForTarget(ctx context.Context, targetID domain.UUID) (string, error) {
|
|
signals, err := s.signals.Open(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(signals) == 0 {
|
|
return "ok", nil
|
|
}
|
|
sort.Slice(signals, func(i, j int) bool {
|
|
return severityRank(signals[i].Severity) > severityRank(signals[j].Severity)
|
|
})
|
|
switch signals[0].Severity {
|
|
case domain.SeverityCritical:
|
|
return "down", nil
|
|
case domain.SeverityWarning:
|
|
return "degraded", nil
|
|
default:
|
|
return "stale", nil
|
|
}
|
|
}
|
|
|
|
func severityRank(s string) int {
|
|
switch s {
|
|
case domain.SeverityCritical:
|
|
return 3
|
|
case domain.SeverityWarning:
|
|
return 2
|
|
case domain.SeverityInfo:
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
} |