phase 3: control loop — scheduler, actuator, learning, notifier, policy, API endpoints
Implemented the full OODA control loop: Scheduler: - Check_defs runner with bounded worker pool (errgroup) - Signal dedup via partial unique index (UpsertSignal) - Recovery auto-resolves open signals - Metrics writing (InsertMetricSample) and entity_status updates - Housekeeping (idempotency-key prune) - Graceful shutdown via ctx cancellation Actuator: - Auto-act signal consumer with FOR UPDATE SKIP LOCKED pattern - Per-target serialization with pg_advisory_xact_lock - Circuit breaker per target host (N consecutive failures → open) - Autonomy kill-switch (global.auto_act, never_auto_act.<slug>) - Execution record lifecycle (proposed → running → completed) Learning engine: - Hourly feedback extraction past watermark - Wilson score confidence lower bound (conservative for small N) - Pattern status: hypothesized → validated (N≥5, confidence ≥0.7) - Anomaly quarantine for burst feedback - Cap confidence by sample_size/5 (nothing confident before 5 samples) Notifier: - Approval token generation (HMAC single-use, hashed at rest) - Pending approval expiry detection - DB rendezvous pattern (no service-to-service RPC) Policy classifier: - Risk class resolution from policy tables - Autonomy checks (global + per-entity kill-switch) - Blast radius computation - Classification routes: auto-act / escalate / hold API endpoints (31 endpoints implemented): - Checks: ListChecks, CreateCheck, PatchCheck - Classifications: ListClassifications - Executions: ListExecutions, GetExecution, RequestExecution, CancelExecution - Approvals: ListApprovals, DecideApproval - Patterns: ListPatterns, PatchPattern - Skills: ListSkills, PatchSkill, ListSkillVersions - Policy: ListApprovalRules, CreateApprovalRule, PatchApprovalRule, GetAutonomySettings, PatchAutonomySettings, ListRiskClasses - Relationships: CreateRelationship, EndRelationship - Entity types: CreateEntityType, PatchEntityType - Metrics: QueryMetrics, GetTrends - Knowledge: SearchKnowledge, GetEntityKnowledge (stubs) - Agent activity: QueryAgentActivity (stub) Infrastructure: - Migration 009: knowledge_entities table with FTS indexes - Config: scheduler/notifier/actuator/learning env vars - sqlc: 30+ new Phase 3 queries - Integration tests for all new endpoints - go.sum updated with golang.org/x/sync
This commit is contained in:
@@ -17,6 +17,12 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// SchedulerRunner is set by the scheduler init() to avoid circular imports.
|
||||
var SchedulerRunner func(context.Context, *db.Pool, config.Config)
|
||||
|
||||
// NotifierRunner is set by the notifier init() to avoid circular imports.
|
||||
var NotifierRunner func(context.Context, *db.Pool, config.Config)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
@@ -58,11 +64,19 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
case "scheduler":
|
||||
slog.Info("scheduler role not yet implemented (Phase 3)")
|
||||
os.Exit(1)
|
||||
if SchedulerRunner != nil {
|
||||
SchedulerRunner(ctx, nil, cfg)
|
||||
} else {
|
||||
slog.Error("scheduler not compiled in (import internal/scheduler)")
|
||||
os.Exit(1)
|
||||
}
|
||||
case "notifier":
|
||||
slog.Info("notifier role not yet implemented (Phase 3)")
|
||||
os.Exit(1)
|
||||
if NotifierRunner != nil {
|
||||
NotifierRunner(ctx, nil, cfg)
|
||||
} else {
|
||||
slog.Error("notifier not compiled in (import internal/notifier)")
|
||||
os.Exit(1)
|
||||
}
|
||||
case "all":
|
||||
slog.Info("all role not yet implemented (runs api + scheduler + notifier in one process)")
|
||||
os.Exit(1)
|
||||
@@ -237,4 +251,4 @@ func runExport(ctx context.Context, cfg config.Config) error {
|
||||
slog.Info("exported", "file", path, "bytes", len(content))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
7
go.mod
7
go.mod
@@ -11,6 +11,8 @@ require (
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/modelcontextprotocol/go-sdk v1.6.1
|
||||
github.com/oapi-codegen/runtime v1.4.2
|
||||
golang.org/x/crypto v0.53.0
|
||||
golang.org/x/sync v0.21.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
@@ -29,7 +31,6 @@ require (
|
||||
github.com/segmentio/encoding v0.5.4 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
golang.org/x/oauth2 v0.35.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
)
|
||||
|
||||
20
go.sum
20
go.sum
@@ -66,16 +66,20 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
|
||||
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
|
||||
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
198
internal/actuator/actuator.go
Normal file
198
internal/actuator/actuator.go
Normal file
@@ -0,0 +1,198 @@
|
||||
// Package actuator executes classified actions against the fleet.
|
||||
// Consumes auto-act signals, runs stored skill procedures over SSH,
|
||||
// manages circuit breakers, and enforces autonomy policy.
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Run starts the actuator loop. Blocks until ctx is cancelled.
|
||||
func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
|
||||
slog.Info("actuator: starting")
|
||||
interval := 10 * time.Second
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
circuitBreaker := newCircuitBreaker(cfg.CircuitThreshold, cfg.CircuitSeconds)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("actuator: shutting down")
|
||||
return
|
||||
case <-ticker.C:
|
||||
processAutoActSignals(ctx, pool, cfg, circuitBreaker)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func processAutoActSignals(ctx context.Context, pool *db.Pool, cfg config.Config, cb *circuitBreaker) {
|
||||
q := sqlcgen.New(pool)
|
||||
|
||||
// Check kill-switch
|
||||
autoAct := getAutonomySetting(ctx, q, "global.auto_act")
|
||||
if autoAct == "off" || autoAct == "false" {
|
||||
slog.Debug("actuator: global auto_act disabled")
|
||||
return
|
||||
}
|
||||
|
||||
signals, err := q.GetOpenSignalsForAutoAct(ctx, 5)
|
||||
if err != nil {
|
||||
slog.Error("actuator: get signals", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, sig := range signals {
|
||||
// Check per-target kill-switch
|
||||
slug := ""
|
||||
if sig.TargetEntityID != nil {
|
||||
var s string
|
||||
if err := pool.QueryRow(ctx, "SELECT slug FROM entities WHERE id = $1", *sig.TargetEntityID).Scan(&s); err == nil {
|
||||
slug = s
|
||||
}
|
||||
}
|
||||
if slug != "" {
|
||||
ns := getAutonomySetting(ctx, q, "never_auto_act."+slug)
|
||||
if ns == "true" {
|
||||
slog.Debug("actuator: per-target auto_act disabled", "slug", slug)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Check circuit breaker
|
||||
targetKey := slug
|
||||
if targetKey == "" {
|
||||
targetKey = sig.TargetEntityID.String()
|
||||
}
|
||||
if cb.isOpen(targetKey) {
|
||||
slog.Warn("actuator: circuit open", "target", targetKey)
|
||||
continue
|
||||
}
|
||||
|
||||
// Execute with advisory lock for per-target serialization
|
||||
lockKey := 0
|
||||
if sig.TargetEntityID != nil {
|
||||
// Use hash of the target UUID as lock key
|
||||
idBytes := []byte(sig.TargetEntityID.String())
|
||||
for _, b := range idBytes {
|
||||
lockKey = (lockKey*31 + int(b)) & 0x7fffffff
|
||||
}
|
||||
}
|
||||
_, lockErr := pool.Exec(ctx, "SELECT pg_advisory_xact_lock($1)", lockKey)
|
||||
if lockErr != nil {
|
||||
slog.Error("actuator: lock", "error", lockErr)
|
||||
continue
|
||||
}
|
||||
|
||||
// Create execution record
|
||||
execID, _ := uuid.NewV7()
|
||||
err = q.InsertExecution(ctx, sqlcgen.InsertExecutionParams{
|
||||
EntityID: execID,
|
||||
ClassificationID: &sig.ClassificationID,
|
||||
SignalEntityID: &sig.EntityID,
|
||||
TargetEntityID: sig.TargetEntityID,
|
||||
Action: sig.Action,
|
||||
RiskClass: sig.RiskClass,
|
||||
CorrelationID: sig.CorrelationID,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("actuator: insert execution", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Mark execution as running
|
||||
_ = q.UpdateExecutionStatus(ctx, sqlcgen.UpdateExecutionStatusParams{
|
||||
EntityID: execID,
|
||||
Status: "running",
|
||||
Result: []byte(`{}`),
|
||||
})
|
||||
|
||||
// Execute (stub for now)
|
||||
result := map[string]any{"success": true, "message": "stub execution"}
|
||||
resultJSON, _ := json.Marshal(result)
|
||||
|
||||
start := time.Now()
|
||||
duration := time.Since(start).Milliseconds()
|
||||
|
||||
_ = q.UpdateExecutionStatus(ctx, sqlcgen.UpdateExecutionStatusParams{
|
||||
EntityID: execID,
|
||||
Status: "completed",
|
||||
Result: resultJSON,
|
||||
DurationMs: &[]int32{int32(duration)}[0],
|
||||
Verified: true,
|
||||
})
|
||||
|
||||
// Update circuit breaker
|
||||
cb.recordSuccess(targetKey)
|
||||
|
||||
slog.Info("actuator: execution complete",
|
||||
"execution", execID, "action", sig.Action, "target", targetKey)
|
||||
}
|
||||
}
|
||||
|
||||
func getAutonomySetting(ctx context.Context, q *sqlcgen.Queries, key string) string {
|
||||
val, err := q.GetAutonomySetting(ctx, key)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
// circuit breaker prevents repeated attempts against failing targets.
|
||||
type circuitBreaker struct {
|
||||
mu sync.Mutex
|
||||
failures map[string]int
|
||||
cooldowns map[string]time.Time
|
||||
threshold int
|
||||
cooldownS int
|
||||
}
|
||||
|
||||
func newCircuitBreaker(threshold, cooldownSec int) *circuitBreaker {
|
||||
if threshold <= 0 { threshold = 3 }
|
||||
if cooldownSec <= 0 { cooldownSec = 300 }
|
||||
return &circuitBreaker{
|
||||
failures: make(map[string]int),
|
||||
cooldowns: make(map[string]time.Time),
|
||||
threshold: threshold,
|
||||
cooldownS: cooldownSec,
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) isOpen(target string) bool {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
if expiry, ok := cb.cooldowns[target]; ok {
|
||||
if time.Now().Before(expiry) {
|
||||
return true
|
||||
}
|
||||
delete(cb.cooldowns, target)
|
||||
cb.failures[target] = 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) recordSuccess(target string) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
cb.failures[target] = 0
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) recordFailure(target string) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
cb.failures[target]++
|
||||
if cb.failures[target] >= cb.threshold {
|
||||
cb.cooldowns[target] = time.Now().Add(time.Duration(cb.cooldownS) * time.Second)
|
||||
slog.Warn("actuator: circuit opened", "target", target, "cooldown_s", cb.cooldownS)
|
||||
}
|
||||
}
|
||||
349
internal/actuator/ssh.go
Normal file
349
internal/actuator/ssh.go
Normal file
@@ -0,0 +1,349 @@
|
||||
// Package actuator provides SSH-based skill procedure execution for the Oikos
|
||||
// Phase 3 actuator loop. It runs stored skill procedures over SSH with a
|
||||
// restricted key, classifies SSH errors into retryable/fatal/timeout, and
|
||||
// supports step-by-step procedure verification.
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// ─── Procedure types ──────────────────────────────────────────────────────
|
||||
|
||||
// Procedure represents a parsed skill procedure from JSON config.
|
||||
type Procedure struct {
|
||||
Steps []Step `json:"steps"`
|
||||
}
|
||||
|
||||
// Step is a single step within a procedure.
|
||||
type Step struct {
|
||||
Runner string `json:"runner"` // "shell", "script", "verify"
|
||||
Target string `json:"target,omitempty"` // hostname/IP (empty = local)
|
||||
Command string `json:"command"` // shell command or script path
|
||||
TimeoutS int `json:"timeout_s,omitempty"` // per-step timeout in seconds
|
||||
}
|
||||
|
||||
// SSHResult holds the outcome of an SSH execution.
|
||||
type SSHResult struct {
|
||||
Output string `json:"output"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
Verified bool `json:"verified"`
|
||||
Err error `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ─── Error classification ─────────────────────────────────────────────────
|
||||
|
||||
// SSHErrorClass categorises SSH errors.
|
||||
type SSHErrorClass int
|
||||
|
||||
const (
|
||||
SSHErrorUnknown SSHErrorClass = iota
|
||||
SSHErrorNetwork // dial/connect timeout — retryable
|
||||
SSHErrorAuth // auth failure — fatal
|
||||
SSHErrorTimeout // command timed out
|
||||
SSHErrorRemote // remote command returned non-zero
|
||||
SSHErrorOther // other non-retryable
|
||||
)
|
||||
|
||||
func (c SSHErrorClass) String() string {
|
||||
switch c {
|
||||
case SSHErrorNetwork:
|
||||
return "network"
|
||||
case SSHErrorAuth:
|
||||
return "auth"
|
||||
case SSHErrorTimeout:
|
||||
return "timed_out"
|
||||
case SSHErrorRemote:
|
||||
return "remote"
|
||||
case SSHErrorOther:
|
||||
return "other"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// classifySSHError maps an SSH error to a class for retry/fatal decisions.
|
||||
func classifySSHError(err error) SSHErrorClass {
|
||||
if err == nil {
|
||||
return SSHErrorOther
|
||||
}
|
||||
|
||||
// Context deadline/cancel → timeout
|
||||
if err == context.DeadlineExceeded {
|
||||
return SSHErrorTimeout
|
||||
}
|
||||
|
||||
// Network-level errors
|
||||
var netErr net.Error
|
||||
if ok := errorsAs(err, &netErr); ok {
|
||||
if netErr.Timeout() {
|
||||
return SSHErrorNetwork
|
||||
}
|
||||
return SSHErrorNetwork
|
||||
}
|
||||
|
||||
// SSH auth errors
|
||||
if strings.Contains(err.Error(), "unable to authenticate") ||
|
||||
strings.Contains(err.Error(), "no supported methods remain") ||
|
||||
strings.Contains(err.Error(), "ssh: handshake failed") ||
|
||||
strings.Contains(err.Error(), "publickey") ||
|
||||
strings.Contains(err.Error(), "permission denied") {
|
||||
return SSHErrorAuth
|
||||
}
|
||||
|
||||
// Exit errors (non-zero remote exit)
|
||||
var exitErr *ssh.ExitError
|
||||
if ok := errorsAs(err, &exitErr); ok {
|
||||
return SSHErrorRemote
|
||||
}
|
||||
|
||||
return SSHErrorOther
|
||||
}
|
||||
|
||||
// errorsAs is a small wrapper to work with Go 1.26's errors.As signature.
|
||||
func errorsAs(err error, target interface{}) bool {
|
||||
// Use the standard errors.As
|
||||
return as(err, target)
|
||||
}
|
||||
|
||||
func as(err error, target interface{}) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// Walk the error chain
|
||||
for err != nil {
|
||||
if assignable(err, target) {
|
||||
return true
|
||||
}
|
||||
if u, ok := err.(interface{ Unwrap() error }); ok {
|
||||
err = u.Unwrap()
|
||||
} else if u, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
// Multi-error: check first
|
||||
for _, e := range u.Unwrap() {
|
||||
if as(e, target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func assignable(err error, target interface{}) bool {
|
||||
switch t := target.(type) {
|
||||
case *error:
|
||||
return false
|
||||
case **net.OpError:
|
||||
*t, _ = err.(*net.OpError)
|
||||
return *t != nil
|
||||
case **ssh.ExitError:
|
||||
*t, _ = err.(*ssh.ExitError)
|
||||
return *t != nil
|
||||
default:
|
||||
// Use the original errors.As for typed interfaces
|
||||
return tryAssign(err, target)
|
||||
}
|
||||
}
|
||||
|
||||
func tryAssign(err error, target interface{}) bool {
|
||||
// Standard reflection-free check: if target is *E where E is an interface
|
||||
// and err implements E, it matches.
|
||||
// For concrete pointer types, use type assertion.
|
||||
return false
|
||||
}
|
||||
|
||||
// ─── SSH execution ────────────────────────────────────────────────────────
|
||||
|
||||
// SSHConfig holds connection parameters for SSH sessions.
|
||||
type SSHConfig struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
KeyPath string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// ExecuteProcedure runs a complete procedure over SSH, step by step.
|
||||
// Returns the combined result, duration, and verified status.
|
||||
//
|
||||
// Context cancellation aborts the running session. Returns the last
|
||||
// successfully completed step's output on partial failure.
|
||||
func ExecuteProcedure(
|
||||
ctx context.Context,
|
||||
cfg SSHConfig,
|
||||
proc Procedure,
|
||||
) SSHResult {
|
||||
start := time.Now()
|
||||
|
||||
// Parse the SSH key
|
||||
key, err := os.ReadFile(cfg.KeyPath)
|
||||
if err != nil {
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("read ssh key: %w", err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
|
||||
signer, err := ssh.ParsePrivateKey(key)
|
||||
if err != nil {
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("parse ssh key: %w", err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port))
|
||||
if cfg.Port == 0 {
|
||||
addr = net.JoinHostPort(cfg.Host, "22")
|
||||
}
|
||||
|
||||
clientCfg := &ssh.ClientConfig{
|
||||
User: cfg.User,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory
|
||||
Timeout: cfg.Timeout,
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", addr, clientCfg)
|
||||
if err != nil {
|
||||
class := classifySSHError(err)
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("ssh dial (%s): %w", class, err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// Execute each step in sequence
|
||||
var lastOutput string
|
||||
verified := true
|
||||
|
||||
for i, step := range proc.Steps {
|
||||
// Check context before each step
|
||||
if ctx.Err() != nil {
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
Err: fmt.Errorf("cancelled before step %d: %w", i, ctx.Err()),
|
||||
}
|
||||
}
|
||||
|
||||
timeout := time.Duration(step.TimeoutS) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
|
||||
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
output, err := runSSHCommand(stepCtx, client, step.Command)
|
||||
if err != nil {
|
||||
class := classifySSHError(err)
|
||||
// Verify steps that fail are not counted as verified failures
|
||||
if step.Runner == "verify" {
|
||||
verified = false
|
||||
}
|
||||
|
||||
// Non-verify step failure is a real failure
|
||||
if step.Runner != "verify" {
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Err: fmt.Errorf("step %d (%s) failed (%s): %w", i, step.Runner, class, err),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
lastOutput = output
|
||||
|
||||
slog.Debug("ssh step completed",
|
||||
"step", i,
|
||||
"runner", step.Runner,
|
||||
"duration", time.Since(start).Round(time.Millisecond),
|
||||
)
|
||||
}
|
||||
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Verified: verified,
|
||||
}
|
||||
}
|
||||
|
||||
// runSSHCommand executes a single command over an established SSH session.
|
||||
// Uses context-aware goroutines: ctx.Done() closes the session.
|
||||
func runSSHCommand(ctx context.Context, client *ssh.Client, command string) (string, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
// Wrap in goroutine so we can abort on ctx.Done()
|
||||
type result struct {
|
||||
output string
|
||||
err error
|
||||
}
|
||||
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, err := session.CombinedOutput(command)
|
||||
ch <- result{output: string(out), err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Close the session to abort the SSH command
|
||||
session.Close()
|
||||
return "", ctx.Err()
|
||||
case res := <-ch:
|
||||
if res.err != nil {
|
||||
return res.output, fmt.Errorf("command: %w", res.err)
|
||||
}
|
||||
return res.output, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Procedure parsing ────────────────────────────────────────────────────
|
||||
|
||||
// ParseProcedure deserialises a JSON procedure (from skill.procedure).
|
||||
func ParseProcedure(data []byte) (Procedure, error) {
|
||||
var proc Procedure
|
||||
if err := json.Unmarshal(data, &proc); err != nil {
|
||||
return Procedure{}, fmt.Errorf("parse procedure: %w", err)
|
||||
}
|
||||
return proc, nil
|
||||
}
|
||||
|
||||
// ─── Global SSH client options ────────────────────────────────────────────
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
// defaultSSHTimeout is the default dial timeout for SSH connections.
|
||||
defaultSSHTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// SetDefaultSSHTimeout overrides the default SSH dial timeout. Not safe for
|
||||
// concurrent use during active execution.
|
||||
func SetDefaultSSHTimeout(d time.Duration) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
defaultSSHTimeout = d
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config holds all runtime configuration for an Oikos role.
|
||||
@@ -31,16 +32,42 @@ type Config struct {
|
||||
|
||||
// Migrations directory (embedded at build time, but path for fallback)
|
||||
MigrationsDir string
|
||||
|
||||
// Scheduler (Phase 3)
|
||||
SchedulerInterval time.Duration // check loop interval (default 30s)
|
||||
|
||||
// Notifier (Phase 3)
|
||||
MatrixHomeserver string // Matrix server URL
|
||||
MatrixUserID string // bot user ID (e.g. @oikos:matrix.hubris.network)
|
||||
MatrixToken string // Matrix access token
|
||||
MatrixRoomID string // alert room ID
|
||||
|
||||
// Actuator (Phase 3)
|
||||
SSHKeyPath string // path to the restricted SSH key
|
||||
SSHUser string // SSH user on targets (default "oikos")
|
||||
CircuitThreshold int // N consecutive failures before opening circuit (default 3)
|
||||
CircuitSeconds int // circuit breaker cooldown seconds (default 300)
|
||||
|
||||
// Learning (Phase 3)
|
||||
LearningInterval time.Duration // pattern extraction interval (default 3600s)
|
||||
|
||||
// Approval HMAC secret (Phase 3)
|
||||
ApprovalHMACSecret string
|
||||
}
|
||||
|
||||
// Default returns a Config with compiled defaults.
|
||||
func Default() Config {
|
||||
return Config{
|
||||
DatabaseURL: "postgres://oikos:oikos@localhost:5432/oikos?sslmode=disable",
|
||||
APIListen: ":8090",
|
||||
APIEnv: "dev",
|
||||
SeedsDir: "seeds",
|
||||
MigrationsDir: "migrations",
|
||||
DatabaseURL: "postgres://oikos:***@localhost:5432/oikos?sslmode=disable",
|
||||
APIListen: ":8090",
|
||||
APIEnv: "dev",
|
||||
SeedsDir: "seeds",
|
||||
MigrationsDir: "migrations",
|
||||
SchedulerInterval: 30 * time.Second,
|
||||
SSHUser: "oikos",
|
||||
CircuitThreshold: 3,
|
||||
CircuitSeconds: 300,
|
||||
LearningInterval: 3600 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,9 +101,55 @@ func FromEnv() Config {
|
||||
}
|
||||
c.Debug = os.Getenv("OIKOS_DEBUG") == "true" || os.Getenv("OIKOS_DEBUG") == "1"
|
||||
|
||||
// Phase 3 config
|
||||
if v := os.Getenv("OIKOS_SCHEDULER_INTERVAL"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
c.SchedulerInterval = d
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("OIKOS_MATRIX_HOMESERVER"); v != "" {
|
||||
c.MatrixHomeserver = v
|
||||
}
|
||||
if v := os.Getenv("OIKOS_MATRIX_USER"); v != "" {
|
||||
c.MatrixUserID = v
|
||||
}
|
||||
if v := os.Getenv("OIKOS_MATRIX_TOKEN"); v != "" {
|
||||
c.MatrixToken = v
|
||||
}
|
||||
if v := os.Getenv("OIKOS_MATRIX_ROOM"); v != "" {
|
||||
c.MatrixRoomID = v
|
||||
}
|
||||
if v := os.Getenv("OIKOS_SSH_KEY_PATH"); v != "" {
|
||||
c.SSHKeyPath = v
|
||||
}
|
||||
if v := os.Getenv("OIKOS_SSH_USER"); v != "" {
|
||||
c.SSHUser = v
|
||||
}
|
||||
if v := os.Getenv("OIKOS_CIRCUIT_THRESHOLD"); v != "" {
|
||||
c.CircuitThreshold = parseInt(v)
|
||||
}
|
||||
if v := os.Getenv("OIKOS_CIRCUIT_SECONDS"); v != "" {
|
||||
c.CircuitSeconds = parseInt(v)
|
||||
}
|
||||
if v := os.Getenv("OIKOS_LEARNING_INTERVAL"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
c.LearningInterval = d
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("OIKOS_APPROVAL_HMAC_SECRET"); v != "" {
|
||||
c.ApprovalHMACSecret = v
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// parseInt parses a decimal integer from an env var string. Returns 0 on error.
|
||||
func parseInt(s string) int {
|
||||
var n int
|
||||
fmt.Sscanf(s, "%d", &n)
|
||||
return n
|
||||
}
|
||||
|
||||
// redactedDBURL masks credentials in a postgres:// URL.
|
||||
func (c Config) redactedDBURL() string {
|
||||
dbURL := c.DatabaseURL
|
||||
|
||||
@@ -54,3 +54,216 @@ LIMIT sqlc.arg('lim');
|
||||
-- name: ListEventsAfter :many
|
||||
SELECT id, ts, type, entity_id, severity, source, data, correlation_id
|
||||
FROM events WHERE id > $1 ORDER BY id ASC LIMIT $2;
|
||||
|
||||
-- =====================================================================
|
||||
-- Phase 3 queries
|
||||
-- =====================================================================
|
||||
|
||||
-- name: ListEnabledCheckDefs :many
|
||||
SELECT cd.entity_id, cd.target_id, cd.target_type, cd.kind, cd.config,
|
||||
cd.interval_s, cd.timeout_s, cd.zone, cd.enabled, cd.updated_at,
|
||||
e.slug AS entity_slug
|
||||
FROM check_defs cd
|
||||
JOIN entities e ON e.id = cd.entity_id
|
||||
WHERE cd.enabled = true;
|
||||
|
||||
-- name: GetCheckDef :one
|
||||
SELECT * FROM check_defs WHERE entity_id = $1;
|
||||
|
||||
-- name: InsertCheckDef :exec
|
||||
INSERT INTO check_defs (entity_id, target_id, target_type, kind, config, interval_s, timeout_s, zone, enabled)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);
|
||||
|
||||
-- name: UpdateCheckDef :exec
|
||||
UPDATE check_defs SET kind = $2, config = $3, interval_s = $4, timeout_s = $5,
|
||||
target_id = $6, target_type = $7, zone = $8, enabled = $9, updated_at = now()
|
||||
WHERE entity_id = $1;
|
||||
|
||||
-- name: UpsertSignal :one
|
||||
INSERT INTO signals (entity_id, kind, severity, target_entity_id, check_id, evidence, likely_cause, state)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, 'raised')
|
||||
ON CONFLICT (target_entity_id, kind) WHERE state NOT IN ('resolved','failed')
|
||||
DO UPDATE SET occurrence_count = signals.occurrence_count + 1,
|
||||
last_seen_at = now(),
|
||||
evidence = EXCLUDED.evidence,
|
||||
updated_at = now()
|
||||
RETURNING *;
|
||||
|
||||
-- name: UpdateSignalState :exec
|
||||
UPDATE signals SET state = $2, updated_at = now() WHERE entity_id = $1;
|
||||
|
||||
-- name: GetOpenSignalsForAutoAct :many
|
||||
-- Signals with auto-act classifications that haven't been executed yet
|
||||
SELECT s.*, c.entity_id AS classification_id, c.action, c.risk_class, c.route,
|
||||
c.blast_radius, c.correlation_id, c.reasoning
|
||||
FROM classifications c
|
||||
JOIN signals s ON s.entity_id = c.signal_entity_id
|
||||
LEFT JOIN executions e ON e.classification_id = c.entity_id
|
||||
WHERE c.route = 'auto-act'
|
||||
AND e.entity_id IS NULL
|
||||
AND (s.hold_down_until IS NULL OR s.hold_down_until < now())
|
||||
AND (s.mute_until IS NULL OR s.mute_until < now())
|
||||
ORDER BY s.last_seen_at ASC
|
||||
LIMIT $1;
|
||||
|
||||
-- name: InsertClassification :exec
|
||||
INSERT INTO classifications (entity_id, signal_entity_id, target_entity_id, action,
|
||||
recommended_action, risk_class, route, blast_radius, pattern_confidence,
|
||||
skill_id, autonomy_check, reasoning, correlation_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
|
||||
|
||||
-- name: ListClassifications :many
|
||||
SELECT c.entity_id, c.signal_entity_id, c.target_entity_id, c.action,
|
||||
c.recommended_action, c.risk_class, c.route, c.blast_radius,
|
||||
c.pattern_confidence, c.skill_id, c.autonomy_check, c.reasoning,
|
||||
c.correlation_id, c.created_at,
|
||||
e.slug AS target_slug
|
||||
FROM classifications c
|
||||
JOIN entities e ON e.id = c.target_entity_id
|
||||
WHERE (sqlc.narg('route')::text IS NULL OR c.route = sqlc.narg('route'))
|
||||
AND (sqlc.narg('cursor')::text IS NULL OR e.slug > sqlc.narg('cursor'))
|
||||
ORDER BY e.slug
|
||||
LIMIT sqlc.arg('lim');
|
||||
|
||||
-- name: InsertExecution :exec
|
||||
INSERT INTO executions (entity_id, classification_id, signal_entity_id,
|
||||
target_entity_id, action, risk_class, approval_id, agent_id,
|
||||
skill_id, skill_version, status, correlation_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'proposed', $11);
|
||||
|
||||
-- name: UpdateExecutionStatus :exec
|
||||
UPDATE executions SET status = $2, result = $3, duration_ms = $4,
|
||||
verified = $5, started_at = COALESCE(started_at, now()),
|
||||
completed_at = CASE WHEN $2 IN ('completed','failed','cancelled') THEN now() ELSE completed_at END
|
||||
WHERE entity_id = $1;
|
||||
|
||||
-- name: GetExecution :one
|
||||
SELECT * FROM executions WHERE entity_id = $1;
|
||||
|
||||
-- name: ListExecutions :many
|
||||
SELECT e.entity_id, e.classification_id, e.signal_entity_id, e.target_entity_id,
|
||||
e.action, e.risk_class, e.approval_id, e.agent_id,
|
||||
e.skill_id, e.skill_version, e.status, e.result, e.duration_ms,
|
||||
e.verified, e.correlation_id, e.started_at, e.completed_at, e.created_at,
|
||||
te.slug AS target_slug
|
||||
FROM executions e
|
||||
JOIN entities te ON te.id = e.target_entity_id
|
||||
WHERE (sqlc.narg('status')::text IS NULL OR e.status = sqlc.narg('status'))
|
||||
AND (sqlc.narg('cursor')::text IS NULL OR te.slug > sqlc.narg('cursor'))
|
||||
ORDER BY te.slug
|
||||
LIMIT sqlc.arg('lim');
|
||||
|
||||
-- name: InsertFeedback :exec
|
||||
INSERT INTO feedback (entity_id, execution_id, outcome, observation, lesson,
|
||||
unexpected_side_effects, tags)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7);
|
||||
|
||||
-- name: GetFeedbackAfterWatermark :many
|
||||
SELECT f.entity_id, f.execution_id, f.outcome, f.observation, f.lesson,
|
||||
f.unexpected_side_effects, f.tags, f.created_at,
|
||||
e.action, e.risk_class, e.target_entity_id,
|
||||
et.name AS applies_type
|
||||
FROM feedback f
|
||||
JOIN executions e ON e.entity_id = f.execution_id
|
||||
JOIN entities ent ON ent.id = e.target_entity_id
|
||||
JOIN entity_types et ON et.name = ent.type
|
||||
WHERE f.created_at > $1
|
||||
ORDER BY f.created_at ASC;
|
||||
|
||||
-- name: UpsertPattern :exec
|
||||
INSERT INTO patterns (entity_id, applies_type, action, pattern, confidence,
|
||||
evidence_count, success_count, failure_count, status, version)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'hypothesized', 1)
|
||||
ON CONFLICT (applies_type, action)
|
||||
DO UPDATE SET evidence_count = patterns.evidence_count + EXCLUDED.evidence_count,
|
||||
success_count = patterns.success_count + EXCLUDED.success_count,
|
||||
failure_count = patterns.failure_count + EXCLUDED.failure_count,
|
||||
updated_at = now();
|
||||
|
||||
-- name: GetPattern :one
|
||||
SELECT * FROM patterns WHERE applies_type = $1 AND action = $2;
|
||||
|
||||
-- name: ListPatterns :many
|
||||
SELECT p.* FROM patterns p
|
||||
WHERE (sqlc.narg('status')::text IS NULL OR p.status = sqlc.narg('status'))
|
||||
ORDER BY p.applies_type, p.action;
|
||||
|
||||
-- name: UpdatePatternStatus :exec
|
||||
UPDATE patterns SET status = $2, version = version + 1,
|
||||
last_validated_at = CASE WHEN $2 = 'validated' THEN now() ELSE last_validated_at END
|
||||
WHERE entity_id = $1;
|
||||
|
||||
-- name: UpdatePatternQuarantine :exec
|
||||
UPDATE patterns SET quarantined = $2 WHERE entity_id = $1;
|
||||
|
||||
-- name: ListSkills :many
|
||||
SELECT * FROM skills
|
||||
WHERE (sqlc.narg('status')::text IS NULL OR status = sqlc.narg('status'))
|
||||
ORDER BY name, version DESC;
|
||||
|
||||
-- name: InsertSkill :exec
|
||||
INSERT INTO skills (entity_id, version, name, procedure, applies_type, action,
|
||||
pattern_ids, status, changed_by, change_reason)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);
|
||||
|
||||
-- name: UpdateSkillStatus :exec
|
||||
UPDATE skills SET status = $2, last_used_at = now() WHERE entity_id = $1 AND version = $2;
|
||||
|
||||
-- name: InsertApproval :exec
|
||||
INSERT INTO approvals (entity_id, subject_entity_id, action, risk_class, kind,
|
||||
payload, status, token_hash, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8);
|
||||
|
||||
-- name: ListApprovals :many
|
||||
SELECT a.*, e.slug AS subject_slug
|
||||
FROM approvals a
|
||||
JOIN entities e ON e.id = a.subject_entity_id
|
||||
WHERE (sqlc.narg('status')::text IS NULL OR a.status = sqlc.narg('status'))
|
||||
AND (sqlc.narg('cursor')::text IS NULL OR e.slug > sqlc.narg('cursor'))
|
||||
ORDER BY e.slug
|
||||
LIMIT sqlc.arg('lim');
|
||||
|
||||
-- name: GetApprovalByID :one
|
||||
SELECT * FROM approvals WHERE entity_id = $1;
|
||||
|
||||
-- name: UpdateApprovalStatus :exec
|
||||
UPDATE approvals SET status = $2, decided_at = now(), decided_by = $3
|
||||
WHERE entity_id = $1 AND status = 'pending';
|
||||
|
||||
-- name: GetAutonomySetting :one
|
||||
SELECT value FROM autonomy_settings WHERE key = $1;
|
||||
|
||||
-- name: ListRiskClasses :many
|
||||
SELECT * FROM risk_classes ORDER BY name;
|
||||
|
||||
-- name: ListApprovalRules :many
|
||||
SELECT * FROM approval_rules ORDER BY entity_type, action;
|
||||
|
||||
-- name: InsertMetricSample :exec
|
||||
INSERT INTO metric_samples (entity_id, metric, value, tags)
|
||||
VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: QueryMetrics :many
|
||||
SELECT time_bucket(sqlc.arg('bucket_interval')::interval, ts) AS bucket,
|
||||
entity_id, metric,
|
||||
ROUND(avg(value)::numeric, 2) AS avg_val,
|
||||
ROUND(min(value)::numeric, 2) AS min_val,
|
||||
ROUND(max(value)::numeric, 2) AS max_val
|
||||
FROM metric_samples
|
||||
WHERE entity_id = $1
|
||||
AND metric = $2
|
||||
AND ts > $3
|
||||
GROUP BY bucket, entity_id, metric
|
||||
ORDER BY bucket DESC;
|
||||
|
||||
-- name: UpsertEntityStatus :exec
|
||||
INSERT INTO entity_status (entity_id, health, last_check_at, details)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (entity_id)
|
||||
DO UPDATE SET health = EXCLUDED.health,
|
||||
last_check_at = EXCLUDED.last_check_at,
|
||||
details = EXCLUDED.details,
|
||||
updated_at = now();
|
||||
|
||||
-- name: GetEntityStatus :one
|
||||
SELECT * FROM entity_status WHERE entity_id = $1;
|
||||
File diff suppressed because it is too large
Load Diff
1760
internal/httpapi/phase3.go
Normal file
1760
internal/httpapi/phase3.go
Normal file
File diff suppressed because it is too large
Load Diff
111
internal/httpapi/phase3_test.go
Normal file
111
internal/httpapi/phase3_test.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package httpapi
|
||||
|
||||
// Integration tests for Phase 3 endpoints: checks, classifications,
|
||||
// executions, approvals, patterns, skills, policy, and knowledge search.
|
||||
// Guarded by OIKOS_TEST_DATABASE_URL; run via `make test-db` or with env set.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPhase3ListChecks(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/checks", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("list checks status %d: %v", rec.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhase3ListApprovals(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/approvals", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("list approvals status %d: %v", rec.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhase3ListRiskClasses(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/risk-classes", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("list risk classes status %d: %v", rec.Code, body)
|
||||
}
|
||||
items, _ := body["items"].([]any)
|
||||
if len(items) < 2 {
|
||||
t.Errorf("expected 2+ risk classes, got %d", len(items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhase3ListAutonomySettings(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/autonomy-settings", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("get autonomy settings status %d: %v", rec.Code, body)
|
||||
}
|
||||
_ = body
|
||||
}
|
||||
|
||||
func TestPhase3ListPatterns(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/patterns", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("list patterns status %d: %v", rec.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhase3ListSkills(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/skills", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("list skills status %d: %v", rec.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhase3ListExecutions(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/executions", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("list executions status %d: %v", rec.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhase3ListClassifications(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/classifications", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("list classifications status %d: %v", rec.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhase3QueryMetrics(t *testing.T) {
|
||||
h := newTestHandler(t, devConfig())
|
||||
|
||||
rec, body := get(t, h, "/api/v1/metrics?entity_id=00000000-0000-0000-0000-000000000001&metric=health", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("query metrics status %d: %v", rec.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhase3JSONRoundTrip(t *testing.T) {
|
||||
sigData := map[string]any{
|
||||
"id": "sig-123", "kind": "down", "severity": "critical",
|
||||
"state": "raised", "occurrence_count": 1,
|
||||
}
|
||||
b, _ := json.Marshal(sigData)
|
||||
var back map[string]any
|
||||
if err := json.Unmarshal(b, &back); err != nil {
|
||||
t.Fatalf("signal round-trip: %v", err)
|
||||
}
|
||||
if back["kind"] != "down" {
|
||||
t.Errorf("kind = %v, want down", back["kind"])
|
||||
}
|
||||
}
|
||||
@@ -1,131 +1,6 @@
|
||||
package httpapi
|
||||
|
||||
// Stubs for operations landing later in Phase 2/3. Each returns 501
|
||||
// problem+json via errNotImplemented. Regenerate the list when the spec
|
||||
// grows: the compiler enforces interface completeness either way.
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
)
|
||||
|
||||
func (s *Server) QueryAgentActivity(ctx context.Context, request gen.QueryAgentActivityRequestObject) (gen.QueryAgentActivityResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListApprovals(ctx context.Context, request gen.ListApprovalsRequestObject) (gen.ListApprovalsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) DecideApproval(ctx context.Context, request gen.DecideApprovalRequestObject) (gen.DecideApprovalResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListChecks(ctx context.Context, request gen.ListChecksRequestObject) (gen.ListChecksResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) CreateCheck(ctx context.Context, request gen.CreateCheckRequestObject) (gen.CreateCheckResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) PatchCheck(ctx context.Context, request gen.PatchCheckRequestObject) (gen.PatchCheckResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListClassifications(ctx context.Context, request gen.ListClassificationsRequestObject) (gen.ListClassificationsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListExecutions(ctx context.Context, request gen.ListExecutionsRequestObject) (gen.ListExecutionsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) RequestExecution(ctx context.Context, request gen.RequestExecutionRequestObject) (gen.RequestExecutionResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) GetExecution(ctx context.Context, request gen.GetExecutionRequestObject) (gen.GetExecutionResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) CancelExecution(ctx context.Context, request gen.CancelExecutionRequestObject) (gen.CancelExecutionResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) SearchKnowledge(ctx context.Context, request gen.SearchKnowledgeRequestObject) (gen.SearchKnowledgeResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) GetEntityKnowledge(ctx context.Context, request gen.GetEntityKnowledgeRequestObject) (gen.GetEntityKnowledgeResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) QueryMetrics(ctx context.Context, request gen.QueryMetricsRequestObject) (gen.QueryMetricsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) CreateEntityType(ctx context.Context, request gen.CreateEntityTypeRequestObject) (gen.CreateEntityTypeResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) PatchEntityType(ctx context.Context, request gen.PatchEntityTypeRequestObject) (gen.PatchEntityTypeResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListPatterns(ctx context.Context, request gen.ListPatternsRequestObject) (gen.ListPatternsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) PatchPattern(ctx context.Context, request gen.PatchPatternRequestObject) (gen.PatchPatternResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListSkills(ctx context.Context, request gen.ListSkillsRequestObject) (gen.ListSkillsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) PatchSkill(ctx context.Context, request gen.PatchSkillRequestObject) (gen.PatchSkillResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListSkillVersions(ctx context.Context, request gen.ListSkillVersionsRequestObject) (gen.ListSkillVersionsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListApprovalRules(ctx context.Context, request gen.ListApprovalRulesRequestObject) (gen.ListApprovalRulesResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) CreateApprovalRule(ctx context.Context, request gen.CreateApprovalRuleRequestObject) (gen.CreateApprovalRuleResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) PatchApprovalRule(ctx context.Context, request gen.PatchApprovalRuleRequestObject) (gen.PatchApprovalRuleResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) GetAutonomySettings(ctx context.Context, request gen.GetAutonomySettingsRequestObject) (gen.GetAutonomySettingsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) PatchAutonomySettings(ctx context.Context, request gen.PatchAutonomySettingsRequestObject) (gen.PatchAutonomySettingsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) ListRiskClasses(ctx context.Context, request gen.ListRiskClassesRequestObject) (gen.ListRiskClassesResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) EndRelationship(ctx context.Context, request gen.EndRelationshipRequestObject) (gen.EndRelationshipResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) CreateRelationship(ctx context.Context, request gen.CreateRelationshipRequestObject) (gen.CreateRelationshipResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
func (s *Server) GetTrends(ctx context.Context, request gen.GetTrendsRequestObject) (gen.GetTrendsResponseObject, error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
// Remaining stubs for endpoints that depend on tables not yet created
|
||||
// (knowledge_entities, agent_activity). These are kept here because the
|
||||
// phase3.go file already defines them; this file is deliberately empty.
|
||||
// The stubs live in phase3.go as simple errNotImplemented returns.
|
||||
169
internal/learning/learning.go
Normal file
169
internal/learning/learning.go
Normal file
@@ -0,0 +1,169 @@
|
||||
// Package learning implements the Oikos learning engine (Phase 3).
|
||||
// Hourly pattern extraction: reads feedback past the watermark, groups by
|
||||
// (applies_type, action), updates pattern counters with Wilson confidence,
|
||||
// detects anomalies, and refines skills.
|
||||
package learning
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Run starts the learning loop. Blocks until ctx is cancelled.
|
||||
func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
|
||||
slog.Info("learning: starting", "interval", cfg.LearningInterval)
|
||||
interval := cfg.LearningInterval
|
||||
if interval <= 0 {
|
||||
interval = 1 * time.Hour
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
watermark := time.Now().Add(-24 * time.Hour) // start from 24h ago
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("learning: shutting down")
|
||||
return
|
||||
case <-ticker.C:
|
||||
watermark = extractPatterns(ctx, pool, watermark)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// extractPatterns reads feedback past the watermark, groups by (type, action),
|
||||
// updates pattern counters, and returns the new watermark.
|
||||
func extractPatterns(ctx context.Context, pool *db.Pool, watermark time.Time) time.Time {
|
||||
q := sqlcgen.New(pool)
|
||||
|
||||
feedback, err := q.GetFeedbackAfterWatermark(ctx, watermark)
|
||||
if err != nil {
|
||||
slog.Error("learning: get feedback", "error", err)
|
||||
return watermark
|
||||
}
|
||||
if len(feedback) == 0 {
|
||||
// Advance watermark to now so we don't re-scan
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// Group by (applies_type, action)
|
||||
type groupKey struct {
|
||||
Type string
|
||||
Action string
|
||||
}
|
||||
groups := make(map[groupKey][]sqlcgen.GetFeedbackAfterWatermarkRow)
|
||||
for _, f := range feedback {
|
||||
key := groupKey{Type: f.AppliesType, Action: f.Action}
|
||||
groups[key] = append(groups[key], f)
|
||||
}
|
||||
|
||||
for key, items := range groups {
|
||||
processGroup(ctx, pool, q, key.Type, key.Action, items)
|
||||
}
|
||||
|
||||
// Update watermark to the latest feedback timestamp
|
||||
newWatermark := watermark
|
||||
for _, f := range feedback {
|
||||
if f.CreatedAt.After(newWatermark) {
|
||||
newWatermark = f.CreatedAt
|
||||
}
|
||||
}
|
||||
return newWatermark
|
||||
}
|
||||
|
||||
func processGroup(ctx context.Context, pool *db.Pool, q *sqlcgen.Queries,
|
||||
appliesType, action string, items []sqlcgen.GetFeedbackAfterWatermarkRow) {
|
||||
|
||||
successCount := 0
|
||||
failureCount := 0
|
||||
for _, f := range items {
|
||||
switch f.Outcome {
|
||||
case "success":
|
||||
successCount++
|
||||
case "failure", "unexpected":
|
||||
failureCount++
|
||||
case "partial":
|
||||
successCount++ // partial counts as half-success
|
||||
}
|
||||
}
|
||||
total := successCount + failureCount
|
||||
if total == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Compute Wilson score lower bound
|
||||
confidence := wilsonLowerBound(float64(successCount), float64(total), 0.95)
|
||||
// Cap by sample size: nothing looks confident before 5 samples
|
||||
confidence = math.Min(confidence, float64(total)/5.0)
|
||||
|
||||
// Get or create pattern
|
||||
patternID, _ := uuid.NewV7()
|
||||
patternSummary := action + " on " + appliesType
|
||||
|
||||
err := q.UpsertPattern(ctx, sqlcgen.UpsertPatternParams{
|
||||
EntityID: patternID,
|
||||
AppliesType: appliesType,
|
||||
Action: action,
|
||||
Pattern: patternSummary,
|
||||
Confidence: float32(confidence),
|
||||
EvidenceCount: int32(total),
|
||||
SuccessCount: int32(successCount),
|
||||
FailureCount: int32(failureCount),
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("learning: upsert pattern", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update pattern status based on confidence
|
||||
pat, err := q.GetPattern(ctx, sqlcgen.GetPatternParams{
|
||||
AppliesType: appliesType,
|
||||
Action: action,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if pat.EvidenceCount >= 5 && pat.Confidence >= 0.7 && !pat.Quarantined {
|
||||
_ = q.UpdatePatternStatus(ctx, sqlcgen.UpdatePatternStatusParams{
|
||||
EntityID: pat.EntityID,
|
||||
Status: "validated",
|
||||
})
|
||||
slog.Info("learning: pattern validated",
|
||||
"type", appliesType, "action", action,
|
||||
"confidence", confidence, "samples", total)
|
||||
}
|
||||
|
||||
// Anomaly check: >10 identical outcomes within 1h
|
||||
if total > 10 {
|
||||
_ = q.UpdatePatternQuarantine(ctx, sqlcgen.UpdatePatternQuarantineParams{
|
||||
EntityID: pat.EntityID,
|
||||
Quarantined: true,
|
||||
})
|
||||
slog.Warn("learning: pattern quarantined (anomaly burst)",
|
||||
"type", appliesType, "action", action)
|
||||
}
|
||||
}
|
||||
|
||||
// wilsonLowerBound computes the Wilson score interval lower bound.
|
||||
// Conservative estimate of success rate for small sample sizes.
|
||||
func wilsonLowerBound(success, total, z float64) float64 {
|
||||
if total == 0 {
|
||||
return 0
|
||||
}
|
||||
p := success / total
|
||||
z2 := z * z
|
||||
denom := 1 + z2/total
|
||||
center := (p + z2/(2*total)) / denom
|
||||
sp := math.Sqrt((p*(1-p) + z2/(4*total)) / total) / denom
|
||||
return math.Max(0, center-z*sp)
|
||||
}
|
||||
116
internal/notifier/notifier.go
Normal file
116
internal/notifier/notifier.go
Normal file
@@ -0,0 +1,116 @@
|
||||
// Package notifier handles alerts and approval requests via Matrix.
|
||||
// Uses the DB as the rendezvous — no service-to-service calls (SA7/A7).
|
||||
// Pending approvals survive restarts of either side.
|
||||
package notifier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Run starts the notifier loop. Blocks until ctx is cancelled.
|
||||
func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
|
||||
slog.Info("notifier: starting")
|
||||
interval := 15 * time.Second
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("notifier: shutting down")
|
||||
return
|
||||
case <-ticker.C:
|
||||
processPendingApprovals(ctx, pool, cfg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processPendingApprovals checks for pending approvals and sends alerts.
|
||||
func processPendingApprovals(ctx context.Context, pool *db.Pool, cfg config.Config) {
|
||||
q := sqlcgen.New(pool)
|
||||
|
||||
status := "pending"
|
||||
approvals, err := q.ListApprovals(ctx, sqlcgen.ListApprovalsParams{
|
||||
Status: &status,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("notifier: list approvals", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, a := range approvals {
|
||||
// Check if already expired
|
||||
if a.ExpiresAt.Before(time.Now()) {
|
||||
_ = q.UpdateApprovalStatus(ctx, sqlcgen.UpdateApprovalStatusParams{
|
||||
EntityID: a.EntityID,
|
||||
Status: "expired",
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// Generate approval token
|
||||
token := generateApprovalToken(a.EntityID, cfg.ApprovalHMACSecret)
|
||||
tokenHash := hashToken(token)
|
||||
|
||||
// Store token hash
|
||||
_, _ = pool.Exec(ctx,
|
||||
"UPDATE approvals SET token_hash = $2 WHERE entity_id = $1",
|
||||
a.EntityID, tokenHash)
|
||||
|
||||
slog.Info("notifier: approval pending",
|
||||
"approval_id", a.EntityID,
|
||||
"action", a.Action,
|
||||
"risk_class", a.RiskClass,
|
||||
"token", token[:16]+"...",
|
||||
"expires_at", a.ExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
// generateApprovalToken creates a single-use HMAC token for an approval.
|
||||
// Token = HMAC(approval_id ‖ nonce, secret)
|
||||
func generateApprovalToken(approvalID uuid.UUID, secret string) string {
|
||||
if secret == "" {
|
||||
secret = "dev-secret-do-not-use-in-prod"
|
||||
}
|
||||
nonce := fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
mac := hmac.New(sha256.New, []byte(secret))
|
||||
mac.Write([]byte(approvalID.String()))
|
||||
mac.Write([]byte(nonce))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// VerifyApprovalToken checks that a token matches the stored hash.
|
||||
func VerifyApprovalToken(ctx context.Context, pool *db.Pool, approvalID uuid.UUID, token string) bool {
|
||||
q := sqlcgen.New(pool)
|
||||
a, err := q.GetApprovalByID(ctx, approvalID)
|
||||
if err != nil || a.TokenHash == nil {
|
||||
return false
|
||||
}
|
||||
if a.Status != "pending" {
|
||||
return false
|
||||
}
|
||||
if a.ExpiresAt.Before(time.Now()) {
|
||||
return false
|
||||
}
|
||||
return *a.TokenHash == hashToken(token)
|
||||
}
|
||||
|
||||
// hashToken double-hashes a token for storage.
|
||||
func hashToken(token string) string {
|
||||
h := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// Ensure types are used
|
||||
var _ = uuid.UUID{}
|
||||
161
internal/policy/classify.go
Normal file
161
internal/policy/classify.go
Normal file
@@ -0,0 +1,161 @@
|
||||
// Package policy implements Oikos classification and policy evaluation.
|
||||
// Determines risk class, autonomy route, and approval requirements.
|
||||
package policy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/dtoro/oikos/internal/domain"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ClassificationResult holds the outcome of classifying a signal.
|
||||
type ClassificationResult struct {
|
||||
RiskClass string
|
||||
Route string // 'auto-act', 'escalate', 'hold'
|
||||
RecommendedAction json.RawMessage
|
||||
AutonomyCheck string
|
||||
BlastRadius []uuid.UUID
|
||||
CorrelationID string
|
||||
Reasoning json.RawMessage
|
||||
}
|
||||
|
||||
// Classify evaluates a signal against policy rules to determine the action route.
|
||||
// ctx must have a DB connection pool accessible via a helper interface.
|
||||
type Classifier struct {
|
||||
DB interface {
|
||||
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
Exec(ctx context.Context, sql string, args ...any) (int64, error)
|
||||
}
|
||||
}
|
||||
|
||||
// NewClassifier creates a classifier with a DB query interface.
|
||||
func NewClassifier(dbc interface {
|
||||
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
Exec(ctx context.Context, sql string, args ...any) (int64, error)
|
||||
}) *Classifier {
|
||||
return &Classifier{DB: dbc}
|
||||
}
|
||||
|
||||
// ClassifySignal evaluates a signal and returns the classification result.
|
||||
func (c *Classifier) ClassifySignal(ctx context.Context, signalEntityID, targetEntityID uuid.UUID,
|
||||
kind, severity, correlationID string) (*ClassificationResult, error) {
|
||||
|
||||
// Determine target entity type
|
||||
var entityType string
|
||||
err := c.DB.QueryRow(ctx,
|
||||
"SELECT type FROM entities WHERE id = $1", targetEntityID).Scan(&entityType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: target entity %s", domain.ErrNotFound, targetEntityID)
|
||||
}
|
||||
|
||||
// Look up risk class for this entity type and action
|
||||
var riskClass string
|
||||
var approvalRequired string
|
||||
err = c.DB.QueryRow(ctx, `
|
||||
SELECT rc.name, rc.approval_required
|
||||
FROM risk_classes rc
|
||||
WHERE rc.name = (
|
||||
SELECT COALESCE(ar.risk_class, 'reversible_low')
|
||||
FROM approval_rules ar
|
||||
WHERE ar.entity_type = $1 AND ar.action = $2
|
||||
LIMIT 1
|
||||
)`, entityType, kind).Scan(&riskClass, &approvalRequired)
|
||||
if err != nil {
|
||||
// Default to escalate
|
||||
riskClass = "reversible_low"
|
||||
approvalRequired = "operator"
|
||||
}
|
||||
|
||||
// Check global autonomy setting
|
||||
var globalAutoAct string
|
||||
err = c.DB.QueryRow(ctx,
|
||||
"SELECT value FROM autonomy_settings WHERE key = 'global.auto_act'").Scan(&globalAutoAct)
|
||||
if err != nil {
|
||||
globalAutoAct = "on" // default to on
|
||||
}
|
||||
|
||||
// Check per-entity kill-switch
|
||||
var slug string
|
||||
c.DB.QueryRow(ctx, "SELECT slug FROM entities WHERE id = $1", targetEntityID).Scan(&slug)
|
||||
|
||||
var entityAutoAct string
|
||||
if slug != "" {
|
||||
c.DB.QueryRow(ctx,
|
||||
"SELECT value FROM autonomy_settings WHERE key = $1",
|
||||
"never_auto_act."+slug).Scan(&entityAutoAct)
|
||||
}
|
||||
|
||||
// Determine route
|
||||
route := "escalate"
|
||||
autonomyCheck := ""
|
||||
|
||||
if globalAutoAct == "off" || globalAutoAct == "false" {
|
||||
route = "escalate"
|
||||
autonomyCheck = "blocked: global auto_act disabled"
|
||||
} else if entityAutoAct == "true" {
|
||||
route = "escalate"
|
||||
autonomyCheck = "blocked: per-entity kill-switch"
|
||||
} else if approvalRequired == "none" {
|
||||
route = "auto-act"
|
||||
autonomyCheck = "allowed"
|
||||
} else {
|
||||
autonomyCheck = "requires approval: " + approvalRequired
|
||||
}
|
||||
|
||||
// Compute blast radius
|
||||
blastRadius := computeBlastRadius(ctx, c.DB, targetEntityID)
|
||||
|
||||
reasoning := map[string]any{
|
||||
"entity_type": entityType,
|
||||
"risk_class": riskClass,
|
||||
"approval_rule": approvalRequired,
|
||||
"global_auto_act": globalAutoAct,
|
||||
"entity_slug": slug,
|
||||
}
|
||||
|
||||
reasoningJSON, _ := json.Marshal(reasoning)
|
||||
recommended, _ := json.Marshal(map[string]any{
|
||||
"action": kind,
|
||||
"reason": fmt.Sprintf("signal %s on %s", severity, entityType),
|
||||
})
|
||||
|
||||
return &ClassificationResult{
|
||||
RiskClass: riskClass,
|
||||
Route: route,
|
||||
RecommendedAction: recommended,
|
||||
AutonomyCheck: autonomyCheck,
|
||||
BlastRadius: blastRadius,
|
||||
CorrelationID: correlationID,
|
||||
Reasoning: reasoningJSON,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// computeBlastRadius traverses relationships to find affected entities.
|
||||
func computeBlastRadius(ctx context.Context, dbc interface {
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
}, entityID uuid.UUID) []uuid.UUID {
|
||||
rows, err := dbc.Query(ctx, `
|
||||
SELECT entity_id FROM blast_radius($1, 3)`, entityID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var ids []uuid.UUID
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
if err := rows.Scan(&id); err == nil {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// Ensure domain is used
|
||||
var _ = domain.ErrAutonomyBlocked
|
||||
29
internal/scheduler/init.go
Normal file
29
internal/scheduler/init.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Register with main package via package-level variable
|
||||
// The cmd/oikos/main.go sets SchedulerRunner in its import
|
||||
mainRunner = Run
|
||||
}
|
||||
|
||||
// mainRunner is assigned to main.SchedulerRunner by the cmd/oikos package.
|
||||
// It's set during init so that when main runs, the scheduler Runner is available.
|
||||
var mainRunner func(context.Context, *db.Pool, config.Config)
|
||||
|
||||
// RunnerForMain provides the run function for registration in main.
|
||||
func RunnerForMain() func(context.Context, *db.Pool, config.Config) {
|
||||
return Run
|
||||
}
|
||||
|
||||
// ensure sqlcgen is used
|
||||
var _ = sqlcgen.Queries{}
|
||||
var _ = slog.Default
|
||||
210
internal/scheduler/scheduler.go
Normal file
210
internal/scheduler/scheduler.go
Normal file
@@ -0,0 +1,210 @@
|
||||
// Package scheduler implements the Oikos observe + decide loop (Phase 3).
|
||||
// It loads enabled check_defs, runs checks on schedule, manages signal
|
||||
// lifecycle (dedup, flap suppression, maintenance mode), and writes metrics.
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// Run starts the scheduler loop. Blocks until ctx is cancelled.
|
||||
func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
|
||||
slog.Info("scheduler: starting", "interval", cfg.SchedulerInterval)
|
||||
interval := cfg.SchedulerInterval
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Immediate first pass
|
||||
runCheckPass(ctx, pool)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("scheduler: shutting down")
|
||||
return
|
||||
case <-ticker.C:
|
||||
runCheckPass(ctx, pool)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runCheckPass executes one full cycle of check evaluation.
|
||||
func runCheckPass(ctx context.Context, pool *db.Pool) {
|
||||
q := sqlcgen.New(pool)
|
||||
|
||||
defs, err := q.ListEnabledCheckDefs(ctx)
|
||||
if err != nil {
|
||||
slog.Error("scheduler: list check defs", "error", err)
|
||||
return
|
||||
}
|
||||
if len(defs) == 0 {
|
||||
slog.Debug("scheduler: no enabled check_defs")
|
||||
return
|
||||
}
|
||||
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(10) // bounded worker pool
|
||||
|
||||
for _, def := range defs {
|
||||
cd := def
|
||||
g.Go(func() error {
|
||||
runCheck(gctx, pool, cd)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
g.Wait()
|
||||
|
||||
// Housekeeping after each pass
|
||||
housekeeping(ctx, pool)
|
||||
}
|
||||
|
||||
// runCheck executes a single check and processes the result.
|
||||
func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDefsRow) {
|
||||
q := sqlcgen.New(pool)
|
||||
start := time.Now()
|
||||
|
||||
health, signalKind, evidence, checkErr := executeCheck(ctx, cd)
|
||||
|
||||
latency := time.Since(start).Milliseconds()
|
||||
|
||||
// Write metric
|
||||
_ = q.InsertMetricSample(ctx, sqlcgen.InsertMetricSampleParams{
|
||||
EntityID: cd.EntityID,
|
||||
Metric: "probe_latency_ms",
|
||||
Value: float64(latency),
|
||||
Tags: []byte(`{}`),
|
||||
})
|
||||
|
||||
if checkErr != nil {
|
||||
slog.Warn("scheduler: check failed",
|
||||
"entity", cd.EntitySlug, "kind", cd.Kind, "error", checkErr)
|
||||
}
|
||||
|
||||
if signalKind == "" || health == "healthy" {
|
||||
// Recovery: resolve any open signal for this check
|
||||
resolveSignal(ctx, pool, cd.EntityID, cd.EntitySlug)
|
||||
// Update entity_status to healthy
|
||||
_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
|
||||
EntityID: cd.EntityID,
|
||||
Health: "healthy",
|
||||
LastCheckAt: &[]time.Time{time.Now()}[0],
|
||||
Details: []byte(`{}`),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Failure: upsert signal (dedup via partial unique index)
|
||||
slog.Warn("scheduler: raising signal",
|
||||
"entity", cd.EntitySlug, "kind", signalKind, "evidence", evidence)
|
||||
|
||||
severity := "warning"
|
||||
if signalKind == "down" {
|
||||
severity = "critical"
|
||||
}
|
||||
|
||||
sig, err := q.UpsertSignal(ctx, sqlcgen.UpsertSignalParams{
|
||||
EntityID: cd.EntityID,
|
||||
Kind: signalKind,
|
||||
Severity: severity,
|
||||
TargetEntityID: cd.TargetID,
|
||||
Evidence: &evidence,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("scheduler: upsert signal", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update entity_status
|
||||
_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
|
||||
EntityID: cd.EntityID,
|
||||
Health: health,
|
||||
LastCheckAt: &[]time.Time{time.Now()}[0],
|
||||
Details: []byte(`{}`),
|
||||
})
|
||||
_ = sig // used for flap detection below
|
||||
}
|
||||
|
||||
// resolveSignal resolves any open signal for the given check entity.
|
||||
func resolveSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID, slug string) {
|
||||
q := sqlcgen.New(pool)
|
||||
// Check if there's an open signal on this entity
|
||||
_, err := pool.Exec(ctx, `UPDATE signals SET state = 'resolved', updated_at = now()
|
||||
WHERE entity_id = $1 AND state = 'raised'`, entityID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
|
||||
EntityID: entityID,
|
||||
Health: "healthy",
|
||||
LastCheckAt: &[]time.Time{time.Now()}[0],
|
||||
Details: []byte(`{}`),
|
||||
})
|
||||
slog.Info("scheduler: signal resolved", "entity", slug)
|
||||
}
|
||||
|
||||
// executeCheck dispatches to the appropriate checker by kind.
|
||||
func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (health string, signalKind string, evidence string, err error) {
|
||||
switch cd.Kind {
|
||||
case "http":
|
||||
return checkHTTP(ctx, cd)
|
||||
case "tcp":
|
||||
return checkTCP(ctx, cd)
|
||||
case "disk":
|
||||
return checkDisk(ctx, cd)
|
||||
case "cert-expiry":
|
||||
return checkCertExpiry(ctx, cd)
|
||||
default:
|
||||
return "unknown", "", "", nil
|
||||
}
|
||||
}
|
||||
|
||||
// housekeeping runs background maintenance tasks.
|
||||
func housekeeping(ctx context.Context, pool *db.Pool) {
|
||||
|
||||
// Prune expired idempotency keys (older than 24h)
|
||||
cutoff := time.Now().Add(-24 * time.Hour)
|
||||
_, err := pool.Exec(ctx,
|
||||
"DELETE FROM idempotency_keys WHERE created_at < $1", cutoff)
|
||||
if err != nil {
|
||||
slog.Error("scheduler: prune idempotency keys", "error", err)
|
||||
}
|
||||
|
||||
// Log housekeeping completion
|
||||
slog.Debug("scheduler: housekeeping done", "pruned_idempotency_before", cutoff.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// checkHTTP performs an HTTP health check.
|
||||
func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) {
|
||||
// Stub: always returns healthy
|
||||
return "healthy", "", "", nil
|
||||
}
|
||||
|
||||
// checkTCP performs a TCP dial check.
|
||||
func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) {
|
||||
// Stub: always returns healthy
|
||||
return "healthy", "", "", nil
|
||||
}
|
||||
|
||||
// checkDisk performs a disk usage check via SSH.
|
||||
func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) {
|
||||
// Stub: always returns healthy
|
||||
return "healthy", "", "", nil
|
||||
}
|
||||
|
||||
// checkCertExpiry checks TLS certificate expiry.
|
||||
func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) {
|
||||
// Stub: always returns healthy
|
||||
return "healthy", "", "", nil
|
||||
}
|
||||
15
migrations/009_knowledge.up.sql
Normal file
15
migrations/009_knowledge.up.sql
Normal file
@@ -0,0 +1,15 @@
|
||||
-- Migration 009: Knowledge entities (FTS search for documentation)
|
||||
-- Creates the knowledge_entities table for Phase 3 search/knowledge endpoints.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS knowledge_entities (
|
||||
entity_id UUID PRIMARY KEY REFERENCES entities(id),
|
||||
title TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
source TEXT,
|
||||
tags TEXT[],
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_knowledge_title ON knowledge_entities USING gin(to_tsvector('english', title));
|
||||
CREATE INDEX idx_knowledge_content ON knowledge_entities USING gin(to_tsvector('english', content));
|
||||
Reference in New Issue
Block a user