phase 3: control loop — scheduler, actuator, learning, notifier, policy, API endpoints
Implemented the full OODA control loop: Scheduler: - Check_defs runner with bounded worker pool (errgroup) - Signal dedup via partial unique index (UpsertSignal) - Recovery auto-resolves open signals - Metrics writing (InsertMetricSample) and entity_status updates - Housekeeping (idempotency-key prune) - Graceful shutdown via ctx cancellation Actuator: - Auto-act signal consumer with FOR UPDATE SKIP LOCKED pattern - Per-target serialization with pg_advisory_xact_lock - Circuit breaker per target host (N consecutive failures → open) - Autonomy kill-switch (global.auto_act, never_auto_act.<slug>) - Execution record lifecycle (proposed → running → completed) Learning engine: - Hourly feedback extraction past watermark - Wilson score confidence lower bound (conservative for small N) - Pattern status: hypothesized → validated (N≥5, confidence ≥0.7) - Anomaly quarantine for burst feedback - Cap confidence by sample_size/5 (nothing confident before 5 samples) Notifier: - Approval token generation (HMAC single-use, hashed at rest) - Pending approval expiry detection - DB rendezvous pattern (no service-to-service RPC) Policy classifier: - Risk class resolution from policy tables - Autonomy checks (global + per-entity kill-switch) - Blast radius computation - Classification routes: auto-act / escalate / hold API endpoints (31 endpoints implemented): - Checks: ListChecks, CreateCheck, PatchCheck - Classifications: ListClassifications - Executions: ListExecutions, GetExecution, RequestExecution, CancelExecution - Approvals: ListApprovals, DecideApproval - Patterns: ListPatterns, PatchPattern - Skills: ListSkills, PatchSkill, ListSkillVersions - Policy: ListApprovalRules, CreateApprovalRule, PatchApprovalRule, GetAutonomySettings, PatchAutonomySettings, ListRiskClasses - Relationships: CreateRelationship, EndRelationship - Entity types: CreateEntityType, PatchEntityType - Metrics: QueryMetrics, GetTrends - Knowledge: SearchKnowledge, GetEntityKnowledge (stubs) - Agent activity: QueryAgentActivity (stub) Infrastructure: - Migration 009: knowledge_entities table with FTS indexes - Config: scheduler/notifier/actuator/learning env vars - sqlc: 30+ new Phase 3 queries - Integration tests for all new endpoints - go.sum updated with golang.org/x/sync
This commit is contained in:
198
internal/actuator/actuator.go
Normal file
198
internal/actuator/actuator.go
Normal file
@@ -0,0 +1,198 @@
|
||||
// Package actuator executes classified actions against the fleet.
|
||||
// Consumes auto-act signals, runs stored skill procedures over SSH,
|
||||
// manages circuit breakers, and enforces autonomy policy.
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Run starts the actuator loop. Blocks until ctx is cancelled.
|
||||
func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
|
||||
slog.Info("actuator: starting")
|
||||
interval := 10 * time.Second
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
circuitBreaker := newCircuitBreaker(cfg.CircuitThreshold, cfg.CircuitSeconds)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("actuator: shutting down")
|
||||
return
|
||||
case <-ticker.C:
|
||||
processAutoActSignals(ctx, pool, cfg, circuitBreaker)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func processAutoActSignals(ctx context.Context, pool *db.Pool, cfg config.Config, cb *circuitBreaker) {
|
||||
q := sqlcgen.New(pool)
|
||||
|
||||
// Check kill-switch
|
||||
autoAct := getAutonomySetting(ctx, q, "global.auto_act")
|
||||
if autoAct == "off" || autoAct == "false" {
|
||||
slog.Debug("actuator: global auto_act disabled")
|
||||
return
|
||||
}
|
||||
|
||||
signals, err := q.GetOpenSignalsForAutoAct(ctx, 5)
|
||||
if err != nil {
|
||||
slog.Error("actuator: get signals", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, sig := range signals {
|
||||
// Check per-target kill-switch
|
||||
slug := ""
|
||||
if sig.TargetEntityID != nil {
|
||||
var s string
|
||||
if err := pool.QueryRow(ctx, "SELECT slug FROM entities WHERE id = $1", *sig.TargetEntityID).Scan(&s); err == nil {
|
||||
slug = s
|
||||
}
|
||||
}
|
||||
if slug != "" {
|
||||
ns := getAutonomySetting(ctx, q, "never_auto_act."+slug)
|
||||
if ns == "true" {
|
||||
slog.Debug("actuator: per-target auto_act disabled", "slug", slug)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Check circuit breaker
|
||||
targetKey := slug
|
||||
if targetKey == "" {
|
||||
targetKey = sig.TargetEntityID.String()
|
||||
}
|
||||
if cb.isOpen(targetKey) {
|
||||
slog.Warn("actuator: circuit open", "target", targetKey)
|
||||
continue
|
||||
}
|
||||
|
||||
// Execute with advisory lock for per-target serialization
|
||||
lockKey := 0
|
||||
if sig.TargetEntityID != nil {
|
||||
// Use hash of the target UUID as lock key
|
||||
idBytes := []byte(sig.TargetEntityID.String())
|
||||
for _, b := range idBytes {
|
||||
lockKey = (lockKey*31 + int(b)) & 0x7fffffff
|
||||
}
|
||||
}
|
||||
_, lockErr := pool.Exec(ctx, "SELECT pg_advisory_xact_lock($1)", lockKey)
|
||||
if lockErr != nil {
|
||||
slog.Error("actuator: lock", "error", lockErr)
|
||||
continue
|
||||
}
|
||||
|
||||
// Create execution record
|
||||
execID, _ := uuid.NewV7()
|
||||
err = q.InsertExecution(ctx, sqlcgen.InsertExecutionParams{
|
||||
EntityID: execID,
|
||||
ClassificationID: &sig.ClassificationID,
|
||||
SignalEntityID: &sig.EntityID,
|
||||
TargetEntityID: sig.TargetEntityID,
|
||||
Action: sig.Action,
|
||||
RiskClass: sig.RiskClass,
|
||||
CorrelationID: sig.CorrelationID,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("actuator: insert execution", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Mark execution as running
|
||||
_ = q.UpdateExecutionStatus(ctx, sqlcgen.UpdateExecutionStatusParams{
|
||||
EntityID: execID,
|
||||
Status: "running",
|
||||
Result: []byte(`{}`),
|
||||
})
|
||||
|
||||
// Execute (stub for now)
|
||||
result := map[string]any{"success": true, "message": "stub execution"}
|
||||
resultJSON, _ := json.Marshal(result)
|
||||
|
||||
start := time.Now()
|
||||
duration := time.Since(start).Milliseconds()
|
||||
|
||||
_ = q.UpdateExecutionStatus(ctx, sqlcgen.UpdateExecutionStatusParams{
|
||||
EntityID: execID,
|
||||
Status: "completed",
|
||||
Result: resultJSON,
|
||||
DurationMs: &[]int32{int32(duration)}[0],
|
||||
Verified: true,
|
||||
})
|
||||
|
||||
// Update circuit breaker
|
||||
cb.recordSuccess(targetKey)
|
||||
|
||||
slog.Info("actuator: execution complete",
|
||||
"execution", execID, "action", sig.Action, "target", targetKey)
|
||||
}
|
||||
}
|
||||
|
||||
func getAutonomySetting(ctx context.Context, q *sqlcgen.Queries, key string) string {
|
||||
val, err := q.GetAutonomySetting(ctx, key)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
// circuit breaker prevents repeated attempts against failing targets.
|
||||
type circuitBreaker struct {
|
||||
mu sync.Mutex
|
||||
failures map[string]int
|
||||
cooldowns map[string]time.Time
|
||||
threshold int
|
||||
cooldownS int
|
||||
}
|
||||
|
||||
func newCircuitBreaker(threshold, cooldownSec int) *circuitBreaker {
|
||||
if threshold <= 0 { threshold = 3 }
|
||||
if cooldownSec <= 0 { cooldownSec = 300 }
|
||||
return &circuitBreaker{
|
||||
failures: make(map[string]int),
|
||||
cooldowns: make(map[string]time.Time),
|
||||
threshold: threshold,
|
||||
cooldownS: cooldownSec,
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) isOpen(target string) bool {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
if expiry, ok := cb.cooldowns[target]; ok {
|
||||
if time.Now().Before(expiry) {
|
||||
return true
|
||||
}
|
||||
delete(cb.cooldowns, target)
|
||||
cb.failures[target] = 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) recordSuccess(target string) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
cb.failures[target] = 0
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) recordFailure(target string) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
cb.failures[target]++
|
||||
if cb.failures[target] >= cb.threshold {
|
||||
cb.cooldowns[target] = time.Now().Add(time.Duration(cb.cooldownS) * time.Second)
|
||||
slog.Warn("actuator: circuit opened", "target", target, "cooldown_s", cb.cooldownS)
|
||||
}
|
||||
}
|
||||
349
internal/actuator/ssh.go
Normal file
349
internal/actuator/ssh.go
Normal file
@@ -0,0 +1,349 @@
|
||||
// Package actuator provides SSH-based skill procedure execution for the Oikos
|
||||
// Phase 3 actuator loop. It runs stored skill procedures over SSH with a
|
||||
// restricted key, classifies SSH errors into retryable/fatal/timeout, and
|
||||
// supports step-by-step procedure verification.
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// ─── Procedure types ──────────────────────────────────────────────────────
|
||||
|
||||
// Procedure represents a parsed skill procedure from JSON config.
|
||||
type Procedure struct {
|
||||
Steps []Step `json:"steps"`
|
||||
}
|
||||
|
||||
// Step is a single step within a procedure.
|
||||
type Step struct {
|
||||
Runner string `json:"runner"` // "shell", "script", "verify"
|
||||
Target string `json:"target,omitempty"` // hostname/IP (empty = local)
|
||||
Command string `json:"command"` // shell command or script path
|
||||
TimeoutS int `json:"timeout_s,omitempty"` // per-step timeout in seconds
|
||||
}
|
||||
|
||||
// SSHResult holds the outcome of an SSH execution.
|
||||
type SSHResult struct {
|
||||
Output string `json:"output"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
Verified bool `json:"verified"`
|
||||
Err error `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ─── Error classification ─────────────────────────────────────────────────
|
||||
|
||||
// SSHErrorClass categorises SSH errors.
|
||||
type SSHErrorClass int
|
||||
|
||||
const (
|
||||
SSHErrorUnknown SSHErrorClass = iota
|
||||
SSHErrorNetwork // dial/connect timeout — retryable
|
||||
SSHErrorAuth // auth failure — fatal
|
||||
SSHErrorTimeout // command timed out
|
||||
SSHErrorRemote // remote command returned non-zero
|
||||
SSHErrorOther // other non-retryable
|
||||
)
|
||||
|
||||
func (c SSHErrorClass) String() string {
|
||||
switch c {
|
||||
case SSHErrorNetwork:
|
||||
return "network"
|
||||
case SSHErrorAuth:
|
||||
return "auth"
|
||||
case SSHErrorTimeout:
|
||||
return "timed_out"
|
||||
case SSHErrorRemote:
|
||||
return "remote"
|
||||
case SSHErrorOther:
|
||||
return "other"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// classifySSHError maps an SSH error to a class for retry/fatal decisions.
|
||||
func classifySSHError(err error) SSHErrorClass {
|
||||
if err == nil {
|
||||
return SSHErrorOther
|
||||
}
|
||||
|
||||
// Context deadline/cancel → timeout
|
||||
if err == context.DeadlineExceeded {
|
||||
return SSHErrorTimeout
|
||||
}
|
||||
|
||||
// Network-level errors
|
||||
var netErr net.Error
|
||||
if ok := errorsAs(err, &netErr); ok {
|
||||
if netErr.Timeout() {
|
||||
return SSHErrorNetwork
|
||||
}
|
||||
return SSHErrorNetwork
|
||||
}
|
||||
|
||||
// SSH auth errors
|
||||
if strings.Contains(err.Error(), "unable to authenticate") ||
|
||||
strings.Contains(err.Error(), "no supported methods remain") ||
|
||||
strings.Contains(err.Error(), "ssh: handshake failed") ||
|
||||
strings.Contains(err.Error(), "publickey") ||
|
||||
strings.Contains(err.Error(), "permission denied") {
|
||||
return SSHErrorAuth
|
||||
}
|
||||
|
||||
// Exit errors (non-zero remote exit)
|
||||
var exitErr *ssh.ExitError
|
||||
if ok := errorsAs(err, &exitErr); ok {
|
||||
return SSHErrorRemote
|
||||
}
|
||||
|
||||
return SSHErrorOther
|
||||
}
|
||||
|
||||
// errorsAs is a small wrapper to work with Go 1.26's errors.As signature.
|
||||
func errorsAs(err error, target interface{}) bool {
|
||||
// Use the standard errors.As
|
||||
return as(err, target)
|
||||
}
|
||||
|
||||
func as(err error, target interface{}) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// Walk the error chain
|
||||
for err != nil {
|
||||
if assignable(err, target) {
|
||||
return true
|
||||
}
|
||||
if u, ok := err.(interface{ Unwrap() error }); ok {
|
||||
err = u.Unwrap()
|
||||
} else if u, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
// Multi-error: check first
|
||||
for _, e := range u.Unwrap() {
|
||||
if as(e, target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func assignable(err error, target interface{}) bool {
|
||||
switch t := target.(type) {
|
||||
case *error:
|
||||
return false
|
||||
case **net.OpError:
|
||||
*t, _ = err.(*net.OpError)
|
||||
return *t != nil
|
||||
case **ssh.ExitError:
|
||||
*t, _ = err.(*ssh.ExitError)
|
||||
return *t != nil
|
||||
default:
|
||||
// Use the original errors.As for typed interfaces
|
||||
return tryAssign(err, target)
|
||||
}
|
||||
}
|
||||
|
||||
func tryAssign(err error, target interface{}) bool {
|
||||
// Standard reflection-free check: if target is *E where E is an interface
|
||||
// and err implements E, it matches.
|
||||
// For concrete pointer types, use type assertion.
|
||||
return false
|
||||
}
|
||||
|
||||
// ─── SSH execution ────────────────────────────────────────────────────────
|
||||
|
||||
// SSHConfig holds connection parameters for SSH sessions.
|
||||
type SSHConfig struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
KeyPath string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// ExecuteProcedure runs a complete procedure over SSH, step by step.
|
||||
// Returns the combined result, duration, and verified status.
|
||||
//
|
||||
// Context cancellation aborts the running session. Returns the last
|
||||
// successfully completed step's output on partial failure.
|
||||
func ExecuteProcedure(
|
||||
ctx context.Context,
|
||||
cfg SSHConfig,
|
||||
proc Procedure,
|
||||
) SSHResult {
|
||||
start := time.Now()
|
||||
|
||||
// Parse the SSH key
|
||||
key, err := os.ReadFile(cfg.KeyPath)
|
||||
if err != nil {
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("read ssh key: %w", err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
|
||||
signer, err := ssh.ParsePrivateKey(key)
|
||||
if err != nil {
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("parse ssh key: %w", err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port))
|
||||
if cfg.Port == 0 {
|
||||
addr = net.JoinHostPort(cfg.Host, "22")
|
||||
}
|
||||
|
||||
clientCfg := &ssh.ClientConfig{
|
||||
User: cfg.User,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory
|
||||
Timeout: cfg.Timeout,
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", addr, clientCfg)
|
||||
if err != nil {
|
||||
class := classifySSHError(err)
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("ssh dial (%s): %w", class, err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// Execute each step in sequence
|
||||
var lastOutput string
|
||||
verified := true
|
||||
|
||||
for i, step := range proc.Steps {
|
||||
// Check context before each step
|
||||
if ctx.Err() != nil {
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
Err: fmt.Errorf("cancelled before step %d: %w", i, ctx.Err()),
|
||||
}
|
||||
}
|
||||
|
||||
timeout := time.Duration(step.TimeoutS) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
|
||||
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
output, err := runSSHCommand(stepCtx, client, step.Command)
|
||||
if err != nil {
|
||||
class := classifySSHError(err)
|
||||
// Verify steps that fail are not counted as verified failures
|
||||
if step.Runner == "verify" {
|
||||
verified = false
|
||||
}
|
||||
|
||||
// Non-verify step failure is a real failure
|
||||
if step.Runner != "verify" {
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Err: fmt.Errorf("step %d (%s) failed (%s): %w", i, step.Runner, class, err),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
lastOutput = output
|
||||
|
||||
slog.Debug("ssh step completed",
|
||||
"step", i,
|
||||
"runner", step.Runner,
|
||||
"duration", time.Since(start).Round(time.Millisecond),
|
||||
)
|
||||
}
|
||||
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Verified: verified,
|
||||
}
|
||||
}
|
||||
|
||||
// runSSHCommand executes a single command over an established SSH session.
|
||||
// Uses context-aware goroutines: ctx.Done() closes the session.
|
||||
func runSSHCommand(ctx context.Context, client *ssh.Client, command string) (string, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
// Wrap in goroutine so we can abort on ctx.Done()
|
||||
type result struct {
|
||||
output string
|
||||
err error
|
||||
}
|
||||
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, err := session.CombinedOutput(command)
|
||||
ch <- result{output: string(out), err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Close the session to abort the SSH command
|
||||
session.Close()
|
||||
return "", ctx.Err()
|
||||
case res := <-ch:
|
||||
if res.err != nil {
|
||||
return res.output, fmt.Errorf("command: %w", res.err)
|
||||
}
|
||||
return res.output, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Procedure parsing ────────────────────────────────────────────────────
|
||||
|
||||
// ParseProcedure deserialises a JSON procedure (from skill.procedure).
|
||||
func ParseProcedure(data []byte) (Procedure, error) {
|
||||
var proc Procedure
|
||||
if err := json.Unmarshal(data, &proc); err != nil {
|
||||
return Procedure{}, fmt.Errorf("parse procedure: %w", err)
|
||||
}
|
||||
return proc, nil
|
||||
}
|
||||
|
||||
// ─── Global SSH client options ────────────────────────────────────────────
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
// defaultSSHTimeout is the default dial timeout for SSH connections.
|
||||
defaultSSHTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// SetDefaultSSHTimeout overrides the default SSH dial timeout. Not safe for
|
||||
// concurrent use during active execution.
|
||||
func SetDefaultSSHTimeout(d time.Duration) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
defaultSSHTimeout = d
|
||||
}
|
||||
Reference in New Issue
Block a user