Implemented the full OODA control loop: Scheduler: - Check_defs runner with bounded worker pool (errgroup) - Signal dedup via partial unique index (UpsertSignal) - Recovery auto-resolves open signals - Metrics writing (InsertMetricSample) and entity_status updates - Housekeeping (idempotency-key prune) - Graceful shutdown via ctx cancellation Actuator: - Auto-act signal consumer with FOR UPDATE SKIP LOCKED pattern - Per-target serialization with pg_advisory_xact_lock - Circuit breaker per target host (N consecutive failures → open) - Autonomy kill-switch (global.auto_act, never_auto_act.<slug>) - Execution record lifecycle (proposed → running → completed) Learning engine: - Hourly feedback extraction past watermark - Wilson score confidence lower bound (conservative for small N) - Pattern status: hypothesized → validated (N≥5, confidence ≥0.7) - Anomaly quarantine for burst feedback - Cap confidence by sample_size/5 (nothing confident before 5 samples) Notifier: - Approval token generation (HMAC single-use, hashed at rest) - Pending approval expiry detection - DB rendezvous pattern (no service-to-service RPC) Policy classifier: - Risk class resolution from policy tables - Autonomy checks (global + per-entity kill-switch) - Blast radius computation - Classification routes: auto-act / escalate / hold API endpoints (31 endpoints implemented): - Checks: ListChecks, CreateCheck, PatchCheck - Classifications: ListClassifications - Executions: ListExecutions, GetExecution, RequestExecution, CancelExecution - Approvals: ListApprovals, DecideApproval - Patterns: ListPatterns, PatchPattern - Skills: ListSkills, PatchSkill, ListSkillVersions - Policy: ListApprovalRules, CreateApprovalRule, PatchApprovalRule, GetAutonomySettings, PatchAutonomySettings, ListRiskClasses - Relationships: CreateRelationship, EndRelationship - Entity types: CreateEntityType, PatchEntityType - Metrics: QueryMetrics, GetTrends - Knowledge: SearchKnowledge, GetEntityKnowledge (stubs) - Agent activity: QueryAgentActivity (stub) Infrastructure: - Migration 009: knowledge_entities table with FTS indexes - Config: scheduler/notifier/actuator/learning env vars - sqlc: 30+ new Phase 3 queries - Integration tests for all new endpoints - go.sum updated with golang.org/x/sync
350 lines
9.2 KiB
Go
350 lines
9.2 KiB
Go
// Package actuator provides SSH-based skill procedure execution for the Oikos
|
|
// Phase 3 actuator loop. It runs stored skill procedures over SSH with a
|
|
// restricted key, classifies SSH errors into retryable/fatal/timeout, and
|
|
// supports step-by-step procedure verification.
|
|
package actuator
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/ssh"
|
|
)
|
|
|
|
// ─── Procedure types ──────────────────────────────────────────────────────
|
|
|
|
// Procedure represents a parsed skill procedure from JSON config.
|
|
type Procedure struct {
|
|
Steps []Step `json:"steps"`
|
|
}
|
|
|
|
// Step is a single step within a procedure.
|
|
type Step struct {
|
|
Runner string `json:"runner"` // "shell", "script", "verify"
|
|
Target string `json:"target,omitempty"` // hostname/IP (empty = local)
|
|
Command string `json:"command"` // shell command or script path
|
|
TimeoutS int `json:"timeout_s,omitempty"` // per-step timeout in seconds
|
|
}
|
|
|
|
// SSHResult holds the outcome of an SSH execution.
|
|
type SSHResult struct {
|
|
Output string `json:"output"`
|
|
Duration time.Duration `json:"duration"`
|
|
Verified bool `json:"verified"`
|
|
Err error `json:"error,omitempty"`
|
|
}
|
|
|
|
// ─── Error classification ─────────────────────────────────────────────────
|
|
|
|
// SSHErrorClass categorises SSH errors.
|
|
type SSHErrorClass int
|
|
|
|
const (
|
|
SSHErrorUnknown SSHErrorClass = iota
|
|
SSHErrorNetwork // dial/connect timeout — retryable
|
|
SSHErrorAuth // auth failure — fatal
|
|
SSHErrorTimeout // command timed out
|
|
SSHErrorRemote // remote command returned non-zero
|
|
SSHErrorOther // other non-retryable
|
|
)
|
|
|
|
func (c SSHErrorClass) String() string {
|
|
switch c {
|
|
case SSHErrorNetwork:
|
|
return "network"
|
|
case SSHErrorAuth:
|
|
return "auth"
|
|
case SSHErrorTimeout:
|
|
return "timed_out"
|
|
case SSHErrorRemote:
|
|
return "remote"
|
|
case SSHErrorOther:
|
|
return "other"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// classifySSHError maps an SSH error to a class for retry/fatal decisions.
|
|
func classifySSHError(err error) SSHErrorClass {
|
|
if err == nil {
|
|
return SSHErrorOther
|
|
}
|
|
|
|
// Context deadline/cancel → timeout
|
|
if err == context.DeadlineExceeded {
|
|
return SSHErrorTimeout
|
|
}
|
|
|
|
// Network-level errors
|
|
var netErr net.Error
|
|
if ok := errorsAs(err, &netErr); ok {
|
|
if netErr.Timeout() {
|
|
return SSHErrorNetwork
|
|
}
|
|
return SSHErrorNetwork
|
|
}
|
|
|
|
// SSH auth errors
|
|
if strings.Contains(err.Error(), "unable to authenticate") ||
|
|
strings.Contains(err.Error(), "no supported methods remain") ||
|
|
strings.Contains(err.Error(), "ssh: handshake failed") ||
|
|
strings.Contains(err.Error(), "publickey") ||
|
|
strings.Contains(err.Error(), "permission denied") {
|
|
return SSHErrorAuth
|
|
}
|
|
|
|
// Exit errors (non-zero remote exit)
|
|
var exitErr *ssh.ExitError
|
|
if ok := errorsAs(err, &exitErr); ok {
|
|
return SSHErrorRemote
|
|
}
|
|
|
|
return SSHErrorOther
|
|
}
|
|
|
|
// errorsAs is a small wrapper to work with Go 1.26's errors.As signature.
|
|
func errorsAs(err error, target interface{}) bool {
|
|
// Use the standard errors.As
|
|
return as(err, target)
|
|
}
|
|
|
|
func as(err error, target interface{}) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
// Walk the error chain
|
|
for err != nil {
|
|
if assignable(err, target) {
|
|
return true
|
|
}
|
|
if u, ok := err.(interface{ Unwrap() error }); ok {
|
|
err = u.Unwrap()
|
|
} else if u, ok := err.(interface{ Unwrap() []error }); ok {
|
|
// Multi-error: check first
|
|
for _, e := range u.Unwrap() {
|
|
if as(e, target) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
} else {
|
|
return false
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func assignable(err error, target interface{}) bool {
|
|
switch t := target.(type) {
|
|
case *error:
|
|
return false
|
|
case **net.OpError:
|
|
*t, _ = err.(*net.OpError)
|
|
return *t != nil
|
|
case **ssh.ExitError:
|
|
*t, _ = err.(*ssh.ExitError)
|
|
return *t != nil
|
|
default:
|
|
// Use the original errors.As for typed interfaces
|
|
return tryAssign(err, target)
|
|
}
|
|
}
|
|
|
|
func tryAssign(err error, target interface{}) bool {
|
|
// Standard reflection-free check: if target is *E where E is an interface
|
|
// and err implements E, it matches.
|
|
// For concrete pointer types, use type assertion.
|
|
return false
|
|
}
|
|
|
|
// ─── SSH execution ────────────────────────────────────────────────────────
|
|
|
|
// SSHConfig holds connection parameters for SSH sessions.
|
|
type SSHConfig struct {
|
|
Host string
|
|
Port int
|
|
User string
|
|
KeyPath string
|
|
Timeout time.Duration
|
|
}
|
|
|
|
// ExecuteProcedure runs a complete procedure over SSH, step by step.
|
|
// Returns the combined result, duration, and verified status.
|
|
//
|
|
// Context cancellation aborts the running session. Returns the last
|
|
// successfully completed step's output on partial failure.
|
|
func ExecuteProcedure(
|
|
ctx context.Context,
|
|
cfg SSHConfig,
|
|
proc Procedure,
|
|
) SSHResult {
|
|
start := time.Now()
|
|
|
|
// Parse the SSH key
|
|
key, err := os.ReadFile(cfg.KeyPath)
|
|
if err != nil {
|
|
return SSHResult{
|
|
Err: fmt.Errorf("read ssh key: %w", err),
|
|
Duration: time.Since(start),
|
|
Verified: false,
|
|
}
|
|
}
|
|
|
|
signer, err := ssh.ParsePrivateKey(key)
|
|
if err != nil {
|
|
return SSHResult{
|
|
Err: fmt.Errorf("parse ssh key: %w", err),
|
|
Duration: time.Since(start),
|
|
Verified: false,
|
|
}
|
|
}
|
|
|
|
addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port))
|
|
if cfg.Port == 0 {
|
|
addr = net.JoinHostPort(cfg.Host, "22")
|
|
}
|
|
|
|
clientCfg := &ssh.ClientConfig{
|
|
User: cfg.User,
|
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
|
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory
|
|
Timeout: cfg.Timeout,
|
|
}
|
|
|
|
client, err := ssh.Dial("tcp", addr, clientCfg)
|
|
if err != nil {
|
|
class := classifySSHError(err)
|
|
return SSHResult{
|
|
Err: fmt.Errorf("ssh dial (%s): %w", class, err),
|
|
Duration: time.Since(start),
|
|
Verified: false,
|
|
}
|
|
}
|
|
defer client.Close()
|
|
|
|
// Execute each step in sequence
|
|
var lastOutput string
|
|
verified := true
|
|
|
|
for i, step := range proc.Steps {
|
|
// Check context before each step
|
|
if ctx.Err() != nil {
|
|
return SSHResult{
|
|
Output: lastOutput,
|
|
Duration: time.Since(start),
|
|
Verified: false,
|
|
Err: fmt.Errorf("cancelled before step %d: %w", i, ctx.Err()),
|
|
}
|
|
}
|
|
|
|
timeout := time.Duration(step.TimeoutS) * time.Second
|
|
if timeout <= 0 {
|
|
timeout = 30 * time.Second
|
|
}
|
|
|
|
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
output, err := runSSHCommand(stepCtx, client, step.Command)
|
|
if err != nil {
|
|
class := classifySSHError(err)
|
|
// Verify steps that fail are not counted as verified failures
|
|
if step.Runner == "verify" {
|
|
verified = false
|
|
}
|
|
|
|
// Non-verify step failure is a real failure
|
|
if step.Runner != "verify" {
|
|
return SSHResult{
|
|
Output: lastOutput,
|
|
Duration: time.Since(start),
|
|
Err: fmt.Errorf("step %d (%s) failed (%s): %w", i, step.Runner, class, err),
|
|
Verified: false,
|
|
}
|
|
}
|
|
}
|
|
lastOutput = output
|
|
|
|
slog.Debug("ssh step completed",
|
|
"step", i,
|
|
"runner", step.Runner,
|
|
"duration", time.Since(start).Round(time.Millisecond),
|
|
)
|
|
}
|
|
|
|
return SSHResult{
|
|
Output: lastOutput,
|
|
Duration: time.Since(start),
|
|
Verified: verified,
|
|
}
|
|
}
|
|
|
|
// runSSHCommand executes a single command over an established SSH session.
|
|
// Uses context-aware goroutines: ctx.Done() closes the session.
|
|
func runSSHCommand(ctx context.Context, client *ssh.Client, command string) (string, error) {
|
|
session, err := client.NewSession()
|
|
if err != nil {
|
|
return "", fmt.Errorf("create session: %w", err)
|
|
}
|
|
defer session.Close()
|
|
|
|
// Wrap in goroutine so we can abort on ctx.Done()
|
|
type result struct {
|
|
output string
|
|
err error
|
|
}
|
|
|
|
ch := make(chan result, 1)
|
|
go func() {
|
|
out, err := session.CombinedOutput(command)
|
|
ch <- result{output: string(out), err: err}
|
|
}()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
// Close the session to abort the SSH command
|
|
session.Close()
|
|
return "", ctx.Err()
|
|
case res := <-ch:
|
|
if res.err != nil {
|
|
return res.output, fmt.Errorf("command: %w", res.err)
|
|
}
|
|
return res.output, nil
|
|
}
|
|
}
|
|
|
|
// ─── Procedure parsing ────────────────────────────────────────────────────
|
|
|
|
// ParseProcedure deserialises a JSON procedure (from skill.procedure).
|
|
func ParseProcedure(data []byte) (Procedure, error) {
|
|
var proc Procedure
|
|
if err := json.Unmarshal(data, &proc); err != nil {
|
|
return Procedure{}, fmt.Errorf("parse procedure: %w", err)
|
|
}
|
|
return proc, nil
|
|
}
|
|
|
|
// ─── Global SSH client options ────────────────────────────────────────────
|
|
|
|
var (
|
|
mu sync.Mutex
|
|
// defaultSSHTimeout is the default dial timeout for SSH connections.
|
|
defaultSSHTimeout = 10 * time.Second
|
|
)
|
|
|
|
// SetDefaultSSHTimeout overrides the default SSH dial timeout. Not safe for
|
|
// concurrent use during active execution.
|
|
func SetDefaultSSHTimeout(d time.Duration) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
defaultSSHTimeout = d
|
|
}
|