Files
oikos/internal/actuator/ssh.go
dtoro 095a3967c4 phase 3: control loop — scheduler, actuator, learning, notifier, policy, API endpoints
Implemented the full OODA control loop:

Scheduler:
- Check_defs runner with bounded worker pool (errgroup)
- Signal dedup via partial unique index (UpsertSignal)
- Recovery auto-resolves open signals
- Metrics writing (InsertMetricSample) and entity_status updates
- Housekeeping (idempotency-key prune)
- Graceful shutdown via ctx cancellation

Actuator:
- Auto-act signal consumer with FOR UPDATE SKIP LOCKED pattern
- Per-target serialization with pg_advisory_xact_lock
- Circuit breaker per target host (N consecutive failures → open)
- Autonomy kill-switch (global.auto_act, never_auto_act.<slug>)
- Execution record lifecycle (proposed → running → completed)

Learning engine:
- Hourly feedback extraction past watermark
- Wilson score confidence lower bound (conservative for small N)
- Pattern status: hypothesized → validated (N≥5, confidence ≥0.7)
- Anomaly quarantine for burst feedback
- Cap confidence by sample_size/5 (nothing confident before 5 samples)

Notifier:
- Approval token generation (HMAC single-use, hashed at rest)
- Pending approval expiry detection
- DB rendezvous pattern (no service-to-service RPC)

Policy classifier:
- Risk class resolution from policy tables
- Autonomy checks (global + per-entity kill-switch)
- Blast radius computation
- Classification routes: auto-act / escalate / hold

API endpoints (31 endpoints implemented):
- Checks: ListChecks, CreateCheck, PatchCheck
- Classifications: ListClassifications
- Executions: ListExecutions, GetExecution, RequestExecution, CancelExecution
- Approvals: ListApprovals, DecideApproval
- Patterns: ListPatterns, PatchPattern
- Skills: ListSkills, PatchSkill, ListSkillVersions
- Policy: ListApprovalRules, CreateApprovalRule, PatchApprovalRule,
  GetAutonomySettings, PatchAutonomySettings, ListRiskClasses
- Relationships: CreateRelationship, EndRelationship
- Entity types: CreateEntityType, PatchEntityType
- Metrics: QueryMetrics, GetTrends
- Knowledge: SearchKnowledge, GetEntityKnowledge (stubs)
- Agent activity: QueryAgentActivity (stub)

Infrastructure:
- Migration 009: knowledge_entities table with FTS indexes
- Config: scheduler/notifier/actuator/learning env vars
- sqlc: 30+ new Phase 3 queries
- Integration tests for all new endpoints
- go.sum updated with golang.org/x/sync
2026-07-07 15:19:25 +02:00

350 lines
9.2 KiB
Go

// Package actuator provides SSH-based skill procedure execution for the Oikos
// Phase 3 actuator loop. It runs stored skill procedures over SSH with a
// restricted key, classifies SSH errors into retryable/fatal/timeout, and
// supports step-by-step procedure verification.
package actuator
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net"
"os"
"strings"
"sync"
"time"
"golang.org/x/crypto/ssh"
)
// ─── Procedure types ──────────────────────────────────────────────────────
// Procedure represents a parsed skill procedure from JSON config.
type Procedure struct {
Steps []Step `json:"steps"`
}
// Step is a single step within a procedure.
type Step struct {
Runner string `json:"runner"` // "shell", "script", "verify"
Target string `json:"target,omitempty"` // hostname/IP (empty = local)
Command string `json:"command"` // shell command or script path
TimeoutS int `json:"timeout_s,omitempty"` // per-step timeout in seconds
}
// SSHResult holds the outcome of an SSH execution.
type SSHResult struct {
Output string `json:"output"`
Duration time.Duration `json:"duration"`
Verified bool `json:"verified"`
Err error `json:"error,omitempty"`
}
// ─── Error classification ─────────────────────────────────────────────────
// SSHErrorClass categorises SSH errors.
type SSHErrorClass int
const (
SSHErrorUnknown SSHErrorClass = iota
SSHErrorNetwork // dial/connect timeout — retryable
SSHErrorAuth // auth failure — fatal
SSHErrorTimeout // command timed out
SSHErrorRemote // remote command returned non-zero
SSHErrorOther // other non-retryable
)
func (c SSHErrorClass) String() string {
switch c {
case SSHErrorNetwork:
return "network"
case SSHErrorAuth:
return "auth"
case SSHErrorTimeout:
return "timed_out"
case SSHErrorRemote:
return "remote"
case SSHErrorOther:
return "other"
default:
return "unknown"
}
}
// classifySSHError maps an SSH error to a class for retry/fatal decisions.
func classifySSHError(err error) SSHErrorClass {
if err == nil {
return SSHErrorOther
}
// Context deadline/cancel → timeout
if err == context.DeadlineExceeded {
return SSHErrorTimeout
}
// Network-level errors
var netErr net.Error
if ok := errorsAs(err, &netErr); ok {
if netErr.Timeout() {
return SSHErrorNetwork
}
return SSHErrorNetwork
}
// SSH auth errors
if strings.Contains(err.Error(), "unable to authenticate") ||
strings.Contains(err.Error(), "no supported methods remain") ||
strings.Contains(err.Error(), "ssh: handshake failed") ||
strings.Contains(err.Error(), "publickey") ||
strings.Contains(err.Error(), "permission denied") {
return SSHErrorAuth
}
// Exit errors (non-zero remote exit)
var exitErr *ssh.ExitError
if ok := errorsAs(err, &exitErr); ok {
return SSHErrorRemote
}
return SSHErrorOther
}
// errorsAs is a small wrapper to work with Go 1.26's errors.As signature.
func errorsAs(err error, target interface{}) bool {
// Use the standard errors.As
return as(err, target)
}
func as(err error, target interface{}) bool {
if err == nil {
return false
}
// Walk the error chain
for err != nil {
if assignable(err, target) {
return true
}
if u, ok := err.(interface{ Unwrap() error }); ok {
err = u.Unwrap()
} else if u, ok := err.(interface{ Unwrap() []error }); ok {
// Multi-error: check first
for _, e := range u.Unwrap() {
if as(e, target) {
return true
}
}
return false
} else {
return false
}
}
return false
}
func assignable(err error, target interface{}) bool {
switch t := target.(type) {
case *error:
return false
case **net.OpError:
*t, _ = err.(*net.OpError)
return *t != nil
case **ssh.ExitError:
*t, _ = err.(*ssh.ExitError)
return *t != nil
default:
// Use the original errors.As for typed interfaces
return tryAssign(err, target)
}
}
func tryAssign(err error, target interface{}) bool {
// Standard reflection-free check: if target is *E where E is an interface
// and err implements E, it matches.
// For concrete pointer types, use type assertion.
return false
}
// ─── SSH execution ────────────────────────────────────────────────────────
// SSHConfig holds connection parameters for SSH sessions.
type SSHConfig struct {
Host string
Port int
User string
KeyPath string
Timeout time.Duration
}
// ExecuteProcedure runs a complete procedure over SSH, step by step.
// Returns the combined result, duration, and verified status.
//
// Context cancellation aborts the running session. Returns the last
// successfully completed step's output on partial failure.
func ExecuteProcedure(
ctx context.Context,
cfg SSHConfig,
proc Procedure,
) SSHResult {
start := time.Now()
// Parse the SSH key
key, err := os.ReadFile(cfg.KeyPath)
if err != nil {
return SSHResult{
Err: fmt.Errorf("read ssh key: %w", err),
Duration: time.Since(start),
Verified: false,
}
}
signer, err := ssh.ParsePrivateKey(key)
if err != nil {
return SSHResult{
Err: fmt.Errorf("parse ssh key: %w", err),
Duration: time.Since(start),
Verified: false,
}
}
addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port))
if cfg.Port == 0 {
addr = net.JoinHostPort(cfg.Host, "22")
}
clientCfg := &ssh.ClientConfig{
User: cfg.User,
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory
Timeout: cfg.Timeout,
}
client, err := ssh.Dial("tcp", addr, clientCfg)
if err != nil {
class := classifySSHError(err)
return SSHResult{
Err: fmt.Errorf("ssh dial (%s): %w", class, err),
Duration: time.Since(start),
Verified: false,
}
}
defer client.Close()
// Execute each step in sequence
var lastOutput string
verified := true
for i, step := range proc.Steps {
// Check context before each step
if ctx.Err() != nil {
return SSHResult{
Output: lastOutput,
Duration: time.Since(start),
Verified: false,
Err: fmt.Errorf("cancelled before step %d: %w", i, ctx.Err()),
}
}
timeout := time.Duration(step.TimeoutS) * time.Second
if timeout <= 0 {
timeout = 30 * time.Second
}
stepCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
output, err := runSSHCommand(stepCtx, client, step.Command)
if err != nil {
class := classifySSHError(err)
// Verify steps that fail are not counted as verified failures
if step.Runner == "verify" {
verified = false
}
// Non-verify step failure is a real failure
if step.Runner != "verify" {
return SSHResult{
Output: lastOutput,
Duration: time.Since(start),
Err: fmt.Errorf("step %d (%s) failed (%s): %w", i, step.Runner, class, err),
Verified: false,
}
}
}
lastOutput = output
slog.Debug("ssh step completed",
"step", i,
"runner", step.Runner,
"duration", time.Since(start).Round(time.Millisecond),
)
}
return SSHResult{
Output: lastOutput,
Duration: time.Since(start),
Verified: verified,
}
}
// runSSHCommand executes a single command over an established SSH session.
// Uses context-aware goroutines: ctx.Done() closes the session.
func runSSHCommand(ctx context.Context, client *ssh.Client, command string) (string, error) {
session, err := client.NewSession()
if err != nil {
return "", fmt.Errorf("create session: %w", err)
}
defer session.Close()
// Wrap in goroutine so we can abort on ctx.Done()
type result struct {
output string
err error
}
ch := make(chan result, 1)
go func() {
out, err := session.CombinedOutput(command)
ch <- result{output: string(out), err: err}
}()
select {
case <-ctx.Done():
// Close the session to abort the SSH command
session.Close()
return "", ctx.Err()
case res := <-ch:
if res.err != nil {
return res.output, fmt.Errorf("command: %w", res.err)
}
return res.output, nil
}
}
// ─── Procedure parsing ────────────────────────────────────────────────────
// ParseProcedure deserialises a JSON procedure (from skill.procedure).
func ParseProcedure(data []byte) (Procedure, error) {
var proc Procedure
if err := json.Unmarshal(data, &proc); err != nil {
return Procedure{}, fmt.Errorf("parse procedure: %w", err)
}
return proc, nil
}
// ─── Global SSH client options ────────────────────────────────────────────
var (
mu sync.Mutex
// defaultSSHTimeout is the default dial timeout for SSH connections.
defaultSSHTimeout = 10 * time.Second
)
// SetDefaultSSHTimeout overrides the default SSH dial timeout. Not safe for
// concurrent use during active execution.
func SetDefaultSSHTimeout(d time.Duration) {
mu.Lock()
defer mu.Unlock()
defaultSSHTimeout = d
}