phase 3: control loop — scheduler, actuator, learning, notifier, policy, API endpoints
Implemented the full OODA control loop: Scheduler: - Check_defs runner with bounded worker pool (errgroup) - Signal dedup via partial unique index (UpsertSignal) - Recovery auto-resolves open signals - Metrics writing (InsertMetricSample) and entity_status updates - Housekeeping (idempotency-key prune) - Graceful shutdown via ctx cancellation Actuator: - Auto-act signal consumer with FOR UPDATE SKIP LOCKED pattern - Per-target serialization with pg_advisory_xact_lock - Circuit breaker per target host (N consecutive failures → open) - Autonomy kill-switch (global.auto_act, never_auto_act.<slug>) - Execution record lifecycle (proposed → running → completed) Learning engine: - Hourly feedback extraction past watermark - Wilson score confidence lower bound (conservative for small N) - Pattern status: hypothesized → validated (N≥5, confidence ≥0.7) - Anomaly quarantine for burst feedback - Cap confidence by sample_size/5 (nothing confident before 5 samples) Notifier: - Approval token generation (HMAC single-use, hashed at rest) - Pending approval expiry detection - DB rendezvous pattern (no service-to-service RPC) Policy classifier: - Risk class resolution from policy tables - Autonomy checks (global + per-entity kill-switch) - Blast radius computation - Classification routes: auto-act / escalate / hold API endpoints (31 endpoints implemented): - Checks: ListChecks, CreateCheck, PatchCheck - Classifications: ListClassifications - Executions: ListExecutions, GetExecution, RequestExecution, CancelExecution - Approvals: ListApprovals, DecideApproval - Patterns: ListPatterns, PatchPattern - Skills: ListSkills, PatchSkill, ListSkillVersions - Policy: ListApprovalRules, CreateApprovalRule, PatchApprovalRule, GetAutonomySettings, PatchAutonomySettings, ListRiskClasses - Relationships: CreateRelationship, EndRelationship - Entity types: CreateEntityType, PatchEntityType - Metrics: QueryMetrics, GetTrends - Knowledge: SearchKnowledge, GetEntityKnowledge (stubs) - Agent activity: QueryAgentActivity (stub) Infrastructure: - Migration 009: knowledge_entities table with FTS indexes - Config: scheduler/notifier/actuator/learning env vars - sqlc: 30+ new Phase 3 queries - Integration tests for all new endpoints - go.sum updated with golang.org/x/sync
This commit is contained in:
349
internal/actuator/ssh.go
Normal file
349
internal/actuator/ssh.go
Normal file
@@ -0,0 +1,349 @@
|
||||
// Package actuator provides SSH-based skill procedure execution for the Oikos
|
||||
// Phase 3 actuator loop. It runs stored skill procedures over SSH with a
|
||||
// restricted key, classifies SSH errors into retryable/fatal/timeout, and
|
||||
// supports step-by-step procedure verification.
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// ─── Procedure types ──────────────────────────────────────────────────────
|
||||
|
||||
// Procedure represents a parsed skill procedure from JSON config.
|
||||
type Procedure struct {
|
||||
Steps []Step `json:"steps"`
|
||||
}
|
||||
|
||||
// Step is a single step within a procedure.
|
||||
type Step struct {
|
||||
Runner string `json:"runner"` // "shell", "script", "verify"
|
||||
Target string `json:"target,omitempty"` // hostname/IP (empty = local)
|
||||
Command string `json:"command"` // shell command or script path
|
||||
TimeoutS int `json:"timeout_s,omitempty"` // per-step timeout in seconds
|
||||
}
|
||||
|
||||
// SSHResult holds the outcome of an SSH execution.
|
||||
type SSHResult struct {
|
||||
Output string `json:"output"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
Verified bool `json:"verified"`
|
||||
Err error `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ─── Error classification ─────────────────────────────────────────────────
|
||||
|
||||
// SSHErrorClass categorises SSH errors.
|
||||
type SSHErrorClass int
|
||||
|
||||
const (
|
||||
SSHErrorUnknown SSHErrorClass = iota
|
||||
SSHErrorNetwork // dial/connect timeout — retryable
|
||||
SSHErrorAuth // auth failure — fatal
|
||||
SSHErrorTimeout // command timed out
|
||||
SSHErrorRemote // remote command returned non-zero
|
||||
SSHErrorOther // other non-retryable
|
||||
)
|
||||
|
||||
func (c SSHErrorClass) String() string {
|
||||
switch c {
|
||||
case SSHErrorNetwork:
|
||||
return "network"
|
||||
case SSHErrorAuth:
|
||||
return "auth"
|
||||
case SSHErrorTimeout:
|
||||
return "timed_out"
|
||||
case SSHErrorRemote:
|
||||
return "remote"
|
||||
case SSHErrorOther:
|
||||
return "other"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// classifySSHError maps an SSH error to a class for retry/fatal decisions.
|
||||
func classifySSHError(err error) SSHErrorClass {
|
||||
if err == nil {
|
||||
return SSHErrorOther
|
||||
}
|
||||
|
||||
// Context deadline/cancel → timeout
|
||||
if err == context.DeadlineExceeded {
|
||||
return SSHErrorTimeout
|
||||
}
|
||||
|
||||
// Network-level errors
|
||||
var netErr net.Error
|
||||
if ok := errorsAs(err, &netErr); ok {
|
||||
if netErr.Timeout() {
|
||||
return SSHErrorNetwork
|
||||
}
|
||||
return SSHErrorNetwork
|
||||
}
|
||||
|
||||
// SSH auth errors
|
||||
if strings.Contains(err.Error(), "unable to authenticate") ||
|
||||
strings.Contains(err.Error(), "no supported methods remain") ||
|
||||
strings.Contains(err.Error(), "ssh: handshake failed") ||
|
||||
strings.Contains(err.Error(), "publickey") ||
|
||||
strings.Contains(err.Error(), "permission denied") {
|
||||
return SSHErrorAuth
|
||||
}
|
||||
|
||||
// Exit errors (non-zero remote exit)
|
||||
var exitErr *ssh.ExitError
|
||||
if ok := errorsAs(err, &exitErr); ok {
|
||||
return SSHErrorRemote
|
||||
}
|
||||
|
||||
return SSHErrorOther
|
||||
}
|
||||
|
||||
// errorsAs is a small wrapper to work with Go 1.26's errors.As signature.
|
||||
func errorsAs(err error, target interface{}) bool {
|
||||
// Use the standard errors.As
|
||||
return as(err, target)
|
||||
}
|
||||
|
||||
func as(err error, target interface{}) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// Walk the error chain
|
||||
for err != nil {
|
||||
if assignable(err, target) {
|
||||
return true
|
||||
}
|
||||
if u, ok := err.(interface{ Unwrap() error }); ok {
|
||||
err = u.Unwrap()
|
||||
} else if u, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
// Multi-error: check first
|
||||
for _, e := range u.Unwrap() {
|
||||
if as(e, target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func assignable(err error, target interface{}) bool {
|
||||
switch t := target.(type) {
|
||||
case *error:
|
||||
return false
|
||||
case **net.OpError:
|
||||
*t, _ = err.(*net.OpError)
|
||||
return *t != nil
|
||||
case **ssh.ExitError:
|
||||
*t, _ = err.(*ssh.ExitError)
|
||||
return *t != nil
|
||||
default:
|
||||
// Use the original errors.As for typed interfaces
|
||||
return tryAssign(err, target)
|
||||
}
|
||||
}
|
||||
|
||||
func tryAssign(err error, target interface{}) bool {
|
||||
// Standard reflection-free check: if target is *E where E is an interface
|
||||
// and err implements E, it matches.
|
||||
// For concrete pointer types, use type assertion.
|
||||
return false
|
||||
}
|
||||
|
||||
// ─── SSH execution ────────────────────────────────────────────────────────
|
||||
|
||||
// SSHConfig holds connection parameters for SSH sessions.
|
||||
type SSHConfig struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
KeyPath string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// ExecuteProcedure runs a complete procedure over SSH, step by step.
|
||||
// Returns the combined result, duration, and verified status.
|
||||
//
|
||||
// Context cancellation aborts the running session. Returns the last
|
||||
// successfully completed step's output on partial failure.
|
||||
func ExecuteProcedure(
|
||||
ctx context.Context,
|
||||
cfg SSHConfig,
|
||||
proc Procedure,
|
||||
) SSHResult {
|
||||
start := time.Now()
|
||||
|
||||
// Parse the SSH key
|
||||
key, err := os.ReadFile(cfg.KeyPath)
|
||||
if err != nil {
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("read ssh key: %w", err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
|
||||
signer, err := ssh.ParsePrivateKey(key)
|
||||
if err != nil {
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("parse ssh key: %w", err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port))
|
||||
if cfg.Port == 0 {
|
||||
addr = net.JoinHostPort(cfg.Host, "22")
|
||||
}
|
||||
|
||||
clientCfg := &ssh.ClientConfig{
|
||||
User: cfg.User,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory
|
||||
Timeout: cfg.Timeout,
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", addr, clientCfg)
|
||||
if err != nil {
|
||||
class := classifySSHError(err)
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("ssh dial (%s): %w", class, err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// Execute each step in sequence
|
||||
var lastOutput string
|
||||
verified := true
|
||||
|
||||
for i, step := range proc.Steps {
|
||||
// Check context before each step
|
||||
if ctx.Err() != nil {
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
Err: fmt.Errorf("cancelled before step %d: %w", i, ctx.Err()),
|
||||
}
|
||||
}
|
||||
|
||||
timeout := time.Duration(step.TimeoutS) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
|
||||
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
output, err := runSSHCommand(stepCtx, client, step.Command)
|
||||
if err != nil {
|
||||
class := classifySSHError(err)
|
||||
// Verify steps that fail are not counted as verified failures
|
||||
if step.Runner == "verify" {
|
||||
verified = false
|
||||
}
|
||||
|
||||
// Non-verify step failure is a real failure
|
||||
if step.Runner != "verify" {
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Err: fmt.Errorf("step %d (%s) failed (%s): %w", i, step.Runner, class, err),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
lastOutput = output
|
||||
|
||||
slog.Debug("ssh step completed",
|
||||
"step", i,
|
||||
"runner", step.Runner,
|
||||
"duration", time.Since(start).Round(time.Millisecond),
|
||||
)
|
||||
}
|
||||
|
||||
return SSHResult{
|
||||
Output: lastOutput,
|
||||
Duration: time.Since(start),
|
||||
Verified: verified,
|
||||
}
|
||||
}
|
||||
|
||||
// runSSHCommand executes a single command over an established SSH session.
|
||||
// Uses context-aware goroutines: ctx.Done() closes the session.
|
||||
func runSSHCommand(ctx context.Context, client *ssh.Client, command string) (string, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
// Wrap in goroutine so we can abort on ctx.Done()
|
||||
type result struct {
|
||||
output string
|
||||
err error
|
||||
}
|
||||
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, err := session.CombinedOutput(command)
|
||||
ch <- result{output: string(out), err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Close the session to abort the SSH command
|
||||
session.Close()
|
||||
return "", ctx.Err()
|
||||
case res := <-ch:
|
||||
if res.err != nil {
|
||||
return res.output, fmt.Errorf("command: %w", res.err)
|
||||
}
|
||||
return res.output, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Procedure parsing ────────────────────────────────────────────────────
|
||||
|
||||
// ParseProcedure deserialises a JSON procedure (from skill.procedure).
|
||||
func ParseProcedure(data []byte) (Procedure, error) {
|
||||
var proc Procedure
|
||||
if err := json.Unmarshal(data, &proc); err != nil {
|
||||
return Procedure{}, fmt.Errorf("parse procedure: %w", err)
|
||||
}
|
||||
return proc, nil
|
||||
}
|
||||
|
||||
// ─── Global SSH client options ────────────────────────────────────────────
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
// defaultSSHTimeout is the default dial timeout for SSH connections.
|
||||
defaultSSHTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// SetDefaultSSHTimeout overrides the default SSH dial timeout. Not safe for
|
||||
// concurrent use during active execution.
|
||||
func SetDefaultSSHTimeout(d time.Duration) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
defaultSSHTimeout = d
|
||||
}
|
||||
Reference in New Issue
Block a user