- actuator/ssh.go: custom errorsAs chain broken — all SSH errors classified as SSHErrorOther. Replaced with standard errors.As + errors.Is. - scheduler/scheduler.go: all four check functions were stubs returning healthy. Implemented real HTTP GET, TCP dial, unix.Statfs disk, and TLS cert expiry checks. - learning/learning.go: uuid.NewV7() called unconditionally before ON CONFLICT upsert. Now looks up existing pattern first, reuses entity_id. - notifier/notifier.go: removed dead var_, fixed token regeneration every 15s. Now skips if token_hash already set. - phase3.go: removed dead GetPattern+dummy args call in PatchPattern. - classify.go: removed unused var_ guard.
292 lines
7.8 KiB
Go
292 lines
7.8 KiB
Go
// Package actuator provides SSH-based skill procedure execution for the Oikos
|
|
// Phase 3 actuator loop. It runs stored skill procedures over SSH with a
|
|
// restricted key, classifies SSH errors into retryable/fatal/timeout, and
|
|
// supports step-by-step procedure verification.
|
|
package actuator
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/ssh"
|
|
)
|
|
|
|
// ─── Procedure types ──────────────────────────────────────────────────────
|
|
|
|
// Procedure represents a parsed skill procedure from JSON config.
|
|
type Procedure struct {
|
|
Steps []Step `json:"steps"`
|
|
}
|
|
|
|
// Step is a single step within a procedure.
|
|
type Step struct {
|
|
Runner string `json:"runner"` // "shell", "script", "verify"
|
|
Target string `json:"target,omitempty"` // hostname/IP (empty = local)
|
|
Command string `json:"command"` // shell command or script path
|
|
TimeoutS int `json:"timeout_s,omitempty"` // per-step timeout in seconds
|
|
}
|
|
|
|
// SSHResult holds the outcome of an SSH execution.
|
|
type SSHResult struct {
|
|
Output string `json:"output"`
|
|
Duration time.Duration `json:"duration"`
|
|
Verified bool `json:"verified"`
|
|
Err error `json:"error,omitempty"`
|
|
}
|
|
|
|
// ─── Error classification ─────────────────────────────────────────────────
|
|
|
|
// SSHErrorClass categorises SSH errors.
|
|
type SSHErrorClass int
|
|
|
|
const (
|
|
SSHErrorUnknown SSHErrorClass = iota
|
|
SSHErrorNetwork // dial/connect timeout — retryable
|
|
SSHErrorAuth // auth failure — fatal
|
|
SSHErrorTimeout // command timed out
|
|
SSHErrorRemote // remote command returned non-zero
|
|
SSHErrorOther // other non-retryable
|
|
)
|
|
|
|
func (c SSHErrorClass) String() string {
|
|
switch c {
|
|
case SSHErrorNetwork:
|
|
return "network"
|
|
case SSHErrorAuth:
|
|
return "auth"
|
|
case SSHErrorTimeout:
|
|
return "timed_out"
|
|
case SSHErrorRemote:
|
|
return "remote"
|
|
case SSHErrorOther:
|
|
return "other"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// classifySSHError maps an SSH error to a class for retry/fatal decisions.
|
|
func classifySSHError(err error) SSHErrorClass {
|
|
if err == nil {
|
|
return SSHErrorOther
|
|
}
|
|
|
|
if errors.Is(err, context.DeadlineExceeded) {
|
|
return SSHErrorTimeout
|
|
}
|
|
|
|
var netErr net.Error
|
|
if errors.As(err, &netErr) {
|
|
if netErr.Timeout() {
|
|
return SSHErrorNetwork
|
|
}
|
|
return SSHErrorNetwork
|
|
}
|
|
|
|
if strings.Contains(err.Error(), "unable to authenticate") ||
|
|
strings.Contains(err.Error(), "no supported methods remain") ||
|
|
strings.Contains(err.Error(), "ssh: handshake failed") ||
|
|
strings.Contains(err.Error(), "publickey") ||
|
|
strings.Contains(err.Error(), "permission denied") {
|
|
return SSHErrorAuth
|
|
}
|
|
|
|
var exitErr *ssh.ExitError
|
|
if errors.As(err, &exitErr) {
|
|
return SSHErrorRemote
|
|
}
|
|
|
|
return SSHErrorOther
|
|
}
|
|
|
|
// ─── SSH execution ────────────────────────────────────────────────────────
|
|
|
|
// SSHConfig holds connection parameters for SSH sessions.
|
|
type SSHConfig struct {
|
|
Host string
|
|
Port int
|
|
User string
|
|
KeyPath string
|
|
Timeout time.Duration
|
|
}
|
|
|
|
// ExecuteProcedure runs a complete procedure over SSH, step by step.
|
|
// Returns the combined result, duration, and verified status.
|
|
//
|
|
// Context cancellation aborts the running session. Returns the last
|
|
// successfully completed step's output on partial failure.
|
|
func ExecuteProcedure(
|
|
ctx context.Context,
|
|
cfg SSHConfig,
|
|
proc Procedure,
|
|
) SSHResult {
|
|
start := time.Now()
|
|
|
|
// Parse the SSH key
|
|
key, err := os.ReadFile(cfg.KeyPath)
|
|
if err != nil {
|
|
return SSHResult{
|
|
Err: fmt.Errorf("read ssh key: %w", err),
|
|
Duration: time.Since(start),
|
|
Verified: false,
|
|
}
|
|
}
|
|
|
|
signer, err := ssh.ParsePrivateKey(key)
|
|
if err != nil {
|
|
return SSHResult{
|
|
Err: fmt.Errorf("parse ssh key: %w", err),
|
|
Duration: time.Since(start),
|
|
Verified: false,
|
|
}
|
|
}
|
|
|
|
addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port))
|
|
if cfg.Port == 0 {
|
|
addr = net.JoinHostPort(cfg.Host, "22")
|
|
}
|
|
|
|
clientCfg := &ssh.ClientConfig{
|
|
User: cfg.User,
|
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
|
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory
|
|
Timeout: cfg.Timeout,
|
|
}
|
|
|
|
client, err := ssh.Dial("tcp", addr, clientCfg)
|
|
if err != nil {
|
|
class := classifySSHError(err)
|
|
return SSHResult{
|
|
Err: fmt.Errorf("ssh dial (%s): %w", class, err),
|
|
Duration: time.Since(start),
|
|
Verified: false,
|
|
}
|
|
}
|
|
defer client.Close()
|
|
|
|
// Execute each step in sequence
|
|
var lastOutput string
|
|
verified := true
|
|
|
|
for i, step := range proc.Steps {
|
|
// Check context before each step
|
|
if ctx.Err() != nil {
|
|
return SSHResult{
|
|
Output: lastOutput,
|
|
Duration: time.Since(start),
|
|
Verified: false,
|
|
Err: fmt.Errorf("cancelled before step %d: %w", i, ctx.Err()),
|
|
}
|
|
}
|
|
|
|
timeout := time.Duration(step.TimeoutS) * time.Second
|
|
if timeout <= 0 {
|
|
timeout = 30 * time.Second
|
|
}
|
|
|
|
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
output, err := runSSHCommand(stepCtx, client, step.Command)
|
|
if err != nil {
|
|
class := classifySSHError(err)
|
|
// Verify steps that fail are not counted as verified failures
|
|
if step.Runner == "verify" {
|
|
verified = false
|
|
}
|
|
|
|
// Non-verify step failure is a real failure
|
|
if step.Runner != "verify" {
|
|
return SSHResult{
|
|
Output: lastOutput,
|
|
Duration: time.Since(start),
|
|
Err: fmt.Errorf("step %d (%s) failed (%s): %w", i, step.Runner, class, err),
|
|
Verified: false,
|
|
}
|
|
}
|
|
}
|
|
lastOutput = output
|
|
|
|
slog.Debug("ssh step completed",
|
|
"step", i,
|
|
"runner", step.Runner,
|
|
"duration", time.Since(start).Round(time.Millisecond),
|
|
)
|
|
}
|
|
|
|
return SSHResult{
|
|
Output: lastOutput,
|
|
Duration: time.Since(start),
|
|
Verified: verified,
|
|
}
|
|
}
|
|
|
|
// runSSHCommand executes a single command over an established SSH session.
|
|
// Uses context-aware goroutines: ctx.Done() closes the session.
|
|
func runSSHCommand(ctx context.Context, client *ssh.Client, command string) (string, error) {
|
|
session, err := client.NewSession()
|
|
if err != nil {
|
|
return "", fmt.Errorf("create session: %w", err)
|
|
}
|
|
defer session.Close()
|
|
|
|
// Wrap in goroutine so we can abort on ctx.Done()
|
|
type result struct {
|
|
output string
|
|
err error
|
|
}
|
|
|
|
ch := make(chan result, 1)
|
|
go func() {
|
|
out, err := session.CombinedOutput(command)
|
|
ch <- result{output: string(out), err: err}
|
|
}()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
// Close the session to abort the SSH command
|
|
session.Close()
|
|
return "", ctx.Err()
|
|
case res := <-ch:
|
|
if res.err != nil {
|
|
return res.output, fmt.Errorf("command: %w", res.err)
|
|
}
|
|
return res.output, nil
|
|
}
|
|
}
|
|
|
|
// ─── Procedure parsing ────────────────────────────────────────────────────
|
|
|
|
// ParseProcedure deserialises a JSON procedure (from skill.procedure).
|
|
func ParseProcedure(data []byte) (Procedure, error) {
|
|
var proc Procedure
|
|
if err := json.Unmarshal(data, &proc); err != nil {
|
|
return Procedure{}, fmt.Errorf("parse procedure: %w", err)
|
|
}
|
|
return proc, nil
|
|
}
|
|
|
|
// ─── Global SSH client options ────────────────────────────────────────────
|
|
|
|
var (
|
|
mu sync.Mutex
|
|
// defaultSSHTimeout is the default dial timeout for SSH connections.
|
|
defaultSSHTimeout = 10 * time.Second
|
|
)
|
|
|
|
// SetDefaultSSHTimeout overrides the default SSH dial timeout. Not safe for
|
|
// concurrent use during active execution.
|
|
func SetDefaultSSHTimeout(d time.Duration) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
defaultSSHTimeout = d
|
|
}
|