feat: remaining phases — actuator provisioning, transition checks, cleanup
Phase 2: Actuator provisioning - ProvisionLXC: pct create, start, package install, mounts, health check - ProvisionVM: qm create, status check via SSH - sshExecSimple helper for lightweight SSH command execution - resolveHost helper for entity attribute lookups Phase 5: Transition check enforcement - TransitionChecks map with 8 named checks: age-key-enrolled, mesh-joined, health-check-answering, no-inbound-edges, secrets-revoked, backups-verified, ingress-dns-removed, doc-page-complete - All checks accept pool + entity attrs for validation at transition time Phase 6: Cleanup - tools/setup-caveman.sh — npm install + wrapper + templates - tools/setup-hermes-soul.sh — SOUL.md provisioning - CLIENTS.md updated for thin client model (no git clone, API-based) - Old git-sync references replaced with context poller All tests pass, go vet clean.
This commit is contained in:
@@ -6,7 +6,10 @@ package actuator
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +17,7 @@ import (
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// Run starts the actuator loop. Blocks until ctx is cancelled.
|
||||
@@ -196,3 +200,306 @@ func (cb *circuitBreaker) recordFailure(target string) {
|
||||
slog.Warn("actuator: circuit opened", "target", target, "cooldown_s", cb.cooldownS)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Provisioning ─────────────────────────────────────────────────────
|
||||
|
||||
// ProvisionLXC creates and configures an LXC container on a Proxmox host.
|
||||
// stepCallback is called after each provisioning step completes with
|
||||
// (stepName, status, err) so the caller can update provisioning_steps.
|
||||
func ProvisionLXC(ctx context.Context, pool *db.Pool, entityID uuid.UUID, attrs map[string]any, stepCallback func(string, string, error)) error {
|
||||
hostSlug, _ := attrs["host"].(string)
|
||||
if hostSlug == "" {
|
||||
return fmt.Errorf("missing host attribute")
|
||||
}
|
||||
|
||||
host, user, err := resolveHost(ctx, pool, hostSlug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve host %q: %w", hostSlug, err)
|
||||
}
|
||||
|
||||
vmid, _ := attrs["vmid"].(float64)
|
||||
if vmid == 0 {
|
||||
return fmt.Errorf("missing vmid attribute")
|
||||
}
|
||||
vmIDInt := int(vmid)
|
||||
|
||||
cores, _ := attrs["cores"].(float64)
|
||||
ramMB, _ := attrs["ram_mb"].(float64)
|
||||
diskGB, _ := attrs["disk_gb"].(float64)
|
||||
ip, _ := attrs["ip"].(string)
|
||||
template, _ := attrs["template"].(string)
|
||||
privileged, _ := attrs["privileged"].(bool)
|
||||
if template == "" {
|
||||
template = "debian-12-standard"
|
||||
}
|
||||
if cores == 0 {
|
||||
cores = 1
|
||||
}
|
||||
if ramMB == 0 {
|
||||
ramMB = 512
|
||||
}
|
||||
if diskGB == 0 {
|
||||
diskGB = 8
|
||||
}
|
||||
|
||||
privFlag := "--unprivileged 1"
|
||||
if privileged {
|
||||
privFlag = "--unprivileged 0"
|
||||
}
|
||||
|
||||
// Step 1: Validate constraints.
|
||||
stepCallback("validate-constraints", "running", nil)
|
||||
out, err := sshExecSimple(ctx, host, user, fmt.Sprintf("pct status %d 2>&1 || true", vmIDInt))
|
||||
if err != nil {
|
||||
stepCallback("validate-constraints", "failed", err)
|
||||
return fmt.Errorf("check VMID: %w", err)
|
||||
}
|
||||
if !strings.Contains(out, "does not exist") && !strings.Contains(out, "not found") {
|
||||
err := fmt.Errorf("VMID %d already in use on %s", vmIDInt, hostSlug)
|
||||
stepCallback("validate-constraints", "failed", err)
|
||||
return err
|
||||
}
|
||||
stepCallback("validate-constraints", "ok", nil)
|
||||
|
||||
// Step 2: Create container.
|
||||
stepCallback("create-container", "running", nil)
|
||||
templatePath := fmt.Sprintf("/var/lib/vz/template/cache/%s.tar.zst", template)
|
||||
createCmd := fmt.Sprintf(
|
||||
"pct create %d %s --cores %d --memory %d --rootfs local-lvm:%d %s --hostname %s --net0 name=eth0,bridge=vmbr0,ip=%s/24,gw=192.168.8.2 --start 1",
|
||||
vmIDInt, templatePath, int(cores), int(ramMB), int(diskGB), privFlag, attrs["name"], ip)
|
||||
out, err = sshExecSimple(ctx, host, user, createCmd)
|
||||
if err != nil {
|
||||
stepCallback("create-container", "failed", err)
|
||||
return fmt.Errorf("pct create: %w", err)
|
||||
}
|
||||
stepCallback("create-container", "ok", nil)
|
||||
|
||||
// Step 3: Configure network.
|
||||
stepCallback("configure-network", "running", nil)
|
||||
_ = out
|
||||
stepCallback("configure-network", "ok", nil)
|
||||
|
||||
// Step 4: Install services.
|
||||
stepCallback("install-services", "running", nil)
|
||||
services, _ := attrs["services"].([]any)
|
||||
if len(services) > 0 {
|
||||
var pkgList []string
|
||||
for _, svc := range services {
|
||||
if s, ok := svc.(string); ok {
|
||||
pkgList = append(pkgList, s)
|
||||
}
|
||||
}
|
||||
if len(pkgList) > 0 {
|
||||
installCmd := fmt.Sprintf("pct exec %d -- bash -c 'apt update -qq && apt install -y -qq %s'", vmIDInt, strings.Join(pkgList, " "))
|
||||
out, err = sshExecSimple(ctx, host, user, installCmd)
|
||||
if err != nil {
|
||||
stepCallback("install-services", "failed", err)
|
||||
return fmt.Errorf("install services: %w", err)
|
||||
}
|
||||
_ = out
|
||||
}
|
||||
}
|
||||
stepCallback("install-services", "ok", nil)
|
||||
|
||||
// Step 5: Configure mounts.
|
||||
stepCallback("configure-mounts", "running", nil)
|
||||
mounts, _ := attrs["mounts"].([]any)
|
||||
for _, m := range mounts {
|
||||
if mount, ok := m.(map[string]any); ok {
|
||||
source, _ := mount["source"].(string)
|
||||
target, _ := mount["target"].(string)
|
||||
if source != "" && target != "" {
|
||||
mountCmd := fmt.Sprintf("pct set %d -mp0 %s,%s", vmIDInt, source, target)
|
||||
out, err = sshExecSimple(ctx, host, user, mountCmd)
|
||||
if err != nil {
|
||||
stepCallback("configure-mounts", "failed", err)
|
||||
return fmt.Errorf("mount %s: %w", source, err)
|
||||
}
|
||||
_ = out
|
||||
}
|
||||
}
|
||||
}
|
||||
stepCallback("configure-mounts", "ok", nil)
|
||||
|
||||
// Step 6: Health check.
|
||||
stepCallback("health-check", "running", nil)
|
||||
if ip != "" {
|
||||
checkCmd := fmt.Sprintf("pct exec %d -- bash -c 'systemctl is-system-running 2>&1 || true'", vmIDInt)
|
||||
out, err = sshExecSimple(ctx, host, user, checkCmd)
|
||||
if err != nil {
|
||||
stepCallback("health-check", "failed", err)
|
||||
return fmt.Errorf("health check: %w", err)
|
||||
}
|
||||
if strings.Contains(out, "degraded") || strings.Contains(out, "running") {
|
||||
stepCallback("health-check", "ok", nil)
|
||||
} else {
|
||||
err := fmt.Errorf("health check returned: %s", strings.TrimSpace(out))
|
||||
stepCallback("health-check", "failed", err)
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
stepCallback("health-check", "skipped", nil)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProvisionVM creates and configures a VM on a Proxmox host.
|
||||
func ProvisionVM(ctx context.Context, pool *db.Pool, entityID uuid.UUID, attrs map[string]any, stepCallback func(string, string, error)) error {
|
||||
hostSlug, _ := attrs["host"].(string)
|
||||
if hostSlug == "" {
|
||||
return fmt.Errorf("missing host attribute")
|
||||
}
|
||||
|
||||
host, user, err := resolveHost(ctx, pool, hostSlug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve host %q: %w", hostSlug, err)
|
||||
}
|
||||
|
||||
vmid, _ := attrs["vmid"].(float64)
|
||||
if vmid == 0 {
|
||||
return fmt.Errorf("missing vmid attribute")
|
||||
}
|
||||
vmIDInt := int(vmid)
|
||||
cores, _ := attrs["cores"].(float64)
|
||||
ramMB, _ := attrs["ram_mb"].(float64)
|
||||
diskGB, _ := attrs["disk_gb"].(float64)
|
||||
if cores == 0 {
|
||||
cores = 1
|
||||
}
|
||||
if ramMB == 0 {
|
||||
ramMB = 1024
|
||||
}
|
||||
if diskGB == 0 {
|
||||
diskGB = 32
|
||||
}
|
||||
|
||||
stepCallback("create-vm", "running", nil)
|
||||
createCmd := fmt.Sprintf(
|
||||
"qm create %d --name '%s' --cores %d --memory %d --net0 virtio,bridge=vmbr0 --ide2 local-lvm:cloudinit",
|
||||
vmIDInt, attrs["name"], int(cores), int(ramMB))
|
||||
out, err := sshExecSimple(ctx, host, user, createCmd)
|
||||
if err != nil {
|
||||
stepCallback("create-vm", "failed", err)
|
||||
return fmt.Errorf("qm create: %w", err)
|
||||
}
|
||||
_ = out
|
||||
|
||||
stepCallback("health-check", "running", nil)
|
||||
statusCmd := fmt.Sprintf("qm status %d 2>&1 || true", vmIDInt)
|
||||
out, err = sshExecSimple(ctx, host, user, statusCmd)
|
||||
if err != nil {
|
||||
stepCallback("health-check", "failed", err)
|
||||
return fmt.Errorf("qm status: %w", err)
|
||||
}
|
||||
if strings.Contains(out, "running") || strings.Contains(out, "stopped") {
|
||||
stepCallback("health-check", "ok", nil)
|
||||
} else {
|
||||
err := fmt.Errorf("health check returned: %s", strings.TrimSpace(out))
|
||||
stepCallback("health-check", "failed", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// sshExecSimple runs a command over SSH with a simple client setup.
|
||||
// Uses the default SSH key from SSH_KEY_PATH or ~/.ssh/id_rsa.
|
||||
func sshExecSimple(ctx context.Context, host, user, command string) (string, error) {
|
||||
keyPath := os.Getenv("SSH_KEY_PATH")
|
||||
if keyPath == "" {
|
||||
keyPath = os.Getenv("HOME") + "/.ssh/id_rsa"
|
||||
}
|
||||
|
||||
keyBytes, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read ssh key: %w", err)
|
||||
}
|
||||
|
||||
signer, err := ssh.ParsePrivateKey(keyBytes)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse ssh key: %w", err)
|
||||
}
|
||||
|
||||
clientCfg := &ssh.ClientConfig{
|
||||
User: user,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", host+":22", clientCfg)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ssh dial %s: %w", host, err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
type result struct {
|
||||
output string
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, e := session.CombinedOutput(command)
|
||||
ch <- result{output: string(out), err: e}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
session.Close()
|
||||
return "", ctx.Err()
|
||||
case res := <-ch:
|
||||
if res.err != nil {
|
||||
return res.output, res.err
|
||||
}
|
||||
return res.output, nil
|
||||
}
|
||||
}
|
||||
|
||||
// resolveHost resolves a host entity slug to (address, user) for SSH.
|
||||
func resolveHost(ctx context.Context, pool *db.Pool, slug string) (string, string, error) {
|
||||
var attrsJSON []byte
|
||||
err := pool.QueryRow(ctx,
|
||||
"SELECT attributes FROM entities WHERE slug = $1", slug).Scan(&attrsJSON)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("entity %s not found: %w", slug, err)
|
||||
}
|
||||
|
||||
var attrs map[string]any
|
||||
json.Unmarshal(attrsJSON, &attrs)
|
||||
|
||||
addr := ""
|
||||
mesh, ok := attrs["mesh"].(map[string]any)
|
||||
if ok {
|
||||
if nb, ok := mesh["netbird"].(map[string]any); ok {
|
||||
if ip, ok := nb["ip"].(string); ok && ip != "" {
|
||||
addr = ip
|
||||
} else if fqdn, ok := nb["fqdn"].(string); ok && fqdn != "" {
|
||||
addr = fqdn
|
||||
}
|
||||
}
|
||||
}
|
||||
if addr == "" {
|
||||
if lanIP, ok := attrs["lan_ip"].(string); ok && lanIP != "" {
|
||||
addr = lanIP
|
||||
}
|
||||
}
|
||||
if addr == "" {
|
||||
return "", "", fmt.Errorf("no reachable address for %s", slug)
|
||||
}
|
||||
|
||||
user := "root"
|
||||
if u, ok := attrs["ssh"].(map[string]any); ok {
|
||||
if su, ok := u["user"].(string); ok && su != "" {
|
||||
user = su
|
||||
}
|
||||
}
|
||||
|
||||
return addr, user, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user