dtoro/oikos
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: Phase 7 — SeedService, SecretsService, ProvisioningService + ssh Provisioner

Browse Source 
Seed ingest/export moves behind ports.SeedRepository (SeedRepo in the
postgres adapter; knowledge ingest absorbed from internal/knowledge,
package deleted). pct_create flow (defaults, template/VMID/gateway
pre-flights, pct create, graph registration) moves from httpapi's
approved-execution path into app.ProvisioningService + the ssh
provisioner adapter; CLI seed/export/secret become adapters over the
services. EntityCreateInput gains EnrolledAt. Plan status corrected:
phases 0-7 shipped, 8 + 9 gates open. VERSION 0.34.1.
This commit is contained in:
David Toro
2026-08-16 09:11:26 +02:00
parent cd44501fa9
commit 60c0432d8b
26 changed files with 1565 additions and 689 deletions
Download Patch File Download Diff File Expand all files Collapse all files

310
internal/httpapi/actuator.go
View File

@@ -7,13 +7,13 @@ import (
"fmt"
"log/slog"
"os"
"strconv"
"strings"
"time"
"github.com/dtoro/oikos/internal/actuator"
"github.com/dtoro/oikos/internal/adapters/postgres"
"github.com/dtoro/oikos/internal/adapters/postgres/sqlcgen"
"github.com/dtoro/oikos/internal/core/app"
"github.com/dtoro/oikos/internal/execlog"
"github.com/dtoro/oikos/internal/observability"
"github.com/google/uuid"
@@ -78,10 +78,6 @@ const sshExecTimeout = 10 * time.Minute
// remote end produced it. Shared implementation lives in internal/actuator
// (actuator.streamWriter / actuator.RunStreaming).
func sshExec(ctx context.Context, host, user, command string) (string, error) {
return sshExecStream(ctx, host, user, command, nil)
}
// sshExecStream runs a command and reports its combined output, forwarding
// each chunk to sink as it arrives. A nil sink behaves exactly as before.
func sshExecStream(ctx context.Context, host, user, command string, sink execlog.Sink) (string, error) {
@@ -224,7 +220,7 @@ func closePlanStepForExecution(ctx context.Context, pool *db.Pool, execID uuid.U
map[string]any{"step_id": stepID, "seq": seq, "status": stepStatus, "execution_id": execID.String()})
}
func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, targetSlug string, actionStr string) {
func (s *Server) executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, targetSlug string, actionStr string) {
slog.Info("httpapi: executing approved action", "execution_id", execID, "target", targetSlug, "action", actionStr)
host, user, wrap, err := resolveRunTarget(ctx, pool, targetSlug)
@@ -320,234 +316,35 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID,
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": err.Error()})
return
}
// Only hostname is required. vmid is optional — when 0 (or later found
// to collide) the VMID guard below assigns a free cluster id.
if cfg.Hostname == "" {
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, `{"error":"pct_create: hostname is required"}`)
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": "missing hostname"})
return
}
if cfg.Cores == 0 {
cfg.Cores = 1
}
if cfg.Memory == 0 {
cfg.Memory = 512
}
if cfg.DiskGB == 0 {
cfg.DiskGB = 8
}
if cfg.Storage == "" {
cfg.Storage = "local-lvm"
}
if cfg.GW == "" {
cfg.GW = "192.168.8.2"
}
if cfg.Nameserver == "" {
cfg.Nameserver = "192.168.8.2"
}
if cfg.Searchdomain == "" {
cfg.Searchdomain = "hubris.network"
}
// Template pre-flight: resolve against what the host actually has
// cached. A hardcoded name (e.g. debian-13) fails opaquely with a raw
// `pct` error when that exact file isn't present. List the cache, then
// either validate the requested template or auto-pick the newest
// debian one; on miss, fail early with the available list so the
// operator/agent can retry with a real name.
cacheList, tplErr := sshExec(ctx, host, user, "ls -1 /var/lib/vz/template/cache/ 2>/dev/null | grep -E '\\.tar\\.(zst|gz|xz)$' || true")
available := []string{}
for _, l := range strings.Split(strings.TrimSpace(cacheList), "\n") {
if l = strings.TrimSpace(l); l != "" {
available = append(available, l)
}
}
if tplErr != nil {
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, jsonErr("list templates on %s: %s", targetSlug, tplErr.Error()))
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": tplErr.Error()})
return
}
cfg.Template = resolveTemplate(cfg.Template, available)
if cfg.Template == "" {
msg := fmt.Sprintf("no usable LXC template on %s. Available: %v", targetSlug, available)
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, jsonErr("%s", msg))
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": msg})
return
}
// VMID collision guard. Proxmox VMIDs are cluster-wide, so the model's
// guess (e.g. 132) can collide with a container on another node — pct
// create then fails with "CT N already exists on node X". Fetch the set
// of in-use VMIDs across the cluster; if the requested id is taken (or
// absent), fall back to the cluster's next free id so provisioning
// still succeeds instead of dead-ending on the operator's approval.
usedRaw, _ := sshExec(ctx, host, user, `pvesh get /cluster/resources --type vm --output-format json 2>/dev/null | grep -o '"vmid":[0-9]*' | grep -o '[0-9]*' || true`)
used := map[int]bool{}
for _, l := range strings.Fields(usedRaw) {
if n, e := strconv.Atoi(strings.TrimSpace(l)); e == nil {
used[n] = true
}
}
if cfg.VMID == 0 || used[cfg.VMID] {
nextRaw, nerr := sshExec(ctx, host, user, `pvesh get /cluster/nextid 2>/dev/null`)
nextID, cerr := strconv.Atoi(strings.TrimSpace(nextRaw))
if nerr != nil || cerr != nil || nextID == 0 {
msg := fmt.Sprintf("VMID %d is already in use on the cluster and could not resolve a free id", cfg.VMID)
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, jsonErr("%s", msg))
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": msg})
return
}
slog.Info("httpapi: pct_create VMID reassigned", "requested", cfg.VMID, "assigned", nextID)
cfg.VMID = nextID
}
privFlag := "--unprivileged 1"
if cfg.Privileged {
privFlag = "--unprivileged 0"
}
nestingFlag := ""
features := []string{}
if cfg.Nesting {
features = append(features, "nesting=1")
}
if cfg.Privileged {
features = append(features, "keyctl=1")
}
if len(features) > 0 {
nestingFlag = fmt.Sprintf(" --features %s", strings.Join(features, ","))
}
if cfg.Bridge == "" {
cfg.Bridge = "vmbr0"
}
// net0: DHCP when no static IP is given (or ip=="dhcp"). Proxmox
// rejects a gateway alongside ip=dhcp, so only add gw for a static IP.
net0 := "name=eth0,bridge=" + cfg.Bridge + ","
isStatic := cfg.IP != "" && !strings.EqualFold(cfg.IP, "dhcp")
if !isStatic {
net0 += "ip=dhcp"
} else {
net0 += "ip=" + cfg.IP
if cfg.GW != "" {
net0 += ",gw=" + cfg.GW
}
}
// Pre-flight: for a static config, ping the gateway from the target
// HOST, on the SPECIFIC BRIDGE being requested, before spending 5+
// minutes creating the container. This is the check that would have
// caught the real TypeType failure immediately instead of after a
// full provision attempt.
//
// Binding to the bridge (`ping -I <bridge>`) matters and was found
// live: a plain unqualified `ping <gw>` from the host can succeed via
// the host's own routing table (multiple routes, possibly through an
// upstream router) even when the *container* — which only gets a
// naive on-link default route via its bridge's veth — can never ARP
// that gateway at all. Confirmed on `strong`: bare `ping 192.168.8.2`
// succeeded (via the host's default route), but a container actually
// attached to vmbr0 showed 100% packet loss trying to reach the same
// address, because vmbr0 doesn't carry that subnet's L2 segment.
// Binding to the bridge interface reproduces what the container will
// actually experience, not what the host's broader routing table can
// reach.
if isStatic && cfg.GW != "" {
pingOut, pingErr := sshExec(ctx, host, user, fmt.Sprintf("ping -I %s -c1 -W2 %s >/dev/null 2>&1 && echo PREFLIGHT_OK || echo PREFLIGHT_FAIL", cfg.Bridge, cfg.GW))
if pingErr != nil || !gatewayPreflightPassed(pingOut) {
msg := fmt.Sprintf(
"gateway %s is not reachable from %s on bridge %s — this almost always means the bridge doesn't carry that subnet on this host (each bridge only reaches the network it's physically wired to). "+
"Do not retry with a different gateway guess in the same subnet: find an existing LXC on this host with an IP in the same /28 and copy its exact bridge+gateway, or use DHCP instead.",
cfg.GW, targetSlug, cfg.Bridge)
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, jsonErr("%s", msg))
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": msg})
return
}
}
templatePath := fmt.Sprintf("/var/lib/vz/template/cache/%s", cfg.Template)
createCmd := fmt.Sprintf(
"pct create %d %s --hostname %s --cores %d --memory %d --rootfs %s:%d %s --net0 %s%s --start 1",
cfg.VMID, templatePath, cfg.Hostname, cfg.Cores, cfg.Memory,
cfg.Storage, cfg.DiskGB, privFlag, net0, nestingFlag)
if cfg.Nameserver != "" {
createCmd += fmt.Sprintf(" --nameserver %s", cfg.Nameserver)
}
if cfg.Searchdomain != "" {
createCmd += fmt.Sprintf(" --searchdomain %s", cfg.Searchdomain)
}
// Add mount points
for i, mp := range cfg.Mounts {
if i < 10 { // pct supports up to mp9
createCmd += fmt.Sprintf(" --mp%d %s", i, mp)
}
}
slog.Info("httpapi: pct_create running", "vmid", cfg.VMID, "hostname", cfg.Hostname, "cmd", createCmd)
output, err = sshExecStream(ctx, host, user, createCmd, sink)
// pct_create is now DELIBERATELY ATOMIC: create + start + register,
// nothing else. It used to also run apt installs and a post_install
// script inline as one black-box multi-minute SSH call — the agent
// got back a single opaque success/fail for the whole thing with no
// way to see (or fix) which step actually broke. That's the opposite
// of what makes an agent able to recover from errors.
//
// Installing packages, running post_install, and verifying the
// service now happen as the agent's OWN follow-up `run` calls against
// the new lxc:<hostname> target — each one is synchronous (in an
// active assent window) or individually gated, so the agent observes
// every step's real output and can diagnose + retry the exact thing
// that failed instead of re-doing the whole container. See SOUL.md
// "After pct_create: you drive the install" and provisionScript's
// surviving role (DNS self-heal) is now something the agent invokes
// itself via `run`, not something baked into this handler.
//
// cfg.Services/cfg.PostInstall are intentionally no longer read here.
// On success, register the entity in the DB with proper relationships
if err == nil {
slug := "lxc:" + cfg.Hostname
var lxcID uuid.UUID
lxcID, _ = uuid.NewV7()
attrs := map[string]any{
"pve_id": fmt.Sprintf("%d", cfg.VMID),
"host": strings.TrimPrefix(targetSlug, "host:"),
"ip": cfg.IP,
}
attrsJSON, _ := json.Marshal(attrs)
_, insErr := pool.Exec(ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, enrolled_at)
VALUES ($1, $2, 'lxc', $3, 'provisioning', $4, now()) ON CONFLICT (slug) DO NOTHING`, lxcID, slug, cfg.Hostname, attrsJSON)
if insErr != nil {
slog.Error("httpapi: pct_create entity insert", "error", insErr, "slug", slug)
}
// Create hosts relationship: Proxmox host → LXC
var hostID uuid.UUID
if err := pool.QueryRow(ctx, "SELECT id FROM entities WHERE slug = $1", targetSlug).Scan(&hostID); err == nil {
_, relErr := pool.Exec(ctx, `INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
VALUES ($1, $2, 'hosts', '{"provisioned_by":"nomos"}'::jsonb, now())`, hostID, lxcID)
if relErr != nil {
slog.Error("httpapi: pct_create relationship insert", "error", relErr, "host", targetSlug, "lxc", slug)
}
}
// Create entity_status row for health tracking
pool.Exec(ctx, `INSERT INTO entity_status (entity_id, health, last_check_at)
VALUES ($1, 'unknown', now()) ON CONFLICT (entity_id) DO NOTHING`, lxcID)
// The flow (spec defaults, template/VMID pre-flights, pct create,
// graph registration) lives in ProvisioningService + the ssh
// provisioner adapter since Phase 7; this handler only maps the
// wire payload and streams the create output to the execution log.
outcome, perr := s.provisioning.CreateLXC(ctx, app.CreateLXCCmd{
HostSlug: targetSlug,
Hostname: cfg.Hostname,
VMID: cfg.VMID,
Cores: cfg.Cores,
MemoryMB: cfg.Memory,
DiskGB: cfg.DiskGB,
IP: cfg.IP,
GW: cfg.GW,
Bridge: cfg.Bridge,
Storage: cfg.Storage,
Template: cfg.Template,
Privileged: bool(cfg.Privileged),
Nesting: bool(cfg.Nesting),
Mounts: cfg.Mounts,
Nameserver: cfg.Nameserver,
Searchdomain: cfg.Searchdomain,
Sink: sink,
})
output = outcome.Output
err = perr
if perr == nil {
emitExecutionEvent(ctx, pool, execID, "executing", map[string]any{
"lxc_slug": slug, "vmid": cfg.VMID, "host": targetSlug,
"lxc_slug": outcome.Slug, "vmid": outcome.VMID, "host": targetSlug,
})
slog.Info("httpapi: pct_create entity registered", "slug", slug, "vmid", cfg.VMID, "host", targetSlug)
}
case "run":
@@ -612,52 +409,3 @@ func jsonErr(format string, args ...any) []byte {
b, _ := json.Marshal(map[string]any{"error": fmt.Sprintf(format, args...)})
return b
}
// resolveTemplate maps a requested template name to one actually present in
// the host's template cache. Exact match wins; a bare distro hint (e.g.
// "debian-13" or "debian") matches by prefix; empty picks the newest debian
// (falling back to any) template available. Returns "" when nothing fits.
// gatewayPreflightPassed interprets the PREFLIGHT_OK/PREFLIGHT_FAIL markers
// from the pct_create gateway pre-flight check. Pulled out as its own
// function (rather than an inline strings.Contains at the call site) so it's
// unit-testable: a prior version checked for "REACHABLE", which is a
// substring of "UNREACHABLE" — the check could never actually fail, and it
// took a live deployment to notice. Exact-match markers plus a test make
// that specific bug class structurally unable to recur silently.
func gatewayPreflightPassed(out string) bool {
return strings.TrimSpace(out) == "PREFLIGHT_OK"
}
func resolveTemplate(requested string, available []string) string {
if len(available) == 0 {
return ""
}
if requested != "" {
for _, a := range available {
if a == requested {
return a
}
}
for _, a := range available {
if strings.HasPrefix(a, requested) {
return a
}
}
}
// Auto-pick: prefer debian, then the lexically-greatest (newest version).
best := ""
for _, a := range available {
if strings.Contains(a, "debian") && a > best {
best = a
}
}
if best != "" {
return best
}
for _, a := range available {
if a > best {
best = a
}
}
return best
}