feat: Phase 7 — SeedService, SecretsService, ProvisioningService + ssh Provisioner

Seed ingest/export moves behind ports.SeedRepository (SeedRepo in the
postgres adapter; knowledge ingest absorbed from internal/knowledge,
package deleted). pct_create flow (defaults, template/VMID/gateway
pre-flights, pct create, graph registration) moves from httpapi's
approved-execution path into app.ProvisioningService + the ssh
provisioner adapter; CLI seed/export/secret become adapters over the
services. EntityCreateInput gains EnrolledAt. Plan status corrected:
phases 0-7 shipped, 8 + 9 gates open. VERSION 0.34.1.
This commit is contained in:
2026-08-16 09:11:26 +02:00
parent cd44501fa9
commit 60c0432d8b
26 changed files with 1565 additions and 689 deletions

View File

@@ -7,13 +7,13 @@ import (
"fmt"
"log/slog"
"os"
"strconv"
"strings"
"time"
"github.com/dtoro/oikos/internal/actuator"
"github.com/dtoro/oikos/internal/adapters/postgres"
"github.com/dtoro/oikos/internal/adapters/postgres/sqlcgen"
"github.com/dtoro/oikos/internal/core/app"
"github.com/dtoro/oikos/internal/execlog"
"github.com/dtoro/oikos/internal/observability"
"github.com/google/uuid"
@@ -78,10 +78,6 @@ const sshExecTimeout = 10 * time.Minute
// remote end produced it. Shared implementation lives in internal/actuator
// (actuator.streamWriter / actuator.RunStreaming).
func sshExec(ctx context.Context, host, user, command string) (string, error) {
return sshExecStream(ctx, host, user, command, nil)
}
// sshExecStream runs a command and reports its combined output, forwarding
// each chunk to sink as it arrives. A nil sink behaves exactly as before.
func sshExecStream(ctx context.Context, host, user, command string, sink execlog.Sink) (string, error) {
@@ -224,7 +220,7 @@ func closePlanStepForExecution(ctx context.Context, pool *db.Pool, execID uuid.U
map[string]any{"step_id": stepID, "seq": seq, "status": stepStatus, "execution_id": execID.String()})
}
func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, targetSlug string, actionStr string) {
func (s *Server) executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, targetSlug string, actionStr string) {
slog.Info("httpapi: executing approved action", "execution_id", execID, "target", targetSlug, "action", actionStr)
host, user, wrap, err := resolveRunTarget(ctx, pool, targetSlug)
@@ -320,234 +316,35 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID,
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": err.Error()})
return
}
// Only hostname is required. vmid is optional — when 0 (or later found
// to collide) the VMID guard below assigns a free cluster id.
if cfg.Hostname == "" {
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, `{"error":"pct_create: hostname is required"}`)
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": "missing hostname"})
return
}
if cfg.Cores == 0 {
cfg.Cores = 1
}
if cfg.Memory == 0 {
cfg.Memory = 512
}
if cfg.DiskGB == 0 {
cfg.DiskGB = 8
}
if cfg.Storage == "" {
cfg.Storage = "local-lvm"
}
if cfg.GW == "" {
cfg.GW = "192.168.8.2"
}
if cfg.Nameserver == "" {
cfg.Nameserver = "192.168.8.2"
}
if cfg.Searchdomain == "" {
cfg.Searchdomain = "hubris.network"
}
// Template pre-flight: resolve against what the host actually has
// cached. A hardcoded name (e.g. debian-13) fails opaquely with a raw
// `pct` error when that exact file isn't present. List the cache, then
// either validate the requested template or auto-pick the newest
// debian one; on miss, fail early with the available list so the
// operator/agent can retry with a real name.
cacheList, tplErr := sshExec(ctx, host, user, "ls -1 /var/lib/vz/template/cache/ 2>/dev/null | grep -E '\\.tar\\.(zst|gz|xz)$' || true")
available := []string{}
for _, l := range strings.Split(strings.TrimSpace(cacheList), "\n") {
if l = strings.TrimSpace(l); l != "" {
available = append(available, l)
}
}
if tplErr != nil {
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, jsonErr("list templates on %s: %s", targetSlug, tplErr.Error()))
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": tplErr.Error()})
return
}
cfg.Template = resolveTemplate(cfg.Template, available)
if cfg.Template == "" {
msg := fmt.Sprintf("no usable LXC template on %s. Available: %v", targetSlug, available)
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, jsonErr("%s", msg))
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": msg})
return
}
// VMID collision guard. Proxmox VMIDs are cluster-wide, so the model's
// guess (e.g. 132) can collide with a container on another node — pct
// create then fails with "CT N already exists on node X". Fetch the set
// of in-use VMIDs across the cluster; if the requested id is taken (or
// absent), fall back to the cluster's next free id so provisioning
// still succeeds instead of dead-ending on the operator's approval.
usedRaw, _ := sshExec(ctx, host, user, `pvesh get /cluster/resources --type vm --output-format json 2>/dev/null | grep -o '"vmid":[0-9]*' | grep -o '[0-9]*' || true`)
used := map[int]bool{}
for _, l := range strings.Fields(usedRaw) {
if n, e := strconv.Atoi(strings.TrimSpace(l)); e == nil {
used[n] = true
}
}
if cfg.VMID == 0 || used[cfg.VMID] {
nextRaw, nerr := sshExec(ctx, host, user, `pvesh get /cluster/nextid 2>/dev/null`)
nextID, cerr := strconv.Atoi(strings.TrimSpace(nextRaw))
if nerr != nil || cerr != nil || nextID == 0 {
msg := fmt.Sprintf("VMID %d is already in use on the cluster and could not resolve a free id", cfg.VMID)
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, jsonErr("%s", msg))
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": msg})
return
}
slog.Info("httpapi: pct_create VMID reassigned", "requested", cfg.VMID, "assigned", nextID)
cfg.VMID = nextID
}
privFlag := "--unprivileged 1"
if cfg.Privileged {
privFlag = "--unprivileged 0"
}
nestingFlag := ""
features := []string{}
if cfg.Nesting {
features = append(features, "nesting=1")
}
if cfg.Privileged {
features = append(features, "keyctl=1")
}
if len(features) > 0 {
nestingFlag = fmt.Sprintf(" --features %s", strings.Join(features, ","))
}
if cfg.Bridge == "" {
cfg.Bridge = "vmbr0"
}
// net0: DHCP when no static IP is given (or ip=="dhcp"). Proxmox
// rejects a gateway alongside ip=dhcp, so only add gw for a static IP.
net0 := "name=eth0,bridge=" + cfg.Bridge + ","
isStatic := cfg.IP != "" && !strings.EqualFold(cfg.IP, "dhcp")
if !isStatic {
net0 += "ip=dhcp"
} else {
net0 += "ip=" + cfg.IP
if cfg.GW != "" {
net0 += ",gw=" + cfg.GW
}
}
// Pre-flight: for a static config, ping the gateway from the target
// HOST, on the SPECIFIC BRIDGE being requested, before spending 5+
// minutes creating the container. This is the check that would have
// caught the real TypeType failure immediately instead of after a
// full provision attempt.
//
// Binding to the bridge (`ping -I <bridge>`) matters and was found
// live: a plain unqualified `ping <gw>` from the host can succeed via
// the host's own routing table (multiple routes, possibly through an
// upstream router) even when the *container* — which only gets a
// naive on-link default route via its bridge's veth — can never ARP
// that gateway at all. Confirmed on `strong`: bare `ping 192.168.8.2`
// succeeded (via the host's default route), but a container actually
// attached to vmbr0 showed 100% packet loss trying to reach the same
// address, because vmbr0 doesn't carry that subnet's L2 segment.
// Binding to the bridge interface reproduces what the container will
// actually experience, not what the host's broader routing table can
// reach.
if isStatic && cfg.GW != "" {
pingOut, pingErr := sshExec(ctx, host, user, fmt.Sprintf("ping -I %s -c1 -W2 %s >/dev/null 2>&1 && echo PREFLIGHT_OK || echo PREFLIGHT_FAIL", cfg.Bridge, cfg.GW))
if pingErr != nil || !gatewayPreflightPassed(pingOut) {
msg := fmt.Sprintf(
"gateway %s is not reachable from %s on bridge %s — this almost always means the bridge doesn't carry that subnet on this host (each bridge only reaches the network it's physically wired to). "+
"Do not retry with a different gateway guess in the same subnet: find an existing LXC on this host with an IP in the same /28 and copy its exact bridge+gateway, or use DHCP instead.",
cfg.GW, targetSlug, cfg.Bridge)
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`,
execID, jsonErr("%s", msg))
emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": msg})
return
}
}
templatePath := fmt.Sprintf("/var/lib/vz/template/cache/%s", cfg.Template)
createCmd := fmt.Sprintf(
"pct create %d %s --hostname %s --cores %d --memory %d --rootfs %s:%d %s --net0 %s%s --start 1",
cfg.VMID, templatePath, cfg.Hostname, cfg.Cores, cfg.Memory,
cfg.Storage, cfg.DiskGB, privFlag, net0, nestingFlag)
if cfg.Nameserver != "" {
createCmd += fmt.Sprintf(" --nameserver %s", cfg.Nameserver)
}
if cfg.Searchdomain != "" {
createCmd += fmt.Sprintf(" --searchdomain %s", cfg.Searchdomain)
}
// Add mount points
for i, mp := range cfg.Mounts {
if i < 10 { // pct supports up to mp9
createCmd += fmt.Sprintf(" --mp%d %s", i, mp)
}
}
slog.Info("httpapi: pct_create running", "vmid", cfg.VMID, "hostname", cfg.Hostname, "cmd", createCmd)
output, err = sshExecStream(ctx, host, user, createCmd, sink)
// pct_create is now DELIBERATELY ATOMIC: create + start + register,
// nothing else. It used to also run apt installs and a post_install
// script inline as one black-box multi-minute SSH call — the agent
// got back a single opaque success/fail for the whole thing with no
// way to see (or fix) which step actually broke. That's the opposite
// of what makes an agent able to recover from errors.
//
// Installing packages, running post_install, and verifying the
// service now happen as the agent's OWN follow-up `run` calls against
// the new lxc:<hostname> target — each one is synchronous (in an
// active assent window) or individually gated, so the agent observes
// every step's real output and can diagnose + retry the exact thing
// that failed instead of re-doing the whole container. See SOUL.md
// "After pct_create: you drive the install" and provisionScript's
// surviving role (DNS self-heal) is now something the agent invokes
// itself via `run`, not something baked into this handler.
//
// cfg.Services/cfg.PostInstall are intentionally no longer read here.
// On success, register the entity in the DB with proper relationships
if err == nil {
slug := "lxc:" + cfg.Hostname
var lxcID uuid.UUID
lxcID, _ = uuid.NewV7()
attrs := map[string]any{
"pve_id": fmt.Sprintf("%d", cfg.VMID),
"host": strings.TrimPrefix(targetSlug, "host:"),
"ip": cfg.IP,
}
attrsJSON, _ := json.Marshal(attrs)
_, insErr := pool.Exec(ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, enrolled_at)
VALUES ($1, $2, 'lxc', $3, 'provisioning', $4, now()) ON CONFLICT (slug) DO NOTHING`, lxcID, slug, cfg.Hostname, attrsJSON)
if insErr != nil {
slog.Error("httpapi: pct_create entity insert", "error", insErr, "slug", slug)
}
// Create hosts relationship: Proxmox host → LXC
var hostID uuid.UUID
if err := pool.QueryRow(ctx, "SELECT id FROM entities WHERE slug = $1", targetSlug).Scan(&hostID); err == nil {
_, relErr := pool.Exec(ctx, `INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
VALUES ($1, $2, 'hosts', '{"provisioned_by":"nomos"}'::jsonb, now())`, hostID, lxcID)
if relErr != nil {
slog.Error("httpapi: pct_create relationship insert", "error", relErr, "host", targetSlug, "lxc", slug)
}
}
// Create entity_status row for health tracking
pool.Exec(ctx, `INSERT INTO entity_status (entity_id, health, last_check_at)
VALUES ($1, 'unknown', now()) ON CONFLICT (entity_id) DO NOTHING`, lxcID)
// The flow (spec defaults, template/VMID pre-flights, pct create,
// graph registration) lives in ProvisioningService + the ssh
// provisioner adapter since Phase 7; this handler only maps the
// wire payload and streams the create output to the execution log.
outcome, perr := s.provisioning.CreateLXC(ctx, app.CreateLXCCmd{
HostSlug: targetSlug,
Hostname: cfg.Hostname,
VMID: cfg.VMID,
Cores: cfg.Cores,
MemoryMB: cfg.Memory,
DiskGB: cfg.DiskGB,
IP: cfg.IP,
GW: cfg.GW,
Bridge: cfg.Bridge,
Storage: cfg.Storage,
Template: cfg.Template,
Privileged: bool(cfg.Privileged),
Nesting: bool(cfg.Nesting),
Mounts: cfg.Mounts,
Nameserver: cfg.Nameserver,
Searchdomain: cfg.Searchdomain,
Sink: sink,
})
output = outcome.Output
err = perr
if perr == nil {
emitExecutionEvent(ctx, pool, execID, "executing", map[string]any{
"lxc_slug": slug, "vmid": cfg.VMID, "host": targetSlug,
"lxc_slug": outcome.Slug, "vmid": outcome.VMID, "host": targetSlug,
})
slog.Info("httpapi: pct_create entity registered", "slug", slug, "vmid", cfg.VMID, "host", targetSlug)
}
case "run":
@@ -612,52 +409,3 @@ func jsonErr(format string, args ...any) []byte {
b, _ := json.Marshal(map[string]any{"error": fmt.Sprintf(format, args...)})
return b
}
// resolveTemplate maps a requested template name to one actually present in
// the host's template cache. Exact match wins; a bare distro hint (e.g.
// "debian-13" or "debian") matches by prefix; empty picks the newest debian
// (falling back to any) template available. Returns "" when nothing fits.
// gatewayPreflightPassed interprets the PREFLIGHT_OK/PREFLIGHT_FAIL markers
// from the pct_create gateway pre-flight check. Pulled out as its own
// function (rather than an inline strings.Contains at the call site) so it's
// unit-testable: a prior version checked for "REACHABLE", which is a
// substring of "UNREACHABLE" — the check could never actually fail, and it
// took a live deployment to notice. Exact-match markers plus a test make
// that specific bug class structurally unable to recur silently.
func gatewayPreflightPassed(out string) bool {
return strings.TrimSpace(out) == "PREFLIGHT_OK"
}
func resolveTemplate(requested string, available []string) string {
if len(available) == 0 {
return ""
}
if requested != "" {
for _, a := range available {
if a == requested {
return a
}
}
for _, a := range available {
if strings.HasPrefix(a, requested) {
return a
}
}
}
// Auto-pick: prefer debian, then the lexically-greatest (newest version).
best := ""
for _, a := range available {
if strings.Contains(a, "debian") && a > best {
best = a
}
}
if best != "" {
return best
}
for _, a := range available {
if a > best {
best = a
}
}
return best
}

View File

@@ -19,6 +19,7 @@ import (
"github.com/dtoro/oikos/internal/config"
"github.com/dtoro/oikos/internal/adapters/postgres"
"github.com/dtoro/oikos/internal/core/app"
"github.com/dtoro/oikos/internal/core/ports/portstest"
"github.com/jackc/pgx/v5"
)
@@ -97,7 +98,14 @@ func newTestHandler(t *testing.T, cfg config.Config) http.Handler {
repo := db.NewEntityRepo(pool)
onto := db.NewOntologyRepo(pool, time.Minute)
return NewHandler(handlerCtx, pool, cfg, app.NewEntityService(repo, onto), repo, db.NewEntityReader(pool), app.NewRelationshipService(db.NewRelRepo(pool), onto))
// Provisioning/seeds get real services over fakes/real repos: tests
// below don't exercise pct_create, but the export endpoint does hit
// SeedService, and a nil would panic.
provisioning := app.NewProvisioningService(
&portstest.FakeProvisioner{}, &portstest.FakeResolver{},
repo, db.NewRelRepo(pool))
seeds := app.NewSeedService(db.NewSeedRepo(pool))
return NewHandler(handlerCtx, pool, cfg, app.NewEntityService(repo, onto), repo, db.NewEntityReader(pool), app.NewRelationshipService(db.NewRelRepo(pool), onto), provisioning, seeds)
}
// testAuthToken is the static bearer token devConfig() configures. There is

View File

@@ -179,7 +179,7 @@ func (s *Server) DecideApproval(ctx context.Context, req gen.DecideApprovalReque
_ = tx.QueryRow(ctx, "SELECT slug FROM entities WHERE id = $1", targetID).Scan(&targetSlug)
safego.Go("httpapi:executeApprovedAction", func() {
executeApprovedAction(context.Background(), s.pool, execID, targetSlug, actionStr)
s.executeApprovedAction(context.Background(), s.pool, execID, targetSlug, actionStr)
})
// Status only — risk_class was set correctly at request time
// (e.g. by policy.ClassifyCommand for `run`); overwriting it to

View File

@@ -4,7 +4,6 @@ import (
"context"
"time"
"github.com/dtoro/oikos/internal/adapters/postgres"
"github.com/dtoro/oikos/internal/httpapi/gen"
)
@@ -68,7 +67,7 @@ func (s *Server) GetFleetHealth(ctx context.Context, req gen.GetFleetHealthReque
}
func (s *Server) ExportSeeds(ctx context.Context, req gen.ExportSeedsRequestObject) (gen.ExportSeedsResponseObject, error) {
exports, err := db.ExportToYAML(ctx, s.pool)
exports, err := s.seeds.Export(ctx)
if err != nil {
return nil, err
}

View File

@@ -43,31 +43,6 @@ func TestFlexBoolUnmarshal(t *testing.T) {
}
}
func TestResolveTemplate(t *testing.T) {
avail := []string{
"debian-12-standard_12.7-1_amd64.tar.zst",
"debian-13-standard_13.0-1_amd64.tar.zst",
"ubuntu-24.04-standard_24.04-2_amd64.tar.zst",
}
cases := []struct {
requested string
want string
}{
{"debian-13-standard_13.0-1_amd64.tar.zst", "debian-13-standard_13.0-1_amd64.tar.zst"}, // exact
{"debian-13", "debian-13-standard_13.0-1_amd64.tar.zst"}, // prefix
{"", "debian-13-standard_13.0-1_amd64.tar.zst"}, // auto newest debian
{"debian-99", "debian-13-standard_13.0-1_amd64.tar.zst"}, // miss prefix → auto debian
}
for _, c := range cases {
if got := resolveTemplate(c.requested, avail); got != c.want {
t.Errorf("resolveTemplate(%q): got %q want %q", c.requested, got, c.want)
}
}
if got := resolveTemplate("debian-13", nil); got != "" {
t.Errorf("empty cache should yield empty, got %q", got)
}
}
// TestJSONErrValidForNastyOutput guards the bug where command output with
// quotes/backslashes/newlines produced invalid JSON, failing the ::jsonb cast
// and silently dropping the execution's final status update.
@@ -87,32 +62,10 @@ func TestJSONErrValidForNastyOutput(t *testing.T) {
}
}
// TestGatewayPreflightPassed guards the exact bug found live: "UNREACHABLE"
// contains "REACHABLE" as a substring, so a strings.Contains(out,"REACHABLE")
// check is true for BOTH outcomes and can never fail. Exact-match only.
func TestGatewayPreflightPassed(t *testing.T) {
cases := []struct {
out string
want bool
}{
{"PREFLIGHT_OK", true},
{"PREFLIGHT_OK\n", true},
{" PREFLIGHT_OK ", true},
{"PREFLIGHT_FAIL", false},
{"PREFLIGHT_FAIL\n", false},
{"", false},
{"some garbage output", false},
// the specific historical bug: a naive substring check on the old
// REACHABLE/UNREACHABLE markers would have called this true.
{"UNREACHABLE", false},
}
for _, c := range cases {
if got := gatewayPreflightPassed(c.out); got != c.want {
t.Errorf("gatewayPreflightPassed(%q) = %v, want %v", c.out, got, c.want)
}
}
}
// resolveTemplate and gatewayPreflightPassed moved to the ssh provisioner
// adapter with the pct flow (Phase 7); their tests live in
// internal/adapters/ssh/provisioner_test.go.
//
// provisionScript and sanitizePkgs were removed when pct_create was made
// atomic (create + start + register only) — installing packages and running
// setup scripts is now the agent's own job via follow-up `run` calls, which

View File

@@ -69,6 +69,11 @@ type Server struct {
entityRepo *db.EntityRepo
readModels ports.ReadModels
relService *app.RelationshipService
// provisioning owns the pct_create flow (Phase 7): spec defaults,
// provisioner dispatch, guest registration in the graph.
provisioning *app.ProvisioningService
// seeds regenerates seed YAMLs for the export endpoint (Phase 7).
seeds *app.SeedService
}
// NewHandler builds the full HTTP handler: /healthz (unauthenticated,
@@ -78,17 +83,19 @@ type Server struct {
// holds a dedicated pooled connection for LISTEN. Callers MUST cancel ctx
// before closing the pool — otherwise the held connection never releases
// and pool.Close() deadlocks.
func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config, entities *app.EntityService, entityRepo *db.EntityRepo, readModels ports.ReadModels, relService *app.RelationshipService) http.Handler {
func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config, entities *app.EntityService, entityRepo *db.EntityRepo, readModels ports.ReadModels, relService *app.RelationshipService, provisioning *app.ProvisioningService, seeds *app.SeedService) http.Handler {
s := &Server{
pool: pool,
cfg: cfg,
entityCache: db.NewEntityCache(60 * time.Second),
sseBroker: newSSEBroker(10000),
sseSubs: make(map[*sseSubscriber]struct{}),
entities: entities,
entityRepo: entityRepo,
readModels: readModels,
relService: relService,
pool: pool,
cfg: cfg,
entityCache: db.NewEntityCache(60 * time.Second),
sseBroker: newSSEBroker(10000),
sseSubs: make(map[*sseSubscriber]struct{}),
entities: entities,
entityRepo: entityRepo,
readModels: readModels,
relService: relService,
provisioning: provisioning,
seeds: seeds,
}
// Wire secrets backend: Infisical primary with SOPS DR fallback.
@@ -937,10 +944,10 @@ main();
// ListenAndServe runs the API server with graceful shutdown on ctx cancel
// (SG4): stop accepting, drain in-flight for up to 30s, then exit.
func ListenAndServe(ctx context.Context, pool *db.Pool, cfg config.Config, entities *app.EntityService, entityRepo *db.EntityRepo, readModels ports.ReadModels, relService *app.RelationshipService) error {
func ListenAndServe(ctx context.Context, pool *db.Pool, cfg config.Config, entities *app.EntityService, entityRepo *db.EntityRepo, readModels ports.ReadModels, relService *app.RelationshipService, provisioning *app.ProvisioningService, seeds *app.SeedService) error {
srv := &http.Server{
Addr: cfg.APIListen,
Handler: NewHandler(ctx, pool, cfg, entities, entityRepo, readModels, relService),
Handler: NewHandler(ctx, pool, cfg, entities, entityRepo, readModels, relService, provisioning, seeds),
ReadHeaderTimeout: 10 * time.Second,
}