dtoro
6487032461
resolveProxmoxHostSlug trusted attributes.host verbatim, so a value polluted with prose — lxc:teddycloud carried host="hubris (confirmed via pct config…)" — became a slug that never resolved, leaving its checks 'down' despite a correct `hosts` edge. Treat an attribute containing whitespace/parens as invalid and fall back to the canonical hosts edge. The audit now reports `polluted_attrs` — entities whose routing-critical attributes carry prose — so this class is visible instead of a silent resolution failure.
304 lines
12 KiB
Go
304 lines
12 KiB
Go
// Package remote resolves how to execute a command on a target entity and
|
|
// turns a plain shell command into whatever must be sent over the SSH
|
|
// connection that reaches it.
|
|
//
|
|
// The canonical access model: a host or workstation is reached by direct SSH
|
|
// to its address; an LXC or VM is NEVER SSH'd into directly — it is reached
|
|
// through its owning Proxmox host via `pct exec` / `qm guest exec`. One SSH
|
|
// credential per host (the host's root key), no per-guest keys, sshd, or
|
|
// lan_ip required for execution. Network probes (http/ping) still hit a
|
|
// guest's lan_ip directly; only command execution host-hops.
|
|
//
|
|
// This is the single resolver shared by the scheduler's check execution and
|
|
// the MCP `run` tool. Previously they diverged — the scheduler SSHed guests
|
|
// directly (broken for headless/keyless/mesh-only guests), while MCP
|
|
// host-hopped (working). Keeping one path keeps them in lockstep.
|
|
package remote
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/dtoro/oikos/internal/db"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
// DefaultUser is the SSH user when an entity declares no ssh.user. The
|
|
// Proxmox hosts and their guests are all administered as root.
|
|
const DefaultUser = "root"
|
|
|
|
// ExecTarget is a resolved execution endpoint: the SSH address and user to
|
|
// connect to, plus Wrap, which rewrites a plain command for transport.
|
|
type ExecTarget struct {
|
|
Host string
|
|
User string
|
|
// Wrap turns a plain shell command into the form that must be sent over
|
|
// the SSH connection to this target: the identity function for a host,
|
|
// `pct exec <id> -- bash -c 'echo <b64> | base64 -d | bash'` for an LXC,
|
|
// the `qm guest exec` equivalent for a VM. The base64 round-trip keeps
|
|
// nested quoting identical across both guest kinds.
|
|
Wrap func(cmd string) string
|
|
}
|
|
|
|
// IsGuest reports whether an entity type is reached via pct/qm exec through a
|
|
// Proxmox host rather than by direct SSH. docker-container is reached via its
|
|
// host's docker socket, not pct, so it is not a guest here.
|
|
func IsGuest(entityType string) bool {
|
|
return entityType == "lxc" || entityType == "vm"
|
|
}
|
|
|
|
// ResolveHost resolves a `host:<slug>` to its reachable network address and
|
|
// SSH user. Address preference: lan_ip, then public_ipv4, then mesh IP, then
|
|
// mesh fqdn. Preferring public_ipv4 over mesh matters because the scheduler
|
|
// container has no mesh interface — a standalone-server with only a mesh IP
|
|
// (netbird-vps) was unreachable, and a public_ipv4 was sitting unused.
|
|
//
|
|
// fallbackUser is used when the entity declares no ssh.user; callers pass
|
|
// their configured default (the scheduler uses "root", the MCP run tool uses
|
|
// its configured OIKOS_SSH_USER).
|
|
func ResolveHost(ctx context.Context, pool *db.Pool, hostSlug, fallbackUser string) (addr, user string, err error) {
|
|
var raw string
|
|
if err = pool.QueryRow(ctx, "SELECT attributes::text FROM entities WHERE slug = $1", hostSlug).Scan(&raw); err != nil {
|
|
return "", "", fmt.Errorf("entity not found: %s", hostSlug)
|
|
}
|
|
var m map[string]any
|
|
if err = json.Unmarshal([]byte(raw), &m); err != nil {
|
|
return "", "", fmt.Errorf("parse attributes for %s: %w", hostSlug, err)
|
|
}
|
|
|
|
if v, ok := m["lan_ip"].(string); ok && v != "" {
|
|
addr = v
|
|
} else if v, ok := m["public_ipv4"].(string); ok && v != "" {
|
|
addr = v
|
|
} else if mesh, ok := m["mesh"].(map[string]any); ok {
|
|
if nb, ok := mesh["netbird"].(map[string]any); ok {
|
|
if v, ok := nb["ip"].(string); ok && v != "" {
|
|
addr = v
|
|
} else if v, ok := nb["fqdn"].(string); ok && v != "" {
|
|
addr = v
|
|
}
|
|
}
|
|
}
|
|
if addr == "" {
|
|
return "", "", fmt.Errorf("no IP found for %s", hostSlug)
|
|
}
|
|
|
|
user = fallbackUser
|
|
if ssh, ok := m["ssh"].(map[string]any); ok {
|
|
if u, ok := ssh["user"].(string); ok && u != "" {
|
|
user = u
|
|
}
|
|
}
|
|
return addr, user, nil
|
|
}
|
|
|
|
// ResolveProxmoxHostSlug resolves the Proxmox host slug that owns a guest.
|
|
// Resolution order: the hostAttr if non-empty (the entity's attributes.host,
|
|
// stored without the "host:" prefix), the `hosts` relationship on the guest
|
|
// (the canonical graph edge), then "hubris" as the documented default.
|
|
//
|
|
// entityID is the guest's entity id; the relationship lookup uses it
|
|
// directly rather than a slug subquery.
|
|
func ResolveProxmoxHostSlug(ctx context.Context, pool *db.Pool, entityID uuid.UUID, hostAttr string) string {
|
|
hostSlug := strings.TrimSpace(hostAttr)
|
|
// Only trust a clean token as a host name. The attribute is operator/
|
|
// agent-writable and has been polluted with prose before ("hubris
|
|
// (confirmed via pct config…)") — using that verbatim produces a slug that
|
|
// never resolves. Treat anything with whitespace or parens as invalid and
|
|
// fall back to the canonical `hosts` edge below.
|
|
if strings.ContainsAny(hostSlug, " \t()") {
|
|
hostSlug = ""
|
|
}
|
|
if hostSlug == "" {
|
|
var relHostSlug string
|
|
if err := pool.QueryRow(ctx, `
|
|
SELECT e.slug FROM relationships r
|
|
JOIN entities e ON e.id = r.source_id
|
|
WHERE r.target_id = $1
|
|
AND r.type = 'hosts' AND r.valid_to IS NULL
|
|
LIMIT 1`, entityID).Scan(&relHostSlug); err == nil && relHostSlug != "" {
|
|
hostSlug = relHostSlug
|
|
}
|
|
}
|
|
if hostSlug == "" {
|
|
hostSlug = "hubris" // documented default Proxmox host when unset
|
|
}
|
|
if !strings.HasPrefix(hostSlug, "host:") {
|
|
hostSlug = "host:" + hostSlug
|
|
}
|
|
return hostSlug
|
|
}
|
|
|
|
// ResolveExecTarget resolves a target slug (host:, lxc:, or vm:) to its
|
|
// execution endpoint. This is the slug-based entry used by the MCP `run` tool.
|
|
func ResolveExecTarget(ctx context.Context, pool *db.Pool, targetSlug, fallbackUser string) (ExecTarget, error) {
|
|
switch {
|
|
case strings.HasPrefix(targetSlug, "host:"):
|
|
addr, user, err := ResolveHost(ctx, pool, targetSlug, fallbackUser)
|
|
if err != nil {
|
|
return ExecTarget{}, err
|
|
}
|
|
return ExecTarget{Host: addr, User: user, Wrap: func(cmd string) string { return cmd }}, nil
|
|
|
|
case strings.HasPrefix(targetSlug, "lxc:"), strings.HasPrefix(targetSlug, "vm:"):
|
|
var (
|
|
id uuid.UUID
|
|
pveID string
|
|
typ string
|
|
hostAttr string
|
|
)
|
|
if err := pool.QueryRow(ctx,
|
|
"SELECT id, type, attributes->>'pve_id', COALESCE(attributes->>'host','') FROM entities WHERE slug = $1",
|
|
targetSlug).Scan(&id, &typ, &pveID, &hostAttr); err != nil || pveID == "" {
|
|
return ExecTarget{}, fmt.Errorf("guest not found or missing pve_id: %s", targetSlug)
|
|
}
|
|
hostSlug := ResolveProxmoxHostSlug(ctx, pool, id, hostAttr)
|
|
addr, user, err := ResolveHost(ctx, pool, hostSlug, fallbackUser)
|
|
if err != nil {
|
|
return ExecTarget{}, err
|
|
}
|
|
return ExecTarget{Host: addr, User: user, Wrap: guestWrap(typ, pveID)}, nil
|
|
}
|
|
return ExecTarget{}, fmt.Errorf("unsupported target %q: must be host:<slug>, lxc:<slug>, or vm:<slug>", targetSlug)
|
|
}
|
|
|
|
// ResolveExecTargetForCheck resolves an execution endpoint keyed by the
|
|
// target's id and type — the data the scheduler has at check-execution time
|
|
// (check_defs carry target_id + target_type, not a slug). Guests route via
|
|
// pct/qm exec; everything else (hosts, workstations, services resolved to
|
|
// their hosting machine) is reached by direct SSH to the entity's own address.
|
|
func ResolveExecTargetForCheck(ctx context.Context, pool *db.Pool, targetID uuid.UUID, targetType, fallbackUser string) (ExecTarget, error) {
|
|
if IsGuest(targetType) {
|
|
return resolveGuest(ctx, pool, targetID, targetType, fallbackUser)
|
|
}
|
|
|
|
// A service (or other non-compute target) has no address of its own — it
|
|
// runs on whatever compute entity provides/hosts it. Resolve that host and
|
|
// route through it: pct if the host is a guest, direct SSH (with the
|
|
// host's correct user) if it's a machine. Previously a service check baked
|
|
// its hosting LXC's lan_ip and SSHed it directly as root, which fails
|
|
// because the scheduler key isn't in each LXC — only on the Proxmox hosts.
|
|
if hostID, hostType, ok := hostingCompute(ctx, pool, targetID); ok {
|
|
if IsGuest(hostType) {
|
|
return resolveGuest(ctx, pool, hostID, hostType, fallbackUser)
|
|
}
|
|
addr, user, err := resolveHostByID(ctx, pool, hostID, fallbackUser)
|
|
if err != nil {
|
|
return ExecTarget{}, err
|
|
}
|
|
return ExecTarget{Host: addr, User: user, Wrap: func(cmd string) string { return cmd }}, nil
|
|
}
|
|
|
|
// No hosting entity found: reach the target directly at its own address
|
|
// (a host/workstation, or a service whose host wasn't resolvable).
|
|
addr, user, err := resolveHostByID(ctx, pool, targetID, fallbackUser)
|
|
if err != nil {
|
|
return ExecTarget{}, err
|
|
}
|
|
return ExecTarget{Host: addr, User: user, Wrap: func(cmd string) string { return cmd }}, nil
|
|
}
|
|
|
|
// hostingCompute walks the provides/runs-on/hosts edges backward from a target
|
|
// to the compute entity that runs it (a service's LXC, an LXC's Proxmox host).
|
|
// Returns the host's id, type, and whether one was found. Most-specific edge
|
|
// first: provides names the runtime container directly.
|
|
func hostingCompute(ctx context.Context, pool *db.Pool, targetID uuid.UUID) (uuid.UUID, string, bool) {
|
|
var hid uuid.UUID
|
|
var htype string
|
|
err := pool.QueryRow(ctx, `
|
|
SELECT e.id, e.type FROM relationships r
|
|
JOIN entities e ON e.id = r.source_id
|
|
WHERE r.target_id = $1 AND r.valid_to IS NULL
|
|
AND r.type IN ('provides','runs-on','hosts')
|
|
ORDER BY CASE r.type WHEN 'provides' THEN 0 WHEN 'runs-on' THEN 1 ELSE 2 END
|
|
LIMIT 1`, targetID).Scan(&hid, &htype)
|
|
if err != nil {
|
|
return uuid.Nil, "", false
|
|
}
|
|
return hid, htype, true
|
|
}
|
|
|
|
// resolveGuest resolves a guest's execution endpoint: the owning Proxmox host
|
|
// (SSH'd directly) with a pct/qm exec wrapper around the command.
|
|
func resolveGuest(ctx context.Context, pool *db.Pool, guestID uuid.UUID, guestType, fallbackUser string) (ExecTarget, error) {
|
|
var pveID, hostAttr string
|
|
if err := pool.QueryRow(ctx,
|
|
"SELECT attributes->>'pve_id', COALESCE(attributes->>'host','') FROM entities WHERE id = $1",
|
|
guestID).Scan(&pveID, &hostAttr); err != nil || pveID == "" {
|
|
return ExecTarget{}, fmt.Errorf("guest %s missing pve_id", guestID)
|
|
}
|
|
hostSlug := ResolveProxmoxHostSlug(ctx, pool, guestID, hostAttr)
|
|
addr, user, err := ResolveHost(ctx, pool, hostSlug, fallbackUser)
|
|
if err != nil {
|
|
return ExecTarget{}, err
|
|
}
|
|
return ExecTarget{Host: addr, User: user, Wrap: guestWrap(guestType, pveID)}, nil
|
|
}
|
|
|
|
// resolveHostByID is ResolveHost keyed by entity id.
|
|
func resolveHostByID(ctx context.Context, pool *db.Pool, id uuid.UUID, fallbackUser string) (addr, user string, err error) {
|
|
var raw string
|
|
if err = pool.QueryRow(ctx, "SELECT attributes::text FROM entities WHERE id = $1", id).Scan(&raw); err != nil {
|
|
return "", "", fmt.Errorf("entity %s not found", id)
|
|
}
|
|
var m map[string]any
|
|
if err = json.Unmarshal([]byte(raw), &m); err != nil {
|
|
return "", "", fmt.Errorf("parse attributes: %w", err)
|
|
}
|
|
for _, key := range []string{"lan_ip", "public_ipv4", "mesh_ip"} {
|
|
if v, ok := m[key].(string); ok && v != "" {
|
|
addr = v
|
|
break
|
|
}
|
|
}
|
|
if addr == "" {
|
|
if mesh, ok := m["mesh"].(map[string]any); ok {
|
|
if nb, ok := mesh["netbird"].(map[string]any); ok {
|
|
if v, ok := nb["ip"].(string); ok && v != "" {
|
|
addr = v
|
|
} else if v, ok := nb["fqdn"].(string); ok && v != "" {
|
|
addr = v
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if addr == "" {
|
|
return "", "", fmt.Errorf("no IP found for entity %s", id)
|
|
}
|
|
user = fallbackUser
|
|
if ssh, ok := m["ssh"].(map[string]any); ok {
|
|
if u, ok := ssh["user"].(string); ok && u != "" {
|
|
user = u
|
|
}
|
|
}
|
|
if u, ok := m["user"].(string); ok && u != "" && user == fallbackUser {
|
|
// Workstations carry their login as a top-level `user` attribute
|
|
// (mac-mini: user: dtoro), not under ssh.user. Take it only when no
|
|
// explicit ssh.user was set, so a host that genuinely wants root still
|
|
// gets root.
|
|
user = u
|
|
}
|
|
return addr, user, nil
|
|
}
|
|
|
|
// guestWrap builds the pct/qm exec wrapper for a guest of the given type.
|
|
func guestWrap(entityType, pveID string) func(cmd string) string {
|
|
if entityType == "vm" {
|
|
return func(cmd string) string {
|
|
b64 := base64.StdEncoding.EncodeToString([]byte(cmd))
|
|
// `qm guest exec` returns JSON; pipe through jq for a clean stdout,
|
|
// falling back to the raw form. Mirrors the LXC base64 round-trip.
|
|
return fmt.Sprintf(
|
|
"qm guest exec %s -- /bin/bash -c 'echo %s | base64 -d | bash' | jq -r '.out // .err // empty' 2>/dev/null || qm guest exec %s -- /bin/bash -c 'echo %s | base64 -d | bash'",
|
|
pveID, b64, pveID, b64)
|
|
}
|
|
}
|
|
return func(cmd string) string {
|
|
b64 := base64.StdEncoding.EncodeToString([]byte(cmd))
|
|
return fmt.Sprintf("pct exec %s -- bash -c 'echo %s | base64 -d | bash'", pveID, b64)
|
|
}
|
|
}
|