// Package remote resolves how to execute a command on a target entity and // turns a plain shell command into whatever must be sent over the SSH // connection that reaches it. // // The canonical access model: a host or workstation is reached by direct SSH // to its address; an LXC or VM is NEVER SSH'd into directly — it is reached // through its owning Proxmox host via `pct exec` / `qm guest exec`. One SSH // credential per host (the host's root key), no per-guest keys, sshd, or // lan_ip required for execution. Network probes (http/ping) still hit a // guest's lan_ip directly; only command execution host-hops. // // This is the single resolver shared by the scheduler's check execution and // the MCP `run` tool. Previously they diverged — the scheduler SSHed guests // directly (broken for headless/keyless/mesh-only guests), while MCP // host-hopped (working). Keeping one path keeps them in lockstep. package remote import ( "context" "encoding/base64" "encoding/json" "fmt" "strings" "github.com/dtoro/oikos/internal/db" "github.com/google/uuid" ) // DefaultUser is the SSH user when an entity declares no ssh.user. The // Proxmox hosts and their guests are all administered as root. const DefaultUser = "root" // ExecTarget is a resolved execution endpoint: the SSH address and user to // connect to, plus Wrap, which rewrites a plain command for transport. type ExecTarget struct { Host string User string // Wrap turns a plain shell command into the form that must be sent over // the SSH connection to this target: the identity function for a host, // `pct exec -- bash -c 'echo | base64 -d | bash'` for an LXC, // the `qm guest exec` equivalent for a VM. The base64 round-trip keeps // nested quoting identical across both guest kinds. Wrap func(cmd string) string } // IsGuest reports whether an entity type is reached via pct/qm exec through a // Proxmox host rather than by direct SSH. docker-container is reached via its // host's docker socket, not pct, so it is not a guest here. func IsGuest(entityType string) bool { return entityType == "lxc" || entityType == "vm" } // ResolveHost resolves a `host:` to its reachable network address and // SSH user. Address preference: lan_ip, then public_ipv4, then mesh IP, then // mesh fqdn. Preferring public_ipv4 over mesh matters because the scheduler // container has no mesh interface — a standalone-server with only a mesh IP // (netbird-vps) was unreachable, and a public_ipv4 was sitting unused. // // fallbackUser is used when the entity declares no ssh.user; callers pass // their configured default (the scheduler uses "root", the MCP run tool uses // its configured OIKOS_SSH_USER). func ResolveHost(ctx context.Context, pool *db.Pool, hostSlug, fallbackUser string) (addr, user string, err error) { var raw string if err = pool.QueryRow(ctx, "SELECT attributes::text FROM entities WHERE slug = $1", hostSlug).Scan(&raw); err != nil { return "", "", fmt.Errorf("entity not found: %s", hostSlug) } var m map[string]any if err = json.Unmarshal([]byte(raw), &m); err != nil { return "", "", fmt.Errorf("parse attributes for %s: %w", hostSlug, err) } if v, ok := m["lan_ip"].(string); ok && v != "" { addr = v } else if v, ok := m["public_ipv4"].(string); ok && v != "" { addr = v } else if mesh, ok := m["mesh"].(map[string]any); ok { if nb, ok := mesh["netbird"].(map[string]any); ok { if v, ok := nb["ip"].(string); ok && v != "" { addr = v } else if v, ok := nb["fqdn"].(string); ok && v != "" { addr = v } } } if addr == "" { return "", "", fmt.Errorf("no IP found for %s", hostSlug) } user = fallbackUser if ssh, ok := m["ssh"].(map[string]any); ok { if u, ok := ssh["user"].(string); ok && u != "" { user = u } } return addr, user, nil } // ResolveProxmoxHostSlug resolves the Proxmox host slug that owns a guest. // Resolution order: the hostAttr if non-empty (the entity's attributes.host, // stored without the "host:" prefix), the `hosts` relationship on the guest // (the canonical graph edge), then "hubris" as the documented default. // // entityID is the guest's entity id; the relationship lookup uses it // directly rather than a slug subquery. func ResolveProxmoxHostSlug(ctx context.Context, pool *db.Pool, entityID uuid.UUID, hostAttr string) string { hostSlug := strings.TrimSpace(hostAttr) if hostSlug == "" { var relHostSlug string if err := pool.QueryRow(ctx, ` SELECT e.slug FROM relationships r JOIN entities e ON e.id = r.source_id WHERE r.target_id = $1 AND r.type = 'hosts' AND r.valid_to IS NULL LIMIT 1`, entityID).Scan(&relHostSlug); err == nil && relHostSlug != "" { hostSlug = relHostSlug } } if hostSlug == "" { hostSlug = "hubris" // documented default Proxmox host when unset } if !strings.HasPrefix(hostSlug, "host:") { hostSlug = "host:" + hostSlug } return hostSlug } // ResolveExecTarget resolves a target slug (host:, lxc:, or vm:) to its // execution endpoint. This is the slug-based entry used by the MCP `run` tool. func ResolveExecTarget(ctx context.Context, pool *db.Pool, targetSlug, fallbackUser string) (ExecTarget, error) { switch { case strings.HasPrefix(targetSlug, "host:"): addr, user, err := ResolveHost(ctx, pool, targetSlug, fallbackUser) if err != nil { return ExecTarget{}, err } return ExecTarget{Host: addr, User: user, Wrap: func(cmd string) string { return cmd }}, nil case strings.HasPrefix(targetSlug, "lxc:"), strings.HasPrefix(targetSlug, "vm:"): var ( id uuid.UUID pveID string typ string hostAttr string ) if err := pool.QueryRow(ctx, "SELECT id, type, attributes->>'pve_id', COALESCE(attributes->>'host','') FROM entities WHERE slug = $1", targetSlug).Scan(&id, &typ, &pveID, &hostAttr); err != nil || pveID == "" { return ExecTarget{}, fmt.Errorf("guest not found or missing pve_id: %s", targetSlug) } hostSlug := ResolveProxmoxHostSlug(ctx, pool, id, hostAttr) addr, user, err := ResolveHost(ctx, pool, hostSlug, fallbackUser) if err != nil { return ExecTarget{}, err } return ExecTarget{Host: addr, User: user, Wrap: guestWrap(typ, pveID)}, nil } return ExecTarget{}, fmt.Errorf("unsupported target %q: must be host:, lxc:, or vm:", targetSlug) } // ResolveExecTargetForCheck resolves an execution endpoint keyed by the // target's id and type — the data the scheduler has at check-execution time // (check_defs carry target_id + target_type, not a slug). Guests route via // pct/qm exec; everything else (hosts, workstations, services resolved to // their hosting machine) is reached by direct SSH to the entity's own address. func ResolveExecTargetForCheck(ctx context.Context, pool *db.Pool, targetID uuid.UUID, targetType, fallbackUser string) (ExecTarget, error) { if IsGuest(targetType) { return resolveGuest(ctx, pool, targetID, targetType, fallbackUser) } // A service (or other non-compute target) has no address of its own — it // runs on whatever compute entity provides/hosts it. Resolve that host and // route through it: pct if the host is a guest, direct SSH (with the // host's correct user) if it's a machine. Previously a service check baked // its hosting LXC's lan_ip and SSHed it directly as root, which fails // because the scheduler key isn't in each LXC — only on the Proxmox hosts. if hostID, hostType, ok := hostingCompute(ctx, pool, targetID); ok { if IsGuest(hostType) { return resolveGuest(ctx, pool, hostID, hostType, fallbackUser) } addr, user, err := resolveHostByID(ctx, pool, hostID, fallbackUser) if err != nil { return ExecTarget{}, err } return ExecTarget{Host: addr, User: user, Wrap: func(cmd string) string { return cmd }}, nil } // No hosting entity found: reach the target directly at its own address // (a host/workstation, or a service whose host wasn't resolvable). addr, user, err := resolveHostByID(ctx, pool, targetID, fallbackUser) if err != nil { return ExecTarget{}, err } return ExecTarget{Host: addr, User: user, Wrap: func(cmd string) string { return cmd }}, nil } // hostingCompute walks the provides/runs-on/hosts edges backward from a target // to the compute entity that runs it (a service's LXC, an LXC's Proxmox host). // Returns the host's id, type, and whether one was found. Most-specific edge // first: provides names the runtime container directly. func hostingCompute(ctx context.Context, pool *db.Pool, targetID uuid.UUID) (uuid.UUID, string, bool) { var hid uuid.UUID var htype string err := pool.QueryRow(ctx, ` SELECT e.id, e.type FROM relationships r JOIN entities e ON e.id = r.source_id WHERE r.target_id = $1 AND r.valid_to IS NULL AND r.type IN ('provides','runs-on','hosts') ORDER BY CASE r.type WHEN 'provides' THEN 0 WHEN 'runs-on' THEN 1 ELSE 2 END LIMIT 1`, targetID).Scan(&hid, &htype) if err != nil { return uuid.Nil, "", false } return hid, htype, true } // resolveGuest resolves a guest's execution endpoint: the owning Proxmox host // (SSH'd directly) with a pct/qm exec wrapper around the command. func resolveGuest(ctx context.Context, pool *db.Pool, guestID uuid.UUID, guestType, fallbackUser string) (ExecTarget, error) { var pveID, hostAttr string if err := pool.QueryRow(ctx, "SELECT attributes->>'pve_id', COALESCE(attributes->>'host','') FROM entities WHERE id = $1", guestID).Scan(&pveID, &hostAttr); err != nil || pveID == "" { return ExecTarget{}, fmt.Errorf("guest %s missing pve_id", guestID) } hostSlug := ResolveProxmoxHostSlug(ctx, pool, guestID, hostAttr) addr, user, err := ResolveHost(ctx, pool, hostSlug, fallbackUser) if err != nil { return ExecTarget{}, err } return ExecTarget{Host: addr, User: user, Wrap: guestWrap(guestType, pveID)}, nil } // resolveHostByID is ResolveHost keyed by entity id. func resolveHostByID(ctx context.Context, pool *db.Pool, id uuid.UUID, fallbackUser string) (addr, user string, err error) { var raw string if err = pool.QueryRow(ctx, "SELECT attributes::text FROM entities WHERE id = $1", id).Scan(&raw); err != nil { return "", "", fmt.Errorf("entity %s not found", id) } var m map[string]any if err = json.Unmarshal([]byte(raw), &m); err != nil { return "", "", fmt.Errorf("parse attributes: %w", err) } for _, key := range []string{"lan_ip", "public_ipv4", "mesh_ip"} { if v, ok := m[key].(string); ok && v != "" { addr = v break } } if addr == "" { if mesh, ok := m["mesh"].(map[string]any); ok { if nb, ok := mesh["netbird"].(map[string]any); ok { if v, ok := nb["ip"].(string); ok && v != "" { addr = v } else if v, ok := nb["fqdn"].(string); ok && v != "" { addr = v } } } } if addr == "" { return "", "", fmt.Errorf("no IP found for entity %s", id) } user = fallbackUser if ssh, ok := m["ssh"].(map[string]any); ok { if u, ok := ssh["user"].(string); ok && u != "" { user = u } } if u, ok := m["user"].(string); ok && u != "" && user == fallbackUser { // Workstations carry their login as a top-level `user` attribute // (mac-mini: user: dtoro), not under ssh.user. Take it only when no // explicit ssh.user was set, so a host that genuinely wants root still // gets root. user = u } return addr, user, nil } // guestWrap builds the pct/qm exec wrapper for a guest of the given type. func guestWrap(entityType, pveID string) func(cmd string) string { if entityType == "vm" { return func(cmd string) string { b64 := base64.StdEncoding.EncodeToString([]byte(cmd)) // `qm guest exec` returns JSON; pipe through jq for a clean stdout, // falling back to the raw form. Mirrors the LXC base64 round-trip. return fmt.Sprintf( "qm guest exec %s -- /bin/bash -c 'echo %s | base64 -d | bash' | jq -r '.out // .err // empty' 2>/dev/null || qm guest exec %s -- /bin/bash -c 'echo %s | base64 -d | bash'", pveID, b64, pveID, b64) } } return func(cmd string) string { b64 := base64.StdEncoding.EncodeToString([]byte(cmd)) return fmt.Sprintf("pct exec %s -- bash -c 'echo %s | base64 -d | bash'", pveID, b64) } }