feat(remote): route LXC/VM checks through the Proxmox host, not direct SSH
The scheduler SSHed each guest directly and assumed a deployed probe script plus working root SSH at the guest's address — false for headless (nfs-export), keyless (teddycloud), mesh-only (rclone), and macOS (mac-mini) targets, which left 49 enabled checks stuck "down" on a healthy fleet. Extract the MCP run tool's resolveExecTarget into a shared internal/remote package and make it the single execution path for both the scheduler and MCP. LXC/VM checks now host-hop via pct exec / qm guest exec through the owning Proxmox host (no per-guest lan_ip, sshd, or authorized key needed); hosts and workstations resolve their address and user live, so mac-mini's `user: dtoro` is honored without a re-seed. Address preference now prefers public_ipv4 over mesh, so netbird-vps is probeable from the scheduler container. cpu_check.sh gains a real Darwin branch (it reported cpu_pct 0 before). checkdefaults.resolveSSHUser reads the top-level `user` attribute too. A machine-target resolution failure is now logged before falling back to baked config, so a broken probe-config is distinguishable from a real outage.
This commit is contained in:
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/observability"
|
||||
"github.com/dtoro/oikos/internal/remote"
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/sys/unix"
|
||||
@@ -111,7 +112,7 @@ func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDef
|
||||
q := sqlcgen.New(pool)
|
||||
start := time.Now()
|
||||
|
||||
result := executeCheck(ctx, cd)
|
||||
result := executeCheck(ctx, pool, cd)
|
||||
|
||||
// Stamp the run before processing the result: due-ness must advance even
|
||||
// when a check fails, or a permanently failing check would be re-run on
|
||||
@@ -277,8 +278,10 @@ type checkResult struct {
|
||||
err error
|
||||
}
|
||||
|
||||
// executeCheck dispatches to the appropriate checker by kind.
|
||||
func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
|
||||
// executeCheck dispatches to the appropriate checker by kind. pool is needed
|
||||
// by the ssh-script path, which resolves the target's execution endpoint
|
||||
// (guests route through their Proxmox host; see internal/remote).
|
||||
func executeCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
|
||||
switch cd.Kind {
|
||||
case "http":
|
||||
return checkHTTP(ctx, cd)
|
||||
@@ -291,7 +294,7 @@ func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) check
|
||||
case "ping":
|
||||
return checkPing(ctx, cd)
|
||||
case "ssh-script":
|
||||
return checkSSHScript(ctx, cd)
|
||||
return checkSSHScript(ctx, pool, cd)
|
||||
case "backup-freshness":
|
||||
return checkBackupFreshness(ctx, cd)
|
||||
default:
|
||||
@@ -706,8 +709,16 @@ func parsePingLatency(output []byte) float64 {
|
||||
return val
|
||||
}
|
||||
|
||||
// checkSSHScript executes an allowlisted script on a remote host via SSH.
|
||||
func checkSSHScript(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
|
||||
// checkSSHScript executes an allowlisted script on a remote target via SSH.
|
||||
//
|
||||
// Routing follows the canonical access model (internal/remote): an LXC or VM
|
||||
// is NEVER SSH'd into directly — it is reached through its Proxmox host via
|
||||
// `pct exec`/`qm guest exec`, so a guest needs no lan_ip, sshd, or authorized
|
||||
// key of its own. Hosts and workstations are reached by direct SSH, resolved
|
||||
// live so a workstation's login (mac-mini: `user: dtoro`) is honored without
|
||||
// a re-seed. Services and other entities fall back to the host address baked
|
||||
// into check config at seed time (their hosting container's address).
|
||||
func checkSSHScript(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
|
||||
cfg := struct {
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
@@ -725,12 +736,6 @@ func checkSSHScript(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) che
|
||||
if cfg.Host == "" || cfg.Script == "" {
|
||||
return checkResult{health: "healthy"}
|
||||
}
|
||||
if cfg.Port == 0 {
|
||||
cfg.Port = 22
|
||||
}
|
||||
if cfg.User == "" {
|
||||
cfg.User = sshUser
|
||||
}
|
||||
|
||||
if !allowlistedScript(cfg.Script) {
|
||||
return checkResult{
|
||||
@@ -753,13 +758,45 @@ func checkSSHScript(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) che
|
||||
// the remote command. The script name itself is allowlisted above.
|
||||
scriptPath += " '" + strings.ReplaceAll(cfg.Args, "'", `'\''`) + "'"
|
||||
}
|
||||
port := strconv.Itoa(cfg.Port)
|
||||
|
||||
output, err := sshExec(ctx, cfg.Host, port, cfg.User, scriptPath, timeout)
|
||||
// Resolve the execution endpoint. Guests host-hop; host/workstation types
|
||||
// resolve their address + user live; everything else uses the baked config.
|
||||
host, port, user := cfg.Host, strconv.Itoa(oru(cfg.Port, 22)), orStr(cfg.User, sshUser)
|
||||
wrap := func(cmd string) string { return cmd }
|
||||
targetType := ""
|
||||
if cd.TargetType != nil {
|
||||
targetType = *cd.TargetType
|
||||
}
|
||||
if cd.TargetID != nil {
|
||||
switch {
|
||||
case remote.IsGuest(targetType):
|
||||
et, err := remote.ResolveExecTargetForCheck(ctx, pool, *cd.TargetID, targetType, sshUser)
|
||||
if err != nil {
|
||||
return checkResult{
|
||||
health: "down", signalKind: "ssh-script",
|
||||
evidence: fmt.Sprintf("route guest %s: %v", cd.EntitySlug, err), err: err,
|
||||
}
|
||||
}
|
||||
host, port, user, wrap = et.Host, "22", et.User, et.Wrap
|
||||
case isMachine(targetType):
|
||||
et, err := remote.ResolveExecTargetForCheck(ctx, pool, *cd.TargetID, targetType, sshUser)
|
||||
if err == nil {
|
||||
host, port, user, wrap = et.Host, "22", et.User, et.Wrap
|
||||
} else {
|
||||
// Log the resolution failure so an opaque ssh "down" doesn't
|
||||
// hide that the real cause was host/user resolution (e.g. a
|
||||
// missing attribute), then fall back to the baked config below.
|
||||
slog.Warn("scheduler: machine target resolution failed, using baked config",
|
||||
"entity", cd.EntitySlug, "target_type", targetType, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output, err := sshExec(ctx, host, port, user, wrap(scriptPath), timeout)
|
||||
if err != nil {
|
||||
return checkResult{
|
||||
health: "down", signalKind: "ssh-script",
|
||||
evidence: fmt.Sprintf("ssh %s:%s %s: %v", cfg.Host, strconv.Itoa(cfg.Port), cfg.Script, err),
|
||||
evidence: fmt.Sprintf("ssh %s:%s %s: %v", host, port, cfg.Script, err),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
@@ -797,6 +834,32 @@ func checkSSHScript(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) che
|
||||
}
|
||||
}
|
||||
|
||||
// isMachine reports whether a target type is a physical/virtual machine that
|
||||
// should be reached by direct SSH at its own resolved address (rather than the
|
||||
// baked hosting-container address a service uses). These are the machine
|
||||
// subtypes in the ontology.
|
||||
func isMachine(entityType string) bool {
|
||||
switch entityType {
|
||||
case "proxmox-host", "standalone-server", "workstation", "appliance":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// oru returns v when nonzero, else def. orStr returns v when non-empty, else def.
|
||||
func oru(v, def int) int {
|
||||
if v != 0 {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
func orStr(v, def string) string {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
var scriptNameRe = regexp.MustCompile(`^[a-z][a-z0-9_-]+\.sh$`)
|
||||
|
||||
func allowlistedScript(name string) bool {
|
||||
|
||||
Reference in New Issue
Block a user