From eb16796bf0519aeee27908550b74baa99fdf40b6 Mon Sep 17 00:00:00 2001 From: dtoro Date: Wed, 29 Jul 2026 23:15:36 +0200 Subject: [PATCH] feat(checks): vm-status probe + matrix cert dial-by-name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VMs declared monitoring [ping], but many block ICMP and lack a guest agent (haos), so ping was the wrong probe — a powered-on VM reported "down". Add a vm-status check: `qm status ` on the VM's Proxmox host, which tests "powered on" without needing the VM's network at all. vm type monitoring is now [vm-status]. matrix.hubris.network is a public hostname (federation) resolving to netbird-vps, not served by the lab Caddy — so its cert-expiry check's dial=caddy IP failed. Drop the dial for matrix; it dials by name (DNS -> public) like wget already proved works. --- internal/checkdefaults/defaults.go | 14 +++++++++ internal/scheduler/scheduler.go | 49 ++++++++++++++++++++++++++++++ seeds/inventory.yaml | 2 +- seeds/ontology.yaml | 4 ++- 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/internal/checkdefaults/defaults.go b/internal/checkdefaults/defaults.go index 5f86e18..be223a6 100644 --- a/internal/checkdefaults/defaults.go +++ b/internal/checkdefaults/defaults.go @@ -32,6 +32,7 @@ const ( KindCapacity = "capacity" KindBackup = "backup-freshness" KindCertExpiry = "cert-expiry" + KindVMStatus = "vm-status" ) // defaultBackupMaxAge is how long a backup target may go without a new @@ -295,6 +296,19 @@ func buildKind(kind string, t Target, attrs map[string]any, host, user string, p config: config, interval: 3600, }}, "" + + case KindVMStatus: + // "Is the VM powered on" via `qm status` on its Proxmox host — the + // right reachability probe for a VM, since many block ICMP and lack a + // guest agent. checkVMStatus re-reads pve_id + host at runtime. + if _, ok := attrs["pve_id"]; !ok { + return nil, "no pve_id to run qm status" + } + return []checkDef{{ + kind: "vm-status", + config: map[string]any{}, + interval: 60, + }}, "" } return nil, "no builder for this kind yet" diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index b36f3d9..31251a1 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -291,6 +291,8 @@ func executeCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledChec return checkDisk(ctx, cd) case "cert-expiry": return checkCertExpiry(ctx, cd) + case "vm-status": + return checkVMStatus(ctx, pool, cd) case "ping": return checkPing(ctx, cd) case "ssh-script": @@ -613,6 +615,53 @@ func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) ch return checkResult{health: "healthy", metrics: metrics} } +// checkVMStatus reports whether a VM is powered on, via `qm status ` +// run on its Proxmox host. This is the right reachability probe for a VM that +// blocks ICMP (haos) and has no guest agent: it doesn't need the VM's network +// at all — "status: running" means the VM is up. The command runs on the host +// (not inside the VM), so it uses the host's address with identity wrap. +func checkVMStatus(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { + if cd.TargetID == nil { + return checkResult{health: "unknown", evidence: "vm-status needs a target VM"} + } + var pveID, hostAttr string + if err := pool.QueryRow(ctx, + "SELECT attributes->>'pve_id', COALESCE(attributes->>'host','') FROM entities WHERE id = $1", + *cd.TargetID).Scan(&pveID, &hostAttr); err != nil || pveID == "" { + return checkResult{health: "unknown", signalKind: "vm-status", + evidence: fmt.Sprintf("vm %s has no pve_id", cd.EntitySlug)} + } + hostSlug := remote.ResolveProxmoxHostSlug(ctx, pool, *cd.TargetID, hostAttr) + addr, user, err := remote.ResolveHost(ctx, pool, hostSlug, sshUser) + if err != nil { + return checkResult{health: "down", signalKind: "vm-status", + evidence: fmt.Sprintf("resolve proxmox host for %s: %v", cd.EntitySlug, err), err: err} + } + + timeout := time.Duration(cd.TimeoutS) * time.Second + if timeout <= 0 { + timeout = 15 * time.Second + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + out, err := sshExec(ctx, addr, "22", user, "qm status "+pveID, timeout) + if err != nil { + return checkResult{health: "down", signalKind: "vm-status", + evidence: fmt.Sprintf("qm status %s on %s: %v", pveID, addr, err), err: err} + } + // `qm status ` prints "status: running" (or stopped/paused). + if strings.Contains(string(out), "status: running") { + return checkResult{health: "healthy"} + } + trimmed := strings.TrimSpace(string(out)) + if trimmed == "" { + trimmed = "(no output)" + } + return checkResult{health: "down", signalKind: "vm-status", + evidence: fmt.Sprintf("%s not running: %s", cd.EntitySlug, trimmed)} +} + // checkPing performs an ICMP ping check using the system ping command. func checkPing(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { cfg := struct { diff --git a/seeds/inventory.yaml b/seeds/inventory.yaml index 147e26b..a4fae62 100644 --- a/seeds/inventory.yaml +++ b/seeds/inventory.yaml @@ -350,7 +350,7 @@ entities: - {slug: "cert:media.hubris.network", type: certificate, name: media.hubris.network, attributes: {dial: "192.168.8.175"}} - {slug: "cert:cloud.hubris.network", type: certificate, name: cloud.hubris.network, attributes: {dial: "192.168.8.175"}} - {slug: "cert:paperless.hubris.network", type: certificate, name: paperless.hubris.network, attributes: {dial: "192.168.8.175"}} - - {slug: "cert:matrix.hubris.network", type: certificate, name: matrix.hubris.network, attributes: {dial: "192.168.8.175"}} + - {slug: "cert:matrix.hubris.network", type: certificate, name: matrix.hubris.network} - {slug: "cert:photos.hubris.network", type: certificate, name: photos.hubris.network, attributes: {dial: "192.168.8.175"}} - {slug: "cert:artifacto.hubris.network", type: certificate, name: artifacto.hubris.network, attributes: {dial: "192.168.8.175"}} - {slug: "cert:trmnl.hubris.network", type: certificate, name: trmnl.hubris.network, attributes: {dial: "192.168.8.175"}} diff --git a/seeds/ontology.yaml b/seeds/ontology.yaml index 4b99396..dceefde 100644 --- a/seeds/ontology.yaml +++ b/seeds/ontology.yaml @@ -272,7 +272,9 @@ entity_types: layer: infrastructure lifecycle: infrastructure description: Virtual machine. - monitoring: [ping] # no guest agent assumed; reachability only + monitoring: [vm-status] # `qm status` from the host: powered-on liveness + # that works even when the VM blocks ICMP and has + # no guest agent (haos). ping is unreliable for VMs. attributes: type: object properties: