feat(checks): vm-status probe + matrix cert dial-by-name

VMs declared monitoring [ping], but many block ICMP and lack a guest agent
(haos), so ping was the wrong probe — a powered-on VM reported "down". Add a
vm-status check: `qm status <pve_id>` on the VM's Proxmox host, which tests
"powered on" without needing the VM's network at all. vm type monitoring is
now [vm-status].

matrix.hubris.network is a public hostname (federation) resolving to
netbird-vps, not served by the lab Caddy — so its cert-expiry check's
dial=caddy IP failed. Drop the dial for matrix; it dials by name (DNS ->
public) like wget already proved works.

internal/checkdefaults/defaults.go

@@ -32,6 +32,7 @@ const (
 	KindCapacity   = "capacity"
 	KindBackup     = "backup-freshness"
 	KindCertExpiry = "cert-expiry"
+	KindVMStatus   = "vm-status"
 )
 
 // defaultBackupMaxAge is how long a backup target may go without a new
@@ -295,6 +296,19 @@ func buildKind(kind string, t Target, attrs map[string]any, host, user string, p
 			config:   config,
 			interval: 3600,
 		}}, ""
+
+	case KindVMStatus:
+		// "Is the VM powered on" via `qm status` on its Proxmox host — the
+		// right reachability probe for a VM, since many block ICMP and lack a
+		// guest agent. checkVMStatus re-reads pve_id + host at runtime.
+		if _, ok := attrs["pve_id"]; !ok {
+			return nil, "no pve_id to run qm status"
+		}
+		return []checkDef{{
+			kind:     "vm-status",
+			config:   map[string]any{},
+			interval: 60,
+		}}, ""
 	}
 
 	return nil, "no builder for this kind yet"

internal/scheduler/scheduler.go

@@ -291,6 +291,8 @@ func executeCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledChec
 		return checkDisk(ctx, cd)
 	case "cert-expiry":
 		return checkCertExpiry(ctx, cd)
+	case "vm-status":
+		return checkVMStatus(ctx, pool, cd)
 	case "ping":
 		return checkPing(ctx, cd)
 	case "ssh-script":
@@ -613,6 +615,53 @@ func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) ch
 	return checkResult{health: "healthy", metrics: metrics}
 }
 
+// checkVMStatus reports whether a VM is powered on, via `qm status <pve_id>`
+// run on its Proxmox host. This is the right reachability probe for a VM that
+// blocks ICMP (haos) and has no guest agent: it doesn't need the VM's network
+// at all — "status: running" means the VM is up. The command runs on the host
+// (not inside the VM), so it uses the host's address with identity wrap.
+func checkVMStatus(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
+	if cd.TargetID == nil {
+		return checkResult{health: "unknown", evidence: "vm-status needs a target VM"}
+	}
+	var pveID, hostAttr string
+	if err := pool.QueryRow(ctx,
+		"SELECT attributes->>'pve_id', COALESCE(attributes->>'host','') FROM entities WHERE id = $1",
+		*cd.TargetID).Scan(&pveID, &hostAttr); err != nil || pveID == "" {
+		return checkResult{health: "unknown", signalKind: "vm-status",
+			evidence: fmt.Sprintf("vm %s has no pve_id", cd.EntitySlug)}
+	}
+	hostSlug := remote.ResolveProxmoxHostSlug(ctx, pool, *cd.TargetID, hostAttr)
+	addr, user, err := remote.ResolveHost(ctx, pool, hostSlug, sshUser)
+	if err != nil {
+		return checkResult{health: "down", signalKind: "vm-status",
+			evidence: fmt.Sprintf("resolve proxmox host for %s: %v", cd.EntitySlug, err), err: err}
+	}
+
+	timeout := time.Duration(cd.TimeoutS) * time.Second
+	if timeout <= 0 {
+		timeout = 15 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	out, err := sshExec(ctx, addr, "22", user, "qm status "+pveID, timeout)
+	if err != nil {
+		return checkResult{health: "down", signalKind: "vm-status",
+			evidence: fmt.Sprintf("qm status %s on %s: %v", pveID, addr, err), err: err}
+	}
+	// `qm status <id>` prints "status: running" (or stopped/paused).
+	if strings.Contains(string(out), "status: running") {
+		return checkResult{health: "healthy"}
+	}
+	trimmed := strings.TrimSpace(string(out))
+	if trimmed == "" {
+		trimmed = "(no output)"
+	}
+	return checkResult{health: "down", signalKind: "vm-status",
+		evidence: fmt.Sprintf("%s not running: %s", cd.EntitySlug, trimmed)}
+}
+
 // checkPing performs an ICMP ping check using the system ping command.
 func checkPing(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
 	cfg := struct {

seeds/inventory.yaml

@@ -350,7 +350,7 @@ entities:
   - {slug: "cert:media.hubris.network", type: certificate, name: media.hubris.network, attributes: {dial: "192.168.8.175"}}
   - {slug: "cert:cloud.hubris.network", type: certificate, name: cloud.hubris.network, attributes: {dial: "192.168.8.175"}}
   - {slug: "cert:paperless.hubris.network", type: certificate, name: paperless.hubris.network, attributes: {dial: "192.168.8.175"}}
-  - {slug: "cert:matrix.hubris.network", type: certificate, name: matrix.hubris.network, attributes: {dial: "192.168.8.175"}}
+  - {slug: "cert:matrix.hubris.network", type: certificate, name: matrix.hubris.network}
   - {slug: "cert:photos.hubris.network", type: certificate, name: photos.hubris.network, attributes: {dial: "192.168.8.175"}}
   - {slug: "cert:artifacto.hubris.network", type: certificate, name: artifacto.hubris.network, attributes: {dial: "192.168.8.175"}}
   - {slug: "cert:trmnl.hubris.network", type: certificate, name: trmnl.hubris.network, attributes: {dial: "192.168.8.175"}}

seeds/ontology.yaml

@@ -272,7 +272,9 @@ entity_types:
     layer: infrastructure
     lifecycle: infrastructure
     description: Virtual machine.
-    monitoring: [ping]            # no guest agent assumed; reachability only
+    monitoring: [vm-status]       # `qm status` from the host: powered-on liveness
+                                  # that works even when the VM blocks ICMP and has
+                                  # no guest agent (haos). ping is unreliable for VMs.
     attributes:
       type: object
       properties: