feat: add corosync quorum health check for proxmox-host entities
Adds pvecm_quorum_check.sh probe script and wires it into the checkdefaults system as a new 'quorum' monitoring kind on proxmox-host entities. Runs every 60s via ssh-script, surfaces unhealthy signal when cluster loses quorum. Closes the monitoring blind spot that let the 2026-08-12 3.5h corosync flapping outage go undetected (ping passed, cluster was non-quorate). Changes: - seeds/ontology.yaml: proxmox-host declares monitoring: [quorum] - internal/checkdefaults/defaults.go: KindQuorum builder - internal/checkdefaults/build_test.go: 2 new test cases - checks/pvecm_quorum_check.sh: new probe (deployed to hubris + strong) - VERSION: 0.30.2 -> 0.31.0
This commit is contained in:
12
checks/pvecm_quorum_check.sh
Normal file
12
checks/pvecm_quorum_check.sh
Normal file
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# pvecm_quorum_check.sh — Proxmox cluster quorum status.
|
||||
# Runs on a PVE host. Fails if the node is not quorate.
|
||||
set -euo pipefail
|
||||
|
||||
# pvecm status exit code is non-zero on non-quorate nodes
|
||||
# (e.g. "Quorate: No — Activity blocked")
|
||||
if pvecm status 2>/dev/null | grep -q 'Quorate.*Yes'; then
|
||||
echo '{"health":"healthy","metrics":{"quorate":1}}'
|
||||
else
|
||||
echo '{"health":"unhealthy","metrics":{"quorate":0}}'
|
||||
fi
|
||||
@@ -94,6 +94,12 @@ func TestBuildKindAllImplementedKinds(t *testing.T) {
|
||||
},
|
||||
{name: "dns without a name skips", kind: KindDNS, target: Target{}, wantSkip: true, wantReason: "no name"},
|
||||
|
||||
{
|
||||
name: "quorum runs pvecm script via ssh", kind: KindQuorum, host: host,
|
||||
wantDefs: 1, wantKind: "ssh-script", wantKey: "script", wantVal: "pvecm_quorum_check.sh", wantInterv: 60,
|
||||
},
|
||||
{name: "quorum no host skips", kind: KindQuorum, wantSkip: true, wantReason: "no address"},
|
||||
|
||||
{name: "unknown kind skips", kind: "telepathy", host: host, wantSkip: true, wantReason: "no builder"},
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ const (
|
||||
KindBackup = "backup-freshness"
|
||||
KindCertExpiry = "cert-expiry"
|
||||
KindVMStatus = "vm-status"
|
||||
KindQuorum = "quorum"
|
||||
KindDNS = "dns"
|
||||
)
|
||||
|
||||
@@ -360,6 +361,15 @@ func buildKind(kind string, t Target, attrs map[string]any, host, user string, p
|
||||
config: map[string]any{},
|
||||
interval: 60,
|
||||
}}, ""
|
||||
|
||||
case KindQuorum:
|
||||
// Proxmox cluster quorum via `pvecm status`. Only meaningful on
|
||||
// proxmox-host entities. Runs every 60s — corosync flaps are
|
||||
// transient and the probe is lightweight (local binary, no network).
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or its host"
|
||||
}
|
||||
return []checkDef{ssh("pvecm_quorum_check.sh")}, ""
|
||||
}
|
||||
|
||||
return nil, "no builder for this kind yet"
|
||||
|
||||
@@ -232,6 +232,7 @@ entity_types:
|
||||
layer: infrastructure
|
||||
lifecycle: infrastructure
|
||||
description: Machine running Proxmox VE.
|
||||
monitoring: [quorum] # corosync quorum check via pvecm status
|
||||
attributes:
|
||||
type: object
|
||||
properties: {pve_version: {type: string}}
|
||||
|
||||
Reference in New Issue
Block a user