feat: add corosync quorum health check for proxmox-host entities
Adds pvecm_quorum_check.sh probe script and wires it into the checkdefaults system as a new 'quorum' monitoring kind on proxmox-host entities. Runs every 60s via ssh-script, surfaces unhealthy signal when cluster loses quorum. Closes the monitoring blind spot that let the 2026-08-12 3.5h corosync flapping outage go undetected (ping passed, cluster was non-quorate). Changes: - seeds/ontology.yaml: proxmox-host declares monitoring: [quorum] - internal/checkdefaults/defaults.go: KindQuorum builder - internal/checkdefaults/build_test.go: 2 new test cases - checks/pvecm_quorum_check.sh: new probe (deployed to hubris + strong) - VERSION: 0.30.2 -> 0.31.0
This commit is contained in:
12
checks/pvecm_quorum_check.sh
Normal file
12
checks/pvecm_quorum_check.sh
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# pvecm_quorum_check.sh — Proxmox cluster quorum status.
|
||||||
|
# Runs on a PVE host. Fails if the node is not quorate.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# pvecm status exit code is non-zero on non-quorate nodes
|
||||||
|
# (e.g. "Quorate: No — Activity blocked")
|
||||||
|
if pvecm status 2>/dev/null | grep -q 'Quorate.*Yes'; then
|
||||||
|
echo '{"health":"healthy","metrics":{"quorate":1}}'
|
||||||
|
else
|
||||||
|
echo '{"health":"unhealthy","metrics":{"quorate":0}}'
|
||||||
|
fi
|
||||||
@@ -94,6 +94,12 @@ func TestBuildKindAllImplementedKinds(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{name: "dns without a name skips", kind: KindDNS, target: Target{}, wantSkip: true, wantReason: "no name"},
|
{name: "dns without a name skips", kind: KindDNS, target: Target{}, wantSkip: true, wantReason: "no name"},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "quorum runs pvecm script via ssh", kind: KindQuorum, host: host,
|
||||||
|
wantDefs: 1, wantKind: "ssh-script", wantKey: "script", wantVal: "pvecm_quorum_check.sh", wantInterv: 60,
|
||||||
|
},
|
||||||
|
{name: "quorum no host skips", kind: KindQuorum, wantSkip: true, wantReason: "no address"},
|
||||||
|
|
||||||
{name: "unknown kind skips", kind: "telepathy", host: host, wantSkip: true, wantReason: "no builder"},
|
{name: "unknown kind skips", kind: "telepathy", host: host, wantSkip: true, wantReason: "no builder"},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ const (
|
|||||||
KindBackup = "backup-freshness"
|
KindBackup = "backup-freshness"
|
||||||
KindCertExpiry = "cert-expiry"
|
KindCertExpiry = "cert-expiry"
|
||||||
KindVMStatus = "vm-status"
|
KindVMStatus = "vm-status"
|
||||||
|
KindQuorum = "quorum"
|
||||||
KindDNS = "dns"
|
KindDNS = "dns"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -360,6 +361,15 @@ func buildKind(kind string, t Target, attrs map[string]any, host, user string, p
|
|||||||
config: map[string]any{},
|
config: map[string]any{},
|
||||||
interval: 60,
|
interval: 60,
|
||||||
}}, ""
|
}}, ""
|
||||||
|
|
||||||
|
case KindQuorum:
|
||||||
|
// Proxmox cluster quorum via `pvecm status`. Only meaningful on
|
||||||
|
// proxmox-host entities. Runs every 60s — corosync flaps are
|
||||||
|
// transient and the probe is lightweight (local binary, no network).
|
||||||
|
if host == "" {
|
||||||
|
return nil, "no address on the entity or its host"
|
||||||
|
}
|
||||||
|
return []checkDef{ssh("pvecm_quorum_check.sh")}, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, "no builder for this kind yet"
|
return nil, "no builder for this kind yet"
|
||||||
|
|||||||
@@ -232,6 +232,7 @@ entity_types:
|
|||||||
layer: infrastructure
|
layer: infrastructure
|
||||||
lifecycle: infrastructure
|
lifecycle: infrastructure
|
||||||
description: Machine running Proxmox VE.
|
description: Machine running Proxmox VE.
|
||||||
|
monitoring: [quorum] # corosync quorum check via pvecm status
|
||||||
attributes:
|
attributes:
|
||||||
type: object
|
type: object
|
||||||
properties: {pve_version: {type: string}}
|
properties: {pve_version: {type: string}}
|
||||||
|
|||||||
Reference in New Issue
Block a user