fix(checks): process_check matches docker containers + prefixed systemd units
process_check.sh ran `systemctl is-active <entity-name>`, but a service's name is a logical label, not its unit/container name — matrix is matrix-synapse.service + element-web/mautrix-* containers, authentik is authentik-server/-worker containers. So every multi-component or docker service reported "inactive" while up (authentik, matrix, photos, house, arr-stack, …). Resolve in order: exact systemd unit, a unit with the name as prefix (matrix -> matrix-synapse.service), or a running docker container whose name contains it. checkdefaults passes a declared probe_unit/systemd_unit/container attribute when set, for precision.
This commit is contained in:
@@ -1,5 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
# process_check.sh — systemd service liveness.
|
||||
# process_check.sh — service liveness.
|
||||
#
|
||||
# A service entity's name is a logical label, rarely the literal systemd unit
|
||||
# or container name. matrix = matrix-synapse.service + element-web/mautrix-*
|
||||
# containers; authentik = authentik-server/-worker containers. So checking
|
||||
# `systemctl is-active matrix` reports "inactive" for a healthy service.
|
||||
#
|
||||
# Resolution order, any hit = healthy:
|
||||
# 1. exact systemd unit `systemctl is-active <name>`
|
||||
# 2. a systemd unit with the name as prefix `<name>*.service`
|
||||
# 3. a running docker container whose name contains <name>
|
||||
# An explicit probe target overrides the label — see checkdefaults, which
|
||||
# passes a `probe_unit`/`container`/`systemd_unit` attribute as $1 when set.
|
||||
set -euo pipefail
|
||||
|
||||
SERVICE="${1:-}"
|
||||
@@ -8,29 +20,31 @@ if [ -z "$SERVICE" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v systemctl >/dev/null 2>&1; then
|
||||
echo '{"health":"unknown","signalKind":"process-check","evidence":"systemctl not found"}'
|
||||
exit 0
|
||||
ok() { echo "{\"health\":\"healthy\"}"; exit 0; }
|
||||
|
||||
# 1. exact systemd unit
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true)
|
||||
[ "$STATE" = "active" ] && ok
|
||||
|
||||
# 2. prefix match: matrix -> matrix-synapse.service, house -> house.service, etc.
|
||||
# --no-legend strips the header/footer so grep can see the unit rows; the
|
||||
# pattern is a systemd unit glob.
|
||||
if systemctl list-units --type=service --state=active --no-legend "$SERVICE*.service" 2>/dev/null \
|
||||
| grep -q '\.service'; then
|
||||
ok
|
||||
fi
|
||||
fi
|
||||
|
||||
# `systemctl is-active` PRINTS the state and exits non-zero when the unit is
|
||||
# not active, so `... || echo unknown` appended a second line and produced
|
||||
# "paperless is inactive\nunknown" — a raw newline inside a JSON string, which
|
||||
# the scheduler rejected as invalid output. head -1 keeps the first line and
|
||||
# the fallback only fires when there was no output at all.
|
||||
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true)
|
||||
[ -z "$STATE" ] && STATE="unknown"
|
||||
# Belt and braces: a unit name or state containing a quote would break the
|
||||
# hand-built JSON below just as thoroughly.
|
||||
# 3. a running docker container whose name contains the label.
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
if docker ps --filter "status=running" --filter "name=$SERVICE" --format '{{.Names}}' 2>/dev/null \
|
||||
| grep -q .; then
|
||||
ok
|
||||
fi
|
||||
fi
|
||||
|
||||
STATE=${STATE:-inactive}
|
||||
STATE=${STATE//\"/}
|
||||
SAFE_SERVICE=${SERVICE//\"/}
|
||||
|
||||
if [ "$STATE" = "active" ]; then
|
||||
echo "{\"health\":\"healthy\"}"
|
||||
else
|
||||
# signalKind is a taxonomy, not a per-service label. Emitting "$SERVICE"
|
||||
# here minted a distinct signal kind for every service (kind=paperless,
|
||||
# kind=qbit, …), which no approval_rule can match and which makes
|
||||
# "how many process checks are failing?" unanswerable.
|
||||
echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE\"}"
|
||||
fi
|
||||
echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE (no active unit/container matched)\"}"
|
||||
|
||||
@@ -202,12 +202,26 @@ func buildKind(kind string, t Target, attrs map[string]any, host, user string, p
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or its host"
|
||||
}
|
||||
if t.Name == "" {
|
||||
// A service's name is a logical label, not usually its systemd unit
|
||||
// or container name (matrix = matrix-synapse.service + containers).
|
||||
// Prefer an explicit probe target when declared; process_check.sh also
|
||||
// matches a unit prefix or a docker container as a fallback.
|
||||
unit := ""
|
||||
for _, key := range []string{"probe_unit", "systemd_unit", "container"} {
|
||||
if v, _ := attrs[key].(string); v != "" {
|
||||
unit = v
|
||||
break
|
||||
}
|
||||
}
|
||||
if unit == "" {
|
||||
unit = t.Name
|
||||
}
|
||||
if unit == "" {
|
||||
return nil, "no name to check a process for"
|
||||
}
|
||||
// process_check.sh takes the unit name as $1 and reports "unknown"
|
||||
// without it.
|
||||
return []checkDef{ssh("process_check.sh", t.Name)}, ""
|
||||
// process_check.sh takes the unit/container name as $1 and reports
|
||||
// "unknown" without it.
|
||||
return []checkDef{ssh("process_check.sh", unit)}, ""
|
||||
|
||||
case KindBackup:
|
||||
// A backup target is checked from the machine that writes to it, so it
|
||||
|
||||
Reference in New Issue
Block a user