fix(checks): process_check matches docker containers + prefixed systemd units
process_check.sh ran `systemctl is-active <entity-name>`, but a service's name is a logical label, not its unit/container name — matrix is matrix-synapse.service + element-web/mautrix-* containers, authentik is authentik-server/-worker containers. So every multi-component or docker service reported "inactive" while up (authentik, matrix, photos, house, arr-stack, …). Resolve in order: exact systemd unit, a unit with the name as prefix (matrix -> matrix-synapse.service), or a running docker container whose name contains it. checkdefaults passes a declared probe_unit/systemd_unit/container attribute when set, for precision.
This commit is contained in:
@@ -1,5 +1,17 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# process_check.sh — systemd service liveness.
|
# process_check.sh — service liveness.
|
||||||
|
#
|
||||||
|
# A service entity's name is a logical label, rarely the literal systemd unit
|
||||||
|
# or container name. matrix = matrix-synapse.service + element-web/mautrix-*
|
||||||
|
# containers; authentik = authentik-server/-worker containers. So checking
|
||||||
|
# `systemctl is-active matrix` reports "inactive" for a healthy service.
|
||||||
|
#
|
||||||
|
# Resolution order, any hit = healthy:
|
||||||
|
# 1. exact systemd unit `systemctl is-active <name>`
|
||||||
|
# 2. a systemd unit with the name as prefix `<name>*.service`
|
||||||
|
# 3. a running docker container whose name contains <name>
|
||||||
|
# An explicit probe target overrides the label — see checkdefaults, which
|
||||||
|
# passes a `probe_unit`/`container`/`systemd_unit` attribute as $1 when set.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
SERVICE="${1:-}"
|
SERVICE="${1:-}"
|
||||||
@@ -8,29 +20,31 @@ if [ -z "$SERVICE" ]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! command -v systemctl >/dev/null 2>&1; then
|
ok() { echo "{\"health\":\"healthy\"}"; exit 0; }
|
||||||
echo '{"health":"unknown","signalKind":"process-check","evidence":"systemctl not found"}'
|
|
||||||
exit 0
|
# 1. exact systemd unit
|
||||||
|
if command -v systemctl >/dev/null 2>&1; then
|
||||||
|
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true)
|
||||||
|
[ "$STATE" = "active" ] && ok
|
||||||
|
|
||||||
|
# 2. prefix match: matrix -> matrix-synapse.service, house -> house.service, etc.
|
||||||
|
# --no-legend strips the header/footer so grep can see the unit rows; the
|
||||||
|
# pattern is a systemd unit glob.
|
||||||
|
if systemctl list-units --type=service --state=active --no-legend "$SERVICE*.service" 2>/dev/null \
|
||||||
|
| grep -q '\.service'; then
|
||||||
|
ok
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# `systemctl is-active` PRINTS the state and exits non-zero when the unit is
|
# 3. a running docker container whose name contains the label.
|
||||||
# not active, so `... || echo unknown` appended a second line and produced
|
if command -v docker >/dev/null 2>&1; then
|
||||||
# "paperless is inactive\nunknown" — a raw newline inside a JSON string, which
|
if docker ps --filter "status=running" --filter "name=$SERVICE" --format '{{.Names}}' 2>/dev/null \
|
||||||
# the scheduler rejected as invalid output. head -1 keeps the first line and
|
| grep -q .; then
|
||||||
# the fallback only fires when there was no output at all.
|
ok
|
||||||
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true)
|
fi
|
||||||
[ -z "$STATE" ] && STATE="unknown"
|
fi
|
||||||
# Belt and braces: a unit name or state containing a quote would break the
|
|
||||||
# hand-built JSON below just as thoroughly.
|
STATE=${STATE:-inactive}
|
||||||
STATE=${STATE//\"/}
|
STATE=${STATE//\"/}
|
||||||
SAFE_SERVICE=${SERVICE//\"/}
|
SAFE_SERVICE=${SERVICE//\"/}
|
||||||
|
echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE (no active unit/container matched)\"}"
|
||||||
if [ "$STATE" = "active" ]; then
|
|
||||||
echo "{\"health\":\"healthy\"}"
|
|
||||||
else
|
|
||||||
# signalKind is a taxonomy, not a per-service label. Emitting "$SERVICE"
|
|
||||||
# here minted a distinct signal kind for every service (kind=paperless,
|
|
||||||
# kind=qbit, …), which no approval_rule can match and which makes
|
|
||||||
# "how many process checks are failing?" unanswerable.
|
|
||||||
echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE\"}"
|
|
||||||
fi
|
|
||||||
|
|||||||
@@ -202,12 +202,26 @@ func buildKind(kind string, t Target, attrs map[string]any, host, user string, p
|
|||||||
if host == "" {
|
if host == "" {
|
||||||
return nil, "no address on the entity or its host"
|
return nil, "no address on the entity or its host"
|
||||||
}
|
}
|
||||||
if t.Name == "" {
|
// A service's name is a logical label, not usually its systemd unit
|
||||||
|
// or container name (matrix = matrix-synapse.service + containers).
|
||||||
|
// Prefer an explicit probe target when declared; process_check.sh also
|
||||||
|
// matches a unit prefix or a docker container as a fallback.
|
||||||
|
unit := ""
|
||||||
|
for _, key := range []string{"probe_unit", "systemd_unit", "container"} {
|
||||||
|
if v, _ := attrs[key].(string); v != "" {
|
||||||
|
unit = v
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if unit == "" {
|
||||||
|
unit = t.Name
|
||||||
|
}
|
||||||
|
if unit == "" {
|
||||||
return nil, "no name to check a process for"
|
return nil, "no name to check a process for"
|
||||||
}
|
}
|
||||||
// process_check.sh takes the unit name as $1 and reports "unknown"
|
// process_check.sh takes the unit/container name as $1 and reports
|
||||||
// without it.
|
// "unknown" without it.
|
||||||
return []checkDef{ssh("process_check.sh", t.Name)}, ""
|
return []checkDef{ssh("process_check.sh", unit)}, ""
|
||||||
|
|
||||||
case KindBackup:
|
case KindBackup:
|
||||||
// A backup target is checked from the machine that writes to it, so it
|
// A backup target is checked from the machine that writes to it, so it
|
||||||
|
|||||||
Reference in New Issue
Block a user