fix(checks): process_check matches docker containers + prefixed systemd units
Some checks failed
ci / build-test (push) Has been cancelled
ci / docker-build (push) Has been cancelled
ci / web (push) Has been cancelled
Desktop App / Build Linux (amd64) (push) Has been cancelled
Desktop App / Attach to Release (push) Has been cancelled

process_check.sh ran `systemctl is-active <entity-name>`, but a service's name
is a logical label, not its unit/container name — matrix is matrix-synapse.service
+ element-web/mautrix-* containers, authentik is authentik-server/-worker
containers. So every multi-component or docker service reported "inactive"
while up (authentik, matrix, photos, house, arr-stack, …).

Resolve in order: exact systemd unit, a unit with the name as prefix
(matrix -> matrix-synapse.service), or a running docker container whose name
contains it. checkdefaults passes a declared probe_unit/systemd_unit/container
attribute when set, for precision.
This commit is contained in:
2026-07-29 22:33:02 +02:00
parent 0929c17cbb
commit e4104eb344
2 changed files with 55 additions and 27 deletions

View File

@@ -1,5 +1,17 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# process_check.sh — systemd service liveness. # process_check.sh — service liveness.
#
# A service entity's name is a logical label, rarely the literal systemd unit
# or container name. matrix = matrix-synapse.service + element-web/mautrix-*
# containers; authentik = authentik-server/-worker containers. So checking
# `systemctl is-active matrix` reports "inactive" for a healthy service.
#
# Resolution order, any hit = healthy:
# 1. exact systemd unit `systemctl is-active <name>`
# 2. a systemd unit with the name as prefix `<name>*.service`
# 3. a running docker container whose name contains <name>
# An explicit probe target overrides the label — see checkdefaults, which
# passes a `probe_unit`/`container`/`systemd_unit` attribute as $1 when set.
set -euo pipefail set -euo pipefail
SERVICE="${1:-}" SERVICE="${1:-}"
@@ -8,29 +20,31 @@ if [ -z "$SERVICE" ]; then
exit 0 exit 0
fi fi
if ! command -v systemctl >/dev/null 2>&1; then ok() { echo "{\"health\":\"healthy\"}"; exit 0; }
echo '{"health":"unknown","signalKind":"process-check","evidence":"systemctl not found"}'
exit 0 # 1. exact systemd unit
if command -v systemctl >/dev/null 2>&1; then
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true)
[ "$STATE" = "active" ] && ok
# 2. prefix match: matrix -> matrix-synapse.service, house -> house.service, etc.
# --no-legend strips the header/footer so grep can see the unit rows; the
# pattern is a systemd unit glob.
if systemctl list-units --type=service --state=active --no-legend "$SERVICE*.service" 2>/dev/null \
| grep -q '\.service'; then
ok
fi
fi fi
# `systemctl is-active` PRINTS the state and exits non-zero when the unit is # 3. a running docker container whose name contains the label.
# not active, so `... || echo unknown` appended a second line and produced if command -v docker >/dev/null 2>&1; then
# "paperless is inactive\nunknown" — a raw newline inside a JSON string, which if docker ps --filter "status=running" --filter "name=$SERVICE" --format '{{.Names}}' 2>/dev/null \
# the scheduler rejected as invalid output. head -1 keeps the first line and | grep -q .; then
# the fallback only fires when there was no output at all. ok
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true) fi
[ -z "$STATE" ] && STATE="unknown" fi
# Belt and braces: a unit name or state containing a quote would break the
# hand-built JSON below just as thoroughly. STATE=${STATE:-inactive}
STATE=${STATE//\"/} STATE=${STATE//\"/}
SAFE_SERVICE=${SERVICE//\"/} SAFE_SERVICE=${SERVICE//\"/}
echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE (no active unit/container matched)\"}"
if [ "$STATE" = "active" ]; then
echo "{\"health\":\"healthy\"}"
else
# signalKind is a taxonomy, not a per-service label. Emitting "$SERVICE"
# here minted a distinct signal kind for every service (kind=paperless,
# kind=qbit, …), which no approval_rule can match and which makes
# "how many process checks are failing?" unanswerable.
echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE\"}"
fi

View File

@@ -202,12 +202,26 @@ func buildKind(kind string, t Target, attrs map[string]any, host, user string, p
if host == "" { if host == "" {
return nil, "no address on the entity or its host" return nil, "no address on the entity or its host"
} }
if t.Name == "" { // A service's name is a logical label, not usually its systemd unit
// or container name (matrix = matrix-synapse.service + containers).
// Prefer an explicit probe target when declared; process_check.sh also
// matches a unit prefix or a docker container as a fallback.
unit := ""
for _, key := range []string{"probe_unit", "systemd_unit", "container"} {
if v, _ := attrs[key].(string); v != "" {
unit = v
break
}
}
if unit == "" {
unit = t.Name
}
if unit == "" {
return nil, "no name to check a process for" return nil, "no name to check a process for"
} }
// process_check.sh takes the unit name as $1 and reports "unknown" // process_check.sh takes the unit/container name as $1 and reports
// without it. // "unknown" without it.
return []checkDef{ssh("process_check.sh", t.Name)}, "" return []checkDef{ssh("process_check.sh", unit)}, ""
case KindBackup: case KindBackup:
// A backup target is checked from the machine that writes to it, so it // A backup target is checked from the machine that writes to it, so it