From e4104eb34461ba02c57d886b3e7ae19493096e98 Mon Sep 17 00:00:00 2001 From: dtoro Date: Wed, 29 Jul 2026 22:33:02 +0200 Subject: [PATCH] fix(checks): process_check matches docker containers + prefixed systemd units MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit process_check.sh ran `systemctl is-active `, but a service's name is a logical label, not its unit/container name — matrix is matrix-synapse.service + element-web/mautrix-* containers, authentik is authentik-server/-worker containers. So every multi-component or docker service reported "inactive" while up (authentik, matrix, photos, house, arr-stack, …). Resolve in order: exact systemd unit, a unit with the name as prefix (matrix -> matrix-synapse.service), or a running docker container whose name contains it. checkdefaults passes a declared probe_unit/systemd_unit/container attribute when set, for precision. --- checks/process_check.sh | 60 ++++++++++++++++++------------ internal/checkdefaults/defaults.go | 22 +++++++++-- 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/checks/process_check.sh b/checks/process_check.sh index e907554..bcc5469 100644 --- a/checks/process_check.sh +++ b/checks/process_check.sh @@ -1,5 +1,17 @@ #!/usr/bin/env bash -# process_check.sh — systemd service liveness. +# process_check.sh — service liveness. +# +# A service entity's name is a logical label, rarely the literal systemd unit +# or container name. matrix = matrix-synapse.service + element-web/mautrix-* +# containers; authentik = authentik-server/-worker containers. So checking +# `systemctl is-active matrix` reports "inactive" for a healthy service. +# +# Resolution order, any hit = healthy: +# 1. exact systemd unit `systemctl is-active ` +# 2. a systemd unit with the name as prefix `*.service` +# 3. a running docker container whose name contains +# An explicit probe target overrides the label — see checkdefaults, which +# passes a `probe_unit`/`container`/`systemd_unit` attribute as $1 when set. set -euo pipefail SERVICE="${1:-}" @@ -8,29 +20,31 @@ if [ -z "$SERVICE" ]; then exit 0 fi -if ! command -v systemctl >/dev/null 2>&1; then - echo '{"health":"unknown","signalKind":"process-check","evidence":"systemctl not found"}' - exit 0 +ok() { echo "{\"health\":\"healthy\"}"; exit 0; } + +# 1. exact systemd unit +if command -v systemctl >/dev/null 2>&1; then + STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true) + [ "$STATE" = "active" ] && ok + + # 2. prefix match: matrix -> matrix-synapse.service, house -> house.service, etc. + # --no-legend strips the header/footer so grep can see the unit rows; the + # pattern is a systemd unit glob. + if systemctl list-units --type=service --state=active --no-legend "$SERVICE*.service" 2>/dev/null \ + | grep -q '\.service'; then + ok + fi fi -# `systemctl is-active` PRINTS the state and exits non-zero when the unit is -# not active, so `... || echo unknown` appended a second line and produced -# "paperless is inactive\nunknown" — a raw newline inside a JSON string, which -# the scheduler rejected as invalid output. head -1 keeps the first line and -# the fallback only fires when there was no output at all. -STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true) -[ -z "$STATE" ] && STATE="unknown" -# Belt and braces: a unit name or state containing a quote would break the -# hand-built JSON below just as thoroughly. +# 3. a running docker container whose name contains the label. +if command -v docker >/dev/null 2>&1; then + if docker ps --filter "status=running" --filter "name=$SERVICE" --format '{{.Names}}' 2>/dev/null \ + | grep -q .; then + ok + fi +fi + +STATE=${STATE:-inactive} STATE=${STATE//\"/} SAFE_SERVICE=${SERVICE//\"/} - -if [ "$STATE" = "active" ]; then - echo "{\"health\":\"healthy\"}" -else - # signalKind is a taxonomy, not a per-service label. Emitting "$SERVICE" - # here minted a distinct signal kind for every service (kind=paperless, - # kind=qbit, …), which no approval_rule can match and which makes - # "how many process checks are failing?" unanswerable. - echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE\"}" -fi +echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE (no active unit/container matched)\"}" diff --git a/internal/checkdefaults/defaults.go b/internal/checkdefaults/defaults.go index 745a245..9614b23 100644 --- a/internal/checkdefaults/defaults.go +++ b/internal/checkdefaults/defaults.go @@ -202,12 +202,26 @@ func buildKind(kind string, t Target, attrs map[string]any, host, user string, p if host == "" { return nil, "no address on the entity or its host" } - if t.Name == "" { + // A service's name is a logical label, not usually its systemd unit + // or container name (matrix = matrix-synapse.service + containers). + // Prefer an explicit probe target when declared; process_check.sh also + // matches a unit prefix or a docker container as a fallback. + unit := "" + for _, key := range []string{"probe_unit", "systemd_unit", "container"} { + if v, _ := attrs[key].(string); v != "" { + unit = v + break + } + } + if unit == "" { + unit = t.Name + } + if unit == "" { return nil, "no name to check a process for" } - // process_check.sh takes the unit name as $1 and reports "unknown" - // without it. - return []checkDef{ssh("process_check.sh", t.Name)}, "" + // process_check.sh takes the unit/container name as $1 and reports + // "unknown" without it. + return []checkDef{ssh("process_check.sh", unit)}, "" case KindBackup: // A backup target is checked from the machine that writes to it, so it