scheduler: add ping + ssh-script check kinds, metrics refactor, 17 host check scripts
Some checks failed
ci / build-test (push) Has been cancelled
ci / docker-build (push) Has been cancelled

- Refactor executeCheck to return checkResult struct with metrics map
- Add ping check kind (ICMP reachability via system ping, macOS+Linux)
- Add ssh-script check kind (remote host exec via SSH, allowlisted scripts)
- Add threshold evaluation (warn/crit per metric from check config JSONB)
- Add inode tracking to disk check
- All 4 existing checks now return structured metrics
- 17 check scripts: cpu, memory, load, swap, disk_usage, disk_smart,
  updates, zfs, process, uptime, oom, journal, time, fd, docker_health,
  caddy_error_rate, backup_freshness
- Auto-deploy via tools/setup-checks.sh -> checks/install.sh on git pull
- Add ping to OpenAPI CheckKind enum and generated Go types
This commit is contained in:
2026-07-08 21:05:53 +02:00
parent cca2ae4621
commit 35feada286
23 changed files with 1126 additions and 64 deletions

View File

@@ -0,0 +1,46 @@
#!/usr/bin/env bash
# caddy_error_rate.sh — 5xx error rate from Caddy JSON access logs.
set -euo pipefail
LOG_DIR=""
if [ -d /var/log/caddy ]; then
LOG_DIR="/var/log/caddy"
elif [ -d /var/lib/caddy/logs ]; then
LOG_DIR="/var/lib/caddy/logs"
elif [ -d /opt/caddy/logs ]; then
LOG_DIR="/opt/caddy/logs"
fi
if [ -z "$LOG_DIR" ]; then
echo '{"health":"healthy"}'
exit 0
fi
CUTOFF=$(date -u -d "5 minutes ago" +%Y-%m-%dT%H:%M 2>/dev/null || date -u -v-5M +%Y-%m-%dT%H:%M 2>/dev/null || echo "")
TOTAL=0
ERR_5XX=0
for LOG in "$LOG_DIR"/access*.log "$LOG_DIR"/access*.json "$LOG_DIR"/*.log 2>/dev/null; do
[ -f "$LOG" ] || continue
[ -r "$LOG" ] || continue
if [ -n "$CUTOFF" ]; then
NEW=$(awk -v cutoff="$CUTOFF" '$0 >= cutoff {print}' "$LOG" 2>/dev/null | wc -l | tr -d ' ' || echo 0)
if [ "$NEW" -gt 0 ]; then
TOTAL=$((TOTAL + NEW))
ERR_5XX=$((ERR_5XX + $(awk -v cutoff="$CUTOFF" '$0 >= cutoff && /"status":5[0-9][0-9]/ {print}' "$LOG" 2>/dev/null | wc -l | tr -d ' ' || echo 0)))
fi
fi
done
if [ "$TOTAL" -gt 100 ]; then
RATE=$(awk "BEGIN {printf \"%.1f\", $ERR_5XX*100/$TOTAL}")
if [ "$(echo "$RATE > 5" | bc 2>/dev/null || echo 0)" = "1" ]; then
echo "{\"health\":\"degraded\",\"signalKind\":\"caddy-errors\",\"evidence\":\"${RATE}% 5xx rate ($ERR_5XX/$TOTAL)\",\"metrics\":{\"caddy_5xx_rate\":$RATE,\"caddy_requests\":$TOTAL,\"caddy_5xx\":$ERR_5XX}}"
exit 0
fi
echo "{\"health\":\"healthy\",\"metrics\":{\"caddy_5xx_rate\":$RATE,\"caddy_requests\":$TOTAL,\"caddy_5xx\":$ERR_5XX}}"
else
echo '{"health":"healthy"}'
fi