scheduler: add ping + ssh-script check kinds, metrics refactor, 17 host check scripts
- Refactor executeCheck to return checkResult struct with metrics map - Add ping check kind (ICMP reachability via system ping, macOS+Linux) - Add ssh-script check kind (remote host exec via SSH, allowlisted scripts) - Add threshold evaluation (warn/crit per metric from check config JSONB) - Add inode tracking to disk check - All 4 existing checks now return structured metrics - 17 check scripts: cpu, memory, load, swap, disk_usage, disk_smart, updates, zfs, process, uptime, oom, journal, time, fd, docker_health, caddy_error_rate, backup_freshness - Auto-deploy via tools/setup-checks.sh -> checks/install.sh on git pull - Add ping to OpenAPI CheckKind enum and generated Go types
This commit is contained in:
46
checks/caddy_error_rate.sh
Normal file
46
checks/caddy_error_rate.sh
Normal file
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# caddy_error_rate.sh — 5xx error rate from Caddy JSON access logs.
|
||||
set -euo pipefail
|
||||
|
||||
LOG_DIR=""
|
||||
if [ -d /var/log/caddy ]; then
|
||||
LOG_DIR="/var/log/caddy"
|
||||
elif [ -d /var/lib/caddy/logs ]; then
|
||||
LOG_DIR="/var/lib/caddy/logs"
|
||||
elif [ -d /opt/caddy/logs ]; then
|
||||
LOG_DIR="/opt/caddy/logs"
|
||||
fi
|
||||
|
||||
if [ -z "$LOG_DIR" ]; then
|
||||
echo '{"health":"healthy"}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CUTOFF=$(date -u -d "5 minutes ago" +%Y-%m-%dT%H:%M 2>/dev/null || date -u -v-5M +%Y-%m-%dT%H:%M 2>/dev/null || echo "")
|
||||
|
||||
TOTAL=0
|
||||
ERR_5XX=0
|
||||
|
||||
for LOG in "$LOG_DIR"/access*.log "$LOG_DIR"/access*.json "$LOG_DIR"/*.log 2>/dev/null; do
|
||||
[ -f "$LOG" ] || continue
|
||||
[ -r "$LOG" ] || continue
|
||||
|
||||
if [ -n "$CUTOFF" ]; then
|
||||
NEW=$(awk -v cutoff="$CUTOFF" '$0 >= cutoff {print}' "$LOG" 2>/dev/null | wc -l | tr -d ' ' || echo 0)
|
||||
if [ "$NEW" -gt 0 ]; then
|
||||
TOTAL=$((TOTAL + NEW))
|
||||
ERR_5XX=$((ERR_5XX + $(awk -v cutoff="$CUTOFF" '$0 >= cutoff && /"status":5[0-9][0-9]/ {print}' "$LOG" 2>/dev/null | wc -l | tr -d ' ' || echo 0)))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$TOTAL" -gt 100 ]; then
|
||||
RATE=$(awk "BEGIN {printf \"%.1f\", $ERR_5XX*100/$TOTAL}")
|
||||
if [ "$(echo "$RATE > 5" | bc 2>/dev/null || echo 0)" = "1" ]; then
|
||||
echo "{\"health\":\"degraded\",\"signalKind\":\"caddy-errors\",\"evidence\":\"${RATE}% 5xx rate ($ERR_5XX/$TOTAL)\",\"metrics\":{\"caddy_5xx_rate\":$RATE,\"caddy_requests\":$TOTAL,\"caddy_5xx\":$ERR_5XX}}"
|
||||
exit 0
|
||||
fi
|
||||
echo "{\"health\":\"healthy\",\"metrics\":{\"caddy_5xx_rate\":$RATE,\"caddy_requests\":$TOTAL,\"caddy_5xx\":$ERR_5XX}}"
|
||||
else
|
||||
echo '{"health":"healthy"}'
|
||||
fi
|
||||
Reference in New Issue
Block a user