scheduler: add ping + ssh-script check kinds, metrics refactor, 17 host check scripts

- Refactor executeCheck to return checkResult struct with metrics map - Add ping check kind (ICMP reachability via system ping, macOS+Linux) - Add ssh-script check kind (remote host exec via SSH, allowlisted scripts) - Add threshold evaluation (warn/crit per metric from check config JSONB) - Add inode tracking to disk check - All 4 existing checks now return structured metrics - 17 check scripts: cpu, memory, load, swap, disk_usage, disk_smart, updates, zfs, process, uptime, oom, journal, time, fd, docker_health, caddy_error_rate, backup_freshness - Auto-deploy via tools/setup-checks.sh -> checks/install.sh on git pull - Add ping to OpenAPI CheckKind enum and generated Go types
2026-07-08 21:05:53 +02:00
parent cca2ae4621
commit 35feada286
23 changed files with 1126 additions and 64 deletions
--- a/checks/backup_freshness.sh
+++ b/checks/backup_freshness.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# backup_freshness.sh — check that backups exist and are recent.
+set -euo pipefail
+
+BACKUP_DIRS="${OIKOS_BACKUP_DIRS:-/var/backups /opt/backups /mnt/backups}"
+GRACE_HOURS="${OIKOS_BACKUP_GRACE:-48}"
+STALE=""
+
+for dir in $BACKUP_DIRS; do
+  [ -d "$dir" ] || continue
+  NEWEST=$(find "$dir" -type f -mmin -$((GRACE_HOURS * 60)) 2>/dev/null | head -1 || true)
+  if [ -z "$NEWEST" ]; then
+    LATEST_TS=$(find "$dir" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | awk '{print $1}' || echo "0")
+    LATEST_HOURS=$(awk "BEGIN {printf \"%.0f\", ($(date +%s) - ${LATEST_TS:-0})/3600}")
+    STALE="$STALE $dir(${LATEST_HOURS}h)"
+  fi
+done
+
+if [ -n "$STALE" ]; then
+  echo "{\"health\":\"degraded\",\"signalKind\":\"backup-stale\",\"evidence\":\"stale backups:$(echo "$STALE" | sed 's/ /, /g')\"}"
+else
+  echo '{"health":"healthy"}'
+fi
--- a/checks/caddy_error_rate.sh
+++ b/checks/caddy_error_rate.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# caddy_error_rate.sh — 5xx error rate from Caddy JSON access logs.
+set -euo pipefail
+
+LOG_DIR=""
+if [ -d /var/log/caddy ]; then
+  LOG_DIR="/var/log/caddy"
+elif [ -d /var/lib/caddy/logs ]; then
+  LOG_DIR="/var/lib/caddy/logs"
+elif [ -d /opt/caddy/logs ]; then
+  LOG_DIR="/opt/caddy/logs"
+fi
+
+if [ -z "$LOG_DIR" ]; then
+  echo '{"health":"healthy"}'
+  exit 0
+fi
+
+CUTOFF=$(date -u -d "5 minutes ago" +%Y-%m-%dT%H:%M 2>/dev/null || date -u -v-5M +%Y-%m-%dT%H:%M 2>/dev/null || echo "")
+
+TOTAL=0
+ERR_5XX=0
+
+for LOG in "$LOG_DIR"/access*.log "$LOG_DIR"/access*.json "$LOG_DIR"/*.log 2>/dev/null; do
+  [ -f "$LOG" ] || continue
+  [ -r "$LOG" ] || continue
+
+  if [ -n "$CUTOFF" ]; then
+    NEW=$(awk -v cutoff="$CUTOFF" '$0 >= cutoff {print}' "$LOG" 2>/dev/null | wc -l | tr -d ' ' || echo 0)
+    if [ "$NEW" -gt 0 ]; then
+      TOTAL=$((TOTAL + NEW))
+      ERR_5XX=$((ERR_5XX + $(awk -v cutoff="$CUTOFF" '$0 >= cutoff && /"status":5[0-9][0-9]/ {print}' "$LOG" 2>/dev/null | wc -l | tr -d ' ' || echo 0)))
+    fi
+  fi
+done
+
+if [ "$TOTAL" -gt 100 ]; then
+  RATE=$(awk "BEGIN {printf \"%.1f\", $ERR_5XX*100/$TOTAL}")
+  if [ "$(echo "$RATE > 5" | bc 2>/dev/null || echo 0)" = "1" ]; then
+    echo "{\"health\":\"degraded\",\"signalKind\":\"caddy-errors\",\"evidence\":\"${RATE}% 5xx rate ($ERR_5XX/$TOTAL)\",\"metrics\":{\"caddy_5xx_rate\":$RATE,\"caddy_requests\":$TOTAL,\"caddy_5xx\":$ERR_5XX}}"
+    exit 0
+  fi
+  echo "{\"health\":\"healthy\",\"metrics\":{\"caddy_5xx_rate\":$RATE,\"caddy_requests\":$TOTAL,\"caddy_5xx\":$ERR_5XX}}"
+else
+  echo '{"health":"healthy"}'
+fi
--- a/checks/cpu_check.sh
+++ b/checks/cpu_check.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# cpu_check.sh — CPU usage % and thermal temperature.
+set -euo pipefail
+
+USAGE=$(top -bn1 2>/dev/null | awk '/^%Cpu/ {print 100 - $8}' || true)
+if [ -z "$USAGE" ]; then
+  CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
+  USAGE=$(awk -v cores="$CORES" '{print ($1+$2+$3)*100/cores}' /proc/loadavg 2>/dev/null || echo "0")
+fi
+
+TEMP=""
+if [ -f /sys/class/thermal/thermal_zone0/temp ]; then
+  TEMP=$(awk '{printf "%.1f", $1/1000}' /sys/class/thermal/thermal_zone0/temp 2>/dev/null || true)
+fi
+
+if [ -n "$TEMP" ]; then
+  echo "{\"health\":\"healthy\",\"metrics\":{\"cpu_pct\":$USAGE,\"cpu_temp\":$TEMP}}"
+else
+  echo "{\"health\":\"healthy\",\"metrics\":{\"cpu_pct\":$USAGE}}"
+fi
--- a/checks/disk_smart_check.sh
+++ b/checks/disk_smart_check.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# disk_smart_check.sh — SMART pre-failure indicators for physical disks.
+set -euo pipefail
+
+if ! command -v smartctl >/dev/null 2>&1; then
+  echo '{"health":"healthy"}'
+  exit 0
+fi
+
+DISKS=$(lsblk -ndo NAME,TYPE 2>/dev/null | awk '$2=="disk"{print "/dev/"$1}' || true)
+if [ -z "$DISKS" ]; then
+  echo '{"health":"healthy"}'
+  exit 0
+fi
+
+FAILED=""
+for dev in $DISKS; do
+  INFO=$(smartctl -H "$dev" 2>/dev/null || true)
+  if ! echo "$INFO" | grep -q "PASSED\|OK"; then
+    MODEL=$(smartctl -i "$dev" 2>/dev/null | awk -F': ' '/Device Model|Product/{print $2; exit}' || echo "$dev")
+    FAILED="$FAILED $MODEL"
+  fi
+done
+
+if [ -n "$FAILED" ]; then
+  echo "{\"health\":\"degraded\",\"signalKind\":\"disk-smart-fail\",\"evidence\":\"SMART check failed for:$(echo "$FAILED" | sed 's/ /, /g')\"}"
+else
+  echo '{"health":"healthy"}'
+fi
--- a/checks/disk_usage_check.sh
+++ b/checks/disk_usage_check.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# disk_usage_check.sh — disk usage and inode usage per mountpoint.
+set -euo pipefail
+
+MOUNTS=$(df -k 2>/dev/null | awk 'NR>1 && $1 ~ /^\// && $NF !~ /^\/(snap|dev|proc|sys|run|private)/ {print $NF}' || true)
+FIRST=1
+
+echo -n '{"health":"healthy","metrics":{'
+for m in $MOUNTS; do
+  LINE=$(df -k "$m" 2>/dev/null | awk 'NR==2 {print $3, $4, $5, $7}' | tr -d '%' || true)
+  if [ -z "$LINE" ]; then continue; fi
+  USED=$(echo "$LINE" | awk '{print $1}')
+  FREE=$(echo "$LINE" | awk '{print $2}')
+  PCT=$(echo "$LINE" | awk '{print $3}')
+
+  INODE_LINE=$(df -i "$m" 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%' || echo "0")
+  INODE_PCT="${INODE_LINE:-0}"
+
+  KEY=$(echo "$m" | sed 's|/|_|g' | sed 's|^_||')
+  [ -z "$KEY" ] && KEY="root"
+
+  if [ $FIRST -eq 0 ]; then echo -n ','; fi
+  FIRST=0
+  echo -n "\"disk_${KEY}_pct\":$PCT,\"inode_${KEY}_pct\":$INODE_PCT"
+done
+echo '}}'
--- a/checks/docker_health_check.sh
+++ b/checks/docker_health_check.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# docker_health_check.sh — detect unhealthy Docker containers.
+set -euo pipefail
+
+if ! command -v docker >/dev/null 2>&1; then
+  echo '{"health":"healthy"}'
+  exit 0
+fi
+
+UNHEALTHY=$(docker ps --filter "health=unhealthy" --format "{{.Names}}" 2>/dev/null || true)
+
+if [ -n "$UNHEALTHY" ]; then
+  COUNT=$(echo "$UNHEALTHY" | wc -l | tr -d ' ')
+  NAMES=$(echo "$UNHEALTHY" | tr '\n' ',' | sed 's/,$//')
+  echo "{\"health\":\"degraded\",\"signalKind\":\"docker-unhealthy\",\"evidence\":\"$COUNT unhealthy container(s): $NAMES\",\"metrics\":{\"docker_unhealthy\":$COUNT}}"
+else
+  TOTAL=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ' || echo 0)
+  echo "{\"health\":\"healthy\",\"metrics\":{\"docker_unhealthy\":0,\"docker_total\":$TOTAL}}"
+fi
--- a/checks/fd_check.sh
+++ b/checks/fd_check.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# fd_check.sh — open file descriptor usage ratio.
+set -euo pipefail
+
+FD_PCT=0
+if [ -r /proc/sys/fs/file-nr ]; then
+  read -r ALLOC _ LIMIT < /proc/sys/fs/file-nr
+  if [ "$LIMIT" -gt 0 ]; then
+    FD_PCT=$(awk "BEGIN {printf \"%.1f\", $ALLOC*100/$LIMIT}")
+  fi
+fi
+
+echo "{\"health\":\"healthy\",\"metrics\":{\"fd_pct\":$FD_PCT}}"
--- a/checks/install.sh
+++ b/checks/install.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# install all check scripts into the canonical directory.
+# Auto-setup hook called by tools/post-pull.sh.
+set -euo pipefail
+
+CLONE_DIR="${HOMELAB_CONTEXT_DIR:-/opt/homelab}"
+CHECK_SRC="$CLONE_DIR/checks"
+CHECK_DST="${OIKOS_CHECK_DIR:-/opt/oikos/checks}"
+
+if [ ! -d "$CHECK_SRC" ]; then
+  exit 0
+fi
+
+mkdir -p "$CHECK_DST"
+
+for script in "$CHECK_SRC"/*.sh; do
+  name=$(basename "$script")
+  if [ "$name" = "install.sh" ]; then continue; fi
+  if [ -f "$CHECK_DST/$name" ]; then
+    if cmp -s "$script" "$CHECK_DST/$name"; then
+      continue
+    fi
+  fi
+  cp "$script" "$CHECK_DST/$name"
+  chmod 755 "$CHECK_DST/$name"
+  echo "[setup-checks] installed $name"
+done
+
+echo "[setup-checks] done"
--- a/checks/journal_check.sh
+++ b/checks/journal_check.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# journal_check.sh — count journal errors in the last check interval.
+set -euo pipefail
+
+ERRORS=0
+
+if command -v journalctl >/dev/null 2>&1; then
+  ERRORS=$(journalctl -p err --since "-5min" --no-pager 2>/dev/null | wc -l | tr -d ' ' || echo 0)
+fi
+
+if [ "$ERRORS" -gt 0 ]; then
+  echo "{\"health\":\"degraded\",\"signalKind\":\"journal-errors\",\"evidence\":\"$ERRORS error entries in last 5min\",\"metrics\":{\"journal_errors\":$ERRORS}}"
+else
+  echo '{"health":"healthy","metrics":{"journal_errors":0}}'
+fi
--- a/checks/load_check.sh
+++ b/checks/load_check.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# load_check.sh — system load average scaled by CPU count.
+set -euo pipefail
+
+LOAD=$(awk '{print $1}' /proc/loadavg 2>/dev/null || sysctl -n vm.loadavg 2>/dev/null | awk '{print $2}' || echo "0")
+CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
+
+echo "{\"health\":\"healthy\",\"metrics\":{\"load1\":$LOAD,\"cores\":$CORES}}"
--- a/checks/memory_check.sh
+++ b/checks/memory_check.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# memory_check.sh — RAM usage percentage.
+set -euo pipefail
+
+TOTAL=1
+AVAIL=1
+
+if [ -r /proc/meminfo ]; then
+  TOTAL=$(awk '/^MemTotal:/ {printf "%d", $2}' /proc/meminfo)
+  AVAIL=$(awk '/^MemAvailable:/ {printf "%d", $2}' /proc/meminfo)
+elif [ "$(uname)" = "Darwin" ]; then
+  MEM=$(vm_stat 2>/dev/null | awk '
+    /page size/ {ps=$8}
+    /Pages free/   {free+=$NF}
+    /Pages active/ {active+=$NF}
+    /Pages wired/  {wired+=$NF}
+    END {printf "%.2f %.2f", ps*(free+active+wired)/1048576, ps*wired/1048576}')
+  TOTAL=$(echo "$MEM" | awk '{printf "%.0f", $1}')
+  AVAIL=$(echo "$MEM" | awk '{printf "%.0f", $1 - $2}')
+fi
+
+if [ "$TOTAL" -eq 0 ]; then TOTAL=1; fi
+if [ "$AVAIL" -lt 0 ]; then AVAIL=0; fi
+
+USED_PCT=$(awk "BEGIN {printf \"%.1f\", (1 - $AVAIL/$TOTAL)*100}")
+
+echo "{\"health\":\"healthy\",\"metrics\":{\"mem_pct\":$USED_PCT}}"
--- a/checks/oom_check.sh
+++ b/checks/oom_check.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# oom_check.sh — detect OOM kills since last boot.
+set -euo pipefail
+
+OOM_COUNT=0
+
+if command -v dmesg >/dev/null 2>&1; then
+  OOM_COUNT=$(dmesg 2>/dev/null | grep -ci 'out of memory\|oom-killer\|Killed process' || echo 0)
+elif command -v journalctl >/dev/null 2>&1; then
+  OOM_COUNT=$(journalctl -k --no-pager 2>/dev/null | grep -ci 'out of memory\|oom-killer\|Killed process' || echo 0)
+fi
+
+if [ "$OOM_COUNT" -gt 0 ]; then
+  echo "{\"health\":\"degraded\",\"signalKind\":\"oom-kills\",\"evidence\":\"$OOM_COUNT OOM events detected since boot\",\"metrics\":{\"oom_count\":$OOM_COUNT}}"
+else
+  echo '{"health":"healthy","metrics":{"oom_count":0}}'
+fi
--- a/checks/process_check.sh
+++ b/checks/process_check.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# process_check.sh — systemd service liveness.
+set -euo pipefail
+
+SERVICE="${1:-}"
+if [ -z "$SERVICE" ]; then
+  echo '{"health":"unknown","signalKind":"process-check","evidence":"no service name provided"}'
+  exit 0
+fi
+
+if ! command -v systemctl >/dev/null 2>&1; then
+  echo '{"health":"unknown","signalKind":"process-check","evidence":"systemctl not found"}'
+  exit 0
+fi
+
+STATE=$(systemctl is-active "$SERVICE" 2>/dev/null || echo "unknown")
+
+if [ "$STATE" = "active" ]; then
+  echo "{\"health\":\"healthy\"}"
+else
+  echo "{\"health\":\"degraded\",\"signalKind\":\"$SERVICE\",\"evidence\":\"$SERVICE is $STATE\"}"
+fi
--- a/checks/swap_check.sh
+++ b/checks/swap_check.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# swap_check.sh — swap usage percentage.
+set -euo pipefail
+
+if [ -r /proc/meminfo ]; then
+  SWAP_TOTAL=$(awk '/^SwapTotal:/ {printf "%d", $2}' /proc/meminfo)
+  SWAP_FREE=$(awk '/^SwapFree:/ {printf "%d", $2}' /proc/meminfo)
+elif [ "$(uname)" = "Darwin" ]; then
+  SP=$(sysctl vm.swapusage 2>/dev/null | awk '{print $4, $9}' | tr -d 'M' || echo "0 0")
+  SWAP_TOTAL=$(echo "$SP" | awk '{printf "%.0f", $1*1024}')
+  SWAP_FREE=$(echo "$SP" | awk '{printf "%.0f", ($1-$2)*1024}')
+else
+  SWAP_TOTAL=0
+  SWAP_FREE=0
+fi
+
+if [ "$SWAP_TOTAL" -eq 0 ]; then
+  echo "{\"health\":\"healthy\",\"metrics\":{\"swap_pct\":0}}"
+else
+  USED_PCT=$(awk "BEGIN {printf \"%.1f\", (1 - $SWAP_FREE/$SWAP_TOTAL)*100}")
+  echo "{\"health\":\"healthy\",\"metrics\":{\"swap_pct\":$USED_PCT}}"
+fi
--- a/checks/time_check.sh
+++ b/checks/time_check.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# time_check.sh — NTP synchronization status and clock drift.
+set -euo pipefail
+
+DRIFT_S=0
+SYNCED=true
+
+if command -v chronyc >/dev/null 2>&1; then
+  TRACKING=$(chronyc tracking 2>/dev/null || true)
+  DRIFT_NS=$(echo "$TRACKING" | awk '/System time/ {print $4}' | sed 's/-//' || echo "0")
+  DRIFT_S=$(awk "BEGIN {printf \"%.6f\", $DRIFT_NS/1e9}")
+  # chronyc returns a very small value when synced (nanoseconds)
+elif command -v timedatectl >/dev/null 2>&1; then
+  STATUS=$(timedatectl show 2>/dev/null || true)
+  if echo "$STATUS" | grep -q "NTPSynchronized=no"; then
+    SYNCED=false
+  fi
+fi
+
+if ! $SYNCED; then
+  echo '{"health":"degraded","signalKind":"time-drift","evidence":"NTP not synchronized"}'
+elif [ "$(echo "$DRIFT_S > 1" | bc 2>/dev/null || echo 0)" = "1" ]; then
+  echo "{\"health\":\"degraded\",\"signalKind\":\"time-drift\",\"evidence\":\"clock drift ${DRIFT_S}s exceeds 1s threshold\",\"metrics\":{\"clock_drift_s\":$DRIFT_S}}"
+else
+  echo "{\"health\":\"healthy\",\"metrics\":{\"clock_drift_s\":$DRIFT_S}}"
+fi
--- a/checks/updates_check.sh
+++ b/checks/updates_check.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# updates_check.sh — pending apt security updates and reboot-required flag.
+set -euo pipefail
+
+SECURITY=0
+REBOOT=0
+
+if command -v apt >/dev/null 2>&1; then
+  apt update -qq >/dev/null 2>&1 || true
+  SECURITY=$(apt list --upgradable 2>/dev/null | grep -c '\-security' || true)
+fi
+
+if [ -f /var/run/reboot-required ]; then
+  REBOOT=1
+fi
+
+echo "{\"health\":\"healthy\",\"metrics\":{\"security_updates\":$SECURITY,\"reboot_required\":$REBOOT}}"
--- a/checks/uptime_check.sh
+++ b/checks/uptime_check.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# uptime_check.sh — detect unexpected reboots by monitoring uptime.
+set -euo pipefail
+
+UPTIME=$(awk '{printf "%.0f", $1}' /proc/uptime 2>/dev/null || sysctl -n kern.boottime 2>/dev/null | awk '{print $4}' | tr -d ',' || echo "0")
+
+echo "{\"health\":\"healthy\",\"metrics\":{\"uptime_seconds\":$UPTIME}}"
--- a/checks/zfs_check.sh
+++ b/checks/zfs_check.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# zfs_check.sh — ZFS pool health.
+set -euo pipefail
+
+if ! command -v zpool >/dev/null 2>&1; then
+  echo '{"health":"healthy"}'
+  exit 0
+fi
+
+STATUS=$(zpool status -x 2>&1 || true)
+SCRUB_OVERDUE=""
+
+if echo "$STATUS" | grep -q "all pools are healthy"; then
+  for pool in $(zpool list -Ho name 2>/dev/null || true); do
+    LAST=$(zpool status "$pool" 2>/dev/null | awk '/scan:/{print $0}' || true)
+    if [ -z "$LAST" ] || echo "$LAST" | grep -q "scrub repaired"; then
+      SCRUB_OVERDUE="$pool"
+      break
+    fi
+  done
+  if [ -n "$SCRUB_OVERDUE" ]; then
+    echo "{\"health\":\"degraded\",\"signalKind\":\"zfs-scrub-overdue\",\"evidence\":\"pool $SCRUB_OVERDUE scrub has errors or is overdue\"}"
+  else
+    echo '{"health":"healthy"}'
+  fi
+else
+  HEALTH=$(echo "$STATUS" | grep "state:" | awk '{print $2}' | head -1)
+  echo "{\"health\":\"degraded\",\"signalKind\":\"zfs-degraded\",\"evidence\":\"pool state: $HEALTH\"}"
+fi