#!/bin/sh
# Oikos watchdog — cron job running every 5 minutes on mac-mini (apps/105).
# Pages the operator via Matrix if the API is unreachable on BOTH paths.
#
# Paths tested (plan O2):
#   LAN direct:  http://<lan-ip>:8090/healthz  (API itself, bypasses Caddy)
#   Mesh/Caddy:  http://<mesh-ip>:8090/healthz  (through Caddy, as workstations see it)
#
# Register: crontab -e → */5 * * * * /opt/oikos/scripts/watchdog.sh
# This cron lives OUTSIDE the Docker stack (orthogonal watch-the-watcher).

set -e

LAN_URL="${LAN_URL:-http://192.168.8.175:8090/healthz}"
MESH_URL="${MESH_URL:-http://100.122.0.10:8090/healthz}"
MATRIX_HOMESERVER="${MATRIX_HOMESERVER:-https://matrix.hubris.network}"
MATRIX_ROOM="${MATRIX_ROOM:-!alerts:hubris.network}"
MATRIX_TOKEN="${MATRIX_TOKEN:-}"
MAX_FAILS=${MAX_FAILS:-3}
FAIL_FILE="/tmp/oikos-watchdog-failures"
FAIL_REASON="/tmp/oikos-watchdog-reason"

health() {
    curl -sf --max-time 5 "$1" > /dev/null 2>&1
}

lan_ok=0
mesh_ok=0

health "$LAN_URL" && lan_ok=1
health "$MESH_URL" && mesh_ok=1

# Only alert if BOTH paths are down (single-path failure is a NetBird/Caddy issue,
# not a stack problem — logged but not paged).
if [ "$lan_ok" -eq 1 ] || [ "$mesh_ok" -eq 1 ]; then
    if [ "$lan_ok" -eq 1 ] && [ "$mesh_ok" -eq 1 ]; then
        echo "0" > "$FAIL_FILE" 2>/dev/null || true
        exit 0
    fi
    # One path down — log but don't page (transient mesh/Caddy issue)
    reason=""
    [ "$lan_ok" -eq 0 ] && reason="LAN path DOWN"
    [ "$mesh_ok" -eq 0 ] && reason="mesh/Caddy path DOWN"
    echo "[oikos-watchdog] partial: $reason ($(date))" | logger -t oikos-watchdog
    exit 0
fi

# Both paths down — increment failure count
reason=""
[ "$lan_ok" -eq 0 ] && reason="LAN"
[ "$mesh_ok" -eq 0 ] && reason="${reason:+$reason + }mesh/Caddy"
reason="${reason} DOWN"

fails=$(cat "$FAIL_FILE" 2>/dev/null || echo 0)
fails=$((fails + 1))
echo "$fails" > "$FAIL_FILE"
echo "$reason" > "$FAIL_REASON"

if [ "$fails" -ge "$MAX_FAILS" ]; then
    prev_reason=$(cat "$FAIL_REASON" 2>/dev/null || echo "unknown")
    msg="ALERT: oikos API is DOWN on BOTH paths — $prev_reason ($MAX_FAILS consecutive failures at $(date))"
    if [ -n "$MATRIX_TOKEN" ]; then
        curl -sf -X POST "$MATRIX_HOMESERVER/_matrix/client/v3/rooms/$MATRIX_ROOM/send/m.room.message" \
            -H "Authorization: Bearer $MATRIX_TOKEN" \
            -H "Content-Type: application/json" \
            -d "{\"msgtype\":\"m.text\",\"body\":\"$msg\"}" > /dev/null 2>&1
    fi
    echo "$msg" | logger -t oikos-watchdog
fi