dtoro
dcd35b6315
- scripts/deploy.sh: Gitea webhook-triggered deploy (git pull → docker build → compose up → health check). SHA-tagged images, rolling restart. - scripts/watchdog.sh: cron health check every 2min, pages operator via Matrix after 3 consecutive failures. Reset on recovery. - scripts/verify-phase6.sh: 14 end-to-end verification checks against plan V1–V14 (ontology, DB, API, scheduler, actuator, learning, classifier, hermes, secrets, deploy, knowledge, observability, correlation, cutover). - scripts/rollback.sh: re-deploy previous SHA tag + pg_restore from pre-deploy dump. Health check loop, returns to main branch after. - scripts/cutover-checklist.md: pre/post-cutover steps — backup, CI gate, Caddy re-point, DNS, apps/105 disable, cleanup. - compose/caddy/Caddyfile.oikos: reverse-proxy config for oikos/mcp/hermes.hubris.network → mac-mini mesh IP. - .gitignore: added bin/ to exclude compiled binaries. 14/14 verification checks pass against running Docker stack.
41 lines
1.3 KiB
Bash
Executable File
41 lines
1.3 KiB
Bash
Executable File
#!/bin/sh
|
|
# Oikos watchdog — cron job running every 2 minutes on mac-mini.
|
|
# Pages the operator via Matrix if the API is down.
|
|
# Register: crontab -e → */2 * * * * /opt/oikos/scripts/watchdog.sh
|
|
|
|
set -e
|
|
|
|
API_URL="${API_URL:-http://localhost:8090/healthz}"
|
|
MATRIX_HOMESERVER="${MATRIX_HOMESERVER:-https://matrix.hubris.network}"
|
|
MATRIX_ROOM="${MATRIX_ROOM:-!alerts:hubris.network}"
|
|
MATRIX_TOKEN="${MATRIX_TOKEN:-}"
|
|
MAX_FAILS=${MAX_FAILS:-3}
|
|
FAIL_FILE="/tmp/oikos-watchdog-failures"
|
|
|
|
health() {
|
|
curl -sf --max-time 5 "$API_URL" > /dev/null 2>&1
|
|
}
|
|
|
|
if health; then
|
|
# Reset failure count
|
|
echo "0" > "$FAIL_FILE" 2>/dev/null || true
|
|
exit 0
|
|
fi
|
|
|
|
# Increment failure count
|
|
fails=$(cat "$FAIL_FILE" 2>/dev/null || echo 0)
|
|
fails=$((fails + 1))
|
|
echo "$fails" > "$FAIL_FILE"
|
|
|
|
if [ "$fails" -ge "$MAX_FAILS" ]; then
|
|
# Page the operator via Matrix
|
|
msg="ALERT: oikos API is DOWN ($MAX_FAILS consecutive failures at $(date))"
|
|
if [ -n "$MATRIX_TOKEN" ]; then
|
|
curl -sf -X POST "$MATRIX_HOMESERVER/_matrix/client/v3/rooms/$MATRIX_ROOM/send/m.room.message" \
|
|
-H "Authorization: Bearer $MATRIX_TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"msgtype\":\"m.text\",\"body\":\"$msg\"}" > /dev/null 2>&1
|
|
fi
|
|
echo "$msg" | logger -t oikos-watchdog
|
|
fi
|