phase 6: deploy pipeline — CI, cutover checklist, watchdog, verification, rollback
- scripts/deploy.sh: Gitea webhook-triggered deploy (git pull → docker build → compose up → health check). SHA-tagged images, rolling restart. - scripts/watchdog.sh: cron health check every 2min, pages operator via Matrix after 3 consecutive failures. Reset on recovery. - scripts/verify-phase6.sh: 14 end-to-end verification checks against plan V1–V14 (ontology, DB, API, scheduler, actuator, learning, classifier, hermes, secrets, deploy, knowledge, observability, correlation, cutover). - scripts/rollback.sh: re-deploy previous SHA tag + pg_restore from pre-deploy dump. Health check loop, returns to main branch after. - scripts/cutover-checklist.md: pre/post-cutover steps — backup, CI gate, Caddy re-point, DNS, apps/105 disable, cleanup. - compose/caddy/Caddyfile.oikos: reverse-proxy config for oikos/mcp/hermes.hubris.network → mac-mini mesh IP. - .gitignore: added bin/ to exclude compiled binaries. 14/14 verification checks pass against running Docker stack.
This commit is contained in:
40
scripts/watchdog.sh
Executable file
40
scripts/watchdog.sh
Executable file
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
# Oikos watchdog — cron job running every 2 minutes on mac-mini.
|
||||
# Pages the operator via Matrix if the API is down.
|
||||
# Register: crontab -e → */2 * * * * /opt/oikos/scripts/watchdog.sh
|
||||
|
||||
set -e
|
||||
|
||||
API_URL="${API_URL:-http://localhost:8090/healthz}"
|
||||
MATRIX_HOMESERVER="${MATRIX_HOMESERVER:-https://matrix.hubris.network}"
|
||||
MATRIX_ROOM="${MATRIX_ROOM:-!alerts:hubris.network}"
|
||||
MATRIX_TOKEN="${MATRIX_TOKEN:-}"
|
||||
MAX_FAILS=${MAX_FAILS:-3}
|
||||
FAIL_FILE="/tmp/oikos-watchdog-failures"
|
||||
|
||||
health() {
|
||||
curl -sf --max-time 5 "$API_URL" > /dev/null 2>&1
|
||||
}
|
||||
|
||||
if health; then
|
||||
# Reset failure count
|
||||
echo "0" > "$FAIL_FILE" 2>/dev/null || true
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Increment failure count
|
||||
fails=$(cat "$FAIL_FILE" 2>/dev/null || echo 0)
|
||||
fails=$((fails + 1))
|
||||
echo "$fails" > "$FAIL_FILE"
|
||||
|
||||
if [ "$fails" -ge "$MAX_FAILS" ]; then
|
||||
# Page the operator via Matrix
|
||||
msg="ALERT: oikos API is DOWN ($MAX_FAILS consecutive failures at $(date))"
|
||||
if [ -n "$MATRIX_TOKEN" ]; then
|
||||
curl -sf -X POST "$MATRIX_HOMESERVER/_matrix/client/v3/rooms/$MATRIX_ROOM/send/m.room.message" \
|
||||
-H "Authorization: Bearer $MATRIX_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"msgtype\":\"m.text\",\"body\":\"$msg\"}" > /dev/null 2>&1
|
||||
fi
|
||||
echo "$msg" | logger -t oikos-watchdog
|
||||
fi
|
||||
Reference in New Issue
Block a user