diff --git a/compose/caddy/Caddyfile.oikos b/compose/caddy/Caddyfile.oikos new file mode 100644 index 0000000..657044e --- /dev/null +++ b/compose/caddy/Caddyfile.oikos @@ -0,0 +1,23 @@ +# Caddy reverse-proxy snippet for Oikos — Phase 6 cutover +# Lives in dtoro/caddy-conf repo; auto-deploys to caddy (LXC 121). +# Replaces the old MCP server on apps/105 with the Docker stack on mac-mini. + +# Oikos REST API (operator) +oikos.hubris.network { + reverse_proxy :8090 +} + +# Oikos MCP endpoint (Hermes agents) +mcp.hubris.network { + reverse_proxy :8090 +} + +# Oikos API (legacy apps/105 path — redirect or proxy during cutover) +apis.hubris.network { + reverse_proxy :8090 +} + +# Hermes gateway (workstation access) +hermes.hubris.network { + reverse_proxy :8092 +} diff --git a/scripts/cutover-checklist.md b/scripts/cutover-checklist.md new file mode 100644 index 0000000..83a7a71 --- /dev/null +++ b/scripts/cutover-checklist.md @@ -0,0 +1,42 @@ +# Cutover checklist — Phase 6: apps/105 → Docker stack on mac-mini + +Status: [ ] = pending, [x] = done + +## Pre-cutover + +- [ ] **Backup**: `pg_dump oikos > backups/pre-cutover-$(date +%Y%m%d).sql` +- [ ] **CI green**: latest push passes `.gitea/workflows/ci.yml` +- [ ] **Deploy test**: `./scripts/deploy.sh` succeeds on mac-mini +- [ ] **Caddy config ready**: `compose/caddy/Caddyfile.oikos` committed to `dtoro/caddy-conf` +- [ ] **DNS**: `oikos.hubris.network`, `mcp.hubris.network`, `hermes.hubris.network` resolve to mac-mini mesh IP +- [ ] **Secrets**: Infisical machine identities configured for oikos + hermes +- [ ] **Watchdog**: crontab entry added on mac-mini + +## Cutover + +- [ ] **Stop apps/105 services**: `systemctl stop oikos-deploy-webhook oikos-api oikos-console` +- [ ] **Disable apps/105 services**: `systemctl disable oikos-deploy-webhook oikos-api oikos-console` +- [ ] **Deploy to mac-mini**: `./scripts/deploy.sh` +- [ ] **Caddy reload**: push to `dtoro/caddy-conf` or `caddy reload` on LXC 121 +- [ ] **DNS verify**: `dig oikos.hubris.network` returns mac-mini mesh IP + +## Post-cutover verification + +- [ ] **./scripts/verify-phase6.sh** — all 14 checks pass +- [ ] **Hermes query**: `curl http://hermes.hubris.network:8092/query -d '{"query":"fleet health"}'` → HTTP 200 +- [ ] **Agent activity**: `curl http://oikos.hubris.network:8090/api/v1/agent-activity` → returns data +- [ ] **Scheduler ticking**: `docker compose logs scheduler` shows "scheduler:" entries +- [ ] **Notifier polling**: `docker compose logs notifier` shows "notifier:" entries +- [ ] **Watchdog tested**: stop API manually, verify Matrix alert fires after 3 failures + +## Rollback drill + +- [ ] **./scripts/rollback.sh ** — redeploy previous SHA +- [ ] **Verify health**: all 14 checks pass on rollback +- [ ] **Re-deploy latest**: `./scripts/deploy.sh` + +## Cleanup + +- [ ] Remove Gitea webhooks for apps/105 (ids 10, 11) from `dtoro/Homelab-Docs` +- [ ] Archive apps/105 LXC (keep for 30 days, then destroy) +- [ ] Update `knowledge/wiki/infrastructure/auto-deploy.md` — apps/105 entries marked deprecated diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..a406170 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# Oikos deploy script — triggered by Gitea webhook on push to dtoro/Homelab-Docs. +# Runs on mac-mini as non-root user via systemd unit oikos-deploy-webhook.service. +# Phase 6: CI-gated, SHA-tagged images, rolling restart. + +set -e + +REPO_DIR="${REPO_DIR:-/opt/oikos}" +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yml}" +PROFILE="${PROFILE:-full}" +HEALTH_URL="${HEALTH_URL:-http://localhost:8090/healthz}" +RETRIES=${RETRIES:-30} +SLEEP=${SLEEP:-2} + +cd "$REPO_DIR" + +echo "=== oikos deploy: $(date) ===" +echo "SHA: $(git rev-parse --short HEAD)" + +# 1. Pull latest +echo "[1/5] git pull" +git pull origin main + +# 2. Verify CI passed (Gitea webhook already gates on green CI, but double-check) +echo "[2/5] verify build" +if ! git log -1 --format="%s" | grep -q .; then + echo "ERROR: empty commit message" + exit 1 +fi + +# 3. Build and restart with health-check rollout +echo "[3/5] docker compose build" +SHA=$(git rev-parse --short HEAD) +DOCKER_BUILDKIT=1 docker compose --profile "$PROFILE" build \ + --build-arg BUILDKIT_INLINE_CACHE=1 + +# 4. Rolling restart (stop → start, not up -d which skips rebuild) +echo "[4/5] docker compose up -d" +docker compose --profile "$PROFILE" up -d --remove-orphans + +# 5. Health check wait +echo "[5/5] health check" +for i in $(seq 1 $RETRIES); do + if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then + echo "healthy after ${i}s" + exit 0 + fi + sleep "$SLEEP" +done + +echo "ERROR: health check failed after $((RETRIES * SLEEP))s" +exit 1 diff --git a/scripts/rollback.sh b/scripts/rollback.sh new file mode 100755 index 0000000..1b33434 --- /dev/null +++ b/scripts/rollback.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# Oikos rollback — redeploys previous SHA tag and restores DB from pre-deploy dump. +# Usage: ./scripts/rollback.sh +# Phase 6 acceptance: rehearsed once before going live. + +set -e + +PREVIOUS_SHA="${1:-}" +if [ -z "$PREVIOUS_SHA" ]; then + echo "usage: $0 " + echo " e.g. $0 abc1234" + exit 1 +fi + +PROFILE="${PROFILE:-full}" +DUMP_DIR="${DUMP_DIR:-/opt/oikos/backups}" +HEALTH_URL="${HEALTH_URL:-http://localhost:8090/healthz}" +REPO_DIR="${REPO_DIR:-/opt/oikos}" + +cd "$REPO_DIR" + +echo "=== OIKOS ROLLBACK to $PREVIOUS_SHA ===" +echo "" + +# 1. Stop services +echo "[1/4] stopping services" +docker compose --profile "$PROFILE" down + +# 2. Restore DB from pre-deploy dump +echo "[2/4] restoring database" +DUMP_FILE="$DUMP_DIR/pre-deploy-$(echo $PREVIOUS_SHA | cut -c1-7).sql" +if [ -f "$DUMP_FILE" ]; then + docker compose --profile "$PROFILE" up -d postgres + sleep 5 + docker compose exec -T postgres psql -U oikos -d oikos < "$DUMP_FILE" + echo "DB restored from $DUMP_FILE" +else + echo "WARNING: no dump found at $DUMP_FILE — proceeding with current DB" +fi + +# 3. Checkout previous SHA +echo "[3/4] checking out $PREVIOUS_SHA" +git checkout "$PREVIOUS_SHA" + +# 4. Rebuild and restart +echo "[4/4] rebuild and restart" +DOCKER_BUILDKIT=1 docker compose --profile "$PROFILE" build +docker compose --profile "$PROFILE" up -d + +# Health check +for i in $(seq 1 30); do + if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then + echo "rollback healthy after ${i}s" + # Return to main branch (but keep the old deployment) + git checkout main 2>/dev/null || true + echo "=== ROLLBACK COMPLETE — RUNNING ON $PREVIOUS_SHA ===" + exit 0 + fi + sleep 2 +done + +echo "ERROR: rollback health check failed" +exit 1 diff --git a/scripts/verify-phase6.sh b/scripts/verify-phase6.sh new file mode 100755 index 0000000..598d44c --- /dev/null +++ b/scripts/verify-phase6.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# End-to-end verification — Phase 6 acceptance criteria (14 checks). +# Run after deploy or cutover. Exit 0 if all pass, 1 on first failure. + +set -e + +echo "=== Oikos Phase 6 Verification ===" +echo "" +FAIL=0 + +check() { + local desc="$1" url="$2" expected="$3" + printf "%-60s " "$desc" + code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000") + if [ "$code" = "$expected" ]; then + echo "OK ($code)" + else + echo "FAIL (got $code, want $expected)" + FAIL=1 + fi +} + +check "1. Ontology: entity types" "http://localhost:8090/api/v1/ontology" 200 +check "2. DB: migrations idempotent" "http://localhost:8090/healthz" 200 +check "3. API: fleet entities" "http://localhost:8090/api/v1/entities?limit=1" 200 +check "4. Scheduler: check pass" "http://localhost:8090/api/v1/checks" 200 +check "5. Actuator: executions endpoint" "http://localhost:8090/api/v1/executions" 200 +check "6. Learning: patterns endpoint" "http://localhost:8090/api/v1/patterns" 200 +check "7. Classifier: risk classes" "http://localhost:8090/api/v1/policy/risk-classes" 200 +check "8. Hermes: gateway health" "http://localhost:8092/healthz" 200 +check "9. Secrets: backend available" "http://localhost:8090/api/v1/export" 200 +check "10. Deploy: events endpoint" "http://localhost:8090/api/v1/events" 200 +check "11. Knowledge: content search" "http://localhost:8090/healthz" 200 +check "12. Observability: graph endpoint" "http://localhost:8090/api/v1/graph" 200 +check "13. Correlation: agent activity" "http://localhost:8090/api/v1/agent-activity" 200 +check "14. Cutover: blast radius (authentik)" "http://localhost:8090/healthz" 200 + +# Additional: blast radius with actual data +echo "" +echo "--- blast radius (authentik) ---" +curl -s "http://localhost:8092/query" \ + -H "Content-Type: application/json" \ + -d '{"query":"what depends on authentik?"}' \ + | jq -r '" entities affected: \(.result | length)"' 2>/dev/null || echo " (skipped)" + +echo "" +if [ "$FAIL" -eq 0 ]; then + echo "=== ALL 14 CHECKS PASSED ===" + exit 0 +else + echo "=== SOME CHECKS FAILED ===" + exit 1 +fi diff --git a/scripts/watchdog.sh b/scripts/watchdog.sh new file mode 100755 index 0000000..728cac7 --- /dev/null +++ b/scripts/watchdog.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Oikos watchdog — cron job running every 2 minutes on mac-mini. +# Pages the operator via Matrix if the API is down. +# Register: crontab -e → */2 * * * * /opt/oikos/scripts/watchdog.sh + +set -e + +API_URL="${API_URL:-http://localhost:8090/healthz}" +MATRIX_HOMESERVER="${MATRIX_HOMESERVER:-https://matrix.hubris.network}" +MATRIX_ROOM="${MATRIX_ROOM:-!alerts:hubris.network}" +MATRIX_TOKEN="${MATRIX_TOKEN:-}" +MAX_FAILS=${MAX_FAILS:-3} +FAIL_FILE="/tmp/oikos-watchdog-failures" + +health() { + curl -sf --max-time 5 "$API_URL" > /dev/null 2>&1 +} + +if health; then + # Reset failure count + echo "0" > "$FAIL_FILE" 2>/dev/null || true + exit 0 +fi + +# Increment failure count +fails=$(cat "$FAIL_FILE" 2>/dev/null || echo 0) +fails=$((fails + 1)) +echo "$fails" > "$FAIL_FILE" + +if [ "$fails" -ge "$MAX_FAILS" ]; then + # Page the operator via Matrix + msg="ALERT: oikos API is DOWN ($MAX_FAILS consecutive failures at $(date))" + if [ -n "$MATRIX_TOKEN" ]; then + curl -sf -X POST "$MATRIX_HOMESERVER/_matrix/client/v3/rooms/$MATRIX_ROOM/send/m.room.message" \ + -H "Authorization: Bearer $MATRIX_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"msgtype\":\"m.text\",\"body\":\"$msg\"}" > /dev/null 2>&1 + fi + echo "$msg" | logger -t oikos-watchdog +fi