#!/bin/sh # Oikos deploy script — triggered by Gitea webhook on push to dtoro/oikos. # Runs on mac-mini as non-root user via launchd unit oikos-deploy-webhook.service. # Phase 6: CI-gated, version-tagged images, rolling restart, pre-deploy pg_dump. # # Plans implemented here: # D1 — CI gate: blocks deploy unless Gitea reports a green run for the SHA. # D2 — versioned images: tags every built image v$VERSION (from VERSION file), # keeps the last 3 tags per service for rollback. # Notify on deploy failure via Matrix. Uses Oikos API to raise an event # so the scheduler picks it up and alerts via the notifier. notify_deploy_failure() { local reason="$1" local sha="${SHA:-unknown}" echo "NOTIFY: deploy failed — $reason" # Try to raise an event through the Oikos API (best-effort, silent failure) if [ -n "${OIKOS_API_TOKEN:-}" ]; then curl -sf -X POST "http://localhost:8090/api/v1/events" \ -H "Authorization: Bearer $OIKOS_API_TOKEN" \ -H "Content-Type: application/json" \ -d "{\"type\":\"deploy.failed\",\"severity\":\"critical\",\"source\":\"webhook\",\"data\":{\"sha\":\"$sha\",\"reason\":\"$reason\"}}" \ >/dev/null 2>&1 || true fi # Also try Matrix directly via the notifier's webhook endpoint if configured if [ -n "${MATRIX_WEBHOOK_URL:-}" ]; then curl -sf -X POST "$MATRIX_WEBHOOK_URL" \ -H "Content-Type: application/json" \ -d "{\"msgtype\":\"m.text\",\"body\":\"🚨 Deploy failed: $reason (sha: $sha)\"}" \ >/dev/null 2>&1 || true fi } set -e REPO_DIR="${REPO_DIR:-$PWD}" COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yml}" PROFILE="${PROFILE:-full}" HEALTH_URL="${HEALTH_URL:-http://localhost:8090/healthz}" DUMP_DIR="${DUMP_DIR:-/opt/oikos/backups}" RETRIES=${RETRIES:-30} SLEEP=${SLEEP:-2} # CI gate (D1). Set GITEA_URL + GITEA_TOKEN to enable; without them the gate # is skipped with a warning (dev/local deploys). Owner/repo default to the # canonical homelab repo but can be overridden or derived from the git remote. GITEA_URL="${GITEA_URL:-}" GITEA_TOKEN="${GITEA_TOKEN:-}" GITEA_OWNER="${GITEA_OWNER:-dtoro}" GITEA_REPO="${GITEA_REPO:-oikos}" CI_POLL_INTERVAL="${CI_POLL_INTERVAL:-15}" CI_TIMEOUT="${CI_TIMEOUT:-1200}" # Serialize deploys: the webhook runs this script in a background goroutine and # the CI gate can hold a deploy open for many minutes, so a second push during # that window would otherwise race on git/pg_dump/compose. mkdir is atomic on # POSIX (no flock dependency — macOS lacks it). The stale-pid check recovers # if a previous deploy was SIGKILLed. LOCKDIR="${LOCKDIR:-/tmp/oikos-deploy.lock}" if ! mkdir "$LOCKDIR" 2>/dev/null; then oldpid=$(cat "$LOCKDIR/pid" 2>/dev/null || echo "") if [ -n "$oldpid" ] && kill -0 "$oldpid" 2>/dev/null; then echo "deploy already in progress (pid $oldpid) — exiting" exit 0 fi echo "removing stale deploy lock (pid ${oldpid:-?} not running)" rm -rf "$LOCKDIR" mkdir "$LOCKDIR" fi echo $$ > "$LOCKDIR/pid" trap 'rc=$?; rm -rf "$LOCKDIR" 2>/dev/null || true; if [ "$_ok" != "1" ]; then notify_deploy_failure "deploy aborted (exit $rc)"; fi' EXIT _ok=0 cd "$REPO_DIR" echo "=== oikos deploy: $(date) ===" # Resolve the SHA we are ABOUT to deploy from the remote (read-only: no working # tree mutation yet) so the CI gate can run before anything is touched. Keep the # full SHA (REMOTE_FULL) for the post-pull equality check; the 12-char form is # only for display and the Gitea status API (which accepts any unique prefix). REMOTE_FULL=$(git ls-remote origin refs/heads/main 2>/dev/null | awk '{print $1}') if [ -z "$REMOTE_FULL" ]; then echo "ERROR: could not resolve origin/main (offline?) — aborting before any change" exit 1 fi REMOTE_SHA=$(printf '%s' "$REMOTE_FULL" | cut -c1-12) echo "remote SHA: $REMOTE_SHA" # ── 1. CI gate (plan D1) ────────────────────────────────────────────────── # Runs BEFORE pg_dump/pull/build: a red or hung pipeline must not leave the # tree half-deployed. On failure|error it refuses; on success it proceeds; on # "no CI signal at all" (Actions unconfigured / token rejected) it warns and # proceeds rather than bricking every deploy. echo "[1/8] verify CI status for $REMOTE_SHA" verify_ci() { sha=$1 if [ -z "$GITEA_URL" ] || [ -z "$GITEA_TOKEN" ]; then echo "SKIP: GITEA_URL/GITEA_TOKEN not set — CI gate disabled. Set both to enforce." return 0 fi # Derive owner/repo from the origin remote when the defaults don't apply. origin=$(git remote get-url origin 2>/dev/null || echo "") seg= case "$origin" in *@*:*) # SSH: git@host:owner/repo.git seg=${origin##*:}; seg=${seg%.git} ;; http://*|https://*) # HTTPS: scheme://host/owner/repo.git seg=${origin#*://}; seg=${seg#*/}; seg=${seg%.git} ;; esac case "$seg" in */*) GITEA_OWNER=${seg%%/*} GITEA_REPO=${seg#*/} ;; esac api="$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO/commits/$sha/status" body=$(mktemp) elapsed=0 saw_ci=0 # became 1 once we observed a real status (pending/success/...) no_signal=0 # consecutive responses with no usable status while [ "$elapsed" -lt "$CI_TIMEOUT" ]; do # Pass the token via curl --config stdin so it never appears in argv # (visible via ps). Don't use -f: we want the HTTP code on 4xx. code=$(printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" | \ curl -sS -o "$body" -w '%{http_code}' --config - "$api" 2>/dev/null) || code="000" state=$(sed -n 's/.*"state"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$body" | head -n1) case "$code" in 200) case "$state" in success) rm -f "$body" echo "CI: green for $sha after ${elapsed}s" return 0 ;; failure|error) rm -f "$body" echo "ERROR: CI $state for $sha — refusing to deploy." echo " See $GITEA_URL/$GITEA_OWNER/$GITEA_REPO/actions" return 1 ;; pending|"") saw_ci=1 no_signal=0 ;; esac ;; 404) # No status checks exist for this commit (Actions not configured # / no runner has reported). Can't gate — warn + proceed. rm -f "$body" echo "WARN: Gitea has no CI status for $sha (404)." echo " Is Gitea Actions configured with a runner for $GITEA_OWNER/$GITEA_REPO?" echo " Proceeding without a gate." return 0 ;; 401|403) rm -f "$body" echo "WARN: GITEA_TOKEN rejected by Gitea ($code) — cannot verify CI." echo " Fix the token to enforce the gate; proceeding without one." return 0 ;; *) # Network blip / 5xx / 000: retry, but count as no-signal. no_signal=$((no_signal + 1)) ;; esac # If we never get a usable signal after ~1 min, assume CI is unreachable # rather than burn the full timeout and brick deploys. if [ "$saw_ci" -eq 0 ] && [ "$no_signal" -ge 4 ]; then rm -f "$body" echo "WARN: no CI signal from Gitea after ${elapsed}s (last code=$code)." echo " CI may be down or misconfigured; proceeding without a gate." return 0 fi sleep "$CI_POLL_INTERVAL" elapsed=$((elapsed + CI_POLL_INTERVAL)) printf '\rCI: waiting (%ss, code=%s state=%s)...' "$elapsed" "$code" "${state:-none}" done rm -f "$body" echo "" echo "ERROR: CI did not reach a terminal state within ${CI_TIMEOUT}s for $sha — refusing to deploy." return 1 } verify_ci "$REMOTE_SHA" || exit 1 # ── 2. Pre-deploy pg_dump for rollback safety (plan O1) ─────────────────── echo "[2/8] pre-deploy pg_dump" DUMP_FILE="$DUMP_DIR/pre-deploy-$REMOTE_SHA.sql" mkdir -p "$DUMP_DIR" docker compose exec -T postgres pg_dump -U oikos oikos > "$DUMP_FILE" 2>/dev/null || \ echo "WARNING: pg_dump failed — rollback will not have a recovery point" # ── 3. Update working tree to the verified commit ───────────────────────── echo "[3/8] git pull (ff-only)" git pull --ff-only origin main # TOCTOU guard: if origin/main advanced during the CI wait + pg_dump, the pull # fast-forwards PAST the SHA we verified without re-checking its CI. Refuse # rather than ship an unverified commit — a retry will verify the new tip. PULLED_FULL=$(git rev-parse HEAD) if [ "$PULLED_FULL" != "$REMOTE_FULL" ]; then echo "ERROR: origin/main advanced during deploy (verified $REMOTE_SHA, now at $(git rev-parse --short HEAD)) — aborting; retry verifies the new tip" exit 1 fi SHA=$(git rev-parse --short HEAD) echo "SHA (deployed): $SHA" # Resolve the deploy version AFTER pull (D2) so the tag matches the code being # built. Compose interpolates $OIKOS_VERSION into each service's image: tag. VERSION_FILE="$REPO_DIR/VERSION" if [ -f "$VERSION_FILE" ]; then OIKOS_VERSION="v$(head -n1 "$VERSION_FILE" | tr -d '[:space:]')" export OIKOS_VERSION echo "VERSION: $OIKOS_VERSION" else echo "WARNING: VERSION file missing — images will use :latest (rollback unavailable)" fi # ── 4. Build version-tagged images (plan D2) ────────────────────────────── echo "[4/8] docker compose build" DOCKER_BUILDKIT=1 docker compose --profile "$PROFILE" build \ --build-arg BUILDKIT_INLINE_CACHE=1 # ── 5. Rolling restart ──────────────────────────────────────────────────── echo "[5/8] docker compose up -d" docker compose --profile "$PROFILE" up -d --remove-orphans # ── 6. Prune old image tags — keep the 3 newest per service so rollback ──── # (OIKOS_VERSION=v0.x.y docker compose up) stays available. The repo list # is derived from compose so it can't drift from the image: names. echo "[6/8] prune old image tags (keep 3)" if [ -n "$OIKOS_VERSION" ]; then images=$(docker compose --profile "$PROFILE" config --images 2>/dev/null || true) if [ -z "$images" ]; then images="oikos-api oikos-scheduler oikos-notifier oikos-migrate oikos-seed oikos-nomos oikos-web" fi printf '%s\n' $images | sed 's/:.*//' | grep '^oikos-' | sort -u | while read -r repo; do docker image ls "$repo" --format '{{.Tag}}' 2>/dev/null | grep '^v' | sort -rV | tail -n +4 | while read -r tag; do docker rmi "$repo:$tag" >/dev/null 2>&1 || true done done fi # ── 7. Health check wait ────────────────────────────────────────────────── echo "[7/8] health check" healthy=0 for i in $(seq 1 $RETRIES); do if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then echo "healthy after ${i}s" healthy=1 break fi sleep "$SLEEP" done if [ "$healthy" -ne 1 ]; then echo "ERROR: health check failed after $((RETRIES * SLEEP))s" exit 1 fi # ── 8. Seed secrets into Infisical (idempotent) ─────────────────────────── echo "[8/8] seed secrets" if [ -f "$REPO_DIR/scripts/seed-secrets.sh" ]; then REPO_DIR="$REPO_DIR" sh "$REPO_DIR/scripts/seed-secrets.sh" || \ echo "WARNING: secret seeding failed" else echo "SKIP: seed-secrets.sh not found" fi # All steps completed successfully — clear failure trap _ok=1 exit 0