dtoro
30ecdc16c2
Two fixes from the deploy pipeline audit:
1. Infisical tag v0.99.1 no longer exists on Docker Hub — bumped to
v0.162.19 (latest available). This was silently breaking the full
deploy pipeline (docker compose up failed on image pull).
2. Deploy failures now notify via two channels:
- Oikos API event (deploy.failed, severity=critical) — picked up by
the scheduler's notifier for Matrix alert
- Matrix webhook URL if MATRIX_WEBHOOK_URL is configured
Uses a trap with _ok flag to catch any non-zero exit path,
including CI gate rejections and health check timeouts.
Webhook now resolves and passes OIKOS_API_TOKEN to deploy.sh.
278 lines
12 KiB
Bash
Executable File
278 lines
12 KiB
Bash
Executable File
#!/bin/sh
|
|
# Oikos deploy script — triggered by Gitea webhook on push to dtoro/oikos.
|
|
# Runs on mac-mini as non-root user via launchd unit oikos-deploy-webhook.service.
|
|
# Phase 6: CI-gated, version-tagged images, rolling restart, pre-deploy pg_dump.
|
|
#
|
|
# Plans implemented here:
|
|
# D1 — CI gate: blocks deploy unless Gitea reports a green run for the SHA.
|
|
# D2 — versioned images: tags every built image v$VERSION (from VERSION file),
|
|
# keeps the last 3 tags per service for rollback.
|
|
|
|
# Notify on deploy failure via Matrix. Uses Oikos API to raise an event
|
|
# so the scheduler picks it up and alerts via the notifier.
|
|
notify_deploy_failure() {
|
|
local reason="$1"
|
|
local sha="${SHA:-unknown}"
|
|
echo "NOTIFY: deploy failed — $reason"
|
|
# Try to raise an event through the Oikos API (best-effort, silent failure)
|
|
if [ -n "${OIKOS_API_TOKEN:-}" ]; then
|
|
curl -sf -X POST "http://localhost:8090/api/v1/events" \
|
|
-H "Authorization: Bearer $OIKOS_API_TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"type\":\"deploy.failed\",\"severity\":\"critical\",\"source\":\"webhook\",\"data\":{\"sha\":\"$sha\",\"reason\":\"$reason\"}}" \
|
|
>/dev/null 2>&1 || true
|
|
fi
|
|
# Also try Matrix directly via the notifier's webhook endpoint if configured
|
|
if [ -n "${MATRIX_WEBHOOK_URL:-}" ]; then
|
|
curl -sf -X POST "$MATRIX_WEBHOOK_URL" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"msgtype\":\"m.text\",\"body\":\"🚨 Deploy failed: $reason (sha: $sha)\"}" \
|
|
>/dev/null 2>&1 || true
|
|
fi
|
|
}
|
|
|
|
set -e
|
|
|
|
REPO_DIR="${REPO_DIR:-$PWD}"
|
|
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yml}"
|
|
PROFILE="${PROFILE:-full}"
|
|
HEALTH_URL="${HEALTH_URL:-http://localhost:8090/healthz}"
|
|
DUMP_DIR="${DUMP_DIR:-/opt/oikos/backups}"
|
|
RETRIES=${RETRIES:-30}
|
|
SLEEP=${SLEEP:-2}
|
|
|
|
# CI gate (D1). Set GITEA_URL + GITEA_TOKEN to enable; without them the gate
|
|
# is skipped with a warning (dev/local deploys). Owner/repo default to the
|
|
# canonical homelab repo but can be overridden or derived from the git remote.
|
|
GITEA_URL="${GITEA_URL:-}"
|
|
GITEA_TOKEN="${GITEA_TOKEN:-}"
|
|
GITEA_OWNER="${GITEA_OWNER:-dtoro}"
|
|
GITEA_REPO="${GITEA_REPO:-oikos}"
|
|
CI_POLL_INTERVAL="${CI_POLL_INTERVAL:-15}"
|
|
CI_TIMEOUT="${CI_TIMEOUT:-1200}"
|
|
|
|
# Serialize deploys: the webhook runs this script in a background goroutine and
|
|
# the CI gate can hold a deploy open for many minutes, so a second push during
|
|
# that window would otherwise race on git/pg_dump/compose. mkdir is atomic on
|
|
# POSIX (no flock dependency — macOS lacks it). The stale-pid check recovers
|
|
# if a previous deploy was SIGKILLed.
|
|
LOCKDIR="${LOCKDIR:-/tmp/oikos-deploy.lock}"
|
|
if ! mkdir "$LOCKDIR" 2>/dev/null; then
|
|
oldpid=$(cat "$LOCKDIR/pid" 2>/dev/null || echo "")
|
|
if [ -n "$oldpid" ] && kill -0 "$oldpid" 2>/dev/null; then
|
|
echo "deploy already in progress (pid $oldpid) — exiting"
|
|
exit 0
|
|
fi
|
|
echo "removing stale deploy lock (pid ${oldpid:-?} not running)"
|
|
rm -rf "$LOCKDIR"
|
|
mkdir "$LOCKDIR"
|
|
fi
|
|
echo $$ > "$LOCKDIR/pid"
|
|
trap 'rc=$?; rm -rf "$LOCKDIR" 2>/dev/null || true; if [ "$_ok" != "1" ]; then notify_deploy_failure "deploy aborted (exit $rc)"; fi' EXIT
|
|
_ok=0
|
|
|
|
cd "$REPO_DIR"
|
|
|
|
echo "=== oikos deploy: $(date) ==="
|
|
|
|
# Resolve the SHA we are ABOUT to deploy from the remote (read-only: no working
|
|
# tree mutation yet) so the CI gate can run before anything is touched. Keep the
|
|
# full SHA (REMOTE_FULL) for the post-pull equality check; the 12-char form is
|
|
# only for display and the Gitea status API (which accepts any unique prefix).
|
|
REMOTE_FULL=$(git ls-remote origin refs/heads/main 2>/dev/null | awk '{print $1}')
|
|
if [ -z "$REMOTE_FULL" ]; then
|
|
echo "ERROR: could not resolve origin/main (offline?) — aborting before any change"
|
|
exit 1
|
|
fi
|
|
REMOTE_SHA=$(printf '%s' "$REMOTE_FULL" | cut -c1-12)
|
|
echo "remote SHA: $REMOTE_SHA"
|
|
|
|
# ── 1. CI gate (plan D1) ──────────────────────────────────────────────────
|
|
# Runs BEFORE pg_dump/pull/build: a red or hung pipeline must not leave the
|
|
# tree half-deployed. On failure|error it refuses; on success it proceeds; on
|
|
# "no CI signal at all" (Actions unconfigured / token rejected) it warns and
|
|
# proceeds rather than bricking every deploy.
|
|
echo "[1/8] verify CI status for $REMOTE_SHA"
|
|
verify_ci() {
|
|
sha=$1
|
|
if [ -z "$GITEA_URL" ] || [ -z "$GITEA_TOKEN" ]; then
|
|
echo "SKIP: GITEA_URL/GITEA_TOKEN not set — CI gate disabled. Set both to enforce."
|
|
return 0
|
|
fi
|
|
# Derive owner/repo from the origin remote when the defaults don't apply.
|
|
origin=$(git remote get-url origin 2>/dev/null || echo "")
|
|
seg=
|
|
case "$origin" in
|
|
*@*:*) # SSH: git@host:owner/repo.git
|
|
seg=${origin##*:}; seg=${seg%.git}
|
|
;;
|
|
http://*|https://*) # HTTPS: scheme://host/owner/repo.git
|
|
seg=${origin#*://}; seg=${seg#*/}; seg=${seg%.git}
|
|
;;
|
|
esac
|
|
case "$seg" in
|
|
*/*)
|
|
GITEA_OWNER=${seg%%/*}
|
|
GITEA_REPO=${seg#*/}
|
|
;;
|
|
esac
|
|
|
|
api="$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO/commits/$sha/status"
|
|
body=$(mktemp)
|
|
elapsed=0
|
|
saw_ci=0 # became 1 once we observed a real status (pending/success/...)
|
|
no_signal=0 # consecutive responses with no usable status
|
|
while [ "$elapsed" -lt "$CI_TIMEOUT" ]; do
|
|
# Pass the token via curl --config stdin so it never appears in argv
|
|
# (visible via ps). Don't use -f: we want the HTTP code on 4xx.
|
|
code=$(printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" | \
|
|
curl -sS -o "$body" -w '%{http_code}' --config - "$api" 2>/dev/null) || code="000"
|
|
state=$(sed -n 's/.*"state"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$body" | head -n1)
|
|
|
|
case "$code" in
|
|
200)
|
|
case "$state" in
|
|
success)
|
|
rm -f "$body"
|
|
echo "CI: green for $sha after ${elapsed}s"
|
|
return 0
|
|
;;
|
|
failure|error)
|
|
rm -f "$body"
|
|
echo "ERROR: CI $state for $sha — refusing to deploy."
|
|
echo " See $GITEA_URL/$GITEA_OWNER/$GITEA_REPO/actions"
|
|
return 1
|
|
;;
|
|
pending|"")
|
|
saw_ci=1
|
|
no_signal=0
|
|
;;
|
|
esac
|
|
;;
|
|
404)
|
|
# No status checks exist for this commit (Actions not configured
|
|
# / no runner has reported). Can't gate — warn + proceed.
|
|
rm -f "$body"
|
|
echo "WARN: Gitea has no CI status for $sha (404)."
|
|
echo " Is Gitea Actions configured with a runner for $GITEA_OWNER/$GITEA_REPO?"
|
|
echo " Proceeding without a gate."
|
|
return 0
|
|
;;
|
|
401|403)
|
|
rm -f "$body"
|
|
echo "WARN: GITEA_TOKEN rejected by Gitea ($code) — cannot verify CI."
|
|
echo " Fix the token to enforce the gate; proceeding without one."
|
|
return 0
|
|
;;
|
|
*)
|
|
# Network blip / 5xx / 000: retry, but count as no-signal.
|
|
no_signal=$((no_signal + 1))
|
|
;;
|
|
esac
|
|
|
|
# If we never get a usable signal after ~1 min, assume CI is unreachable
|
|
# rather than burn the full timeout and brick deploys.
|
|
if [ "$saw_ci" -eq 0 ] && [ "$no_signal" -ge 4 ]; then
|
|
rm -f "$body"
|
|
echo "WARN: no CI signal from Gitea after ${elapsed}s (last code=$code)."
|
|
echo " CI may be down or misconfigured; proceeding without a gate."
|
|
return 0
|
|
fi
|
|
|
|
sleep "$CI_POLL_INTERVAL"
|
|
elapsed=$((elapsed + CI_POLL_INTERVAL))
|
|
printf '\rCI: waiting (%ss, code=%s state=%s)...' "$elapsed" "$code" "${state:-none}"
|
|
done
|
|
rm -f "$body"
|
|
echo ""
|
|
echo "ERROR: CI did not reach a terminal state within ${CI_TIMEOUT}s for $sha — refusing to deploy."
|
|
return 1
|
|
}
|
|
verify_ci "$REMOTE_SHA" || exit 1
|
|
|
|
# ── 2. Pre-deploy pg_dump for rollback safety (plan O1) ───────────────────
|
|
echo "[2/8] pre-deploy pg_dump"
|
|
DUMP_FILE="$DUMP_DIR/pre-deploy-$REMOTE_SHA.sql"
|
|
mkdir -p "$DUMP_DIR"
|
|
docker compose exec -T postgres pg_dump -U oikos oikos > "$DUMP_FILE" 2>/dev/null || \
|
|
echo "WARNING: pg_dump failed — rollback will not have a recovery point"
|
|
|
|
# ── 3. Update working tree to the verified commit ─────────────────────────
|
|
echo "[3/8] git pull (ff-only)"
|
|
git pull --ff-only origin main
|
|
# TOCTOU guard: if origin/main advanced during the CI wait + pg_dump, the pull
|
|
# fast-forwards PAST the SHA we verified without re-checking its CI. Refuse
|
|
# rather than ship an unverified commit — a retry will verify the new tip.
|
|
PULLED_FULL=$(git rev-parse HEAD)
|
|
if [ "$PULLED_FULL" != "$REMOTE_FULL" ]; then
|
|
echo "ERROR: origin/main advanced during deploy (verified $REMOTE_SHA, now at $(git rev-parse --short HEAD)) — aborting; retry verifies the new tip"
|
|
exit 1
|
|
fi
|
|
SHA=$(git rev-parse --short HEAD)
|
|
echo "SHA (deployed): $SHA"
|
|
|
|
# Resolve the deploy version AFTER pull (D2) so the tag matches the code being
|
|
# built. Compose interpolates $OIKOS_VERSION into each service's image: tag.
|
|
VERSION_FILE="$REPO_DIR/VERSION"
|
|
if [ -f "$VERSION_FILE" ]; then
|
|
OIKOS_VERSION="v$(head -n1 "$VERSION_FILE" | tr -d '[:space:]')"
|
|
export OIKOS_VERSION
|
|
echo "VERSION: $OIKOS_VERSION"
|
|
else
|
|
echo "WARNING: VERSION file missing — images will use :latest (rollback unavailable)"
|
|
fi
|
|
|
|
# ── 4. Build version-tagged images (plan D2) ──────────────────────────────
|
|
echo "[4/8] docker compose build"
|
|
DOCKER_BUILDKIT=1 docker compose --profile "$PROFILE" build \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1
|
|
|
|
# ── 5. Rolling restart ────────────────────────────────────────────────────
|
|
echo "[5/8] docker compose up -d"
|
|
docker compose --profile "$PROFILE" up -d --remove-orphans
|
|
|
|
# ── 6. Prune old image tags — keep the 3 newest per service so rollback ────
|
|
# (OIKOS_VERSION=v0.x.y docker compose up) stays available. The repo list
|
|
# is derived from compose so it can't drift from the image: names.
|
|
echo "[6/8] prune old image tags (keep 3)"
|
|
if [ -n "$OIKOS_VERSION" ]; then
|
|
images=$(docker compose --profile "$PROFILE" config --images 2>/dev/null || true)
|
|
if [ -z "$images" ]; then
|
|
images="oikos-api oikos-scheduler oikos-notifier oikos-migrate oikos-seed oikos-nomos oikos-web"
|
|
fi
|
|
printf '%s\n' $images | sed 's/:.*//' | grep '^oikos-' | sort -u | while read -r repo; do
|
|
docker image ls "$repo" --format '{{.Tag}}' 2>/dev/null | grep '^v' | sort -rV | tail -n +4 | while read -r tag; do
|
|
docker rmi "$repo:$tag" >/dev/null 2>&1 || true
|
|
done
|
|
done
|
|
fi
|
|
|
|
# ── 7. Health check wait ──────────────────────────────────────────────────
|
|
echo "[7/8] health check"
|
|
healthy=0
|
|
for i in $(seq 1 $RETRIES); do
|
|
if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then
|
|
echo "healthy after ${i}s"
|
|
healthy=1
|
|
break
|
|
fi
|
|
sleep "$SLEEP"
|
|
done
|
|
if [ "$healthy" -ne 1 ]; then
|
|
echo "ERROR: health check failed after $((RETRIES * SLEEP))s"
|
|
exit 1
|
|
fi
|
|
|
|
# ── 8. Seed secrets into Infisical (idempotent) ───────────────────────────
|
|
echo "[8/8] seed secrets"
|
|
if [ -f "$REPO_DIR/scripts/seed-secrets.sh" ]; then
|
|
REPO_DIR="$REPO_DIR" sh "$REPO_DIR/scripts/seed-secrets.sh" || \
|
|
echo "WARNING: secret seeding failed"
|
|
else
|
|
echo "SKIP: seed-secrets.sh not found"
|
|
fi
|
|
|
|
# All steps completed successfully — clear failure trap
|
|
_ok=1
|
|
exit 0
|