0.28.0 — operational hardening (plan D1–D5): CI deploy gate, versioned images, rate limiting, resource limits, health probes
D1: deploy.sh CI gate — read-only SHA via git ls-remote, Gitea commit-status
poll, portable mkdir deploy lock (macOS, no flock), TOCTOU guard, token
passed via curl --config - (not argv), graceful misconfig tolerance.
D2: version-tagged images — OIKOS_VERSION=v$VERSION, keep-last-3 prune derived
from 'docker compose config --images'; VERSION read after pull.
D3: per-IP rate limiting — new internal/httpapi/ratelimit.go (x/time/rate),
rightmost-XFF, /healthz exempt, ctx-driven sweep; disabled by default.
D4: mem_limit/cpus on all 10 compose services.
D5: staleness-aware health probes — new internal/health package wired into
scheduler (:8093) and notifier (:8094); nomos already had :8092.
Two /review passes hardened the deploy lock, TOCTOU guard, token hygiene,
and XFF handling.
This commit is contained in:
@@ -1,7 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Oikos deploy script — triggered by Gitea webhook on push to dtoro/oikos.
|
||||
# Runs on mac-mini as non-root user via systemd unit oikos-deploy-webhook.service.
|
||||
# Phase 6: CI-gated, SHA-tagged images, rolling restart, pre-deploy pg_dump.
|
||||
# Runs on mac-mini as non-root user via launchd unit oikos-deploy-webhook.service.
|
||||
# Phase 6: CI-gated, version-tagged images, rolling restart, pre-deploy pg_dump.
|
||||
#
|
||||
# Plans implemented here:
|
||||
# D1 — CI gate: blocks deploy unless Gitea reports a green run for the SHA.
|
||||
# D2 — versioned images: tags every built image v$VERSION (from VERSION file),
|
||||
# keeps the last 3 tags per service for rollback.
|
||||
|
||||
set -e
|
||||
|
||||
@@ -13,54 +18,234 @@ DUMP_DIR="${DUMP_DIR:-/opt/oikos/backups}"
|
||||
RETRIES=${RETRIES:-30}
|
||||
SLEEP=${SLEEP:-2}
|
||||
|
||||
# CI gate (D1). Set GITEA_URL + GITEA_TOKEN to enable; without them the gate
|
||||
# is skipped with a warning (dev/local deploys). Owner/repo default to the
|
||||
# canonical homelab repo but can be overridden or derived from the git remote.
|
||||
GITEA_URL="${GITEA_URL:-}"
|
||||
GITEA_TOKEN="${GITEA_TOKEN:-}"
|
||||
GITEA_OWNER="${GITEA_OWNER:-dtoro}"
|
||||
GITEA_REPO="${GITEA_REPO:-oikos}"
|
||||
CI_POLL_INTERVAL="${CI_POLL_INTERVAL:-15}"
|
||||
CI_TIMEOUT="${CI_TIMEOUT:-1200}"
|
||||
|
||||
# Serialize deploys: the webhook runs this script in a background goroutine and
|
||||
# the CI gate can hold a deploy open for many minutes, so a second push during
|
||||
# that window would otherwise race on git/pg_dump/compose. mkdir is atomic on
|
||||
# POSIX (no flock dependency — macOS lacks it). The stale-pid check recovers
|
||||
# if a previous deploy was SIGKILLed.
|
||||
LOCKDIR="${LOCKDIR:-/tmp/oikos-deploy.lock}"
|
||||
if ! mkdir "$LOCKDIR" 2>/dev/null; then
|
||||
oldpid=$(cat "$LOCKDIR/pid" 2>/dev/null || echo "")
|
||||
if [ -n "$oldpid" ] && kill -0 "$oldpid" 2>/dev/null; then
|
||||
echo "deploy already in progress (pid $oldpid) — exiting"
|
||||
exit 0
|
||||
fi
|
||||
echo "removing stale deploy lock (pid ${oldpid:-?} not running)"
|
||||
rm -rf "$LOCKDIR"
|
||||
mkdir "$LOCKDIR"
|
||||
fi
|
||||
echo $$ > "$LOCKDIR/pid"
|
||||
trap 'rm -rf "$LOCKDIR" 2>/dev/null || true' EXIT INT TERM
|
||||
|
||||
cd "$REPO_DIR"
|
||||
|
||||
echo "=== oikos deploy: $(date) ==="
|
||||
SHA=$(git rev-parse --short HEAD)
|
||||
echo "SHA: $SHA"
|
||||
|
||||
# 1. Pre-deploy pg_dump for rollback safety (plan O1)
|
||||
echo "[1/6] pre-deploy pg_dump"
|
||||
DUMP_FILE="$DUMP_DIR/pre-deploy-$SHA.sql"
|
||||
# Resolve the SHA we are ABOUT to deploy from the remote (read-only: no working
|
||||
# tree mutation yet) so the CI gate can run before anything is touched. Keep the
|
||||
# full SHA (REMOTE_FULL) for the post-pull equality check; the 12-char form is
|
||||
# only for display and the Gitea status API (which accepts any unique prefix).
|
||||
REMOTE_FULL=$(git ls-remote origin refs/heads/main 2>/dev/null | awk '{print $1}')
|
||||
if [ -z "$REMOTE_FULL" ]; then
|
||||
echo "ERROR: could not resolve origin/main (offline?) — aborting before any change"
|
||||
exit 1
|
||||
fi
|
||||
REMOTE_SHA=$(printf '%s' "$REMOTE_FULL" | cut -c1-12)
|
||||
echo "remote SHA: $REMOTE_SHA"
|
||||
|
||||
# ── 1. CI gate (plan D1) ──────────────────────────────────────────────────
|
||||
# Runs BEFORE pg_dump/pull/build: a red or hung pipeline must not leave the
|
||||
# tree half-deployed. On failure|error it refuses; on success it proceeds; on
|
||||
# "no CI signal at all" (Actions unconfigured / token rejected) it warns and
|
||||
# proceeds rather than bricking every deploy.
|
||||
echo "[1/8] verify CI status for $REMOTE_SHA"
|
||||
verify_ci() {
|
||||
sha=$1
|
||||
if [ -z "$GITEA_URL" ] || [ -z "$GITEA_TOKEN" ]; then
|
||||
echo "SKIP: GITEA_URL/GITEA_TOKEN not set — CI gate disabled. Set both to enforce."
|
||||
return 0
|
||||
fi
|
||||
# Derive owner/repo from the origin remote when the defaults don't apply.
|
||||
origin=$(git remote get-url origin 2>/dev/null || echo "")
|
||||
seg=
|
||||
case "$origin" in
|
||||
*@*:*) # SSH: git@host:owner/repo.git
|
||||
seg=${origin##*:}; seg=${seg%.git}
|
||||
;;
|
||||
http://*|https://*) # HTTPS: scheme://host/owner/repo.git
|
||||
seg=${origin#*://}; seg=${seg#*/}; seg=${seg%.git}
|
||||
;;
|
||||
esac
|
||||
case "$seg" in
|
||||
*/*)
|
||||
GITEA_OWNER=${seg%%/*}
|
||||
GITEA_REPO=${seg#*/}
|
||||
;;
|
||||
esac
|
||||
|
||||
api="$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO/commits/$sha/status"
|
||||
body=$(mktemp)
|
||||
elapsed=0
|
||||
saw_ci=0 # became 1 once we observed a real status (pending/success/...)
|
||||
no_signal=0 # consecutive responses with no usable status
|
||||
while [ "$elapsed" -lt "$CI_TIMEOUT" ]; do
|
||||
# Pass the token via curl --config stdin so it never appears in argv
|
||||
# (visible via ps). Don't use -f: we want the HTTP code on 4xx.
|
||||
code=$(printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" | \
|
||||
curl -sS -o "$body" -w '%{http_code}' --config - "$api" 2>/dev/null) || code="000"
|
||||
state=$(sed -n 's/.*"state"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$body" | head -n1)
|
||||
|
||||
case "$code" in
|
||||
200)
|
||||
case "$state" in
|
||||
success)
|
||||
rm -f "$body"
|
||||
echo "CI: green for $sha after ${elapsed}s"
|
||||
return 0
|
||||
;;
|
||||
failure|error)
|
||||
rm -f "$body"
|
||||
echo "ERROR: CI $state for $sha — refusing to deploy."
|
||||
echo " See $GITEA_URL/$GITEA_OWNER/$GITEA_REPO/actions"
|
||||
return 1
|
||||
;;
|
||||
pending|"")
|
||||
saw_ci=1
|
||||
no_signal=0
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
404)
|
||||
# No status checks exist for this commit (Actions not configured
|
||||
# / no runner has reported). Can't gate — warn + proceed.
|
||||
rm -f "$body"
|
||||
echo "WARN: Gitea has no CI status for $sha (404)."
|
||||
echo " Is Gitea Actions configured with a runner for $GITEA_OWNER/$GITEA_REPO?"
|
||||
echo " Proceeding without a gate."
|
||||
return 0
|
||||
;;
|
||||
401|403)
|
||||
rm -f "$body"
|
||||
echo "WARN: GITEA_TOKEN rejected by Gitea ($code) — cannot verify CI."
|
||||
echo " Fix the token to enforce the gate; proceeding without one."
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
# Network blip / 5xx / 000: retry, but count as no-signal.
|
||||
no_signal=$((no_signal + 1))
|
||||
;;
|
||||
esac
|
||||
|
||||
# If we never get a usable signal after ~1 min, assume CI is unreachable
|
||||
# rather than burn the full timeout and brick deploys.
|
||||
if [ "$saw_ci" -eq 0 ] && [ "$no_signal" -ge 4 ]; then
|
||||
rm -f "$body"
|
||||
echo "WARN: no CI signal from Gitea after ${elapsed}s (last code=$code)."
|
||||
echo " CI may be down or misconfigured; proceeding without a gate."
|
||||
return 0
|
||||
fi
|
||||
|
||||
sleep "$CI_POLL_INTERVAL"
|
||||
elapsed=$((elapsed + CI_POLL_INTERVAL))
|
||||
printf '\rCI: waiting (%ss, code=%s state=%s)...' "$elapsed" "$code" "${state:-none}"
|
||||
done
|
||||
rm -f "$body"
|
||||
echo ""
|
||||
echo "ERROR: CI did not reach a terminal state within ${CI_TIMEOUT}s for $sha — refusing to deploy."
|
||||
return 1
|
||||
}
|
||||
verify_ci "$REMOTE_SHA" || exit 1
|
||||
|
||||
# ── 2. Pre-deploy pg_dump for rollback safety (plan O1) ───────────────────
|
||||
echo "[2/8] pre-deploy pg_dump"
|
||||
DUMP_FILE="$DUMP_DIR/pre-deploy-$REMOTE_SHA.sql"
|
||||
mkdir -p "$DUMP_DIR"
|
||||
docker compose exec -T postgres pg_dump -U oikos oikos > "$DUMP_FILE" 2>/dev/null || \
|
||||
echo "WARNING: pg_dump failed — rollback will not have a recovery point"
|
||||
|
||||
# 2. Pull latest
|
||||
echo "[2/6] git pull"
|
||||
git pull origin main
|
||||
|
||||
# 3. Verify CI passed
|
||||
echo "[3/6] verify build"
|
||||
if ! git log -1 --format="%s" | grep -q .; then
|
||||
echo "ERROR: empty commit message"
|
||||
# ── 3. Update working tree to the verified commit ─────────────────────────
|
||||
echo "[3/8] git pull (ff-only)"
|
||||
git pull --ff-only origin main
|
||||
# TOCTOU guard: if origin/main advanced during the CI wait + pg_dump, the pull
|
||||
# fast-forwards PAST the SHA we verified without re-checking its CI. Refuse
|
||||
# rather than ship an unverified commit — a retry will verify the new tip.
|
||||
PULLED_FULL=$(git rev-parse HEAD)
|
||||
if [ "$PULLED_FULL" != "$REMOTE_FULL" ]; then
|
||||
echo "ERROR: origin/main advanced during deploy (verified $REMOTE_SHA, now at $(git rev-parse --short HEAD)) — aborting; retry verifies the new tip"
|
||||
exit 1
|
||||
fi
|
||||
SHA=$(git rev-parse --short HEAD)
|
||||
echo "SHA (deployed): $SHA"
|
||||
|
||||
# 4. Build and restart with health-check rollout
|
||||
echo "[4/6] docker compose build"
|
||||
# Resolve the deploy version AFTER pull (D2) so the tag matches the code being
|
||||
# built. Compose interpolates $OIKOS_VERSION into each service's image: tag.
|
||||
VERSION_FILE="$REPO_DIR/VERSION"
|
||||
if [ -f "$VERSION_FILE" ]; then
|
||||
OIKOS_VERSION="v$(head -n1 "$VERSION_FILE" | tr -d '[:space:]')"
|
||||
export OIKOS_VERSION
|
||||
echo "VERSION: $OIKOS_VERSION"
|
||||
else
|
||||
echo "WARNING: VERSION file missing — images will use :latest (rollback unavailable)"
|
||||
fi
|
||||
|
||||
# ── 4. Build version-tagged images (plan D2) ──────────────────────────────
|
||||
echo "[4/8] docker compose build"
|
||||
DOCKER_BUILDKIT=1 docker compose --profile "$PROFILE" build \
|
||||
--build-arg BUILDKIT_INLINE_CACHE=1
|
||||
|
||||
# 5. Rolling restart
|
||||
echo "[5/6] docker compose up -d"
|
||||
# ── 5. Rolling restart ────────────────────────────────────────────────────
|
||||
echo "[5/8] docker compose up -d"
|
||||
docker compose --profile "$PROFILE" up -d --remove-orphans
|
||||
|
||||
# 6. Health check wait
|
||||
echo "[6/6] health check"
|
||||
# ── 6. Prune old image tags — keep the 3 newest per service so rollback ────
|
||||
# (OIKOS_VERSION=v0.x.y docker compose up) stays available. The repo list
|
||||
# is derived from compose so it can't drift from the image: names.
|
||||
echo "[6/8] prune old image tags (keep 3)"
|
||||
if [ -n "$OIKOS_VERSION" ]; then
|
||||
images=$(docker compose --profile "$PROFILE" config --images 2>/dev/null || true)
|
||||
if [ -z "$images" ]; then
|
||||
images="oikos-api oikos-scheduler oikos-notifier oikos-migrate oikos-seed oikos-nomos oikos-web"
|
||||
fi
|
||||
printf '%s\n' $images | sed 's/:.*//' | grep '^oikos-' | sort -u | while read -r repo; do
|
||||
docker image ls "$repo" --format '{{.Tag}}' 2>/dev/null | grep '^v' | sort -rV | tail -n +4 | while read -r tag; do
|
||||
docker rmi "$repo:$tag" >/dev/null 2>&1 || true
|
||||
done
|
||||
done
|
||||
fi
|
||||
|
||||
# ── 7. Health check wait ──────────────────────────────────────────────────
|
||||
echo "[7/8] health check"
|
||||
healthy=0
|
||||
for i in $(seq 1 $RETRIES); do
|
||||
if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then
|
||||
echo "healthy after ${i}s"
|
||||
healthy=1
|
||||
break
|
||||
fi
|
||||
sleep "$SLEEP"
|
||||
done
|
||||
if [ "$healthy" -ne 1 ]; then
|
||||
echo "ERROR: health check failed after $((RETRIES * SLEEP))s"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 7. Seed secrets into Infisical (idempotent)
|
||||
echo "[7/7] seed secrets"
|
||||
# ── 8. Seed secrets into Infisical (idempotent) ───────────────────────────
|
||||
echo "[8/8] seed secrets"
|
||||
if [ -f "$REPO_DIR/scripts/seed-secrets.sh" ]; then
|
||||
REPO_DIR="$REPO_DIR" sh "$REPO_DIR/scripts/seed-secrets.sh" || \
|
||||
echo "WARNING: secret seeding failed"
|
||||
else
|
||||
echo "SKIP: seed-secrets.sh not found"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user