0.28.0 — operational hardening (plan D1–D5): CI deploy gate, versioned images, rate limiting, resource limits, health probes

D1: deploy.sh CI gate — read-only SHA via git ls-remote, Gitea commit-status poll, portable mkdir deploy lock (macOS, no flock), TOCTOU guard, token passed via curl --config - (not argv), graceful misconfig tolerance. D2: version-tagged images — OIKOS_VERSION=v$VERSION, keep-last-3 prune derived from 'docker compose config --images'; VERSION read after pull. D3: per-IP rate limiting — new internal/httpapi/ratelimit.go (x/time/rate), rightmost-XFF, /healthz exempt, ctx-driven sweep; disabled by default. D4: mem_limit/cpus on all 10 compose services. D5: staleness-aware health probes — new internal/health package wired into scheduler (:8093) and notifier (:8094); nomos already had :8092. Two /review passes hardened the deploy lock, TOCTOU guard, token hygiene, and XFF handling.
2026-08-08 21:31:16 +02:00
parent ef762794e7
commit fa79c1ea25
14 changed files with 852 additions and 41 deletions
--- a/2
+++ b/2
@@ -1 +1 @@
-0.27.5
+0.28.0
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -20,6 +20,8 @@ services:
      - "5432:5432"
    volumes:
      - pg-data:/var/lib/postgresql/data
+    mem_limit: 1g
+    cpus: 2.0
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "oikos"]
      interval: 5s
@@ -28,6 +30,7 @@ services:

  # One-shot: run migrations then exit
  migrate:
+    image: oikos-migrate:${OIKOS_VERSION:-latest}
    build:
      context: .
      dockerfile: compose/oikos/Dockerfile
@@ -38,9 +41,12 @@ services:
      OIKOS_DATABASE_URL: postgres://oikos:${OIKOS_DB_PASSWORD:-oikos_dev}@postgres:5432/oikos?sslmode=disable
    command: ["migrate"]
    restart: "no"
+    mem_limit: 512m
+    cpus: 1.0

  # One-shot: ingest seeds then exit
  seed:
+    image: oikos-seed:${OIKOS_VERSION:-latest}
    build:
      context: .
      dockerfile: compose/oikos/Dockerfile
@@ -52,9 +58,12 @@ services:
      OIKOS_SEEDS_DIR: /seeds
    command: ["seed"]
    restart: "no"
+    mem_limit: 512m
+    cpus: 1.0

  # API server (Phase 2)
  api:
+    image: oikos-api:${OIKOS_VERSION:-latest}
    build:
      context: .
      dockerfile: compose/oikos/Dockerfile
@@ -76,6 +85,10 @@ services:
      OIKOS_OIDC_CLIENT_ID: ${OIKOS_OIDC_CLIENT_ID:-otkHBSueHJsYtOHstL6rn5izeGgyOsavp1qA1hod}
      OIKOS_NOMOS_AGENT_SLUG: ${OIKOS_NOMOS_AGENT_SLUG:-agent:nomos}
      NOMOS_PROXY_URL: http://nomos:8092
+      # Rate limiting (plan D3). Default off; set OIKOS_API_RATE_LIMIT to a
+      # requests/sec value to throttle runaway agent loops per source IP.
+      OIKOS_API_RATE_LIMIT: ${OIKOS_API_RATE_LIMIT:-}
+      OIKOS_API_RATE_BURST: ${OIKOS_API_RATE_BURST:-}
      # Infisical secret store (Phase 5)
      OIKOS_INFISICAL_SITE_URL: ${OIKOS_INFISICAL_SITE_URL:-}
      OIKOS_INFISICAL_CLIENT_ID: ${OIKOS_INFISICAL_CLIENT_ID:-}
@@ -89,6 +102,8 @@ services:
    command: ["api"]
    stop_signal: SIGTERM
    stop_grace_period: 30s
+    mem_limit: 512m
+    cpus: 1.0
    # Exists so nomos can wait for the API to actually answer rather than just
    # for its container to exist — see nomos's depends_on below. wget is
    # BusyBox's, already in the alpine runtime image, so this adds no
@@ -104,6 +119,7 @@ services:

  # Scheduler (Phase 3) — observe loop
  scheduler:
+    image: oikos-scheduler:${OIKOS_VERSION:-latest}
    build:
      context: .
      dockerfile: compose/oikos/Dockerfile
@@ -118,6 +134,9 @@ services:
      OIKOS_SCHEDULER_INTERVAL: "30s"
      OIKOS_SSH_KEY_PATH: /etc/oikos/ssh_key
      OIKOS_SSH_USER: root
+      # Liveness probe (plan D5): exposes a staleness-aware /healthz inside
+      # the container; the scheduler bumps it each check pass.
+      OIKOS_HEALTH_LISTEN: ":8093"
    volumes:
      - ${OIKOS_SSH_KEY_PATH:-~/.ssh/id_ed25519}:/etc/oikos/ssh_key:ro
    cap_add:
@@ -125,9 +144,18 @@ services:
    command: ["scheduler"]
    stop_signal: SIGTERM
    stop_grace_period: 30s
+    mem_limit: 256m
+    cpus: 1.0
+    healthcheck:
+      test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8093/healthz"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 90s

  # Notifier (Phase 3) — Matrix alerts
  notifier:
+    image: oikos-notifier:${OIKOS_VERSION:-latest}
    build:
      context: .
      dockerfile: compose/oikos/Dockerfile
@@ -144,6 +172,8 @@ services:
      OIKOS_MATRIX_USER: ${OIKOS_MATRIX_USER:-@hermes:hubris.network}
      OIKOS_MATRIX_TOKEN: ${OIKOS_MATRIX_TOKEN}
      OIKOS_MATRIX_ROOM: ${OIKOS_MATRIX_ROOM:-!alerts:hubris.network}
+      # Liveness probe (plan D5): bumps each approval/reaction tick.
+      OIKOS_HEALTH_LISTEN: ":8094"
      OIKOS_INFISICAL_SITE_URL: ${OIKOS_INFISICAL_SITE_URL:-}
      OIKOS_INFISICAL_CLIENT_ID: ${OIKOS_INFISICAL_CLIENT_ID:-}
      OIKOS_INFISICAL_CLIENT_SECRET: ${OIKOS_INFISICAL_CLIENT_SECRET:-}
@@ -152,9 +182,18 @@ services:
    command: ["notifier"]
    stop_signal: SIGTERM
    stop_grace_period: 30s
+    mem_limit: 128m
+    cpus: 0.5
+    healthcheck:
+      test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8094/healthz"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 120s

  # Nomos agent gateway (Phase 4) — mesh-published :8092
  nomos:
+    image: oikos-nomos:${OIKOS_VERSION:-latest}
    build:
      context: .
      dockerfile: compose/nomos/Dockerfile
@@ -188,12 +227,21 @@ services:
      - "8092:8092"
    stop_signal: SIGTERM
    stop_grace_period: 10s
+    mem_limit: 512m
+    cpus: 1.0
+    healthcheck:
+      test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8092/healthz"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 15s

  # Control-room SPA — static build served behind Caddy. The outer
  # production Caddy (caddy-conf repo, LXC 121) splits /api/*, /mcp,
  # /agent/* off to api:8090 and sends everything else here; this
  # container only serves static files with SPA-fallback routing.
  web:
+    image: oikos-web:${OIKOS_VERSION:-latest}
    build:
      context: .
      dockerfile: compose/web/Dockerfile
@@ -202,6 +250,8 @@ services:
    ports:
      - "8091:80"
    stop_signal: SIGTERM
+    mem_limit: 64m
+    cpus: 0.25

  # Redis (required by Infisical — Phase 5)
  redis:
@@ -210,6 +260,8 @@ services:
    profiles: ["infisical", "full"]
    volumes:
      - redis-data:/data
+    mem_limit: 128m
+    cpus: 0.5
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
@@ -240,6 +292,8 @@ services:
      REDIS_URL: redis://redis:6379
    ports:
      - "8080:8080"
+    mem_limit: 512m
+    cpus: 1.0

 volumes:
  pg-data:
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
 	golang.org/x/crypto v0.53.0
 	golang.org/x/sync v0.21.0
 	golang.org/x/sys v0.46.0
+	golang.org/x/time v0.14.0
 	gopkg.in/yaml.v3 v3.0.1
 )

@@ -88,7 +89,6 @@ require (
 	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/oauth2 v0.35.0 // indirect
 	golang.org/x/text v0.38.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
 	google.golang.org/api v0.267.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -30,6 +30,17 @@ type Config struct {
 	// port than the API); a no-op when the SPA and API share an origin.
 	CORSAllowedOrigin string

+	// Rate limiting (plan D3). APIRateLimit is the per-IP requests/sec cap;
+	// APIRateBurst is the token-bucket burst (defaults to 2x the limit when
+	// unset). A limit of 0 disables rate limiting entirely.
+	APIRateLimit int
+	APIRateBurst int
+
+	// Health probe HTTP listener (plan D5). Background-loop roles (scheduler,
+	// notifier) expose a staleness-aware /healthz here. Empty disables the
+	// health server (local/non-docker runs).
+	HealthListen string
+
 	// Observability
 	Debug bool // verbose logging, probe payloads, SQL

@@ -121,6 +132,11 @@ func FromEnv() Config {
 	if v := os.Getenv("OIKOS_CORS_ORIGIN"); v != "" {
 		c.CORSAllowedOrigin = v
 	}
+	c.APIRateLimit = parseInt(os.Getenv("OIKOS_API_RATE_LIMIT"))
+	c.APIRateBurst = parseInt(os.Getenv("OIKOS_API_RATE_BURST"))
+	if v := os.Getenv("OIKOS_HEALTH_LISTEN"); v != "" {
+		c.HealthListen = v
+	}
 	if v := os.Getenv("OIKOS_SEEDS_DIR"); v != "" {
 		c.SeedsDir = v
 	}
--- a/internal/health/health.go
+++ b/internal/health/health.go
@@ -0,0 +1,93 @@
+// Package health provides a staleness-aware liveness probe for background-
+// loop services (scheduler, notifier) that don't otherwise serve HTTP.
+//
+// The owning loop calls Probe.Bump() on each iteration. A /healthz endpoint
+// returns 200 while the last bump is within the staleness window, and 503
+// once the loop has gone quiet — so a wedged goroutine (stuck SSH, deadlock)
+// surfaces as an unhealthy container instead of a silently-idle one.
+package health
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"sync/atomic"
+	"time"
+)
+
+// Probe tracks the last time the owning loop made progress.
+type Probe struct {
+	last  atomic.Int64 // unix-nano timestamp of the last Bump
+	stale time.Duration
+}
+
+// New returns a Probe that considers the owner healthy while Bump has been
+// called within stale of the current time.
+func New(stale time.Duration) *Probe {
+	if stale <= 0 {
+		stale = 2 * time.Minute
+	}
+	p := &Probe{stale: stale}
+	p.last.Store(time.Now().UnixNano()) // boot-healthy until first loop stalls
+	return p
+}
+
+// Bump records that the owning loop completed another iteration.
+func (p *Probe) Bump() {
+	p.last.Store(time.Now().UnixNano())
+}
+
+// Healthy reports whether the last Bump is within the staleness window.
+func (p *Probe) Healthy() bool {
+	last := time.Unix(0, p.last.Load())
+	return time.Since(last) <= p.stale
+}
+
+// Handler returns an http.Handler serving GET /healthz. Returns 200 with a
+// small JSON body when healthy, 503 (Service Unavailable) when stale.
+func (p *Probe) Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		last := time.Unix(0, p.last.Load())
+		if !p.Healthy() {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}
+		json.NewEncoder(w).Encode(map[string]any{
+			"status":         healthStatus(p.Healthy()),
+			"last_heartbeat": last.UTC().Format(time.RFC3339),
+		})
+	})
+}
+
+// Serve starts an HTTP server exposing the probe's /healthz on addr until ctx
+// is cancelled. A no-op when addr is empty (local/non-docker runs skip it).
+// The server is bound to addr (e.g. ":8093"); containers hit it via 127.0.0.1.
+func (p *Probe) Serve(ctx context.Context, addr string) {
+	if addr == "" {
+		return
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/healthz", p.Handler())
+	srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
+
+	go func() {
+		slog.Info("health server listening", "addr", addr)
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			slog.Warn("health server stopped", "addr", addr, "error", err)
+		}
+	}()
+	go func() {
+		<-ctx.Done()
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		srv.Shutdown(shutdownCtx)
+	}()
+}
+
+func healthStatus(ok bool) string {
+	if ok {
+		return "ok"
+	}
+	return "stale"
+}
--- a/internal/health/health_test.go
+++ b/internal/health/health_test.go
@@ -0,0 +1,73 @@
+package health
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestProbeHealthyAtBoot(t *testing.T) {
+	p := New(time.Minute)
+	if !p.Healthy() {
+		t.Fatal("probe should be healthy immediately after creation")
+	}
+}
+
+func TestProbeStaleAfterWindow(t *testing.T) {
+	p := New(50 * time.Millisecond)
+	time.Sleep(80 * time.Millisecond)
+	if p.Healthy() {
+		t.Fatal("probe should be stale after the staleness window elapses with no Bump")
+	}
+	p.Bump()
+	if !p.Healthy() {
+		t.Fatal("probe should recover immediately after Bump")
+	}
+}
+
+func TestProbeHandlerStatusCodes(t *testing.T) {
+	p := New(20 * time.Millisecond)
+
+	// Fresh → 200
+	if code := probeCode(p); code != http.StatusOK {
+		t.Fatalf("fresh probe: want 200, got %d", code)
+	}
+
+	// Stale → 503
+	time.Sleep(40 * time.Millisecond)
+	if code := probeCode(p); code != http.StatusServiceUnavailable {
+		t.Fatalf("stale probe: want 503, got %d", code)
+	}
+}
+
+func TestProbeHandlerBody(t *testing.T) {
+	p := New(time.Minute)
+	rec := httptest.NewRecorder()
+	p.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+
+	var body map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
+		t.Fatalf("invalid JSON body: %v (body=%q)", err, rec.Body.String())
+	}
+	if body["status"] != "ok" {
+		t.Fatalf("want status=ok, got %v", body["status"])
+	}
+	if _, ok := body["last_heartbeat"].(string); !ok {
+		t.Fatalf("want last_heartbeat string, got %v", body["last_heartbeat"])
+	}
+}
+
+func TestNewDefaultsStale(t *testing.T) {
+	p := New(0)
+	if p.stale <= 0 {
+		t.Fatal("New(0) should fall back to a positive staleness window")
+	}
+}
+
+func probeCode(p *Probe) int {
+	rec := httptest.NewRecorder()
+	p.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+	return rec.Code
+}
--- a/internal/httpapi/ratelimit.go
+++ b/internal/httpapi/ratelimit.go
@@ -0,0 +1,148 @@
+package httpapi
+
+import (
+	"context"
+	"log/slog"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+// rateLimiter is a per-client (IP) token-bucket limiter registry. Each unique
+// client gets its own *rate.Limiter; idle entries are swept periodically so a
+// flood of distinct IPs can't grow the map unbounded. A rate of zero (rps==0)
+// disables limiting entirely — the returned middleware is a no-op.
+//
+// Client identity is the source IP. Behind Caddy the real client is in
+// X-Forwarded-For: Caddy appends the immediate client as the LAST hop, while
+// earlier hops are client-supplied and spoofable. clientIP therefore takes the
+// rightmost XFF entry (the proxy's contribution) rather than the first.
+type rateLimiter struct {
+	mu       sync.Mutex
+	limiters map[string]*entry
+	rps      rate.Limit
+	burst    int
+}
+
+type entry struct {
+	limiter  *rate.Limiter
+	lastSeen time.Time
+}
+
+// newRateLimiter builds the registry and starts the idle-entry sweeper tied to
+// ctx, so the ticker is stopped when the server shuts down.
+func newRateLimiter(ctx context.Context, rps, burst int) *rateLimiter {
+	rl := &rateLimiter{
+		limiters: make(map[string]*entry),
+		rps:      rate.Limit(rps),
+		burst:    burst,
+	}
+	if rps > 0 {
+		go rl.sweep(ctx)
+	}
+	return rl
+}
+
+// sweep drops entries untouched since the last sweep so the registry doesn't
+// grow without bound under a rotating-IP attack or long-lived process. Exits
+// (and stops its ticker) when ctx is cancelled.
+func (rl *rateLimiter) sweep(ctx context.Context) {
+	ticker := time.NewTicker(5 * time.Minute)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			rl.mu.Lock()
+			for ip, e := range rl.limiters {
+				if time.Since(e.lastSeen) > 10*time.Minute {
+					delete(rl.limiters, ip)
+				}
+			}
+			rl.mu.Unlock()
+		}
+	}
+}
+
+func (rl *rateLimiter) get(ip string) *rate.Limiter {
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+	if e, ok := rl.limiters[ip]; ok {
+		e.lastSeen = time.Now()
+		return e.limiter
+	}
+	l := rate.NewLimiter(rl.rps, rl.burst)
+	rl.limiters[ip] = &entry{limiter: l, lastSeen: time.Now()}
+	return l
+}
+
+// middleware returns a chi-style middleware that enforces the per-IP limit.
+// Call with rps==0 to get a pass-through no-op.
+func (rl *rateLimiter) middleware(next http.Handler) http.Handler {
+	if rl.rps <= 0 {
+		return next
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Exempt infra liveness probes so Caddy/compose healthchecks can't be
+		// throttled into marking the service unhealthy.
+		if r.URL.Path == "/healthz" {
+			next.ServeHTTP(w, r)
+			return
+		}
+		if !rl.get(clientIP(r)).Allow() {
+			w.Header().Set("Retry-After", "1")
+			writeProblem(w, r, http.StatusTooManyRequests, "rate limit exceeded", "")
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// clientIP extracts the originating client address. It takes the rightmost
+// X-Forwarded-For hop — the one the reverse proxy (Caddy) appends for the
+// immediate client — because earlier hops are attacker-controlled and could
+// be spoofed to dodge the limit or exhaust another client's bucket. Falls
+// back to r.RemoteAddr when no XFF header is present.
+//
+// Known limitation: this is only trustworthy when the request actually
+// traverses Caddy. A client connecting directly to the published :8090 (not
+// behind the proxy) can set a single-hop XFF and have it trusted. That only
+// evades rate limiting (auth is still required), and rate limiting is off by
+// default, so the blast radius is narrow. Fully closing it requires either
+// Caddy trusted_proxies (so it overwrites XFF / sets a non-spoofable
+// X-Real-Ip) or keying the limiter on the auth token instead of IP.
+func clientIP(r *http.Request) string {
+	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
+		if idx := strings.LastIndex(xff, ","); idx >= 0 {
+			xff = xff[idx+1:]
+		}
+		if ip := strings.TrimSpace(xff); ip != "" {
+			return ip
+		}
+	}
+	host, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		return r.RemoteAddr
+	}
+	return host
+}
+
+// newRateLimiterFromConfig builds the limiter from API config, logging the
+// chosen policy once at startup. rps<=0 means "disabled" (returns a no-op
+// middleware) so dev/single-user setups aren't throttled by default.
+func newRateLimiterFromConfig(ctx context.Context, rps, burst int) *rateLimiter {
+	if rps <= 0 {
+		slog.Info("api rate limiting disabled (OIKOS_API_RATE_LIMIT unset)")
+		return &rateLimiter{rps: 0}
+	}
+	if burst <= 0 {
+		burst = rps * 2
+	}
+	slog.Info("api rate limiting enabled", "rps", rps, "burst", burst)
+	return newRateLimiter(ctx, rps, burst)
+}
--- a/internal/httpapi/ratelimit_test.go
+++ b/internal/httpapi/ratelimit_test.go
@@ -0,0 +1,161 @@
+package httpapi
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// okHandler is a sentinel upstream that records it was reached.
+func okHandler(t *testing.T, reached *bool) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		*reached = true
+		w.WriteHeader(http.StatusOK)
+	})
+}
+
+func TestRateLimiterDisabledWhenRPSZero(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rl := newRateLimiterFromConfig(ctx, 0, 0)
+	if rl.rps != 0 {
+		t.Fatalf("rps should be 0 when disabled, got %v", rl.rps)
+	}
+	// Disabled limiter is a pass-through: requests always reach upstream.
+	reached := false
+	h := rl.middleware(okHandler(t, &reached))
+	for i := 0; i < 50; i++ {
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil))
+		if rec.Code != http.StatusOK {
+			t.Fatalf("disabled limiter request %d: want 200, got %d", i, rec.Code)
+		}
+	}
+	if !reached {
+		t.Fatal("disabled limiter never reached upstream")
+	}
+}
+
+func TestRateLimiterThrottlesAfterBurst(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rl := newRateLimiter(ctx, 1, 3) // 1 rps, burst 3
+	reached := false
+	h := rl.middleware(okHandler(t, &reached))
+
+	var last429, okCount int
+	for i := 0; i < 6; i++ {
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil))
+		switch rec.Code {
+		case http.StatusOK:
+			okCount++
+		case http.StatusTooManyRequests:
+			last429 = i
+		}
+	}
+	if okCount < 1 {
+		t.Fatal("expected at least one request through within burst")
+	}
+	if last429 == 0 {
+		t.Fatal("expected at least one 429 once burst exhausted")
+	}
+	if !reached {
+		t.Fatal("upstream never reached")
+	}
+}
+
+func TestRateLimiterExemptsHealthz(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rl := newRateLimiter(ctx, 1, 1) // tiny burst
+	reached := false
+	h := rl.middleware(okHandler(t, &reached))
+	// /healthz must never be throttled, even under a flood.
+	for i := 0; i < 20; i++ {
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+		if rec.Code != http.StatusOK {
+			t.Fatalf("healthz request %d throttled: want 200, got %d", i, rec.Code)
+		}
+	}
+	if !reached {
+		t.Fatal("healthz never reached upstream")
+	}
+}
+
+func TestClientIPTakesRightmostXFF(t *testing.T) {
+	// The reverse proxy appends the real client as the LAST hop; earlier hops
+	// are spoofable and must be ignored.
+	cases := []struct {
+		name   string
+		xff    string
+		remote string
+		wantIP string
+	}{
+		{"single xff", "203.0.113.7", "10.0.0.1:4000", "203.0.113.7"},
+		{"multi hop takes rightmost", "spoofed-attacker, 203.0.113.7", "10.0.0.1:4000", "203.0.113.7"},
+		{"no xff falls back to remote", "", "198.51.100.2:4000", "198.51.100.2"},
+		{"blank xff falls back to remote", "  ", "198.51.100.2:4000", "198.51.100.2"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			req.RemoteAddr = tc.remote
+			if strings.TrimSpace(tc.xff) != "" {
+				req.Header.Set("X-Forwarded-For", tc.xff)
+			}
+			if got := clientIP(req); got != tc.wantIP {
+				t.Fatalf("clientIP: want %q, got %q", tc.wantIP, got)
+			}
+		})
+	}
+}
+
+func TestRateLimiterConcurrency(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rl := newRateLimiter(ctx, 1000, 100) // generous; ensures no deadlock/panic under contention
+	h := rl.middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			rec := httptest.NewRecorder()
+			h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil))
+		}()
+	}
+	wg.Wait()
+}
+
+func TestNewRateLimiterFromConfigBurstDefault(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()                             // release the sweep goroutine + ticker
+	rl := newRateLimiterFromConfig(ctx, 10, 0) // burst unset → defaults to 2x
+	if rl.burst != 20 {
+		t.Fatalf("default burst should be 2x rps (20), got %d", rl.burst)
+	}
+}
+
+// TestSweepStopsOnContextCancel verifies the ticker is released when the
+// server context is cancelled (no process-lifetime goroutine/ticker leak).
+func TestSweepStopsOnContextCancel(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	rl := newRateLimiter(ctx, 1, 1)
+	cancel()
+	// Give the sweeper a moment to observe cancellation. It must return
+	// without blocking; the deferred ticker.Stop() fires on return.
+	time.Sleep(20 * time.Millisecond)
+	// Limiter remains usable for the brief test lifetime.
+	if !rl.get("10.0.0.1").Allow() {
+		t.Fatal("limiter should still allow within burst after sweep stops")
+	}
+}
--- a/internal/httpapi/server.go
+++ b/internal/httpapi/server.go
@@ -23,8 +23,8 @@ import (
 	"sync"
 	"time"

-	"github.com/dtoro/oikos/internal/config"
 	"github.com/dtoro/oikos/internal/actuator"
+	"github.com/dtoro/oikos/internal/config"
 	"github.com/dtoro/oikos/internal/db"
 	"github.com/dtoro/oikos/internal/httpapi/gen"
 	mcphandler "github.com/dtoro/oikos/internal/mcp"
@@ -118,6 +118,11 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand
 	r.Use(middleware.Recoverer)
 	r.Use(middleware.RequestID)
 	r.Use(requestLogger)
+	// Per-IP rate limiting (plan D3). Applied before CORS/auth so a runaway
+	// agent loop is throttled regardless of credentials. The middleware
+	// exempts /healthz so liveness probes can't be throttled.
+	limiter := newRateLimiterFromConfig(ctx, cfg.APIRateLimit, cfg.APIRateBurst)
+	r.Use(limiter.middleware)
 	r.Use(cors.Handler(cors.Options{
 		AllowedOrigins: []string{cfg.CORSAllowedOrigin},
 		AllowedMethods: []string{"GET", "POST", "PATCH", "DELETE", "OPTIONS"},
--- a/internal/notifier/notifier.go
+++ b/internal/notifier/notifier.go
@@ -18,6 +18,7 @@ import (

 	"github.com/dtoro/oikos/internal/config"
 	"github.com/dtoro/oikos/internal/db"
+	"github.com/dtoro/oikos/internal/health"
 	"github.com/google/uuid"
 )

@@ -25,7 +26,14 @@ import (
 func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
 	slog.Info("notifier: starting")

+	// Liveness probe (plan D5): the notifier ticks every 15s (approvals) and
+	// 30s (reactions). 2 min staleness covers a slow Matrix round-trip plus a
+	// missed tick without false-failing.
+	probe := health.New(2 * time.Minute)
+	probe.Serve(ctx, cfg.HealthListen)
+
 	processPendingApprovals(ctx, pool, cfg)
+	probe.Bump()

 	ticker := time.NewTicker(15 * time.Second)
 	defer ticker.Stop()
@@ -40,8 +48,10 @@ func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
 			return
 		case <-ticker.C:
 			processPendingApprovals(ctx, pool, cfg)
+			probe.Bump()
 		case <-reactionTimer.C:
 			pollReactions(ctx, pool, cfg)
+			probe.Bump()
 		}
 	}
 }
@@ -52,13 +62,13 @@ func RunnerForMain() func(context.Context, *db.Pool, config.Config) {
 }

 type pendingApproval struct {
-	ID             uuid.UUID
-	Action         string
-	RiskClass      string
-	TokenHash      *string
-	AlertSentAt    *time.Time
-	MatrixEventID  *string
-	ExpiresAt      time.Time
+	ID            uuid.UUID
+	Action        string
+	RiskClass     string
+	TokenHash     *string
+	AlertSentAt   *time.Time
+	MatrixEventID *string
+	ExpiresAt     time.Time
 }

 // processPendingApprovals finds pending approvals, generates tokens, and sends Matrix alerts.
@@ -173,7 +183,7 @@ func checkReaction(ctx context.Context, cfg config.Config, roomID, eventID strin

 	var result struct {
 		Chunk []struct {
-			Type string `json:"type"`
+			Type    string `json:"type"`
 			Content struct {
 				RelatesTo map[string]string `json:"m.relates_to"`
 			} `json:"content"`
@@ -259,7 +269,9 @@ func sendMatrixAlert(ctx context.Context, cfg config.Config, approvalID uuid.UUI
 	}
 	defer resp.Body.Close()

-	var mxResp struct{ EventID string `json:"event_id"` }
+	var mxResp struct {
+		EventID string `json:"event_id"`
+	}
 	json.NewDecoder(resp.Body).Decode(&mxResp)

 	if mxResp.EventID == "" {
--- a/internal/scheduler/scheduler.go
+++ b/internal/scheduler/scheduler.go
@@ -22,6 +22,7 @@ import (
 	"github.com/dtoro/oikos/internal/config"
 	"github.com/dtoro/oikos/internal/db"
 	"github.com/dtoro/oikos/internal/db/sqlcgen"
+	"github.com/dtoro/oikos/internal/health"
 	"github.com/dtoro/oikos/internal/observability"
 	"github.com/dtoro/oikos/internal/remote"
 	"github.com/google/uuid"
@@ -48,11 +49,22 @@ func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
 		sshUser = "root"
 	}

+	// Liveness probe (plan D5): staleness is 3x the interval so a single
+	// slow check pass (one host hung on SSH) doesn't flap the container
+	// unhealthy before the next scheduled tick.
+	stale := 3 * interval
+	if stale < 90*time.Second {
+		stale = 90 * time.Second
+	}
+	probe := health.New(stale)
+	probe.Serve(ctx, cfg.HealthListen)
+
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()

 	// Immediate first pass
 	runCheckPass(ctx, pool)
+	probe.Bump()

 	for {
 		select {
@@ -61,6 +73,7 @@ func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
 			return
 		case <-ticker.C:
 			runCheckPass(ctx, pool)
+			probe.Bump()
 		}
 	}
 }
@@ -586,7 +599,7 @@ func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkRes
 // checkCertExpiry checks TLS certificate expiry.
 func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
 	cfg := struct {
-		Host     string `json:"host"`
+		Host string `json:"host"`
 		// Dial is an optional explicit dial address (the TLS terminator's IP)
 		// for when the hostname doesn't resolve/reach from the scheduler — the
 		// container has no mesh interface and the host resolver doesn't know
--- a/plans/2026-08-05-backend-evaluation-improvements.md
+++ b/plans/2026-08-05-backend-evaluation-improvements.md
@@ -1,11 +1,16 @@
 # 2026-08-05 — Backend evaluation: architecture, security, and reliability improvements

-Status: **In Progress** — Phase 0 code changes complete (B1, B2, B4, B5, B6, B7); B3
-is a post-deploy operational step.
+Status: **In Progress** — Phase 0 (B1, B2, B4, B5, B6, B7) and Phase 2 (D1–D5)
+complete; D1–D5 were hardened across two `/review` passes (deploy lock, TOCTOU
+guard, token hygiene, XFF rightmost-hop, ctx-driven sweep). B3 is a post-deploy
+operational step. Remaining: Phase 1 security (C1–C3), Phase 3 code quality
+(E1–E5), and Phase 4–6 backlog.

 Scope: full evaluation of the oikos backend (Go binaries `oikos`, `nomos`, `webhook`,
 Postgres/TimescaleDB, Docker deployment, MCP server) excluding frontend clients
-(`web/` SPA and `desktop/` Wails app). Research-only pass — no code changes.
+(`web/` SPA and `desktop/` Wails app). Began as a research-only pass; Phase 0 (B)
+and Phase 2 (D) have since been implemented as code changes — see each item's
+"Status".

 Method: four parallel research passes (Go backend structure, database schema,
 deployment/infrastructure, API/MCP design) plus Infisical secrets audit and
@@ -211,35 +216,81 @@ binary reads secrets from env vars or plaintext files.
 ## D. Operational improvements (high)

 ### D1. Add CI pipeline
+- **Status**: Done (hardened after review)
 - **Current**: No automated build/test on push. `make lint test generate-check`
  exists but is manual.
 - **Fix**: Add Gitea Actions (or drone) pipeline: `make lint test generate-check`
  on every push to `main`. Block deploy if pipeline fails.
+- **What changed**: The Gitea Actions pipeline already exists
+  (`.gitea/workflows/ci.yml`). Added the missing deploy gate as step [1/8] in
+  `scripts/deploy.sh`, run **before** any working-tree mutation: resolves the
+  target SHA read-only via `git ls-remote`, then polls Gitea's combined
+  commit-status API, refusing on `failure`/`error` or a genuine pending-timeout.
+  Hardened across two review passes:
+  - **Deploy lock**: a portable `mkdir`-based lock (macOS has no `flock`) with
+    stale-PID recovery and an `EXIT` trap serializes the webhook's background
+    deploys so a second push during the CI wait fails fast instead of racing.
+  - **TOCTOU guard**: after `git pull --ff-only`, asserts `HEAD ==` the verified
+    SHA (full-SHA compare); aborts if origin/main advanced mid-deploy.
+  - **Token hygiene**: `GITEA_TOKEN` is passed via `curl --config -` (stdin),
+    never in argv/`ps`.
+  - **Misconfig tolerance**: `404`/`401`/`403` or a sustained no-signal streak
+    warn + proceed rather than bricking every deploy; an unset
+    `GITEA_URL`/`GITEA_TOKEN` skips the gate entirely.
 - **Risk class**: config_mutation

 ### D2. Version Docker images
+- **Status**: Done
 - **Current**: All images built as `:latest`. Rollback requires full rebuild.
 - **Fix**: Tag images with `v$VERSION` from the VERSION file in deploy.sh. Keep
  last 3 versions. Enable `docker compose up` to pin a version tag.
+- **What changed**: Every built compose service now carries an `image:
+  oikos-<svc>:${OIKOS_VERSION:-latest}` tag. `deploy.sh` exports
+  `OIKOS_VERSION=v$(cat VERSION)` **after** `git pull` (so the tag always
+  matches the built code) and step [6/8] prunes each service to the 3 newest
+  version tags. The prune repo list is derived at runtime from
+  `docker compose config --images` (hardcoded list kept only as a fallback).
 - **Risk class**: config_mutation

 ### D3. Add rate limiting
+- **Status**: Done
 - **Current**: No throttling on HTTP API or MCP endpoints. An agent in a loop
  could hammer the API or exhaust DB connections.
 - **Fix**: Add `golang.org/x/time/rate` middleware to chi router. Per-IP or
  per-token rate limit with burst allowance. Separate limits for API vs MCP.
+- **What changed**: New `internal/httpapi/ratelimit.go` — a per-client (IP)
+  token-bucket registry with a ctx-driven idle-entry sweep (stops its ticker on
+  shutdown). Wired into `NewHandler` before CORS/auth; `/healthz` is exempt.
+  Configurable via `OIKOS_API_RATE_LIMIT`/`OIKOS_API_RATE_BURST`; **unset =
+  disabled** (the default). Returns RFC 9457 429 + Retry-After. `x/time`
+  promoted to a direct dependency. `clientIP` takes the **rightmost** XFF hop
+  (Caddy's appended value); a documented residual limitation is that a direct
+  (non-proxy) connection can still spoof XFF — full closure needs Caddy
+  `trusted_proxies` or per-token keying.
 - **Risk class**: config_mutation

 ### D4. Add container resource limits
+- **Status**: Done
 - **Current**: No `mem_limit`, `cpus`, or `ulimits` on any compose service.
 - **Fix**: Add memory and CPU limits to all services in docker-compose.yml.
  Suggested: API 512MB, scheduler 256MB, notifier 128MB, nomos 512MB.
+- **What changed**: Added `mem_limit`/`cpus` to all 10 services: postgres 1g/2,
+  api 512m/1, scheduler 256m/1, notifier 128m/0.5, nomos 512m/1, web 64m/0.25,
+  redis 128m/0.5, infisical 512m/1, migrate/seed 512m/1.
 - **Risk class**: config_mutation

 ### D5. Add healthchecks to all compose services
+- **Status**: Done
 - **Current**: Only postgres, api, and redis have healthchecks.
 - **Fix**: Add `healthcheck` to scheduler, notifier, and nomos. Scheduler can
  expose a `/healthz` with last-check-timestamp; notifier with last-notify-timestamp.
+- **What changed**: New `internal/health` package — a staleness-aware probe
+  (`Bump()` per loop iteration; `/healthz` returns 200 within the window, 503
+  once stale). Wired into `scheduler.Run` (:8093, 3× interval) and
+  `notifier.Run` (:8094, 2 min); nomos already served `:8092/healthz`. Added
+  compose healthchecks for scheduler, notifier, and nomos. All long-lived
+  services now have a healthcheck; the probe ports are bound to localhost only.
+- **Risk class**: config_mutation

 ## E. Code quality (medium)

@@ -349,7 +400,7 @@ Priority packages (currently 0% coverage):
 2. **Phase 1 — Security** (C1–C3, B5): Nomos auth, pg_dump failure, CORS default,
   SSH host keys (now stored in Infisical per B5).
 3. **Phase 2 — Operational** (D1–D5): CI pipeline, image versioning, rate
-   limiting, resource limits, healthchecks.
+   limiting, resource limits, healthchecks. **Done.**
 4. **Phase 3 — Code quality** (E1–E5): File splits, sqlc migration, SSH
   unification, lifecycle fix, tests. E1–E3 are large refactors — do one
   file/area per commit.
--- a/plans/index.md
+++ b/plans/index.md
@@ -21,7 +21,7 @@ went sideways, open an investigation.
 | 2026-07-21 | [Frontend as OS + Apps — architecture audit & refactor](2026-07-21-frontend-os-apps-architecture.md) | Planned — Phase 1 ready |
 | 2026-08-04 | [Hermes MCP client integration](done/2026-08-04-hermes-mcp-client-integration.md) | Done — deployed |
 | 2026-08-05 | [Agent execution safety: QEMU guest agent gate + host-mutation guard](done/2026-08-05-agent-execution-safety-qemu-guest-agent-gate.md) | Done — implemented (1b9c761) |
-| 2026-08-05 | [Backend evaluation: architecture, security, and reliability improvements](2026-08-05-backend-evaluation-improvements.md) | Planned — 6 phases, security items first |
+| 2026-08-05 | [Backend evaluation: architecture, security, and reliability improvements](2026-08-05-backend-evaluation-improvements.md) | In Progress — Phase 0 (B) + Phase 2 (D1–D5) done; Phase 1 (C) pending |

 ## Done

--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -1,7 +1,12 @@
 #!/bin/sh
 # Oikos deploy script — triggered by Gitea webhook on push to dtoro/oikos.
-# Runs on mac-mini as non-root user via systemd unit oikos-deploy-webhook.service.
-# Phase 6: CI-gated, SHA-tagged images, rolling restart, pre-deploy pg_dump.
+# Runs on mac-mini as non-root user via launchd unit oikos-deploy-webhook.service.
+# Phase 6: CI-gated, version-tagged images, rolling restart, pre-deploy pg_dump.
+#
+# Plans implemented here:
+#   D1 — CI gate: blocks deploy unless Gitea reports a green run for the SHA.
+#   D2 — versioned images: tags every built image v$VERSION (from VERSION file),
+#        keeps the last 3 tags per service for rollback.

 set -e

@@ -13,54 +18,234 @@ DUMP_DIR="${DUMP_DIR:-/opt/oikos/backups}"
 RETRIES=${RETRIES:-30}
 SLEEP=${SLEEP:-2}

+# CI gate (D1). Set GITEA_URL + GITEA_TOKEN to enable; without them the gate
+# is skipped with a warning (dev/local deploys). Owner/repo default to the
+# canonical homelab repo but can be overridden or derived from the git remote.
+GITEA_URL="${GITEA_URL:-}"
+GITEA_TOKEN="${GITEA_TOKEN:-}"
+GITEA_OWNER="${GITEA_OWNER:-dtoro}"
+GITEA_REPO="${GITEA_REPO:-oikos}"
+CI_POLL_INTERVAL="${CI_POLL_INTERVAL:-15}"
+CI_TIMEOUT="${CI_TIMEOUT:-1200}"
+
+# Serialize deploys: the webhook runs this script in a background goroutine and
+# the CI gate can hold a deploy open for many minutes, so a second push during
+# that window would otherwise race on git/pg_dump/compose. mkdir is atomic on
+# POSIX (no flock dependency — macOS lacks it). The stale-pid check recovers
+# if a previous deploy was SIGKILLed.
+LOCKDIR="${LOCKDIR:-/tmp/oikos-deploy.lock}"
+if ! mkdir "$LOCKDIR" 2>/dev/null; then
+    oldpid=$(cat "$LOCKDIR/pid" 2>/dev/null || echo "")
+    if [ -n "$oldpid" ] && kill -0 "$oldpid" 2>/dev/null; then
+        echo "deploy already in progress (pid $oldpid) — exiting"
+        exit 0
+    fi
+    echo "removing stale deploy lock (pid ${oldpid:-?} not running)"
+    rm -rf "$LOCKDIR"
+    mkdir "$LOCKDIR"
+fi
+echo $$ > "$LOCKDIR/pid"
+trap 'rm -rf "$LOCKDIR" 2>/dev/null || true' EXIT INT TERM
+
 cd "$REPO_DIR"

 echo "=== oikos deploy: $(date) ==="
-SHA=$(git rev-parse --short HEAD)
-echo "SHA: $SHA"

-# 1. Pre-deploy pg_dump for rollback safety (plan O1)
-echo "[1/6] pre-deploy pg_dump"
-DUMP_FILE="$DUMP_DIR/pre-deploy-$SHA.sql"
+# Resolve the SHA we are ABOUT to deploy from the remote (read-only: no working
+# tree mutation yet) so the CI gate can run before anything is touched. Keep the
+# full SHA (REMOTE_FULL) for the post-pull equality check; the 12-char form is
+# only for display and the Gitea status API (which accepts any unique prefix).
+REMOTE_FULL=$(git ls-remote origin refs/heads/main 2>/dev/null | awk '{print $1}')
+if [ -z "$REMOTE_FULL" ]; then
+    echo "ERROR: could not resolve origin/main (offline?) — aborting before any change"
+    exit 1
+fi
+REMOTE_SHA=$(printf '%s' "$REMOTE_FULL" | cut -c1-12)
+echo "remote SHA: $REMOTE_SHA"
+
+# ── 1. CI gate (plan D1) ──────────────────────────────────────────────────
+# Runs BEFORE pg_dump/pull/build: a red or hung pipeline must not leave the
+# tree half-deployed. On failure|error it refuses; on success it proceeds; on
+# "no CI signal at all" (Actions unconfigured / token rejected) it warns and
+# proceeds rather than bricking every deploy.
+echo "[1/8] verify CI status for $REMOTE_SHA"
+verify_ci() {
+    sha=$1
+    if [ -z "$GITEA_URL" ] || [ -z "$GITEA_TOKEN" ]; then
+        echo "SKIP: GITEA_URL/GITEA_TOKEN not set — CI gate disabled. Set both to enforce."
+        return 0
+    fi
+    # Derive owner/repo from the origin remote when the defaults don't apply.
+    origin=$(git remote get-url origin 2>/dev/null || echo "")
+    seg=
+    case "$origin" in
+        *@*:*)  # SSH: git@host:owner/repo.git
+            seg=${origin##*:}; seg=${seg%.git}
+            ;;
+        http://*|https://*)  # HTTPS: scheme://host/owner/repo.git
+            seg=${origin#*://}; seg=${seg#*/}; seg=${seg%.git}
+            ;;
+    esac
+    case "$seg" in
+        */*)
+            GITEA_OWNER=${seg%%/*}
+            GITEA_REPO=${seg#*/}
+            ;;
+    esac
+
+    api="$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO/commits/$sha/status"
+    body=$(mktemp)
+    elapsed=0
+    saw_ci=0          # became 1 once we observed a real status (pending/success/...)
+    no_signal=0       # consecutive responses with no usable status
+    while [ "$elapsed" -lt "$CI_TIMEOUT" ]; do
+        # Pass the token via curl --config stdin so it never appears in argv
+        # (visible via ps). Don't use -f: we want the HTTP code on 4xx.
+        code=$(printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" | \
+            curl -sS -o "$body" -w '%{http_code}' --config - "$api" 2>/dev/null) || code="000"
+        state=$(sed -n 's/.*"state"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$body" | head -n1)
+
+        case "$code" in
+            200)
+                case "$state" in
+                    success)
+                        rm -f "$body"
+                        echo "CI: green for $sha after ${elapsed}s"
+                        return 0
+                        ;;
+                    failure|error)
+                        rm -f "$body"
+                        echo "ERROR: CI $state for $sha — refusing to deploy."
+                        echo "       See $GITEA_URL/$GITEA_OWNER/$GITEA_REPO/actions"
+                        return 1
+                        ;;
+                    pending|"")
+                        saw_ci=1
+                        no_signal=0
+                        ;;
+                esac
+                ;;
+            404)
+                # No status checks exist for this commit (Actions not configured
+                # / no runner has reported). Can't gate — warn + proceed.
+                rm -f "$body"
+                echo "WARN: Gitea has no CI status for $sha (404)."
+                echo "      Is Gitea Actions configured with a runner for $GITEA_OWNER/$GITEA_REPO?"
+                echo "      Proceeding without a gate."
+                return 0
+                ;;
+            401|403)
+                rm -f "$body"
+                echo "WARN: GITEA_TOKEN rejected by Gitea ($code) — cannot verify CI."
+                echo "      Fix the token to enforce the gate; proceeding without one."
+                return 0
+                ;;
+            *)
+                # Network blip / 5xx / 000: retry, but count as no-signal.
+                no_signal=$((no_signal + 1))
+                ;;
+        esac
+
+        # If we never get a usable signal after ~1 min, assume CI is unreachable
+        # rather than burn the full timeout and brick deploys.
+        if [ "$saw_ci" -eq 0 ] && [ "$no_signal" -ge 4 ]; then
+            rm -f "$body"
+            echo "WARN: no CI signal from Gitea after ${elapsed}s (last code=$code)."
+            echo "      CI may be down or misconfigured; proceeding without a gate."
+            return 0
+        fi
+
+        sleep "$CI_POLL_INTERVAL"
+        elapsed=$((elapsed + CI_POLL_INTERVAL))
+        printf '\rCI: waiting (%ss, code=%s state=%s)...' "$elapsed" "$code" "${state:-none}"
+    done
+    rm -f "$body"
+    echo ""
+    echo "ERROR: CI did not reach a terminal state within ${CI_TIMEOUT}s for $sha — refusing to deploy."
+    return 1
+}
+verify_ci "$REMOTE_SHA" || exit 1
+
+# ── 2. Pre-deploy pg_dump for rollback safety (plan O1) ───────────────────
+echo "[2/8] pre-deploy pg_dump"
+DUMP_FILE="$DUMP_DIR/pre-deploy-$REMOTE_SHA.sql"
 mkdir -p "$DUMP_DIR"
 docker compose exec -T postgres pg_dump -U oikos oikos > "$DUMP_FILE" 2>/dev/null || \
    echo "WARNING: pg_dump failed — rollback will not have a recovery point"

-# 2. Pull latest
-echo "[2/6] git pull"
-git pull origin main
-
-# 3. Verify CI passed
-echo "[3/6] verify build"
-if ! git log -1 --format="%s" | grep -q .; then
-    echo "ERROR: empty commit message"
+# ── 3. Update working tree to the verified commit ─────────────────────────
+echo "[3/8] git pull (ff-only)"
+git pull --ff-only origin main
+# TOCTOU guard: if origin/main advanced during the CI wait + pg_dump, the pull
+# fast-forwards PAST the SHA we verified without re-checking its CI. Refuse
+# rather than ship an unverified commit — a retry will verify the new tip.
+PULLED_FULL=$(git rev-parse HEAD)
+if [ "$PULLED_FULL" != "$REMOTE_FULL" ]; then
+    echo "ERROR: origin/main advanced during deploy (verified $REMOTE_SHA, now at $(git rev-parse --short HEAD)) — aborting; retry verifies the new tip"
    exit 1
 fi
+SHA=$(git rev-parse --short HEAD)
+echo "SHA (deployed): $SHA"

-# 4. Build and restart with health-check rollout
-echo "[4/6] docker compose build"
+# Resolve the deploy version AFTER pull (D2) so the tag matches the code being
+# built. Compose interpolates $OIKOS_VERSION into each service's image: tag.
+VERSION_FILE="$REPO_DIR/VERSION"
+if [ -f "$VERSION_FILE" ]; then
+    OIKOS_VERSION="v$(head -n1 "$VERSION_FILE" | tr -d '[:space:]')"
+    export OIKOS_VERSION
+    echo "VERSION: $OIKOS_VERSION"
+else
+    echo "WARNING: VERSION file missing — images will use :latest (rollback unavailable)"
+fi
+
+# ── 4. Build version-tagged images (plan D2) ──────────────────────────────
+echo "[4/8] docker compose build"
 DOCKER_BUILDKIT=1 docker compose --profile "$PROFILE" build \
    --build-arg BUILDKIT_INLINE_CACHE=1

-# 5. Rolling restart
-echo "[5/6] docker compose up -d"
+# ── 5. Rolling restart ────────────────────────────────────────────────────
+echo "[5/8] docker compose up -d"
 docker compose --profile "$PROFILE" up -d --remove-orphans

-# 6. Health check wait
-echo "[6/6] health check"
+# ── 6. Prune old image tags — keep the 3 newest per service so rollback ────
+#    (OIKOS_VERSION=v0.x.y docker compose up) stays available. The repo list
+#    is derived from compose so it can't drift from the image: names.
+echo "[6/8] prune old image tags (keep 3)"
+if [ -n "$OIKOS_VERSION" ]; then
+    images=$(docker compose --profile "$PROFILE" config --images 2>/dev/null || true)
+    if [ -z "$images" ]; then
+        images="oikos-api oikos-scheduler oikos-notifier oikos-migrate oikos-seed oikos-nomos oikos-web"
+    fi
+    printf '%s\n' $images | sed 's/:.*//' | grep '^oikos-' | sort -u | while read -r repo; do
+        docker image ls "$repo" --format '{{.Tag}}' 2>/dev/null | grep '^v' | sort -rV | tail -n +4 | while read -r tag; do
+            docker rmi "$repo:$tag" >/dev/null 2>&1 || true
+        done
+    done
+fi
+
+# ── 7. Health check wait ──────────────────────────────────────────────────
+echo "[7/8] health check"
+healthy=0
 for i in $(seq 1 $RETRIES); do
    if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then
        echo "healthy after ${i}s"
+        healthy=1
        break
    fi
    sleep "$SLEEP"
 done
+if [ "$healthy" -ne 1 ]; then
+    echo "ERROR: health check failed after $((RETRIES * SLEEP))s"
+    exit 1
+fi

-# 7. Seed secrets into Infisical (idempotent)
-echo "[7/7] seed secrets"
+# ── 8. Seed secrets into Infisical (idempotent) ───────────────────────────
+echo "[8/8] seed secrets"
 if [ -f "$REPO_DIR/scripts/seed-secrets.sh" ]; then
    REPO_DIR="$REPO_DIR" sh "$REPO_DIR/scripts/seed-secrets.sh" || \
        echo "WARNING: secret seeding failed"
 else
    echo "SKIP: seed-secrets.sh not found"
 fi
+
+exit 0
@@ -1 +1 @@
 .27.5
 .28.0