0.28.0 — operational hardening (plan D1–D5): CI deploy gate, versioned images, rate limiting, resource limits, health probes

D1: deploy.sh CI gate — read-only SHA via git ls-remote, Gitea commit-status
    poll, portable mkdir deploy lock (macOS, no flock), TOCTOU guard, token
    passed via curl --config - (not argv), graceful misconfig tolerance.
D2: version-tagged images — OIKOS_VERSION=v$VERSION, keep-last-3 prune derived
    from 'docker compose config --images'; VERSION read after pull.
D3: per-IP rate limiting — new internal/httpapi/ratelimit.go (x/time/rate),
    rightmost-XFF, /healthz exempt, ctx-driven sweep; disabled by default.
D4: mem_limit/cpus on all 10 compose services.
D5: staleness-aware health probes — new internal/health package wired into
    scheduler (:8093) and notifier (:8094); nomos already had :8092.

Two /review passes hardened the deploy lock, TOCTOU guard, token hygiene,
and XFF handling.

internal/httpapi/ratelimit.go

@@ -0,0 +1,148 @@
+package httpapi
+
+import (
+	"context"
+	"log/slog"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+// rateLimiter is a per-client (IP) token-bucket limiter registry. Each unique
+// client gets its own *rate.Limiter; idle entries are swept periodically so a
+// flood of distinct IPs can't grow the map unbounded. A rate of zero (rps==0)
+// disables limiting entirely — the returned middleware is a no-op.
+//
+// Client identity is the source IP. Behind Caddy the real client is in
+// X-Forwarded-For: Caddy appends the immediate client as the LAST hop, while
+// earlier hops are client-supplied and spoofable. clientIP therefore takes the
+// rightmost XFF entry (the proxy's contribution) rather than the first.
+type rateLimiter struct {
+	mu       sync.Mutex
+	limiters map[string]*entry
+	rps      rate.Limit
+	burst    int
+}
+
+type entry struct {
+	limiter  *rate.Limiter
+	lastSeen time.Time
+}
+
+// newRateLimiter builds the registry and starts the idle-entry sweeper tied to
+// ctx, so the ticker is stopped when the server shuts down.
+func newRateLimiter(ctx context.Context, rps, burst int) *rateLimiter {
+	rl := &rateLimiter{
+		limiters: make(map[string]*entry),
+		rps:      rate.Limit(rps),
+		burst:    burst,
+	}
+	if rps > 0 {
+		go rl.sweep(ctx)
+	}
+	return rl
+}
+
+// sweep drops entries untouched since the last sweep so the registry doesn't
+// grow without bound under a rotating-IP attack or long-lived process. Exits
+// (and stops its ticker) when ctx is cancelled.
+func (rl *rateLimiter) sweep(ctx context.Context) {
+	ticker := time.NewTicker(5 * time.Minute)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			rl.mu.Lock()
+			for ip, e := range rl.limiters {
+				if time.Since(e.lastSeen) > 10*time.Minute {
+					delete(rl.limiters, ip)
+				}
+			}
+			rl.mu.Unlock()
+		}
+	}
+}
+
+func (rl *rateLimiter) get(ip string) *rate.Limiter {
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+	if e, ok := rl.limiters[ip]; ok {
+		e.lastSeen = time.Now()
+		return e.limiter
+	}
+	l := rate.NewLimiter(rl.rps, rl.burst)
+	rl.limiters[ip] = &entry{limiter: l, lastSeen: time.Now()}
+	return l
+}
+
+// middleware returns a chi-style middleware that enforces the per-IP limit.
+// Call with rps==0 to get a pass-through no-op.
+func (rl *rateLimiter) middleware(next http.Handler) http.Handler {
+	if rl.rps <= 0 {
+		return next
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Exempt infra liveness probes so Caddy/compose healthchecks can't be
+		// throttled into marking the service unhealthy.
+		if r.URL.Path == "/healthz" {
+			next.ServeHTTP(w, r)
+			return
+		}
+		if !rl.get(clientIP(r)).Allow() {
+			w.Header().Set("Retry-After", "1")
+			writeProblem(w, r, http.StatusTooManyRequests, "rate limit exceeded", "")
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// clientIP extracts the originating client address. It takes the rightmost
+// X-Forwarded-For hop — the one the reverse proxy (Caddy) appends for the
+// immediate client — because earlier hops are attacker-controlled and could
+// be spoofed to dodge the limit or exhaust another client's bucket. Falls
+// back to r.RemoteAddr when no XFF header is present.
+//
+// Known limitation: this is only trustworthy when the request actually
+// traverses Caddy. A client connecting directly to the published :8090 (not
+// behind the proxy) can set a single-hop XFF and have it trusted. That only
+// evades rate limiting (auth is still required), and rate limiting is off by
+// default, so the blast radius is narrow. Fully closing it requires either
+// Caddy trusted_proxies (so it overwrites XFF / sets a non-spoofable
+// X-Real-Ip) or keying the limiter on the auth token instead of IP.
+func clientIP(r *http.Request) string {
+	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
+		if idx := strings.LastIndex(xff, ","); idx >= 0 {
+			xff = xff[idx+1:]
+		}
+		if ip := strings.TrimSpace(xff); ip != "" {
+			return ip
+		}
+	}
+	host, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		return r.RemoteAddr
+	}
+	return host
+}
+
+// newRateLimiterFromConfig builds the limiter from API config, logging the
+// chosen policy once at startup. rps<=0 means "disabled" (returns a no-op
+// middleware) so dev/single-user setups aren't throttled by default.
+func newRateLimiterFromConfig(ctx context.Context, rps, burst int) *rateLimiter {
+	if rps <= 0 {
+		slog.Info("api rate limiting disabled (OIKOS_API_RATE_LIMIT unset)")
+		return &rateLimiter{rps: 0}
+	}
+	if burst <= 0 {
+		burst = rps * 2
+	}
+	slog.Info("api rate limiting enabled", "rps", rps, "burst", burst)
+	return newRateLimiter(ctx, rps, burst)
+}

internal/httpapi/ratelimit_test.go

@@ -0,0 +1,161 @@
+package httpapi
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// okHandler is a sentinel upstream that records it was reached.
+func okHandler(t *testing.T, reached *bool) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		*reached = true
+		w.WriteHeader(http.StatusOK)
+	})
+}
+
+func TestRateLimiterDisabledWhenRPSZero(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rl := newRateLimiterFromConfig(ctx, 0, 0)
+	if rl.rps != 0 {
+		t.Fatalf("rps should be 0 when disabled, got %v", rl.rps)
+	}
+	// Disabled limiter is a pass-through: requests always reach upstream.
+	reached := false
+	h := rl.middleware(okHandler(t, &reached))
+	for i := 0; i < 50; i++ {
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil))
+		if rec.Code != http.StatusOK {
+			t.Fatalf("disabled limiter request %d: want 200, got %d", i, rec.Code)
+		}
+	}
+	if !reached {
+		t.Fatal("disabled limiter never reached upstream")
+	}
+}
+
+func TestRateLimiterThrottlesAfterBurst(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rl := newRateLimiter(ctx, 1, 3) // 1 rps, burst 3
+	reached := false
+	h := rl.middleware(okHandler(t, &reached))
+
+	var last429, okCount int
+	for i := 0; i < 6; i++ {
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil))
+		switch rec.Code {
+		case http.StatusOK:
+			okCount++
+		case http.StatusTooManyRequests:
+			last429 = i
+		}
+	}
+	if okCount < 1 {
+		t.Fatal("expected at least one request through within burst")
+	}
+	if last429 == 0 {
+		t.Fatal("expected at least one 429 once burst exhausted")
+	}
+	if !reached {
+		t.Fatal("upstream never reached")
+	}
+}
+
+func TestRateLimiterExemptsHealthz(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rl := newRateLimiter(ctx, 1, 1) // tiny burst
+	reached := false
+	h := rl.middleware(okHandler(t, &reached))
+	// /healthz must never be throttled, even under a flood.
+	for i := 0; i < 20; i++ {
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+		if rec.Code != http.StatusOK {
+			t.Fatalf("healthz request %d throttled: want 200, got %d", i, rec.Code)
+		}
+	}
+	if !reached {
+		t.Fatal("healthz never reached upstream")
+	}
+}
+
+func TestClientIPTakesRightmostXFF(t *testing.T) {
+	// The reverse proxy appends the real client as the LAST hop; earlier hops
+	// are spoofable and must be ignored.
+	cases := []struct {
+		name   string
+		xff    string
+		remote string
+		wantIP string
+	}{
+		{"single xff", "203.0.113.7", "10.0.0.1:4000", "203.0.113.7"},
+		{"multi hop takes rightmost", "spoofed-attacker, 203.0.113.7", "10.0.0.1:4000", "203.0.113.7"},
+		{"no xff falls back to remote", "", "198.51.100.2:4000", "198.51.100.2"},
+		{"blank xff falls back to remote", "  ", "198.51.100.2:4000", "198.51.100.2"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			req.RemoteAddr = tc.remote
+			if strings.TrimSpace(tc.xff) != "" {
+				req.Header.Set("X-Forwarded-For", tc.xff)
+			}
+			if got := clientIP(req); got != tc.wantIP {
+				t.Fatalf("clientIP: want %q, got %q", tc.wantIP, got)
+			}
+		})
+	}
+}
+
+func TestRateLimiterConcurrency(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rl := newRateLimiter(ctx, 1000, 100) // generous; ensures no deadlock/panic under contention
+	h := rl.middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			rec := httptest.NewRecorder()
+			h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil))
+		}()
+	}
+	wg.Wait()
+}
+
+func TestNewRateLimiterFromConfigBurstDefault(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()                             // release the sweep goroutine + ticker
+	rl := newRateLimiterFromConfig(ctx, 10, 0) // burst unset → defaults to 2x
+	if rl.burst != 20 {
+		t.Fatalf("default burst should be 2x rps (20), got %d", rl.burst)
+	}
+}
+
+// TestSweepStopsOnContextCancel verifies the ticker is released when the
+// server context is cancelled (no process-lifetime goroutine/ticker leak).
+func TestSweepStopsOnContextCancel(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	rl := newRateLimiter(ctx, 1, 1)
+	cancel()
+	// Give the sweeper a moment to observe cancellation. It must return
+	// without blocking; the deferred ticker.Stop() fires on return.
+	time.Sleep(20 * time.Millisecond)
+	// Limiter remains usable for the brief test lifetime.
+	if !rl.get("10.0.0.1").Allow() {
+		t.Fatal("limiter should still allow within burst after sweep stops")
+	}
+}

internal/httpapi/server.go

@@ -23,8 +23,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/dtoro/oikos/internal/config"
 	"github.com/dtoro/oikos/internal/actuator"
+	"github.com/dtoro/oikos/internal/config"
 	"github.com/dtoro/oikos/internal/db"
 	"github.com/dtoro/oikos/internal/httpapi/gen"
 	mcphandler "github.com/dtoro/oikos/internal/mcp"
@@ -118,6 +118,11 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand
 	r.Use(middleware.Recoverer)
 	r.Use(middleware.RequestID)
 	r.Use(requestLogger)
+	// Per-IP rate limiting (plan D3). Applied before CORS/auth so a runaway
+	// agent loop is throttled regardless of credentials. The middleware
+	// exempts /healthz so liveness probes can't be throttled.
+	limiter := newRateLimiterFromConfig(ctx, cfg.APIRateLimit, cfg.APIRateBurst)
+	r.Use(limiter.middleware)
 	r.Use(cors.Handler(cors.Options{
 		AllowedOrigins: []string{cfg.CORSAllowedOrigin},
 		AllowedMethods: []string{"GET", "POST", "PATCH", "DELETE", "OPTIONS"},