0.28.0 — operational hardening (plan D1–D5): CI deploy gate, versioned images, rate limiting, resource limits, health probes

D1: deploy.sh CI gate — read-only SHA via git ls-remote, Gitea commit-status
    poll, portable mkdir deploy lock (macOS, no flock), TOCTOU guard, token
    passed via curl --config - (not argv), graceful misconfig tolerance.
D2: version-tagged images — OIKOS_VERSION=v$VERSION, keep-last-3 prune derived
    from 'docker compose config --images'; VERSION read after pull.
D3: per-IP rate limiting — new internal/httpapi/ratelimit.go (x/time/rate),
    rightmost-XFF, /healthz exempt, ctx-driven sweep; disabled by default.
D4: mem_limit/cpus on all 10 compose services.
D5: staleness-aware health probes — new internal/health package wired into
    scheduler (:8093) and notifier (:8094); nomos already had :8092.

Two /review passes hardened the deploy lock, TOCTOU guard, token hygiene,
and XFF handling.

internal/health/health.go

@@ -0,0 +1,93 @@
+// Package health provides a staleness-aware liveness probe for background-
+// loop services (scheduler, notifier) that don't otherwise serve HTTP.
+//
+// The owning loop calls Probe.Bump() on each iteration. A /healthz endpoint
+// returns 200 while the last bump is within the staleness window, and 503
+// once the loop has gone quiet — so a wedged goroutine (stuck SSH, deadlock)
+// surfaces as an unhealthy container instead of a silently-idle one.
+package health
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"sync/atomic"
+	"time"
+)
+
+// Probe tracks the last time the owning loop made progress.
+type Probe struct {
+	last  atomic.Int64 // unix-nano timestamp of the last Bump
+	stale time.Duration
+}
+
+// New returns a Probe that considers the owner healthy while Bump has been
+// called within stale of the current time.
+func New(stale time.Duration) *Probe {
+	if stale <= 0 {
+		stale = 2 * time.Minute
+	}
+	p := &Probe{stale: stale}
+	p.last.Store(time.Now().UnixNano()) // boot-healthy until first loop stalls
+	return p
+}
+
+// Bump records that the owning loop completed another iteration.
+func (p *Probe) Bump() {
+	p.last.Store(time.Now().UnixNano())
+}
+
+// Healthy reports whether the last Bump is within the staleness window.
+func (p *Probe) Healthy() bool {
+	last := time.Unix(0, p.last.Load())
+	return time.Since(last) <= p.stale
+}
+
+// Handler returns an http.Handler serving GET /healthz. Returns 200 with a
+// small JSON body when healthy, 503 (Service Unavailable) when stale.
+func (p *Probe) Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		last := time.Unix(0, p.last.Load())
+		if !p.Healthy() {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}
+		json.NewEncoder(w).Encode(map[string]any{
+			"status":         healthStatus(p.Healthy()),
+			"last_heartbeat": last.UTC().Format(time.RFC3339),
+		})
+	})
+}
+
+// Serve starts an HTTP server exposing the probe's /healthz on addr until ctx
+// is cancelled. A no-op when addr is empty (local/non-docker runs skip it).
+// The server is bound to addr (e.g. ":8093"); containers hit it via 127.0.0.1.
+func (p *Probe) Serve(ctx context.Context, addr string) {
+	if addr == "" {
+		return
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/healthz", p.Handler())
+	srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
+
+	go func() {
+		slog.Info("health server listening", "addr", addr)
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			slog.Warn("health server stopped", "addr", addr, "error", err)
+		}
+	}()
+	go func() {
+		<-ctx.Done()
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		srv.Shutdown(shutdownCtx)
+	}()
+}
+
+func healthStatus(ok bool) string {
+	if ok {
+		return "ok"
+	}
+	return "stale"
+}

internal/health/health_test.go

@@ -0,0 +1,73 @@
+package health
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestProbeHealthyAtBoot(t *testing.T) {
+	p := New(time.Minute)
+	if !p.Healthy() {
+		t.Fatal("probe should be healthy immediately after creation")
+	}
+}
+
+func TestProbeStaleAfterWindow(t *testing.T) {
+	p := New(50 * time.Millisecond)
+	time.Sleep(80 * time.Millisecond)
+	if p.Healthy() {
+		t.Fatal("probe should be stale after the staleness window elapses with no Bump")
+	}
+	p.Bump()
+	if !p.Healthy() {
+		t.Fatal("probe should recover immediately after Bump")
+	}
+}
+
+func TestProbeHandlerStatusCodes(t *testing.T) {
+	p := New(20 * time.Millisecond)
+
+	// Fresh → 200
+	if code := probeCode(p); code != http.StatusOK {
+		t.Fatalf("fresh probe: want 200, got %d", code)
+	}
+
+	// Stale → 503
+	time.Sleep(40 * time.Millisecond)
+	if code := probeCode(p); code != http.StatusServiceUnavailable {
+		t.Fatalf("stale probe: want 503, got %d", code)
+	}
+}
+
+func TestProbeHandlerBody(t *testing.T) {
+	p := New(time.Minute)
+	rec := httptest.NewRecorder()
+	p.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+
+	var body map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
+		t.Fatalf("invalid JSON body: %v (body=%q)", err, rec.Body.String())
+	}
+	if body["status"] != "ok" {
+		t.Fatalf("want status=ok, got %v", body["status"])
+	}
+	if _, ok := body["last_heartbeat"].(string); !ok {
+		t.Fatalf("want last_heartbeat string, got %v", body["last_heartbeat"])
+	}
+}
+
+func TestNewDefaultsStale(t *testing.T) {
+	p := New(0)
+	if p.stale <= 0 {
+		t.Fatal("New(0) should fall back to a positive staleness window")
+	}
+}
+
+func probeCode(p *Probe) int {
+	rec := httptest.NewRecorder()
+	p.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+	return rec.Code
+}