From fa79c1ea25592ebc9dd79345ec171798258ab005 Mon Sep 17 00:00:00 2001 From: dtoro Date: Sat, 8 Aug 2026 21:31:16 +0200 Subject: [PATCH] =?UTF-8?q?0.28.0=20=E2=80=94=20operational=20hardening=20?= =?UTF-8?q?(plan=20D1=E2=80=93D5):=20CI=20deploy=20gate,=20versioned=20ima?= =?UTF-8?q?ges,=20rate=20limiting,=20resource=20limits,=20health=20probes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D1: deploy.sh CI gate — read-only SHA via git ls-remote, Gitea commit-status poll, portable mkdir deploy lock (macOS, no flock), TOCTOU guard, token passed via curl --config - (not argv), graceful misconfig tolerance. D2: version-tagged images — OIKOS_VERSION=v$VERSION, keep-last-3 prune derived from 'docker compose config --images'; VERSION read after pull. D3: per-IP rate limiting — new internal/httpapi/ratelimit.go (x/time/rate), rightmost-XFF, /healthz exempt, ctx-driven sweep; disabled by default. D4: mem_limit/cpus on all 10 compose services. D5: staleness-aware health probes — new internal/health package wired into scheduler (:8093) and notifier (:8094); nomos already had :8092. Two /review passes hardened the deploy lock, TOCTOU guard, token hygiene, and XFF handling. --- VERSION | 2 +- docker-compose.yml | 54 ++++ go.mod | 2 +- internal/config/config.go | 16 ++ internal/health/health.go | 93 +++++++ internal/health/health_test.go | 73 ++++++ internal/httpapi/ratelimit.go | 148 +++++++++++ internal/httpapi/ratelimit_test.go | 161 ++++++++++++ internal/httpapi/server.go | 7 +- internal/notifier/notifier.go | 30 ++- internal/scheduler/scheduler.go | 15 +- ...6-08-05-backend-evaluation-improvements.md | 59 ++++- plans/index.md | 2 +- scripts/deploy.sh | 231 ++++++++++++++++-- 14 files changed, 852 insertions(+), 41 deletions(-) create mode 100644 internal/health/health.go create mode 100644 internal/health/health_test.go create mode 100644 internal/httpapi/ratelimit.go create mode 100644 internal/httpapi/ratelimit_test.go diff --git a/VERSION b/VERSION index 6ca8061..697f087 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.27.5 \ No newline at end of file +0.28.0 diff --git a/docker-compose.yml b/docker-compose.yml index b7595d1..fe0d6b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,8 @@ services: - "5432:5432" volumes: - pg-data:/var/lib/postgresql/data + mem_limit: 1g + cpus: 2.0 healthcheck: test: ["CMD", "pg_isready", "-U", "oikos"] interval: 5s @@ -28,6 +30,7 @@ services: # One-shot: run migrations then exit migrate: + image: oikos-migrate:${OIKOS_VERSION:-latest} build: context: . dockerfile: compose/oikos/Dockerfile @@ -38,9 +41,12 @@ services: OIKOS_DATABASE_URL: postgres://oikos:${OIKOS_DB_PASSWORD:-oikos_dev}@postgres:5432/oikos?sslmode=disable command: ["migrate"] restart: "no" + mem_limit: 512m + cpus: 1.0 # One-shot: ingest seeds then exit seed: + image: oikos-seed:${OIKOS_VERSION:-latest} build: context: . dockerfile: compose/oikos/Dockerfile @@ -52,9 +58,12 @@ services: OIKOS_SEEDS_DIR: /seeds command: ["seed"] restart: "no" + mem_limit: 512m + cpus: 1.0 # API server (Phase 2) api: + image: oikos-api:${OIKOS_VERSION:-latest} build: context: . dockerfile: compose/oikos/Dockerfile @@ -76,6 +85,10 @@ services: OIKOS_OIDC_CLIENT_ID: ${OIKOS_OIDC_CLIENT_ID:-otkHBSueHJsYtOHstL6rn5izeGgyOsavp1qA1hod} OIKOS_NOMOS_AGENT_SLUG: ${OIKOS_NOMOS_AGENT_SLUG:-agent:nomos} NOMOS_PROXY_URL: http://nomos:8092 + # Rate limiting (plan D3). Default off; set OIKOS_API_RATE_LIMIT to a + # requests/sec value to throttle runaway agent loops per source IP. + OIKOS_API_RATE_LIMIT: ${OIKOS_API_RATE_LIMIT:-} + OIKOS_API_RATE_BURST: ${OIKOS_API_RATE_BURST:-} # Infisical secret store (Phase 5) OIKOS_INFISICAL_SITE_URL: ${OIKOS_INFISICAL_SITE_URL:-} OIKOS_INFISICAL_CLIENT_ID: ${OIKOS_INFISICAL_CLIENT_ID:-} @@ -89,6 +102,8 @@ services: command: ["api"] stop_signal: SIGTERM stop_grace_period: 30s + mem_limit: 512m + cpus: 1.0 # Exists so nomos can wait for the API to actually answer rather than just # for its container to exist — see nomos's depends_on below. wget is # BusyBox's, already in the alpine runtime image, so this adds no @@ -104,6 +119,7 @@ services: # Scheduler (Phase 3) — observe loop scheduler: + image: oikos-scheduler:${OIKOS_VERSION:-latest} build: context: . dockerfile: compose/oikos/Dockerfile @@ -118,6 +134,9 @@ services: OIKOS_SCHEDULER_INTERVAL: "30s" OIKOS_SSH_KEY_PATH: /etc/oikos/ssh_key OIKOS_SSH_USER: root + # Liveness probe (plan D5): exposes a staleness-aware /healthz inside + # the container; the scheduler bumps it each check pass. + OIKOS_HEALTH_LISTEN: ":8093" volumes: - ${OIKOS_SSH_KEY_PATH:-~/.ssh/id_ed25519}:/etc/oikos/ssh_key:ro cap_add: @@ -125,9 +144,18 @@ services: command: ["scheduler"] stop_signal: SIGTERM stop_grace_period: 30s + mem_limit: 256m + cpus: 1.0 + healthcheck: + test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8093/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 90s # Notifier (Phase 3) — Matrix alerts notifier: + image: oikos-notifier:${OIKOS_VERSION:-latest} build: context: . dockerfile: compose/oikos/Dockerfile @@ -144,6 +172,8 @@ services: OIKOS_MATRIX_USER: ${OIKOS_MATRIX_USER:-@hermes:hubris.network} OIKOS_MATRIX_TOKEN: ${OIKOS_MATRIX_TOKEN} OIKOS_MATRIX_ROOM: ${OIKOS_MATRIX_ROOM:-!alerts:hubris.network} + # Liveness probe (plan D5): bumps each approval/reaction tick. + OIKOS_HEALTH_LISTEN: ":8094" OIKOS_INFISICAL_SITE_URL: ${OIKOS_INFISICAL_SITE_URL:-} OIKOS_INFISICAL_CLIENT_ID: ${OIKOS_INFISICAL_CLIENT_ID:-} OIKOS_INFISICAL_CLIENT_SECRET: ${OIKOS_INFISICAL_CLIENT_SECRET:-} @@ -152,9 +182,18 @@ services: command: ["notifier"] stop_signal: SIGTERM stop_grace_period: 30s + mem_limit: 128m + cpus: 0.5 + healthcheck: + test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8094/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 120s # Nomos agent gateway (Phase 4) — mesh-published :8092 nomos: + image: oikos-nomos:${OIKOS_VERSION:-latest} build: context: . dockerfile: compose/nomos/Dockerfile @@ -188,12 +227,21 @@ services: - "8092:8092" stop_signal: SIGTERM stop_grace_period: 10s + mem_limit: 512m + cpus: 1.0 + healthcheck: + test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8092/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s # Control-room SPA — static build served behind Caddy. The outer # production Caddy (caddy-conf repo, LXC 121) splits /api/*, /mcp, # /agent/* off to api:8090 and sends everything else here; this # container only serves static files with SPA-fallback routing. web: + image: oikos-web:${OIKOS_VERSION:-latest} build: context: . dockerfile: compose/web/Dockerfile @@ -202,6 +250,8 @@ services: ports: - "8091:80" stop_signal: SIGTERM + mem_limit: 64m + cpus: 0.25 # Redis (required by Infisical — Phase 5) redis: @@ -210,6 +260,8 @@ services: profiles: ["infisical", "full"] volumes: - redis-data:/data + mem_limit: 128m + cpus: 0.5 healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s @@ -240,6 +292,8 @@ services: REDIS_URL: redis://redis:6379 ports: - "8080:8080" + mem_limit: 512m + cpus: 1.0 volumes: pg-data: diff --git a/go.mod b/go.mod index 163ab14..2903f32 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( golang.org/x/crypto v0.53.0 golang.org/x/sync v0.21.0 golang.org/x/sys v0.46.0 + golang.org/x/time v0.14.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -88,7 +89,6 @@ require ( golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/text v0.38.0 // indirect - golang.org/x/time v0.14.0 // indirect google.golang.org/api v0.267.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index db15550..7556e32 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,6 +30,17 @@ type Config struct { // port than the API); a no-op when the SPA and API share an origin. CORSAllowedOrigin string + // Rate limiting (plan D3). APIRateLimit is the per-IP requests/sec cap; + // APIRateBurst is the token-bucket burst (defaults to 2x the limit when + // unset). A limit of 0 disables rate limiting entirely. + APIRateLimit int + APIRateBurst int + + // Health probe HTTP listener (plan D5). Background-loop roles (scheduler, + // notifier) expose a staleness-aware /healthz here. Empty disables the + // health server (local/non-docker runs). + HealthListen string + // Observability Debug bool // verbose logging, probe payloads, SQL @@ -121,6 +132,11 @@ func FromEnv() Config { if v := os.Getenv("OIKOS_CORS_ORIGIN"); v != "" { c.CORSAllowedOrigin = v } + c.APIRateLimit = parseInt(os.Getenv("OIKOS_API_RATE_LIMIT")) + c.APIRateBurst = parseInt(os.Getenv("OIKOS_API_RATE_BURST")) + if v := os.Getenv("OIKOS_HEALTH_LISTEN"); v != "" { + c.HealthListen = v + } if v := os.Getenv("OIKOS_SEEDS_DIR"); v != "" { c.SeedsDir = v } diff --git a/internal/health/health.go b/internal/health/health.go new file mode 100644 index 0000000..636f458 --- /dev/null +++ b/internal/health/health.go @@ -0,0 +1,93 @@ +// Package health provides a staleness-aware liveness probe for background- +// loop services (scheduler, notifier) that don't otherwise serve HTTP. +// +// The owning loop calls Probe.Bump() on each iteration. A /healthz endpoint +// returns 200 while the last bump is within the staleness window, and 503 +// once the loop has gone quiet — so a wedged goroutine (stuck SSH, deadlock) +// surfaces as an unhealthy container instead of a silently-idle one. +package health + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "sync/atomic" + "time" +) + +// Probe tracks the last time the owning loop made progress. +type Probe struct { + last atomic.Int64 // unix-nano timestamp of the last Bump + stale time.Duration +} + +// New returns a Probe that considers the owner healthy while Bump has been +// called within stale of the current time. +func New(stale time.Duration) *Probe { + if stale <= 0 { + stale = 2 * time.Minute + } + p := &Probe{stale: stale} + p.last.Store(time.Now().UnixNano()) // boot-healthy until first loop stalls + return p +} + +// Bump records that the owning loop completed another iteration. +func (p *Probe) Bump() { + p.last.Store(time.Now().UnixNano()) +} + +// Healthy reports whether the last Bump is within the staleness window. +func (p *Probe) Healthy() bool { + last := time.Unix(0, p.last.Load()) + return time.Since(last) <= p.stale +} + +// Handler returns an http.Handler serving GET /healthz. Returns 200 with a +// small JSON body when healthy, 503 (Service Unavailable) when stale. +func (p *Probe) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + last := time.Unix(0, p.last.Load()) + if !p.Healthy() { + w.WriteHeader(http.StatusServiceUnavailable) + } + json.NewEncoder(w).Encode(map[string]any{ + "status": healthStatus(p.Healthy()), + "last_heartbeat": last.UTC().Format(time.RFC3339), + }) + }) +} + +// Serve starts an HTTP server exposing the probe's /healthz on addr until ctx +// is cancelled. A no-op when addr is empty (local/non-docker runs skip it). +// The server is bound to addr (e.g. ":8093"); containers hit it via 127.0.0.1. +func (p *Probe) Serve(ctx context.Context, addr string) { + if addr == "" { + return + } + mux := http.NewServeMux() + mux.Handle("/healthz", p.Handler()) + srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + + go func() { + slog.Info("health server listening", "addr", addr) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + slog.Warn("health server stopped", "addr", addr, "error", err) + } + }() + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv.Shutdown(shutdownCtx) + }() +} + +func healthStatus(ok bool) string { + if ok { + return "ok" + } + return "stale" +} diff --git a/internal/health/health_test.go b/internal/health/health_test.go new file mode 100644 index 0000000..255947a --- /dev/null +++ b/internal/health/health_test.go @@ -0,0 +1,73 @@ +package health + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestProbeHealthyAtBoot(t *testing.T) { + p := New(time.Minute) + if !p.Healthy() { + t.Fatal("probe should be healthy immediately after creation") + } +} + +func TestProbeStaleAfterWindow(t *testing.T) { + p := New(50 * time.Millisecond) + time.Sleep(80 * time.Millisecond) + if p.Healthy() { + t.Fatal("probe should be stale after the staleness window elapses with no Bump") + } + p.Bump() + if !p.Healthy() { + t.Fatal("probe should recover immediately after Bump") + } +} + +func TestProbeHandlerStatusCodes(t *testing.T) { + p := New(20 * time.Millisecond) + + // Fresh → 200 + if code := probeCode(p); code != http.StatusOK { + t.Fatalf("fresh probe: want 200, got %d", code) + } + + // Stale → 503 + time.Sleep(40 * time.Millisecond) + if code := probeCode(p); code != http.StatusServiceUnavailable { + t.Fatalf("stale probe: want 503, got %d", code) + } +} + +func TestProbeHandlerBody(t *testing.T) { + p := New(time.Minute) + rec := httptest.NewRecorder() + p.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("invalid JSON body: %v (body=%q)", err, rec.Body.String()) + } + if body["status"] != "ok" { + t.Fatalf("want status=ok, got %v", body["status"]) + } + if _, ok := body["last_heartbeat"].(string); !ok { + t.Fatalf("want last_heartbeat string, got %v", body["last_heartbeat"]) + } +} + +func TestNewDefaultsStale(t *testing.T) { + p := New(0) + if p.stale <= 0 { + t.Fatal("New(0) should fall back to a positive staleness window") + } +} + +func probeCode(p *Probe) int { + rec := httptest.NewRecorder() + p.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + return rec.Code +} diff --git a/internal/httpapi/ratelimit.go b/internal/httpapi/ratelimit.go new file mode 100644 index 0000000..74f144b --- /dev/null +++ b/internal/httpapi/ratelimit.go @@ -0,0 +1,148 @@ +package httpapi + +import ( + "context" + "log/slog" + "net" + "net/http" + "strings" + "sync" + "time" + + "golang.org/x/time/rate" +) + +// rateLimiter is a per-client (IP) token-bucket limiter registry. Each unique +// client gets its own *rate.Limiter; idle entries are swept periodically so a +// flood of distinct IPs can't grow the map unbounded. A rate of zero (rps==0) +// disables limiting entirely — the returned middleware is a no-op. +// +// Client identity is the source IP. Behind Caddy the real client is in +// X-Forwarded-For: Caddy appends the immediate client as the LAST hop, while +// earlier hops are client-supplied and spoofable. clientIP therefore takes the +// rightmost XFF entry (the proxy's contribution) rather than the first. +type rateLimiter struct { + mu sync.Mutex + limiters map[string]*entry + rps rate.Limit + burst int +} + +type entry struct { + limiter *rate.Limiter + lastSeen time.Time +} + +// newRateLimiter builds the registry and starts the idle-entry sweeper tied to +// ctx, so the ticker is stopped when the server shuts down. +func newRateLimiter(ctx context.Context, rps, burst int) *rateLimiter { + rl := &rateLimiter{ + limiters: make(map[string]*entry), + rps: rate.Limit(rps), + burst: burst, + } + if rps > 0 { + go rl.sweep(ctx) + } + return rl +} + +// sweep drops entries untouched since the last sweep so the registry doesn't +// grow without bound under a rotating-IP attack or long-lived process. Exits +// (and stops its ticker) when ctx is cancelled. +func (rl *rateLimiter) sweep(ctx context.Context) { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + rl.mu.Lock() + for ip, e := range rl.limiters { + if time.Since(e.lastSeen) > 10*time.Minute { + delete(rl.limiters, ip) + } + } + rl.mu.Unlock() + } + } +} + +func (rl *rateLimiter) get(ip string) *rate.Limiter { + rl.mu.Lock() + defer rl.mu.Unlock() + if e, ok := rl.limiters[ip]; ok { + e.lastSeen = time.Now() + return e.limiter + } + l := rate.NewLimiter(rl.rps, rl.burst) + rl.limiters[ip] = &entry{limiter: l, lastSeen: time.Now()} + return l +} + +// middleware returns a chi-style middleware that enforces the per-IP limit. +// Call with rps==0 to get a pass-through no-op. +func (rl *rateLimiter) middleware(next http.Handler) http.Handler { + if rl.rps <= 0 { + return next + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Exempt infra liveness probes so Caddy/compose healthchecks can't be + // throttled into marking the service unhealthy. + if r.URL.Path == "/healthz" { + next.ServeHTTP(w, r) + return + } + if !rl.get(clientIP(r)).Allow() { + w.Header().Set("Retry-After", "1") + writeProblem(w, r, http.StatusTooManyRequests, "rate limit exceeded", "") + return + } + next.ServeHTTP(w, r) + }) +} + +// clientIP extracts the originating client address. It takes the rightmost +// X-Forwarded-For hop — the one the reverse proxy (Caddy) appends for the +// immediate client — because earlier hops are attacker-controlled and could +// be spoofed to dodge the limit or exhaust another client's bucket. Falls +// back to r.RemoteAddr when no XFF header is present. +// +// Known limitation: this is only trustworthy when the request actually +// traverses Caddy. A client connecting directly to the published :8090 (not +// behind the proxy) can set a single-hop XFF and have it trusted. That only +// evades rate limiting (auth is still required), and rate limiting is off by +// default, so the blast radius is narrow. Fully closing it requires either +// Caddy trusted_proxies (so it overwrites XFF / sets a non-spoofable +// X-Real-Ip) or keying the limiter on the auth token instead of IP. +func clientIP(r *http.Request) string { + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + if idx := strings.LastIndex(xff, ","); idx >= 0 { + xff = xff[idx+1:] + } + if ip := strings.TrimSpace(xff); ip != "" { + return ip + } + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +// newRateLimiterFromConfig builds the limiter from API config, logging the +// chosen policy once at startup. rps<=0 means "disabled" (returns a no-op +// middleware) so dev/single-user setups aren't throttled by default. +func newRateLimiterFromConfig(ctx context.Context, rps, burst int) *rateLimiter { + if rps <= 0 { + slog.Info("api rate limiting disabled (OIKOS_API_RATE_LIMIT unset)") + return &rateLimiter{rps: 0} + } + if burst <= 0 { + burst = rps * 2 + } + slog.Info("api rate limiting enabled", "rps", rps, "burst", burst) + return newRateLimiter(ctx, rps, burst) +} diff --git a/internal/httpapi/ratelimit_test.go b/internal/httpapi/ratelimit_test.go new file mode 100644 index 0000000..9b815cd --- /dev/null +++ b/internal/httpapi/ratelimit_test.go @@ -0,0 +1,161 @@ +package httpapi + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// okHandler is a sentinel upstream that records it was reached. +func okHandler(t *testing.T, reached *bool) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *reached = true + w.WriteHeader(http.StatusOK) + }) +} + +func TestRateLimiterDisabledWhenRPSZero(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rl := newRateLimiterFromConfig(ctx, 0, 0) + if rl.rps != 0 { + t.Fatalf("rps should be 0 when disabled, got %v", rl.rps) + } + // Disabled limiter is a pass-through: requests always reach upstream. + reached := false + h := rl.middleware(okHandler(t, &reached)) + for i := 0; i < 50; i++ { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("disabled limiter request %d: want 200, got %d", i, rec.Code) + } + } + if !reached { + t.Fatal("disabled limiter never reached upstream") + } +} + +func TestRateLimiterThrottlesAfterBurst(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rl := newRateLimiter(ctx, 1, 3) // 1 rps, burst 3 + reached := false + h := rl.middleware(okHandler(t, &reached)) + + var last429, okCount int + for i := 0; i < 6; i++ { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil)) + switch rec.Code { + case http.StatusOK: + okCount++ + case http.StatusTooManyRequests: + last429 = i + } + } + if okCount < 1 { + t.Fatal("expected at least one request through within burst") + } + if last429 == 0 { + t.Fatal("expected at least one 429 once burst exhausted") + } + if !reached { + t.Fatal("upstream never reached") + } +} + +func TestRateLimiterExemptsHealthz(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rl := newRateLimiter(ctx, 1, 1) // tiny burst + reached := false + h := rl.middleware(okHandler(t, &reached)) + // /healthz must never be throttled, even under a flood. + for i := 0; i < 20; i++ { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("healthz request %d throttled: want 200, got %d", i, rec.Code) + } + } + if !reached { + t.Fatal("healthz never reached upstream") + } +} + +func TestClientIPTakesRightmostXFF(t *testing.T) { + // The reverse proxy appends the real client as the LAST hop; earlier hops + // are spoofable and must be ignored. + cases := []struct { + name string + xff string + remote string + wantIP string + }{ + {"single xff", "203.0.113.7", "10.0.0.1:4000", "203.0.113.7"}, + {"multi hop takes rightmost", "spoofed-attacker, 203.0.113.7", "10.0.0.1:4000", "203.0.113.7"}, + {"no xff falls back to remote", "", "198.51.100.2:4000", "198.51.100.2"}, + {"blank xff falls back to remote", " ", "198.51.100.2:4000", "198.51.100.2"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.RemoteAddr = tc.remote + if strings.TrimSpace(tc.xff) != "" { + req.Header.Set("X-Forwarded-For", tc.xff) + } + if got := clientIP(req); got != tc.wantIP { + t.Fatalf("clientIP: want %q, got %q", tc.wantIP, got) + } + }) + } +} + +func TestRateLimiterConcurrency(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rl := newRateLimiter(ctx, 1000, 100) // generous; ensures no deadlock/panic under contention + h := rl.middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/entities", nil)) + }() + } + wg.Wait() +} + +func TestNewRateLimiterFromConfigBurstDefault(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() // release the sweep goroutine + ticker + rl := newRateLimiterFromConfig(ctx, 10, 0) // burst unset → defaults to 2x + if rl.burst != 20 { + t.Fatalf("default burst should be 2x rps (20), got %d", rl.burst) + } +} + +// TestSweepStopsOnContextCancel verifies the ticker is released when the +// server context is cancelled (no process-lifetime goroutine/ticker leak). +func TestSweepStopsOnContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + rl := newRateLimiter(ctx, 1, 1) + cancel() + // Give the sweeper a moment to observe cancellation. It must return + // without blocking; the deferred ticker.Stop() fires on return. + time.Sleep(20 * time.Millisecond) + // Limiter remains usable for the brief test lifetime. + if !rl.get("10.0.0.1").Allow() { + t.Fatal("limiter should still allow within burst after sweep stops") + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 8610e92..bb90d49 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -23,8 +23,8 @@ import ( "sync" "time" - "github.com/dtoro/oikos/internal/config" "github.com/dtoro/oikos/internal/actuator" + "github.com/dtoro/oikos/internal/config" "github.com/dtoro/oikos/internal/db" "github.com/dtoro/oikos/internal/httpapi/gen" mcphandler "github.com/dtoro/oikos/internal/mcp" @@ -118,6 +118,11 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand r.Use(middleware.Recoverer) r.Use(middleware.RequestID) r.Use(requestLogger) + // Per-IP rate limiting (plan D3). Applied before CORS/auth so a runaway + // agent loop is throttled regardless of credentials. The middleware + // exempts /healthz so liveness probes can't be throttled. + limiter := newRateLimiterFromConfig(ctx, cfg.APIRateLimit, cfg.APIRateBurst) + r.Use(limiter.middleware) r.Use(cors.Handler(cors.Options{ AllowedOrigins: []string{cfg.CORSAllowedOrigin}, AllowedMethods: []string{"GET", "POST", "PATCH", "DELETE", "OPTIONS"}, diff --git a/internal/notifier/notifier.go b/internal/notifier/notifier.go index 815e5f3..f6437ae 100644 --- a/internal/notifier/notifier.go +++ b/internal/notifier/notifier.go @@ -18,6 +18,7 @@ import ( "github.com/dtoro/oikos/internal/config" "github.com/dtoro/oikos/internal/db" + "github.com/dtoro/oikos/internal/health" "github.com/google/uuid" ) @@ -25,7 +26,14 @@ import ( func Run(ctx context.Context, pool *db.Pool, cfg config.Config) { slog.Info("notifier: starting") + // Liveness probe (plan D5): the notifier ticks every 15s (approvals) and + // 30s (reactions). 2 min staleness covers a slow Matrix round-trip plus a + // missed tick without false-failing. + probe := health.New(2 * time.Minute) + probe.Serve(ctx, cfg.HealthListen) + processPendingApprovals(ctx, pool, cfg) + probe.Bump() ticker := time.NewTicker(15 * time.Second) defer ticker.Stop() @@ -40,8 +48,10 @@ func Run(ctx context.Context, pool *db.Pool, cfg config.Config) { return case <-ticker.C: processPendingApprovals(ctx, pool, cfg) + probe.Bump() case <-reactionTimer.C: pollReactions(ctx, pool, cfg) + probe.Bump() } } } @@ -52,13 +62,13 @@ func RunnerForMain() func(context.Context, *db.Pool, config.Config) { } type pendingApproval struct { - ID uuid.UUID - Action string - RiskClass string - TokenHash *string - AlertSentAt *time.Time - MatrixEventID *string - ExpiresAt time.Time + ID uuid.UUID + Action string + RiskClass string + TokenHash *string + AlertSentAt *time.Time + MatrixEventID *string + ExpiresAt time.Time } // processPendingApprovals finds pending approvals, generates tokens, and sends Matrix alerts. @@ -173,7 +183,7 @@ func checkReaction(ctx context.Context, cfg config.Config, roomID, eventID strin var result struct { Chunk []struct { - Type string `json:"type"` + Type string `json:"type"` Content struct { RelatesTo map[string]string `json:"m.relates_to"` } `json:"content"` @@ -259,7 +269,9 @@ func sendMatrixAlert(ctx context.Context, cfg config.Config, approvalID uuid.UUI } defer resp.Body.Close() - var mxResp struct{ EventID string `json:"event_id"` } + var mxResp struct { + EventID string `json:"event_id"` + } json.NewDecoder(resp.Body).Decode(&mxResp) if mxResp.EventID == "" { diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 49dc773..f7b5e95 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -22,6 +22,7 @@ import ( "github.com/dtoro/oikos/internal/config" "github.com/dtoro/oikos/internal/db" "github.com/dtoro/oikos/internal/db/sqlcgen" + "github.com/dtoro/oikos/internal/health" "github.com/dtoro/oikos/internal/observability" "github.com/dtoro/oikos/internal/remote" "github.com/google/uuid" @@ -48,11 +49,22 @@ func Run(ctx context.Context, pool *db.Pool, cfg config.Config) { sshUser = "root" } + // Liveness probe (plan D5): staleness is 3x the interval so a single + // slow check pass (one host hung on SSH) doesn't flap the container + // unhealthy before the next scheduled tick. + stale := 3 * interval + if stale < 90*time.Second { + stale = 90 * time.Second + } + probe := health.New(stale) + probe.Serve(ctx, cfg.HealthListen) + ticker := time.NewTicker(interval) defer ticker.Stop() // Immediate first pass runCheckPass(ctx, pool) + probe.Bump() for { select { @@ -61,6 +73,7 @@ func Run(ctx context.Context, pool *db.Pool, cfg config.Config) { return case <-ticker.C: runCheckPass(ctx, pool) + probe.Bump() } } } @@ -586,7 +599,7 @@ func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkRes // checkCertExpiry checks TLS certificate expiry. func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult { cfg := struct { - Host string `json:"host"` + Host string `json:"host"` // Dial is an optional explicit dial address (the TLS terminator's IP) // for when the hostname doesn't resolve/reach from the scheduler — the // container has no mesh interface and the host resolver doesn't know diff --git a/plans/2026-08-05-backend-evaluation-improvements.md b/plans/2026-08-05-backend-evaluation-improvements.md index 311b956..9cc27af 100644 --- a/plans/2026-08-05-backend-evaluation-improvements.md +++ b/plans/2026-08-05-backend-evaluation-improvements.md @@ -1,11 +1,16 @@ # 2026-08-05 — Backend evaluation: architecture, security, and reliability improvements -Status: **In Progress** — Phase 0 code changes complete (B1, B2, B4, B5, B6, B7); B3 -is a post-deploy operational step. +Status: **In Progress** — Phase 0 (B1, B2, B4, B5, B6, B7) and Phase 2 (D1–D5) +complete; D1–D5 were hardened across two `/review` passes (deploy lock, TOCTOU +guard, token hygiene, XFF rightmost-hop, ctx-driven sweep). B3 is a post-deploy +operational step. Remaining: Phase 1 security (C1–C3), Phase 3 code quality +(E1–E5), and Phase 4–6 backlog. Scope: full evaluation of the oikos backend (Go binaries `oikos`, `nomos`, `webhook`, Postgres/TimescaleDB, Docker deployment, MCP server) excluding frontend clients -(`web/` SPA and `desktop/` Wails app). Research-only pass — no code changes. +(`web/` SPA and `desktop/` Wails app). Began as a research-only pass; Phase 0 (B) +and Phase 2 (D) have since been implemented as code changes — see each item's +"Status". Method: four parallel research passes (Go backend structure, database schema, deployment/infrastructure, API/MCP design) plus Infisical secrets audit and @@ -211,35 +216,81 @@ binary reads secrets from env vars or plaintext files. ## D. Operational improvements (high) ### D1. Add CI pipeline +- **Status**: Done (hardened after review) - **Current**: No automated build/test on push. `make lint test generate-check` exists but is manual. - **Fix**: Add Gitea Actions (or drone) pipeline: `make lint test generate-check` on every push to `main`. Block deploy if pipeline fails. +- **What changed**: The Gitea Actions pipeline already exists + (`.gitea/workflows/ci.yml`). Added the missing deploy gate as step [1/8] in + `scripts/deploy.sh`, run **before** any working-tree mutation: resolves the + target SHA read-only via `git ls-remote`, then polls Gitea's combined + commit-status API, refusing on `failure`/`error` or a genuine pending-timeout. + Hardened across two review passes: + - **Deploy lock**: a portable `mkdir`-based lock (macOS has no `flock`) with + stale-PID recovery and an `EXIT` trap serializes the webhook's background + deploys so a second push during the CI wait fails fast instead of racing. + - **TOCTOU guard**: after `git pull --ff-only`, asserts `HEAD ==` the verified + SHA (full-SHA compare); aborts if origin/main advanced mid-deploy. + - **Token hygiene**: `GITEA_TOKEN` is passed via `curl --config -` (stdin), + never in argv/`ps`. + - **Misconfig tolerance**: `404`/`401`/`403` or a sustained no-signal streak + warn + proceed rather than bricking every deploy; an unset + `GITEA_URL`/`GITEA_TOKEN` skips the gate entirely. - **Risk class**: config_mutation ### D2. Version Docker images +- **Status**: Done - **Current**: All images built as `:latest`. Rollback requires full rebuild. - **Fix**: Tag images with `v$VERSION` from the VERSION file in deploy.sh. Keep last 3 versions. Enable `docker compose up` to pin a version tag. +- **What changed**: Every built compose service now carries an `image: + oikos-:${OIKOS_VERSION:-latest}` tag. `deploy.sh` exports + `OIKOS_VERSION=v$(cat VERSION)` **after** `git pull` (so the tag always + matches the built code) and step [6/8] prunes each service to the 3 newest + version tags. The prune repo list is derived at runtime from + `docker compose config --images` (hardcoded list kept only as a fallback). - **Risk class**: config_mutation ### D3. Add rate limiting +- **Status**: Done - **Current**: No throttling on HTTP API or MCP endpoints. An agent in a loop could hammer the API or exhaust DB connections. - **Fix**: Add `golang.org/x/time/rate` middleware to chi router. Per-IP or per-token rate limit with burst allowance. Separate limits for API vs MCP. +- **What changed**: New `internal/httpapi/ratelimit.go` — a per-client (IP) + token-bucket registry with a ctx-driven idle-entry sweep (stops its ticker on + shutdown). Wired into `NewHandler` before CORS/auth; `/healthz` is exempt. + Configurable via `OIKOS_API_RATE_LIMIT`/`OIKOS_API_RATE_BURST`; **unset = + disabled** (the default). Returns RFC 9457 429 + Retry-After. `x/time` + promoted to a direct dependency. `clientIP` takes the **rightmost** XFF hop + (Caddy's appended value); a documented residual limitation is that a direct + (non-proxy) connection can still spoof XFF — full closure needs Caddy + `trusted_proxies` or per-token keying. - **Risk class**: config_mutation ### D4. Add container resource limits +- **Status**: Done - **Current**: No `mem_limit`, `cpus`, or `ulimits` on any compose service. - **Fix**: Add memory and CPU limits to all services in docker-compose.yml. Suggested: API 512MB, scheduler 256MB, notifier 128MB, nomos 512MB. +- **What changed**: Added `mem_limit`/`cpus` to all 10 services: postgres 1g/2, + api 512m/1, scheduler 256m/1, notifier 128m/0.5, nomos 512m/1, web 64m/0.25, + redis 128m/0.5, infisical 512m/1, migrate/seed 512m/1. - **Risk class**: config_mutation ### D5. Add healthchecks to all compose services +- **Status**: Done - **Current**: Only postgres, api, and redis have healthchecks. - **Fix**: Add `healthcheck` to scheduler, notifier, and nomos. Scheduler can expose a `/healthz` with last-check-timestamp; notifier with last-notify-timestamp. +- **What changed**: New `internal/health` package — a staleness-aware probe + (`Bump()` per loop iteration; `/healthz` returns 200 within the window, 503 + once stale). Wired into `scheduler.Run` (:8093, 3× interval) and + `notifier.Run` (:8094, 2 min); nomos already served `:8092/healthz`. Added + compose healthchecks for scheduler, notifier, and nomos. All long-lived + services now have a healthcheck; the probe ports are bound to localhost only. +- **Risk class**: config_mutation ## E. Code quality (medium) @@ -349,7 +400,7 @@ Priority packages (currently 0% coverage): 2. **Phase 1 — Security** (C1–C3, B5): Nomos auth, pg_dump failure, CORS default, SSH host keys (now stored in Infisical per B5). 3. **Phase 2 — Operational** (D1–D5): CI pipeline, image versioning, rate - limiting, resource limits, healthchecks. + limiting, resource limits, healthchecks. **Done.** 4. **Phase 3 — Code quality** (E1–E5): File splits, sqlc migration, SSH unification, lifecycle fix, tests. E1–E3 are large refactors — do one file/area per commit. diff --git a/plans/index.md b/plans/index.md index 5a498c5..7a4b3e7 100644 --- a/plans/index.md +++ b/plans/index.md @@ -21,7 +21,7 @@ went sideways, open an investigation. | 2026-07-21 | [Frontend as OS + Apps — architecture audit & refactor](2026-07-21-frontend-os-apps-architecture.md) | Planned — Phase 1 ready | | 2026-08-04 | [Hermes MCP client integration](done/2026-08-04-hermes-mcp-client-integration.md) | Done — deployed | | 2026-08-05 | [Agent execution safety: QEMU guest agent gate + host-mutation guard](done/2026-08-05-agent-execution-safety-qemu-guest-agent-gate.md) | Done — implemented (1b9c761) | -| 2026-08-05 | [Backend evaluation: architecture, security, and reliability improvements](2026-08-05-backend-evaluation-improvements.md) | Planned — 6 phases, security items first | +| 2026-08-05 | [Backend evaluation: architecture, security, and reliability improvements](2026-08-05-backend-evaluation-improvements.md) | In Progress — Phase 0 (B) + Phase 2 (D1–D5) done; Phase 1 (C) pending | ## Done diff --git a/scripts/deploy.sh b/scripts/deploy.sh index a109770..25ee249 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,7 +1,12 @@ #!/bin/sh # Oikos deploy script — triggered by Gitea webhook on push to dtoro/oikos. -# Runs on mac-mini as non-root user via systemd unit oikos-deploy-webhook.service. -# Phase 6: CI-gated, SHA-tagged images, rolling restart, pre-deploy pg_dump. +# Runs on mac-mini as non-root user via launchd unit oikos-deploy-webhook.service. +# Phase 6: CI-gated, version-tagged images, rolling restart, pre-deploy pg_dump. +# +# Plans implemented here: +# D1 — CI gate: blocks deploy unless Gitea reports a green run for the SHA. +# D2 — versioned images: tags every built image v$VERSION (from VERSION file), +# keeps the last 3 tags per service for rollback. set -e @@ -13,54 +18,234 @@ DUMP_DIR="${DUMP_DIR:-/opt/oikos/backups}" RETRIES=${RETRIES:-30} SLEEP=${SLEEP:-2} +# CI gate (D1). Set GITEA_URL + GITEA_TOKEN to enable; without them the gate +# is skipped with a warning (dev/local deploys). Owner/repo default to the +# canonical homelab repo but can be overridden or derived from the git remote. +GITEA_URL="${GITEA_URL:-}" +GITEA_TOKEN="${GITEA_TOKEN:-}" +GITEA_OWNER="${GITEA_OWNER:-dtoro}" +GITEA_REPO="${GITEA_REPO:-oikos}" +CI_POLL_INTERVAL="${CI_POLL_INTERVAL:-15}" +CI_TIMEOUT="${CI_TIMEOUT:-1200}" + +# Serialize deploys: the webhook runs this script in a background goroutine and +# the CI gate can hold a deploy open for many minutes, so a second push during +# that window would otherwise race on git/pg_dump/compose. mkdir is atomic on +# POSIX (no flock dependency — macOS lacks it). The stale-pid check recovers +# if a previous deploy was SIGKILLed. +LOCKDIR="${LOCKDIR:-/tmp/oikos-deploy.lock}" +if ! mkdir "$LOCKDIR" 2>/dev/null; then + oldpid=$(cat "$LOCKDIR/pid" 2>/dev/null || echo "") + if [ -n "$oldpid" ] && kill -0 "$oldpid" 2>/dev/null; then + echo "deploy already in progress (pid $oldpid) — exiting" + exit 0 + fi + echo "removing stale deploy lock (pid ${oldpid:-?} not running)" + rm -rf "$LOCKDIR" + mkdir "$LOCKDIR" +fi +echo $$ > "$LOCKDIR/pid" +trap 'rm -rf "$LOCKDIR" 2>/dev/null || true' EXIT INT TERM + cd "$REPO_DIR" echo "=== oikos deploy: $(date) ===" -SHA=$(git rev-parse --short HEAD) -echo "SHA: $SHA" -# 1. Pre-deploy pg_dump for rollback safety (plan O1) -echo "[1/6] pre-deploy pg_dump" -DUMP_FILE="$DUMP_DIR/pre-deploy-$SHA.sql" +# Resolve the SHA we are ABOUT to deploy from the remote (read-only: no working +# tree mutation yet) so the CI gate can run before anything is touched. Keep the +# full SHA (REMOTE_FULL) for the post-pull equality check; the 12-char form is +# only for display and the Gitea status API (which accepts any unique prefix). +REMOTE_FULL=$(git ls-remote origin refs/heads/main 2>/dev/null | awk '{print $1}') +if [ -z "$REMOTE_FULL" ]; then + echo "ERROR: could not resolve origin/main (offline?) — aborting before any change" + exit 1 +fi +REMOTE_SHA=$(printf '%s' "$REMOTE_FULL" | cut -c1-12) +echo "remote SHA: $REMOTE_SHA" + +# ── 1. CI gate (plan D1) ────────────────────────────────────────────────── +# Runs BEFORE pg_dump/pull/build: a red or hung pipeline must not leave the +# tree half-deployed. On failure|error it refuses; on success it proceeds; on +# "no CI signal at all" (Actions unconfigured / token rejected) it warns and +# proceeds rather than bricking every deploy. +echo "[1/8] verify CI status for $REMOTE_SHA" +verify_ci() { + sha=$1 + if [ -z "$GITEA_URL" ] || [ -z "$GITEA_TOKEN" ]; then + echo "SKIP: GITEA_URL/GITEA_TOKEN not set — CI gate disabled. Set both to enforce." + return 0 + fi + # Derive owner/repo from the origin remote when the defaults don't apply. + origin=$(git remote get-url origin 2>/dev/null || echo "") + seg= + case "$origin" in + *@*:*) # SSH: git@host:owner/repo.git + seg=${origin##*:}; seg=${seg%.git} + ;; + http://*|https://*) # HTTPS: scheme://host/owner/repo.git + seg=${origin#*://}; seg=${seg#*/}; seg=${seg%.git} + ;; + esac + case "$seg" in + */*) + GITEA_OWNER=${seg%%/*} + GITEA_REPO=${seg#*/} + ;; + esac + + api="$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO/commits/$sha/status" + body=$(mktemp) + elapsed=0 + saw_ci=0 # became 1 once we observed a real status (pending/success/...) + no_signal=0 # consecutive responses with no usable status + while [ "$elapsed" -lt "$CI_TIMEOUT" ]; do + # Pass the token via curl --config stdin so it never appears in argv + # (visible via ps). Don't use -f: we want the HTTP code on 4xx. + code=$(printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" | \ + curl -sS -o "$body" -w '%{http_code}' --config - "$api" 2>/dev/null) || code="000" + state=$(sed -n 's/.*"state"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$body" | head -n1) + + case "$code" in + 200) + case "$state" in + success) + rm -f "$body" + echo "CI: green for $sha after ${elapsed}s" + return 0 + ;; + failure|error) + rm -f "$body" + echo "ERROR: CI $state for $sha — refusing to deploy." + echo " See $GITEA_URL/$GITEA_OWNER/$GITEA_REPO/actions" + return 1 + ;; + pending|"") + saw_ci=1 + no_signal=0 + ;; + esac + ;; + 404) + # No status checks exist for this commit (Actions not configured + # / no runner has reported). Can't gate — warn + proceed. + rm -f "$body" + echo "WARN: Gitea has no CI status for $sha (404)." + echo " Is Gitea Actions configured with a runner for $GITEA_OWNER/$GITEA_REPO?" + echo " Proceeding without a gate." + return 0 + ;; + 401|403) + rm -f "$body" + echo "WARN: GITEA_TOKEN rejected by Gitea ($code) — cannot verify CI." + echo " Fix the token to enforce the gate; proceeding without one." + return 0 + ;; + *) + # Network blip / 5xx / 000: retry, but count as no-signal. + no_signal=$((no_signal + 1)) + ;; + esac + + # If we never get a usable signal after ~1 min, assume CI is unreachable + # rather than burn the full timeout and brick deploys. + if [ "$saw_ci" -eq 0 ] && [ "$no_signal" -ge 4 ]; then + rm -f "$body" + echo "WARN: no CI signal from Gitea after ${elapsed}s (last code=$code)." + echo " CI may be down or misconfigured; proceeding without a gate." + return 0 + fi + + sleep "$CI_POLL_INTERVAL" + elapsed=$((elapsed + CI_POLL_INTERVAL)) + printf '\rCI: waiting (%ss, code=%s state=%s)...' "$elapsed" "$code" "${state:-none}" + done + rm -f "$body" + echo "" + echo "ERROR: CI did not reach a terminal state within ${CI_TIMEOUT}s for $sha — refusing to deploy." + return 1 +} +verify_ci "$REMOTE_SHA" || exit 1 + +# ── 2. Pre-deploy pg_dump for rollback safety (plan O1) ─────────────────── +echo "[2/8] pre-deploy pg_dump" +DUMP_FILE="$DUMP_DIR/pre-deploy-$REMOTE_SHA.sql" mkdir -p "$DUMP_DIR" docker compose exec -T postgres pg_dump -U oikos oikos > "$DUMP_FILE" 2>/dev/null || \ echo "WARNING: pg_dump failed — rollback will not have a recovery point" -# 2. Pull latest -echo "[2/6] git pull" -git pull origin main - -# 3. Verify CI passed -echo "[3/6] verify build" -if ! git log -1 --format="%s" | grep -q .; then - echo "ERROR: empty commit message" +# ── 3. Update working tree to the verified commit ───────────────────────── +echo "[3/8] git pull (ff-only)" +git pull --ff-only origin main +# TOCTOU guard: if origin/main advanced during the CI wait + pg_dump, the pull +# fast-forwards PAST the SHA we verified without re-checking its CI. Refuse +# rather than ship an unverified commit — a retry will verify the new tip. +PULLED_FULL=$(git rev-parse HEAD) +if [ "$PULLED_FULL" != "$REMOTE_FULL" ]; then + echo "ERROR: origin/main advanced during deploy (verified $REMOTE_SHA, now at $(git rev-parse --short HEAD)) — aborting; retry verifies the new tip" exit 1 fi +SHA=$(git rev-parse --short HEAD) +echo "SHA (deployed): $SHA" -# 4. Build and restart with health-check rollout -echo "[4/6] docker compose build" +# Resolve the deploy version AFTER pull (D2) so the tag matches the code being +# built. Compose interpolates $OIKOS_VERSION into each service's image: tag. +VERSION_FILE="$REPO_DIR/VERSION" +if [ -f "$VERSION_FILE" ]; then + OIKOS_VERSION="v$(head -n1 "$VERSION_FILE" | tr -d '[:space:]')" + export OIKOS_VERSION + echo "VERSION: $OIKOS_VERSION" +else + echo "WARNING: VERSION file missing — images will use :latest (rollback unavailable)" +fi + +# ── 4. Build version-tagged images (plan D2) ────────────────────────────── +echo "[4/8] docker compose build" DOCKER_BUILDKIT=1 docker compose --profile "$PROFILE" build \ --build-arg BUILDKIT_INLINE_CACHE=1 -# 5. Rolling restart -echo "[5/6] docker compose up -d" +# ── 5. Rolling restart ──────────────────────────────────────────────────── +echo "[5/8] docker compose up -d" docker compose --profile "$PROFILE" up -d --remove-orphans -# 6. Health check wait -echo "[6/6] health check" +# ── 6. Prune old image tags — keep the 3 newest per service so rollback ──── +# (OIKOS_VERSION=v0.x.y docker compose up) stays available. The repo list +# is derived from compose so it can't drift from the image: names. +echo "[6/8] prune old image tags (keep 3)" +if [ -n "$OIKOS_VERSION" ]; then + images=$(docker compose --profile "$PROFILE" config --images 2>/dev/null || true) + if [ -z "$images" ]; then + images="oikos-api oikos-scheduler oikos-notifier oikos-migrate oikos-seed oikos-nomos oikos-web" + fi + printf '%s\n' $images | sed 's/:.*//' | grep '^oikos-' | sort -u | while read -r repo; do + docker image ls "$repo" --format '{{.Tag}}' 2>/dev/null | grep '^v' | sort -rV | tail -n +4 | while read -r tag; do + docker rmi "$repo:$tag" >/dev/null 2>&1 || true + done + done +fi + +# ── 7. Health check wait ────────────────────────────────────────────────── +echo "[7/8] health check" +healthy=0 for i in $(seq 1 $RETRIES); do if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then echo "healthy after ${i}s" + healthy=1 break fi sleep "$SLEEP" done +if [ "$healthy" -ne 1 ]; then + echo "ERROR: health check failed after $((RETRIES * SLEEP))s" + exit 1 +fi -# 7. Seed secrets into Infisical (idempotent) -echo "[7/7] seed secrets" +# ── 8. Seed secrets into Infisical (idempotent) ─────────────────────────── +echo "[8/8] seed secrets" if [ -f "$REPO_DIR/scripts/seed-secrets.sh" ]; then REPO_DIR="$REPO_DIR" sh "$REPO_DIR/scripts/seed-secrets.sh" || \ echo "WARNING: secret seeding failed" else echo "SKIP: seed-secrets.sh not found" fi + +exit 0