dtoro
fa79c1ea25
D1: deploy.sh CI gate — read-only SHA via git ls-remote, Gitea commit-status
poll, portable mkdir deploy lock (macOS, no flock), TOCTOU guard, token
passed via curl --config - (not argv), graceful misconfig tolerance.
D2: version-tagged images — OIKOS_VERSION=v$VERSION, keep-last-3 prune derived
from 'docker compose config --images'; VERSION read after pull.
D3: per-IP rate limiting — new internal/httpapi/ratelimit.go (x/time/rate),
rightmost-XFF, /healthz exempt, ctx-driven sweep; disabled by default.
D4: mem_limit/cpus on all 10 compose services.
D5: staleness-aware health probes — new internal/health package wired into
scheduler (:8093) and notifier (:8094); nomos already had :8092.
Two /review passes hardened the deploy lock, TOCTOU guard, token hygiene,
and XFF handling.
94 lines
2.8 KiB
Go
94 lines
2.8 KiB
Go
// Package health provides a staleness-aware liveness probe for background-
|
|
// loop services (scheduler, notifier) that don't otherwise serve HTTP.
|
|
//
|
|
// The owning loop calls Probe.Bump() on each iteration. A /healthz endpoint
|
|
// returns 200 while the last bump is within the staleness window, and 503
|
|
// once the loop has gone quiet — so a wedged goroutine (stuck SSH, deadlock)
|
|
// surfaces as an unhealthy container instead of a silently-idle one.
|
|
package health
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"net/http"
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
// Probe tracks the last time the owning loop made progress.
|
|
type Probe struct {
|
|
last atomic.Int64 // unix-nano timestamp of the last Bump
|
|
stale time.Duration
|
|
}
|
|
|
|
// New returns a Probe that considers the owner healthy while Bump has been
|
|
// called within stale of the current time.
|
|
func New(stale time.Duration) *Probe {
|
|
if stale <= 0 {
|
|
stale = 2 * time.Minute
|
|
}
|
|
p := &Probe{stale: stale}
|
|
p.last.Store(time.Now().UnixNano()) // boot-healthy until first loop stalls
|
|
return p
|
|
}
|
|
|
|
// Bump records that the owning loop completed another iteration.
|
|
func (p *Probe) Bump() {
|
|
p.last.Store(time.Now().UnixNano())
|
|
}
|
|
|
|
// Healthy reports whether the last Bump is within the staleness window.
|
|
func (p *Probe) Healthy() bool {
|
|
last := time.Unix(0, p.last.Load())
|
|
return time.Since(last) <= p.stale
|
|
}
|
|
|
|
// Handler returns an http.Handler serving GET /healthz. Returns 200 with a
|
|
// small JSON body when healthy, 503 (Service Unavailable) when stale.
|
|
func (p *Probe) Handler() http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
last := time.Unix(0, p.last.Load())
|
|
if !p.Healthy() {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
}
|
|
json.NewEncoder(w).Encode(map[string]any{
|
|
"status": healthStatus(p.Healthy()),
|
|
"last_heartbeat": last.UTC().Format(time.RFC3339),
|
|
})
|
|
})
|
|
}
|
|
|
|
// Serve starts an HTTP server exposing the probe's /healthz on addr until ctx
|
|
// is cancelled. A no-op when addr is empty (local/non-docker runs skip it).
|
|
// The server is bound to addr (e.g. ":8093"); containers hit it via 127.0.0.1.
|
|
func (p *Probe) Serve(ctx context.Context, addr string) {
|
|
if addr == "" {
|
|
return
|
|
}
|
|
mux := http.NewServeMux()
|
|
mux.Handle("/healthz", p.Handler())
|
|
srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
|
|
|
go func() {
|
|
slog.Info("health server listening", "addr", addr)
|
|
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
|
slog.Warn("health server stopped", "addr", addr, "error", err)
|
|
}
|
|
}()
|
|
go func() {
|
|
<-ctx.Done()
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
srv.Shutdown(shutdownCtx)
|
|
}()
|
|
}
|
|
|
|
func healthStatus(ok bool) string {
|
|
if ok {
|
|
return "ok"
|
|
}
|
|
return "stale"
|
|
}
|