0.27.0 — Infisical hardening: runtime refresh, audit logging, CLI verify/audit, startup verification, SSH host key verification, interface consolidation

VERSION

@@ -1 +1 @@
-0.26.0
+0.27.0

cmd/nomos/agent.go

@@ -66,9 +66,9 @@ type agent struct {
 	queue *messageQueue
 }
 
-func newAgent(ctx context.Context, clients *mcpClientPool, st *store, agentSlug string) (*agent, error) {
+func newAgent(ctx context.Context, clients *mcpClientPool, st *store, agentSlug string, openrouterAPIKey string) (*agent, error) {
 	system := loadSoul()
-	apiKey := os.Getenv("OPENROUTER_API_KEY")
+	apiKey := openrouterAPIKey
 	model := os.Getenv("NOMOS_MODEL")
 	if model == "" {
 		// v4-pro over v4-flash: the flash tier over-narrates, occasionally

cmd/nomos/main.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/dtoro/oikos/internal/safego"
+	"github.com/dtoro/oikos/internal/secrets"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
 )
@@ -46,6 +47,36 @@ func main() {
 		databaseURL = os.Getenv("OIKOS_DATABASE_URL")
 	}
 
+	// Resolve secrets from Infisical, falling back to env vars.
+	// The MCP token and OpenRouter key are fetched once at startup and
+	// injected via os.Setenv so downstream code (newAgent) picks them up
+	// without signature changes.
+	sec := secrets.NewManagerFromConfig(
+		os.Getenv("OIKOS_INFISICAL_SITE_URL"),
+		os.Getenv("OIKOS_INFISICAL_CLIENT_ID"),
+		os.Getenv("OIKOS_INFISICAL_CLIENT_SECRET"),
+		os.Getenv("OIKOS_INFISICAL_PROJECT_ID"),
+		os.Getenv("OIKOS_INFISICAL_ENV"),
+		os.Getenv("OIKOS_SECRETS_DIR"),
+	)
+	var openrouterAPIKey string
+	var secretsResolved int
+	if sec != nil {
+		resCtx, resCancel := context.WithTimeout(context.Background(), 10*time.Second)
+		if v := secrets.ResolveSecret(resCtx, sec, "mcp/bearer-token", ""); v != "" {
+			mcpToken = v
+			secretsResolved++
+		}
+		openrouterAPIKey = secrets.ResolveSecret(resCtx, sec, "openrouter/api-key", os.Getenv("OPENROUTER_API_KEY"))
+		if openrouterAPIKey != "" && openrouterAPIKey != os.Getenv("OPENROUTER_API_KEY") {
+			secretsResolved++
+		}
+		resCancel()
+		if secretsResolved > 0 {
+			slog.Info("nomos: secrets resolved from Infisical", "count", secretsResolved)
+		}
+	}
+
 	switch os.Args[1] {
 	case "serve":
 		ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
@@ -75,7 +106,7 @@ func main() {
 			defer st.close()
 		}
 
-		nAgent, err := newAgent(ctx, clientPool, st, agentSlug)
+		nAgent, err := newAgent(ctx, clientPool, st, agentSlug, openrouterAPIKey)
 		if err != nil {
 			slog.Error("nomos: agent init", "error", err)
 			os.Exit(1)

cmd/oikos/main.go

@@ -37,12 +37,39 @@ func main() {
 	logger := observability.NewLogger(cfg.Debug)
 	slog.SetDefault(logger)
 
-	slog.Info("starting oikos", "role", role, "config", cfg)
-
 	ctx, cancel := signal.NotifyContext(context.Background(),
 		syscall.SIGTERM, syscall.SIGINT)
 	defer cancel()
 
+	// Resolve secrets from Infisical, overlaying env-derived config values.
+	// If Infisical is not configured, env vars are used as-is (no change).
+	sec := secrets.NewManagerFromConfig(
+		cfg.InfisicalSiteURL,
+		cfg.InfisicalClientID,
+		cfg.InfisicalClientSecret,
+		cfg.InfisicalProjectID,
+		cfg.InfisicalEnv,
+		cfg.SecretsDir,
+	)
+	if sec != nil {
+		overlays := secrets.ConfigOverlays(map[string]func(string){
+			"matrix/token":         func(v string) { cfg.MatrixToken = v },
+			"approval/hmac-secret": func(v string) { cfg.ApprovalHMACSecret = v },
+			"mcp/bearer-token":     func(v string) { cfg.MCPBearerToken = v },
+			"api/token":            func(v string) { cfg.APIToken = v },
+			"oidc/client-secret":   func(v string) { cfg.OIDCClientSecret = v },
+		})
+		n := secrets.OverlayConfig(ctx, sec, overlays)
+		slog.Info("secrets resolved from Infisical", "count", n)
+
+		secrets.VerifyExpectedSecrets(ctx, sec, []string{
+			"matrix/token", "approval/hmac-secret", "mcp/bearer-token",
+			"api/token", "oidc/client-secret", "openrouter/api-key", "webhook/hmac-secret",
+		})
+	}
+
+	slog.Info("starting oikos", "role", role, "config", cfg)
+
 	switch role {
 	case "migrate":
 		if err := runMigrate(ctx, cfg); err != nil {
@@ -115,7 +142,7 @@ Roles:
   scheduler  Run the observe loop
   notifier   Run the notification service (Matrix alerts)
   all        Run all roles in one process (dev mode)
-  secret     Secret management (Infisical: get, set, list, migrate, export-sops)
+  secret     Secret management (Infisical: get, set, list, verify, audit, migrate, export-sops)
   knowledge  Convert wiki to knowledge seed (one-shot)
   version    Print version info
 
@@ -336,6 +363,12 @@ func runSecret(ctx context.Context, cfg config.Config) {
 			fmt.Println(k)
 		}
 
+	case "verify":
+		runSecretVerify(ctx, cfg)
+
+	case "audit":
+		runSecretAudit(ctx, cfg)
+
 	case "migrate", "export-sops":
 		runSecretLegacy(ctx, cfg, sub)
 
@@ -345,6 +378,93 @@ func runSecret(ctx context.Context, cfg config.Config) {
 	}
 }
 
+// expectedSecrets is the set of keys that should exist in Infisical
+// for a fully-migrated deployment.
+var expectedSecrets = []string{
+	"matrix/token",
+	"approval/hmac-secret",
+	"mcp/bearer-token",
+	"api/token",
+	"oidc/client-secret",
+	"openrouter/api-key",
+	"webhook/hmac-secret",
+}
+
+// runSecretVerify checks that all expected secrets are present in Infisical.
+func runSecretVerify(ctx context.Context, cfg config.Config) {
+	backend := newInfisicalBackendOrFail(cfg)
+	keys, err := backend.List(ctx)
+	if err != nil {
+		slog.Error("verify: list", "error", err)
+		os.Exit(1)
+	}
+	keySet := make(map[string]struct{}, len(keys))
+	for _, k := range keys {
+		keySet[k] = struct{}{}
+	}
+
+	missing := 0
+	for _, exp := range expectedSecrets {
+		if _, ok := keySet[exp]; !ok {
+			fmt.Printf("MISSING: %s\n", exp)
+			missing++
+		} else {
+			fmt.Printf("OK:      %s\n", exp)
+		}
+	}
+
+	fmt.Printf("\n%d/%d present, %d missing\n", len(expectedSecrets)-missing, len(expectedSecrets), missing)
+	if missing > 0 {
+		os.Exit(1)
+	}
+}
+
+// runSecretAudit resolves all expected secrets from Infisical and prints
+// a diff against current env-derived values. Values are truncated for safety.
+func runSecretAudit(ctx context.Context, cfg config.Config) {
+	backend := newInfisicalBackendOrFail(cfg)
+
+	envValues := map[string]string{
+		"matrix/token":         cfg.MatrixToken,
+		"approval/hmac-secret": cfg.ApprovalHMACSecret,
+		"mcp/bearer-token":     cfg.MCPBearerToken,
+		"api/token":            cfg.APIToken,
+		"oidc/client-secret":   cfg.OIDCClientSecret,
+	}
+
+	fmt.Println("key                           infisical  env        status")
+	fmt.Println(strings.Repeat("-", 72))
+
+	for _, key := range expectedSecrets {
+		infVal, infErr := backend.Get(ctx, key)
+		envVal := envValues[key]
+
+		if infErr != nil {
+			fmt.Printf("%-29s ERROR      %-10s NOT-IN-INFISICAL\n", key, trunc(envVal, 8))
+			continue
+		}
+		if envVal == "" {
+			fmt.Printf("%-29s %-10s (empty)   INFISICAL-ONLY\n", key, trunc(infVal, 8))
+			continue
+		}
+		if infVal == envVal {
+			fmt.Printf("%-29s %-10s %-10s MATCH\n", key, trunc(infVal, 8), trunc(envVal, 8))
+		} else {
+			fmt.Printf("%-29s %-10s %-10s DRIFT\n", key, trunc(infVal, 8), trunc(envVal, 8))
+		}
+	}
+}
+
+func trunc(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	if n > 1 {
+		return s[:n-1] + "…"
+	}
+	return s[:n]
+}
+
 // newInfisicalBackendOrFail creates an Infisical backend from config or exits.
 func newInfisicalBackendOrFail(cfg config.Config) *secrets.InfisicalBackend {
 	if cfg.InfisicalSiteURL == "" {

cmd/webhook/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"crypto/hmac"
 	"crypto/sha256"
 	"encoding/hex"
@@ -12,26 +13,29 @@ import (
 	"os/exec"
 	"time"
 
+	"github.com/dtoro/oikos/internal/secrets"
 	"github.com/dtoro/oikos/internal/safego"
 )
 
 func main() {
+	ctx := context.Background()
+
 	port := os.Getenv("WEBHOOK_LISTEN")
 	if port == "" {
 		port = ":9797"
 	}
 
-	secret := os.Getenv("WEBHOOK_HMAC_SECRET")
-	if secret == "" {
-		fmt.Fprintln(os.Stderr, "WEBHOOK_HMAC_SECRET must be set")
-		os.Exit(1)
-	}
-
 	repoDir := os.Getenv("WEBHOOK_REPO_DIR")
 	if repoDir == "" {
 		repoDir = os.Getenv("HOME") + "/Projects/oikos"
 	}
 
+	secret := resolveWebhookHMAC(ctx)
+	if secret == "" {
+		fmt.Fprintln(os.Stderr, "WEBHOOK_HMAC_SECRET must be set (env var or Infisical webhook/hmac-secret)")
+		os.Exit(1)
+	}
+
 	mux := http.NewServeMux()
 	mux.HandleFunc("/deploy", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPost {
@@ -94,3 +98,18 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+// resolveWebhookHMAC fetches the webhook HMAC secret from Infisical,
+// falling back to the WEBHOOK_HMAC_SECRET env var.
+func resolveWebhookHMAC(ctx context.Context) string {
+	sec := secrets.NewManagerFromConfig(
+		os.Getenv("OIKOS_INFISICAL_SITE_URL"),
+		os.Getenv("OIKOS_INFISICAL_CLIENT_ID"),
+		os.Getenv("OIKOS_INFISICAL_CLIENT_SECRET"),
+		os.Getenv("OIKOS_INFISICAL_PROJECT_ID"),
+		os.Getenv("OIKOS_INFISICAL_ENV"),
+		os.Getenv("OIKOS_SECRETS_DIR"),
+	)
+	envFallback := os.Getenv("WEBHOOK_HMAC_SECRET")
+	return secrets.ResolveSecret(ctx, sec, "webhook/hmac-secret", envFallback)
+}

internal/actuator/actuator.go

@@ -424,7 +424,7 @@ func sshExecSimple(ctx context.Context, host, user, command string) (string, err
 	clientCfg := &ssh.ClientConfig{
 		User:            user,
 		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		HostKeyCallback: HostKeyCallback(),
 		Timeout:         10 * time.Second,
 	}
 

internal/actuator/hostkeys.go

@@ -0,0 +1,129 @@
+package actuator
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log/slog"
+	"net"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+)
+
+var (
+	hostKeyMu    sync.RWMutex
+	hostKeyCache map[string]ssh.PublicKey
+	hostKeyOnce sync.Once
+	hostKeySrc   HostKeySource
+)
+
+// HostKeySource provides storage for SSH host public keys.
+type HostKeySource interface {
+	GetHostKey(ctx context.Context, hostname string) (string, error)
+	SetHostKey(ctx context.Context, hostname string, key string) error
+}
+
+// SetHostKeySource sets the host key source. Must be called before
+// any SSH connections. A nil source enables TOFU-only mode (keys
+// accepted in memory but not persisted).
+func SetHostKeySource(src HostKeySource) {
+	hostKeyMu.Lock()
+	defer hostKeyMu.Unlock()
+	hostKeySrc = src
+}
+
+// HostKeyCallback returns an ssh.HostKeyCallback that verifies host keys.
+// Known keys are verified (MITM detection). Unknown keys are accepted
+// via TOFU and optionally persisted to the source.
+func HostKeyCallback() ssh.HostKeyCallback {
+	return hostKeyVerify
+}
+
+func hostKeyVerify(hostname string, remote net.Addr, key ssh.PublicKey) error {
+	hostKeyOnce.Do(func() {
+		hostKeyCache = make(map[string]ssh.PublicKey)
+	})
+
+	normalized := hostWithoutPort(hostname)
+
+	hostKeyMu.RLock()
+	known, exists := hostKeyCache[normalized]
+	hostKeyMu.RUnlock()
+
+	if exists {
+		if bytes.Equal(key.Marshal(), known.Marshal()) {
+			return nil
+		}
+		return fmt.Errorf("SSH HOST KEY CHANGED for %s (possible MITM)", normalized)
+	}
+
+	hostKeyMu.Lock()
+	hostKeyCache[normalized] = key
+	hostKeyMu.Unlock()
+
+	slog.Info("ssh: accepting new host key (TOFU)", "host", normalized)
+
+	if hostKeySrc != nil {
+		go persistHostKey(normalized, key)
+	}
+
+	return nil
+}
+
+func persistHostKey(hostname string, key ssh.PublicKey) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	keyBase64 := key.Type() + " " + string(key.Marshal())
+	if err := hostKeySrc.SetHostKey(ctx, "ssh/host-keys/"+hostname, keyBase64); err != nil {
+		slog.Warn("ssh: failed to persist host key", "host", hostname, "error", err)
+	}
+}
+
+// LoadHostKeys pre-loads known host keys from the source into the
+// in-memory cache. Call at startup to avoid TOFU on first connection.
+// The source should return key lines in the format "key-type base64-data".
+func LoadHostKeys(ctx context.Context, hostnames []string, src HostKeySource) {
+	if src == nil {
+		return
+	}
+	SetHostKeySource(src)
+
+	hostKeyMu.Lock()
+	defer hostKeyMu.Unlock()
+
+	if hostKeyCache == nil {
+		hostKeyCache = make(map[string]ssh.PublicKey)
+	}
+
+	loaded := 0
+	for _, hostname := range hostnames {
+		keyData, err := src.GetHostKey(ctx, "ssh/host-keys/"+hostname)
+		if err != nil {
+			slog.Debug("ssh: no stored key for host", "host", hostname, "error", err)
+			continue
+		}
+		pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(keyData))
+		if err != nil {
+			slog.Warn("ssh: invalid stored key for host", "host", hostname, "error", err)
+			continue
+		}
+		hostKeyCache[hostname] = pubKey
+		loaded++
+	}
+
+	if loaded > 0 {
+		slog.Info("ssh: loaded host keys from Infisical", "count", loaded)
+	}
+}
+
+func hostWithoutPort(hostname string) string {
+	for i := len(hostname) - 1; i >= 0; i-- {
+		if hostname[i] == ':' {
+			return hostname[:i]
+		}
+	}
+	return hostname
+}

internal/actuator/hostkeys_infisical.go

@@ -0,0 +1,56 @@
+package actuator
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/dtoro/oikos/internal/db"
+	"github.com/dtoro/oikos/internal/secrets"
+)
+
+// InfisicalHostKeySource implements HostKeySource backed by Infisical.
+type InfisicalHostKeySource struct {
+	sec secrets.Backend
+}
+
+// NewInfisicalHostKeySource creates a HostKeySource that reads/writes
+// SSH host public keys from Infisical under the `ssh/host-keys/` prefix.
+func NewInfisicalHostKeySource(sec secrets.Backend) *InfisicalHostKeySource {
+	return &InfisicalHostKeySource{sec: sec}
+}
+
+func (s *InfisicalHostKeySource) GetHostKey(ctx context.Context, path string) (string, error) {
+	val, err := s.sec.Get(ctx, path)
+	if err != nil {
+		return "", err
+	}
+	return val, nil
+}
+
+func (s *InfisicalHostKeySource) SetHostKey(ctx context.Context, path string, key string) error {
+	return s.sec.Set(ctx, path, key)
+}
+
+// ResolveSSHHosts queries the DB for active proxmox-host and standalone-server
+// entities, returning their slugs as SSH host identifiers.
+func ResolveSSHHosts(ctx context.Context, pool *db.Pool) []string {
+	rows, err := pool.Query(ctx, `
+		SELECT slug FROM entities
+		WHERE type IN ('proxmox-host', 'standalone-server')
+		  AND state = 'active'
+		ORDER BY slug`)
+	if err != nil {
+		slog.Warn("ssh: failed to list hosts", "error", err)
+		return nil
+	}
+	defer rows.Close()
+
+	var hosts []string
+	for rows.Next() {
+		var slug string
+		if rows.Scan(&slug) == nil {
+			hosts = append(hosts, slug)
+		}
+	}
+	return hosts
+}

internal/actuator/ssh.go

@@ -157,7 +157,7 @@ func ExecuteProcedure(
 	clientCfg := &ssh.ClientConfig{
 		User:            cfg.User,
 		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory
+		HostKeyCallback: HostKeyCallback(),
 		Timeout:         cfg.Timeout,
 	}
 

internal/httpapi/server.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/dtoro/oikos/internal/config"
+	"github.com/dtoro/oikos/internal/actuator"
 	"github.com/dtoro/oikos/internal/db"
 	"github.com/dtoro/oikos/internal/httpapi/gen"
 	mcphandler "github.com/dtoro/oikos/internal/mcp"
@@ -53,22 +54,12 @@ type actor struct {
 type Server struct {
 	pool           *db.Pool
 	cfg            config.Config
-	secretsManager secretsBackend
+	secretsManager secrets.Backend
 	sseBroker      *sseBroker
 	sseSubs        map[*sseSubscriber]struct{}
 	sseMu          sync.Mutex
 }
 
-// secretsBackend is a minimal interface for secrets operations used by the
-// HTTP API (enrollment key storage, listing, retrieval). Compatible with
-// internal/secrets.Backend. If Infisical is configured, a real backend is
-// wired in; otherwise the field stays nil and all guarded paths are no-ops.
-type secretsBackend interface {
-	Get(ctx context.Context, key string) (string, error)
-	Set(ctx context.Context, key string, value string) error
-	List(ctx context.Context) ([]string, error)
-}
-
 // NewHandler builds the full HTTP handler: /healthz (unauthenticated,
 // SG18) + the OpenAPI surface under /api/v1 behind bearer auth.
 //
@@ -84,7 +75,7 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand
 		sseSubs:   make(map[*sseSubscriber]struct{}),
 	}
 
-	// Wire Infisical backend when configured.
+	// Wire secrets backend: Infisical primary with SOPS DR fallback.
 	if cfg.InfisicalSiteURL != "" {
 		infCfg := secrets.InfisicalConfig{
 			SiteURL:      cfg.InfisicalSiteURL,
@@ -97,8 +88,24 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand
 		if infCfg.Env == "" {
 			infCfg.Env = "dev"
 		}
-		s.secretsManager = secrets.NewInfisicalBackend(infCfg)
+		primary := secrets.NewInfisicalBackend(infCfg)
-		slog.Info("secrets backend wired", "backend", "infisical", "site", cfg.InfisicalSiteURL)
+		var fallback secrets.Backend
+		if cfg.SecretsDir != "" {
+			fallback = secrets.NewSOPSBackend(cfg.SecretsDir)
+		}
+		s.secretsManager = secrets.NewManager(primary, fallback)
+		slog.Info("secrets backend wired", "backend", "infisical+sops", "site", cfg.InfisicalSiteURL)
+
+		// Start background secret refresh loop.
+		if mgr, ok := s.secretsManager.(*secrets.Manager); ok {
+			mgr.StartRefreshLoop(ctx)
+		}
+
+		// Pre-load SSH host keys from Infisical for host verification.
+		if hosts := actuator.ResolveSSHHosts(ctx, pool); len(hosts) > 0 {
+			hkSrc := actuator.NewInfisicalHostKeySource(s.secretsManager)
+			actuator.LoadHostKeys(ctx, hosts, hkSrc)
+		}
 	}
 
 	// Start background SSE listener, tied to ctx for clean shutdown.

internal/mcp/secrets_tools_test.go

@@ -5,11 +5,11 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/dtoro/oikos/internal/secrets"
 	"github.com/google/uuid"
 	"github.com/modelcontextprotocol/go-sdk/mcp"
 )
 
-// mockSecretBackend implements secretBackend for testing.
 type mockSecretBackend struct {
 	data map[string]string
 }
@@ -17,7 +17,7 @@ type mockSecretBackend struct {
 func (m *mockSecretBackend) Get(ctx context.Context, key string) (string, error) {
 	v, ok := m.data[key]
 	if !ok {
-		return "", &secretErr{msg: "secret not found: " + key}
+		return "", secrets.ErrNotFound
 	}
 	return v, nil
 }
@@ -35,12 +35,10 @@ func (m *mockSecretBackend) List(ctx context.Context) ([]string, error) {
 	return keys, nil
 }
 
-type secretErr struct{ msg string }
+func (m *mockSecretBackend) Name() string { return "mock" }
-
-func (e *secretErr) Error() string { return e.msg }
 
 // findToolHandler locates a tool's handler from allTools by name.
-func findToolHandler(t *testing.T, pool interface{}, name string, sec secretBackend) func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+func findToolHandler(t *testing.T, pool interface{}, name string, sec secrets.Backend) func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
 	t.Helper()
 	for _, r := range allTools(nil, uuid.Nil, sec) {
 		if r.tool.Name == name {
@@ -51,7 +49,7 @@ func findToolHandler(t *testing.T, pool interface{}, name string, sec secretBack
 	return nil
 }
 
-func callToolJSON(t *testing.T, name string, sec secretBackend, args map[string]any) any {
+func callToolJSON(t *testing.T, name string, sec secrets.Backend, args map[string]any) any {
 	t.Helper()
 	handler := findToolHandler(t, nil, name, sec)
 	argBytes, _ := json.Marshal(args)
@@ -74,7 +72,7 @@ func callToolJSON(t *testing.T, name string, sec secretBackend, args map[string]
 	return out
 }
 
-func callToolText(t *testing.T, name string, sec secretBackend, args map[string]any) string {
+func callToolText(t *testing.T, name string, sec secrets.Backend, args map[string]any) string {
 	t.Helper()
 	handler := findToolHandler(t, nil, name, sec)
 	argBytes, _ := json.Marshal(args)

internal/mcp/server.go

@@ -19,12 +19,14 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dtoro/oikos/internal/actuator"
 	"github.com/dtoro/oikos/internal/db"
 	"github.com/dtoro/oikos/internal/db/sqlcgen"
 	"github.com/dtoro/oikos/internal/execlog"
 	"github.com/dtoro/oikos/internal/observability"
 	"github.com/dtoro/oikos/internal/policy"
 	"github.com/dtoro/oikos/internal/remote"
+	"github.com/dtoro/oikos/internal/secrets"
 	"github.com/google/jsonschema-go/jsonschema"
 	"github.com/google/uuid"
 	"github.com/modelcontextprotocol/go-sdk/mcp"
@@ -47,18 +49,9 @@ func objSchema(props ...prop) *jsonschema.Schema {
 	return s
 }
 
-// secretBackend is the interface MCP tools use to access the secrets store.
-// Defined here to avoid importing the full secrets package (which brings in
-// the Infisical SDK). Mirrors the subset of secrets.Backend used by tools.
-type secretBackend interface {
-	Get(ctx context.Context, key string) (string, error)
-	Set(ctx context.Context, key string, value string) error
-	List(ctx context.Context) ([]string, error)
-}
-
 // NewHandler creates an http.Handler that serves the Oikos MCP server.
 // agentID is the Nomos agent entity UUID; tool calls are logged to agent_activity.
-func NewHandler(pool *db.Pool, token string, agentID uuid.UUID, sec secretBackend) http.Handler {
+func NewHandler(pool *db.Pool, token string, agentID uuid.UUID, sec secrets.Backend) http.Handler {
 	s := newServer(pool, agentID, sec)
 	handler := mcp.NewStreamableHTTPHandler(func(r *http.Request) *mcp.Server {
 		if token != "" {
@@ -74,7 +67,7 @@ func NewHandler(pool *db.Pool, token string, agentID uuid.UUID, sec secretBacken
 // toolHandler is the function signature registered via AddTool.
 type toolHandler = mcp.ToolHandler
 
-func newServer(pool *db.Pool, agentID uuid.UUID, sec secretBackend) *mcp.Server {
+func newServer(pool *db.Pool, agentID uuid.UUID, sec secrets.Backend) *mcp.Server {
 	s := mcp.NewServer(&mcp.Implementation{Name: "oikos", Version: "dev"}, &mcp.ServerOptions{
 		Logger: slog.Default(),
 	})
@@ -499,7 +492,7 @@ func sshExecStream(ctx context.Context, host, user, command string, sink execlog
 	cfg := &ssh.ClientConfig{
 		User:            user,
 		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		HostKeyCallback: actuator.HostKeyCallback(),
 		Timeout:         10 * time.Second,
 	}
 

internal/mcp/tools.go

@@ -14,6 +14,7 @@ import (
 	"github.com/dtoro/oikos/internal/db/sqlcgen"
 	"github.com/dtoro/oikos/internal/observability"
 	"github.com/dtoro/oikos/internal/policy"
+	"github.com/dtoro/oikos/internal/secrets"
 	"github.com/google/uuid"
 	"github.com/modelcontextprotocol/go-sdk/mcp"
 )
@@ -29,7 +30,7 @@ type toolReg struct {
 // allTools returns every MCP tool registration. Tool definitions, schemas,
 // descriptions, and handler bodies are kept verbatim from the former inline
 // newServer registrations.
-func allTools(pool *db.Pool, agentID uuid.UUID, sec secretBackend) []toolReg {
+func allTools(pool *db.Pool, agentID uuid.UUID, sec secrets.Backend) []toolReg {
 	return []toolReg{
 		{tool: &mcp.Tool{Name: "ping", Description: "Lightweight connectivity check. Returns server identity, no DB hit.",
 			InputSchema: objSchema(),

internal/secrets/backend.go

@@ -1,10 +1,9 @@
-// Package secrets abstracts secret retrieval across backends (SOPS, Infisical).
-// Phase 5: SOPS → Infisical migration with SOPS DR fallback.
 package secrets
 
 import (
 	"context"
 	"errors"
+	"log/slog"
 	"sync"
 	"time"
 )
@@ -14,24 +13,23 @@ var ErrBackendUnavailable = errors.New("secret backend unavailable")
 
 // Backend is the interface for retrieving and storing secrets.
 type Backend interface {
-	// Get retrieves a secret value by path/key.
 	Get(ctx context.Context, key string) (string, error)
-	// List returns all secret keys available in this backend.
 	List(ctx context.Context) ([]string, error)
-	// Set stores a secret value. Used during migration.
 	Set(ctx context.Context, key string, value string) error
-	// Name returns a human-readable backend identifier.
 	Name() string
 }
 
 // Manager holds a primary and fallback backend. If the primary fails,
-// it falls back to the secondary.
+// it falls back to the secondary. Supports periodic background refresh
+// of cached secrets from the primary backend.
 type Manager struct {
-	primary   Backend
+	primary     Backend
-	fallback  Backend
+	fallback    Backend
-	cache     map[string]cachedSecret
+	cache       map[string]cachedSecret
-	mu        sync.RWMutex
+	mu          sync.RWMutex
-	cacheTTL  time.Duration
+	cacheTTL    time.Duration
+	refreshMu   sync.Mutex
+	lastRefresh time.Time
 }
 
 type cachedSecret struct {
@@ -39,21 +37,97 @@ type cachedSecret struct {
 	expiresAt time.Time
 }
 
+// ManagerRefreshInterval controls how often cached secrets are re-fetched
+// from the primary backend in the background. Zero disables background refresh.
+var ManagerRefreshInterval = 5 * time.Minute
+
 // NewManager creates a secret manager with primary and fallback backends.
 func NewManager(primary, fallback Backend) *Manager {
 	return &Manager{
-		primary:  primary,
+		primary:     primary,
-		fallback: fallback,
+		fallback:    fallback,
-		cache:    make(map[string]cachedSecret),
+		cache:       make(map[string]cachedSecret),
-		cacheTTL: 5 * time.Minute,
+		cacheTTL:    5 * time.Minute,
+		lastRefresh: time.Now(),
 	}
 }
 
+// StartRefreshLoop starts a background goroutine that periodically refreshes
+// cached secrets from the primary backend. Call from the server's main
+// goroutine. The loop runs until ctx is cancelled.
+func (m *Manager) StartRefreshLoop(ctx context.Context) {
+	if ManagerRefreshInterval <= 0 {
+		return
+	}
+	slog.Info("secrets: background refresh loop started",
+		"interval", ManagerRefreshInterval)
+	ticker := time.NewTicker(ManagerRefreshInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			slog.Info("secrets: refresh loop stopped")
+			return
+		case <-ticker.C:
+			m.refreshAll(ctx)
+		}
+	}
+}
+
+// refreshAll re-fetches all cached secrets from the primary backend.
+// Keys not found in the primary are left cached (they may be in fallback).
+// Logs a summary line.
+func (m *Manager) refreshAll(ctx context.Context) {
+	m.refreshMu.Lock()
+	defer m.refreshMu.Unlock()
+
+	if m.primary == nil {
+		return
+	}
+
+	m.mu.RLock()
+	keys := make([]string, 0, len(m.cache))
+	for k := range m.cache {
+		keys = append(keys, k)
+	}
+	m.mu.RUnlock()
+
+	if len(keys) == 0 {
+		return
+	}
+
+	refreshed, stale, failed := 0, 0, 0
+	for _, key := range keys {
+		val, err := m.primary.Get(ctx, key)
+		if err != nil {
+			failed++
+			continue
+		}
+		m.mu.RLock()
+		cached, ok := m.cache[key]
+		m.mu.RUnlock()
+		if !ok || val != cached.value {
+			m.mu.Lock()
+			m.cache[key] = cachedSecret{value: val, expiresAt: time.Now().Add(m.cacheTTL)}
+			m.mu.Unlock()
+			refreshed++
+		} else {
+			stale++
+		}
+	}
+
+	m.lastRefresh = time.Now()
+	slog.Info("secrets: background refresh complete",
+		"refreshed", refreshed, "stale", stale, "failed", failed,
+		"cached", len(keys))
+}
+
 // Get retrieves a secret from primary, falling back to secondary on error.
 func (m *Manager) Get(ctx context.Context, key string) (string, error) {
 	m.mu.RLock()
 	if cached, ok := m.cache[key]; ok && time.Now().Before(cached.expiresAt) {
 		m.mu.RUnlock()
+		slog.Debug("secret: cache hit", "key", key)
 		return cached.value, nil
 	}
 	m.mu.RUnlock()
@@ -63,16 +137,20 @@ func (m *Manager) Get(ctx context.Context, key string) (string, error) {
 		m.mu.Lock()
 		m.cache[key] = cachedSecret{value: val, expiresAt: time.Now().Add(m.cacheTTL)}
 		m.mu.Unlock()
+		slog.Debug("secret: fetched from primary", "key", key)
 		return val, nil
 	}
 
 	if m.fallback != nil {
 		val, fallbackErr := m.fallback.Get(ctx, key)
 		if fallbackErr == nil {
+			slog.Warn("secret: primary failed, using fallback",
+				"key", key, "primary_error", err)
 			return val, nil
 		}
 	}
 
+	slog.Warn("secret: not found in any backend", "key", key, "error", err)
 	return "", err
 }
 
@@ -85,11 +163,17 @@ func (m *Manager) List(ctx context.Context) ([]string, error) {
 	return keys, err
 }
 
-// Set stores a secret in the primary backend (used during migration).
+// Set stores a secret in the primary backend and invalidates the cache.
 func (m *Manager) Set(ctx context.Context, key string, value string) error {
+	m.InvalidateCache()
 	return m.primary.Set(ctx, key, value)
 }
 
+// Name returns the primary backend name.
+func (m *Manager) Name() string {
+	return m.primary.Name()
+}
+
 // PrimaryName returns the name of the primary backend.
 func (m *Manager) PrimaryName() string {
 	return m.primary.Name()
@@ -101,3 +185,14 @@ func (m *Manager) InvalidateCache() {
 	m.cache = make(map[string]cachedSecret)
 	m.mu.Unlock()
 }
+
+// LastRefresh returns the timestamp of the last background refresh.
+func (m *Manager) LastRefresh() time.Time {
+	return m.lastRefresh
+}
+
+// secretOverlay maps an Infisical key to a config setter function.
+type secretOverlay struct {
+	infisicalKey string
+	apply        func(value string)
+}

internal/secrets/config_overlay.go

@@ -0,0 +1,118 @@
+package secrets
+
+import (
+	"context"
+	"log/slog"
+)
+
+// NewManagerFromConfig creates a secrets Manager from the Infisical connection
+// parameters in cfg, with an optional SOPS fallback from cfg.SecretsDir.
+// Returns nil if Infisical is not configured.
+func NewManagerFromConfig(siteURL, clientID, clientSecret, projectID, env, secretsDir string) *Manager {
+	if siteURL == "" {
+		return nil
+	}
+	infCfg := InfisicalConfig{
+		SiteURL:      siteURL,
+		ClientID:     clientID,
+		ClientSecret: clientSecret,
+		ProjectID:    projectID,
+		SecretPath:   "/",
+		Env:          env,
+	}
+	if infCfg.Env == "" {
+		infCfg.Env = "dev"
+	}
+	primary := NewInfisicalBackend(infCfg)
+	var fallback Backend
+	if secretsDir != "" {
+		fallback = NewSOPSBackend(secretsDir)
+	}
+	return NewManager(primary, fallback)
+}
+
+// VerifyExpectedSecrets checks that a list of expected keys are present
+// in the backend. Logs a summary and returns the count of missing keys.
+// Use at startup to detect incomplete Infisical migration.
+func VerifyExpectedSecrets(ctx context.Context, sec Backend, expected []string) int {
+	keys, err := sec.List(ctx)
+	if err != nil {
+		slog.Warn("secrets: cannot verify expected secrets, list failed", "error", err)
+		return len(expected)
+	}
+	keySet := make(map[string]struct{}, len(keys))
+	for _, k := range keys {
+		keySet[k] = struct{}{}
+	}
+	missing := 0
+	for _, exp := range expected {
+		if _, ok := keySet[exp]; !ok {
+			missing++
+			slog.Warn("secrets: expected key missing from Infisical", "key", exp)
+		}
+	}
+	if missing == 0 {
+		slog.Info("secrets: all expected keys present", "count", len(expected))
+	} else {
+		slog.Warn("secrets: some expected keys missing from Infisical",
+			"missing", missing, "total", len(expected))
+	}
+	return missing
+}
+
+// OverlayConfig fetches secrets from the backend and returns a function that
+// applies them to config fields. Each entry maps an Infisical key to a setter;
+// if the key is found and non-empty, the setter is called; if not found or
+// empty, the env-derived value is left unchanged and a warning is logged.
+// Returns the number of secrets resolved from Infisical (useful for logging).
+func OverlayConfig(ctx context.Context, sec Backend, overlays []secretOverlay) int {
+	resolved := 0
+	for _, o := range overlays {
+		val, err := sec.Get(ctx, o.infisicalKey)
+		if err != nil {
+			slog.Warn("secret not resolved from Infisical, using env fallback",
+				"key", o.infisicalKey, "error", err)
+			continue
+		}
+		if val == "" {
+			slog.Warn("Infisical returned empty value, keeping env-derived value",
+				"key", o.infisicalKey)
+			continue
+		}
+		o.apply(val)
+		resolved++
+	}
+	return resolved
+}
+
+// ConfigOverlays returns the standard set of Infisical → config overlays for
+// the oikos binary. Each overlay is attempted at startup; if the key exists
+// in Infisical, it overrides the env-derived value.
+func ConfigOverlays(cfg map[string]func(string)) []secretOverlay {
+	overlays := make([]secretOverlay, 0, len(cfg))
+	for key, setter := range cfg {
+		overlays = append(overlays, secretOverlay{infisicalKey: key, apply: setter})
+	}
+	return overlays
+}
+
+// ResolveSecret attempts to fetch a single secret from the backend. Returns
+// the Infisical value if found and non-empty, otherwise falls back to the
+// env-derived value. Warnings are logged for failures.
+func ResolveSecret(ctx context.Context, sec Backend, infisicalKey, fallback string) string {
+	if sec == nil {
+		return fallback
+	}
+	val, err := sec.Get(ctx, infisicalKey)
+	if err != nil {
+		slog.Warn("secret not resolved from Infisical, using env fallback",
+			"key", infisicalKey, "error", err)
+		return fallback
+	}
+	if val == "" {
+		slog.Warn("Infisical returned empty value, keeping env-derived value",
+			"key", infisicalKey)
+		return fallback
+	}
+	return val
+}

internal/secrets/infisical.go

@@ -3,7 +3,6 @@ package secrets
 import (
 	"context"
 	"fmt"
-	"os"
 	"strings"
 	"sync"
 
@@ -54,14 +53,8 @@ func (b *InfisicalBackend) connect() error {
 
 	clientID := b.cfg.ClientID
 	clientSecret := b.cfg.ClientSecret
-	if clientID == "" {
-		clientID = os.Getenv("INFISICAL_CLIENT_ID")
-	}
-	if clientSecret == "" {
-		clientSecret = os.Getenv("INFISICAL_CLIENT_SECRET")
-	}
 	if clientID == "" || clientSecret == "" {
-		return fmt.Errorf("%w: INFISICAL_CLIENT_ID and INFISICAL_CLIENT_SECRET not set", ErrBackendUnavailable)
+		return fmt.Errorf("%w: OIKOS_INFISICAL_CLIENT_ID and OIKOS_INFISICAL_CLIENT_SECRET not set", ErrBackendUnavailable)
 	}
 
 	_, err := b.client.Auth().UniversalAuthLogin(clientID, clientSecret)

internal/secrets/sops.go

@@ -3,6 +3,7 @@ package secrets
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"os"
 	"os/exec"
 	"strings"
@@ -45,11 +46,13 @@ func (s *SOPSBackend) load() error {
 		cmd := exec.Command("sops", "-d", path)
 		out, err := cmd.Output()
 		if err != nil {
+			slog.Warn("sops: decryption failed", "file", entry.Name(), "error", err)
 			continue // skip unreadable files
 		}
 
 		var data map[string]any
 		if err := yaml.Unmarshal(out, &data); err != nil {
+			slog.Warn("sops: invalid yaml after decryption", "file", entry.Name(), "error", err)
 			continue
 		}
 

plans/2026-08-05-backend-evaluation-improvements.md

@@ -0,0 +1,442 @@
+# 2026-08-05 — Backend evaluation: architecture, security, and reliability improvements
+
+Status: **In Progress** — Phase 0 code changes complete (B1, B2, B4, B5, B6, B7); B3
+is a post-deploy operational step.
+
+Scope: full evaluation of the oikos backend (Go binaries `oikos`, `nomos`, `webhook`,
+Postgres/TimescaleDB, Docker deployment, MCP server) excluding frontend clients
+(`web/` SPA and `desktop/` Wails app). Research-only pass — no code changes.
+
+Method: four parallel research passes (Go backend structure, database schema,
+deployment/infrastructure, API/MCP design) plus Infisical secrets audit and
+dependency analysis of `go.mod`.
+
+---
+
+## A. Summary of findings
+
+The backend is well-architected with strong fundamentals: contract-first API
+(oapi-codegen), type-safe SQL (sqlc + pgx), TimescaleDB observability, policy-
+governed autonomy, and a sophisticated ontology-driven data model. The main gaps
+are secret management (Infisical is wired but barely used — 3 of 4 binaries
+read secrets from env/plaintext), operational maturity (CI, image tagging, backup
+reliability), security hardening (SSH host keys, unauthenticated endpoints), and
+code hygiene (monolithic files, mixed SQL access patterns).
+
+| Category | Grade | Notes |
+|----------|-------|-------|
+| Tech stack | A | Go + pgx + sqlc + TimescaleDB + chi + slog — all correct choices |
+| Data model | A- | Dual-entity pattern, temporal relationships, partial indexes, 6 state machines |
+| API design | B+ | Contract-first with ~50 MCP tools, RFC 9457 errors; no rate limiting |
+| Secret management | **D** | Infisical SDK wired but only in API/MCP tools path; nomos, webhook, scheduler, notifier all read plaintext env vars. 7 production secrets in `.env`, HMAC in world-readable plist. SOPS fallback is dead code. |
+| Security | C | OIDC+static token auth is good, but SSH host keys disabled, unauthenticated nomos endpoint, HMAC secret in world-readable plist |
+| Deployment | C+ | Multi-stage builds, pre-deploy backups, but no CI, no versioned images, silent backup failures |
+| Code quality | B | Good error handling, panic safety, doc; but 3 files over 1100 lines, mixed raw/sqlc SQL |
+| Performance | B | Appropriate for scale; SSH check storm risk, no query caching |
+| Observability | B- | TimescaleDB hypertables + SSE + slog; no Prometheus/Grafana, no OTel tracing |
+| Testing | C | `make test` exists but many core packages (scheduler, domain, actuator, policy) have 0% coverage |
+
+## B. Infisical consolidation (critical)
+
+Infisical is deployed (Redis + Infisical service in compose, Go SDK in go.mod,
+`internal/secrets/infisical.go` fully implemented) but severely underutilized.
+Only the API server's MCP tools path creates an Infisical backend. Every other
+binary reads secrets from env vars or plaintext files.
+
+### Current wiring map
+
+| Binary / role | Uses Infisical? | Secrets read from env/plaintext |
+|---------------|----------------|-------------------------------|
+| oikos `api` role | Yes (MCP tools only) | `OIKOS_DATABASE_URL`, `OIKOS_MCP_BEARER_TOKEN`, `OIKOS_API_TOKEN`, `OIKOS_OIDC_CLIENT_SECRET` |
+| oikos `scheduler` role | **No** | `OIKOS_SSH_KEY_PATH`, `OIKOS_DATABASE_URL` |
+| oikos `notifier` role | **No** | `OIKOS_MATRIX_TOKEN`, `OIKOS_APPROVAL_HMAC_SECRET`, `OIKOS_DATABASE_URL` |
+| nomos | **No** | `OPENROUTER_API_KEY`, `OIKOS_MCP_BEARER_TOKEN`, `DATABASE_URL` |
+| webhook | **No** | `WEBHOOK_HMAC_SECRET` (also hardcoded in plist) |
+
+### B1. Wire Infisical into all binaries at startup
+
+- **Goal**: Every binary fetches its secrets from Infisical at startup instead of
+  relying on env vars. Bootstrap-only env vars (`INFISICAL_CLIENT_ID`,
+  `INFISICAL_CLIENT_SECRET`, `INFISICAL_SITE_URL`, `INFISICAL_PROJECT_ID`,
+  `OIKOS_DATABASE_URL`) remain as env vars (chicken-egg).
+- **Approach**: Add a `secrets.InitFromEnv(ctx)` call to each binary's `main()` that
+  creates a `secrets.Manager` (primary Infisical + SOPS fallback). Store the
+  manager in a package-level var or pass it through the initialization chain.
+- **Files to change**:
+  - `cmd/oikos/main.go` — create Manager in `runWithPool`, pass to scheduler and
+    notifier runners alongside cfg and pool
+  - `cmd/nomos/main.go` — create Manager at startup, fetch `OPENROUTER_API_KEY`
+    and `OIKOS_MCP_BEARER_TOKEN` from Infisical before creating the agent
+  - `cmd/webhook/main.go` — create Manager at startup, fetch `WEBHOOK_HMAC_SECRET`
+    from Infisical
+- **Secrets to migrate into Infisical** (move from env vars / `.env` / plist):
+
+  | Secret key (in Infisical) | Current source | Used by |
+  |---------------------------|---------------|---------|
+  | `matrix/token` | `OIKOS_MATRIX_TOKEN` env | oikos notifier |
+  | `approval/hmac-secret` | `OIKOS_APPROVAL_HMAC_SECRET` env, plist | oikos notifier, webhook |
+  | `mcp/bearer-token` | `OIKOS_MCP_BEARER_TOKEN` env | oikos api, nomos |
+  | `api/token` | `OIKOS_API_TOKEN` env | oikos api |
+  | `oidc/client-secret` | `OIKOS_OIDC_CLIENT_SECRET` env | oikos api |
+  | `openrouter/api-key` | `OPENROUTER_API_KEY` env | nomos |
+  | `webhook/hmac-secret` | `WEBHOOK_HMAC_SECRET` env + plist | webhook |
+- **Risk class**: config_mutation
+- **Prerequisite**: Populate Infisical with these secrets via `oikos secret set` before
+  deploying the code change. Existing `.env` values serve as the source of truth
+  for the initial migration.
+
+### B2. Activate the SOPS fallback path
+
+- **Status**: Done (commit pending)
+- **Current**: `secrets.NewManager(infisical, sops)` is only used in tests.
+  `httpapi/server.go` creates `InfisicalBackend` directly — if Infisical is down,
+  there is no fallback.
+- **Fix**: Use `secrets.NewManager()` in production everywhere so the SOPS DR
+  fallback actually works when Infisical is unreachable. The Manager's cache
+  (5min TTL) already masks transient Infisical blips.
+- **What changed**: `httpapi/server.go` now creates `secrets.NewManager(infisical,
+  sopsFallback)` instead of bare `secrets.NewInfisicalBackend`. SOPS backend
+  is created from `cfg.SecretsDir` when set.
+- **Risk class**: config_mutation
+
+### B3. Remove `.env` plaintext secrets after migration
+
+- **Current**: `.env` contains 7 production secrets in plaintext on mac-mini disk.
+- **Fix**: After B1 is deployed and all binaries read from Infisical, strip
+  secrets from `.env` leaving only non-secret config (`OIKOS_API_LISTEN`,
+  `OIKOS_SCHEDULER_INTERVAL`, etc.). Bootstrap env vars
+  (`OIKOS_DATABASE_URL`, `OIKOS_INFISICAL_*`) stay — they're the trust anchor.
+- **Risk class**: config_mutation
+
+### B4. Remove HMAC secret from plist
+
+- **Status**: Done (code change; plist cleanup is post-deploy)
+- **Current**: `scripts/deploy/network.hubris.oikos-deploy-webhook.plist` line 14
+  has `WEBHOOK_HMAC_SECRET` hardcoded in plaintext. World-readable.
+- **Fix**: After B1 (webhook reads from Infisical), remove the `EnvironmentVariables`
+  `WEBHOOK_HMAC_SECRET` entry from the plist entirely. The webhook binary will
+  fetch it from Infisical at startup.
+- **Risk class**: config_mutation
+- **Supersedes**: Original plan item B3 (HMAC secret in plist) — same issue,
+  now resolved via Infisical instead of file permissions workarounds.
+
+### B5. Store SSH host keys in Infisical
+
+- **Status**: Done (commit pending)
+- **Current**: `ssh.InsecureIgnoreHostKey()` in 3 code paths
+  (`internal/actuator/ssh.go:160`, `internal/actuator/actuator.go:427`,
+  `internal/mcp/server.go`).
+- **Fix**: Store Proxmox host public keys in Infisical under
+  `ssh/host-keys/{hostname}`. Actuator reads them at connection init and builds
+  a `knownhosts` callback. For dynamic targets, implement TOFU (trust-on-first-
+  use) writing back to Infisical.
+- **What changed**:
+  - `internal/actuator/hostkeys.go` — `HostKeyCallback()` returns an
+    `ssh.HostKeyCallback` that verifies against cached keys (MITM detection)
+    and accepts unknown hosts via TOFU, persisting new keys to Infisical.
+  - `internal/actuator/hostkeys_infisical.go` — `InfisicalHostKeySource`
+    implements `HostKeySource` over `secrets.Backend`; `ResolveSSHHosts()`
+    queries DB for active proxmox-host/standalone-server slugs.
+  - `internal/actuator/ssh.go:160` — replaced `InsecureIgnoreHostKey()` with
+    `HostKeyCallback()`.
+  - `internal/actuator/actuator.go:427` — replaced `InsecureIgnoreHostKey()` with
+    `HostKeyCallback()`.
+  - `internal/mcp/server.go:494` — replaced `InsecureIgnoreHostKey()` with
+    `actuator.HostKeyCallback()`.
+  - `internal/httpapi/server.go` — pre-loads SSH host keys from Infisical at
+    startup (queries active hosts, loads their keys from Infisical).
+- **Post-deploy step**: On first deploy, TOFU will accept all current host
+  keys and store them in Infisical under `ssh/host-keys/{slug}`. Verify the
+  stored keys are correct by checking `oikos secret list`. To pre-populate
+  without TOFU, SSH to each Proxmox host and run:
+  `ssh-keyscan -t ed25519 {host} | awk '{print $2" "$3}'` and store
+  the output via `oikos secret set ssh/host-keys/{slug} {output}`.
+- **Risk class**: config_mutation (initial pin) / destructive (if keys change)
+
+### B6. Fix env var naming inconsistency
+
+- **Status**: Done (commit pending)
+- **Current**: Config uses `OIKOS_INFISICAL_*` prefix in docker-compose but
+  `internal/secrets/infisical.go` lines 58–62 falls back to bare `INFISICAL_*`
+  (without OIKOS prefix). Two naming conventions for the same bootstrap vars.
+- **Fix**: Standardize on `OIKOS_INFISICAL_*` everywhere. Remove the bare
+  `INFISICAL_*` fallback in infisical.go.
+- **What changed**: Removed the `os.Getenv("INFISICAL_CLIENT_ID")` and
+  `os.Getenv("INFISICAL_CLIENT_SECRET")` fallbacks in `infisical.go connect()`.
+  Removed unused `os` import. Error message updated to reference
+  `OIKOS_INFISICAL_*` names.
+- **Risk class**: config_mutation
+
+### B7. Align `secretsBackend` interface with `secrets.Backend`
+
+- **Status**: Done (commit pending)
+- **Current**: `internal/httpapi/server.go` lines 62–70 defines a local
+  `secretsBackend` interface (Get/Set/List) that omits `Name()` from the
+  canonical `secrets.Backend`.
+- **Fix**: Use `secrets.Backend` directly in httpapi, or embed it in the local
+  interface.
+- **What changed**:
+  - Removed `secretsBackend` interface from `httpapi/server.go`; `Server.secretsManager`
+    now uses `secrets.Backend` directly.
+  - Removed `secretBackend` interface from `mcp/server.go`; `NewHandler` and
+    `newServer` now accept `secrets.Backend`.
+  - Updated `mcp/tools.go` `allTools()` signature to accept `secrets.Backend`.
+  - Updated `mcp/secrets_tools_test.go` mock to implement `secrets.Backend`
+    (added `Name()` method, uses `secrets.ErrNotFound` instead of custom error).
+- **Risk class**: read_only
+
+## C. Security fixes (critical, non-Infisical)
+
+### C1. nomos.hubris.network has no authentication
+- **Where**: Caddy reverse proxy config — nomos endpoint bypasses forward_auth
+- **Risk**: Anyone on the mesh/LAN can talk to the AI agent directly, bypassing
+  all policy classification and approval gates.
+- **Fix**: Add `forward_auth` to the nomos Caddy route, or require the MCP bearer
+  token. At minimum, add a shared secret via Caddy `basicauth`.
+- **Risk class**: config_mutation
+
+### C2. pg_dump failure is silently ignored
+- **Where**: `scripts/deploy.sh` — `pg_dump ... || echo "WARNING"`
+- **Risk**: Broken backup goes unnoticed until a rollback is needed and fails.
+- **Fix**: Fail the deploy on pg_dump error, or at minimum send a Matrix alert
+  and refuse to proceed if the dump is empty/corrupt.
+- **Risk class**: config_mutation
+
+### C3. CORS defaults to `*`
+- **Where**: `internal/httpapi/server.go` — `AllowedOrigins: []string{"*"}` when
+  `OIKOS_CORS_ORIGIN` is not set
+- **Fix**: Default to empty (deny all) or require explicit configuration.
+- **Risk class**: config_mutation
+
+## D. Operational improvements (high)
+
+### D1. Add CI pipeline
+- **Current**: No automated build/test on push. `make lint test generate-check`
+  exists but is manual.
+- **Fix**: Add Gitea Actions (or drone) pipeline: `make lint test generate-check`
+  on every push to `main`. Block deploy if pipeline fails.
+- **Risk class**: config_mutation
+
+### D2. Version Docker images
+- **Current**: All images built as `:latest`. Rollback requires full rebuild.
+- **Fix**: Tag images with `v$VERSION` from the VERSION file in deploy.sh. Keep
+  last 3 versions. Enable `docker compose up` to pin a version tag.
+- **Risk class**: config_mutation
+
+### D3. Add rate limiting
+- **Current**: No throttling on HTTP API or MCP endpoints. An agent in a loop
+  could hammer the API or exhaust DB connections.
+- **Fix**: Add `golang.org/x/time/rate` middleware to chi router. Per-IP or
+  per-token rate limit with burst allowance. Separate limits for API vs MCP.
+- **Risk class**: config_mutation
+
+### D4. Add container resource limits
+- **Current**: No `mem_limit`, `cpus`, or `ulimits` on any compose service.
+- **Fix**: Add memory and CPU limits to all services in docker-compose.yml.
+  Suggested: API 512MB, scheduler 256MB, notifier 128MB, nomos 512MB.
+- **Risk class**: config_mutation
+
+### D5. Add healthchecks to all compose services
+- **Current**: Only postgres, api, and redis have healthchecks.
+- **Fix**: Add `healthcheck` to scheduler, notifier, and nomos. Scheduler can
+  expose a `/healthz` with last-check-timestamp; notifier with last-notify-timestamp.
+
+## E. Code quality (medium)
+
+### E1. Split monolithic files
+| File | Lines | Split target |
+|------|-------|-------------|
+| `internal/mcp/tools.go` | 1774 | `entity_tools.go`, `ops_tools.go`, `knowledge_tools.go`, `analysis_tools.go` |
+| `internal/httpapi/impl.go` | 1533 | Handler group files by domain (entities, executions, approvals, metrics, etc.) |
+| `cmd/nomos/main.go` | 1127 | `server.go`, `workers.go`, `mcp.go` (partially done — `agent.go` and `store.go` exist) |
+
+### E2. Migrate raw pool.Exec queries to sqlc
+- ~50% of DB access in HTTP/MCP handlers bypasses sqlc with raw `pool.Exec`/`pool.QueryRow`.
+- Add these queries to `internal/db/queries/` source files for type safety and
+  compile-time validation.
+
+### E3. Unify SSH implementations
+- Scheduler uses `os/exec ssh` (system binary), MCP/actuator uses `crypto/ssh`.
+- Unify on `crypto/ssh` throughout for consistency, testability, and connection
+  multiplexing (single TCP connection, multiple sessions).
+- Consider a shared SSH pool in `internal/actuator/` used by both scheduler and MCP.
+
+### E4. Fix lifecycle attribute check
+- `internal/db/lifecycle.go`: `checkPrecondition` uses `strings.Contains(attrs, want)`
+  on raw JSONB text, bypassing the GIN index.
+- Parse attributes properly and use `@>` or `?` JSONB operators.
+
+### E5. Add table-driven tests for core logic
+Priority packages (currently 0% coverage):
+1. `internal/policy` — risk classification rules (table-driven with seed policy.yaml cases)
+2. `internal/domain` — lifecycle state machine transitions
+3. `internal/scheduler` — check dispatch and signal resolution
+4. `internal/ontology` — monitoring resolution and type tree traversal
+5. `internal/checkdefaults` — check derivation from monitoring specs
+
+## F. Performance (medium)
+
+### F1. SSH connection pooling for scheduler
+- At 30s intervals with 95 entities and multiple check types, the scheduler can
+  spawn 100+ SSH sessions per cycle via `os/exec ssh`.
+- Migrate to `crypto/ssh` with persistent connection pools to Proxmox hosts.
+  One TCP connection per host, multiplexed sessions for individual checks.
+
+### F2. Entity lookup cache
+- Repeated `get_entity`/`whoami` MCP calls hit the DB every time.
+- Add an in-memory TTL cache (hashicorp/golang-lru, already in go.mod) with
+  60s TTL for entity lookups. Invalidate on write.
+
+### F3. Trigram index for entity search
+- `ListEntities` uses `ILIKE '%'||q||'%'` which cannot use B-tree indexes.
+- Add GIN trigram indexes on `entities.slug` and `entities.name`.
+- Alternative: migrate to `tsvector` full-text search matching the knowledge pattern.
+
+### F4. Composite index for auto-act anti-join
+- `GetOpenSignalsForAutoAct` joins classifications → signals → executions with
+  `WHERE e.entity_id IS NULL`. No composite index on `(classification_id, entity_id)`.
+- Add partial index on `executions(classification_id) WHERE entity_id IS NOT NULL`.
+
+## G. Observability (low)
+
+### G1. Add OpenTelemetry tracing
+- OTel SDK is already in go.mod as indirect dependency.
+- Instrument HTTP handlers, MCP tools, and DB queries with spans.
+- Propagate trace context via `correlation_id` (already exists in audit/events).
+
+### G2. Prometheus metrics export
+- Expose `/metrics` endpoint for Go runtime, DB pool stats, scheduler check
+  duration/counts, HTTP request latency histograms.
+- Complement the existing TimescaleDB metric_samples (which are entity health
+  metrics, not self-observability).
+
+### G3. Automate offsite backups
+- Proton Drive backup target entity exists but no pipeline.
+- Add `rclone cron` to `pg_dump | zstd | rclone sync` to Proton Drive.
+- Weekly backup verification (restore to test DB, run `make test-db`).
+
+## H. Infrastructure (low)
+
+### H1. Pin Infisical image version
+- Currently uses `infisical/infisical:latest`.
+- Pin to a specific version tag.
+
+### H2. Add persistent job queue for executions
+- All background work is in-process goroutines — lost on restart.
+- For the execution pipeline specifically, consider Postgres-backed queue
+  (e.g., `river` or custom `pending_executions` poll with advisory lock).
+- Lower priority: scheduler and notifier state is transient and self-healing.
+
+### H3. Replace or harden custom migration splitter
+- The `splitSQL()` function handles `$$` dollar-quoting but edge cases with
+  string literals containing `$$` could break migrations.
+- Add test cases for nested quoting, or adopt golang-migrate.
+
+### H4. Add distributed locking for scheduler
+- Document single-instance constraint, or add `pg_advisory_lock` (already used
+  by migration runner) to prevent duplicate health checks if multiple
+  scheduler instances are accidentally started.
+
+---
+
+## Execution order
+
+1. **Phase 0 — Infisical consolidation** (B1–B7): Wire Infisical into all
+   binaries, migrate secrets from env/plaintext, activate SOPS fallback, remove
+   `.env` secrets and plist HMAC. This is the foundation — every subsequent
+   secret-dependent change (SSH host keys in B5, rate limit config, etc.) goes
+   through Infisical. **Do this first.**
+2. **Phase 1 — Security** (C1–C3, B5): Nomos auth, pg_dump failure, CORS default,
+   SSH host keys (now stored in Infisical per B5).
+3. **Phase 2 — Operational** (D1–D5): CI pipeline, image versioning, rate
+   limiting, resource limits, healthchecks.
+4. **Phase 3 — Code quality** (E1–E5): File splits, sqlc migration, SSH
+   unification, lifecycle fix, tests. E1–E3 are large refactors — do one
+   file/area per commit.
+5. **Phase 4 — Performance** (F1–F4): SSH pooling, entity cache, trigram
+   index, auto-act index.
+6. **Phase 5 — Observability** (G1–G3): OTel tracing, Prometheus, offsite backups.
+7. **Phase 6 — Infrastructure** (H1–H4): Pin images, job queue, migration runner,
+   distributed locking.
+
+Phase 0 is the gate. Once all secrets flow through Infisical, phases 1–2 can
+proceed. Phases 3–4 should wait for CI (D1) so refactors are validated.
+Phases 5–6 are backlog.
+
+---
+
+## Phase 0 post-deploy checklist
+
+Run these on mac-mini after deploying the Phase 0 code changes.
+
+### Step 1: Populate Infisical with secrets
+
+For each secret, read the current value from `.env` and store it in Infisical:
+
+```bash
+# Values from .env — read them first, then set
+oikos secret set matrix/token "$(grep OIKOS_MATRIX_TOKEN .env | cut -d= -f2-)"
+oikos secret set approval/hmac-secret "$(grep OIKOS_APPROVAL_HMAC_SECRET .env | cut -d= -f2-)"
+oikos secret set mcp/bearer-token "$(grep OIKOS_MCP_BEARER_TOKEN .env | cut -d= -f2-)"
+oikos secret set api/token "$(grep OIKOS_API_TOKEN .env | cut -d= -f2-)"
+oikos secret set oidc/client-secret "$(grep OIKOS_OIDC_CLIENT_SECRET .env | cut -d= -f2-)"
+oikos secret set openrouter/api-key "$(grep OPENROUTER_API_KEY .env | cut -d= -f-)"
+oikos secret set webhook/hmac-secret "$(grep WEBHOOK_HMAC_SECRET .env | cut -d= -f2-)"
+```
+
+Verify: `oikos secret list` should show all 8 keys.
+
+### Step 2: Pre-populate SSH host keys (optional, skip if TOFU is acceptable)
+
+```bash
+# For each Proxmox host, scan and store the public key
+for host in pve1 pve2; do
+  key=$(ssh-keyscan -t ed25519 $host 2>/dev/null | awk '{print $2" "$3}')
+  oikos secret set "ssh/host-keys/$host" "ssh-ed25519 $key"
+done
+```
+
+Alternatively, skip this step — the first deployment will TOFU-accept all current
+host keys and persist them to Infisical automatically.
+
+### Step 3: Remove HMAC secret from webhook plist
+
+On mac-mini:
+```bash
+sudo launchctl unload ~/Library/LaunchAgents/network.hubris.oikos-deploy-webhook.plist
+# Edit the plist: remove the <key>WEBHOOK_HMAC_SECRET</key> block
+sudo launchctl load ~/Library/LaunchAgents/network.hubris.oikos-deploy-webhook.plist
+```
+
+### Step 4: Strip secrets from .env
+
+Edit `.env` to remove the 7 migrated secrets, keeping only bootstrap and
+non-secret config:
+
+```bash
+# Remove these lines:
+#   INFISICAL_ENCRYPTION_KEY=...
+#   OIKOS_MATRIX_TOKEN=...
+#   OIKOS_INFISICAL_CLIENT_ID=...
+#   OIKOS_INFISICAL_CLIENT_SECRET=...
+#   OIKOS_INFISICAL_PROJECT_ID=...
+#   OPENROUTER_API_KEY=...
+#   OIKOS_MCP_BEARER_TOKEN=...
+
+# Keep these (bootstrap / non-secret):
+#   OIKOS_DATABASE_URL=...
+#   OIKOS_API_LISTEN=...
+#   OIKOS_INFISICAL_SITE_URL=...
+#   OIKOS_INFISICAL_ENV=...
+```
+
+### Step 5: Verify
+
+1. `oikos secret list` — 8 keys + SSH host keys
+2. `docker compose logs api | grep "secrets resolved"` — should show count=5
+3. Trigger a test deploy — webhook should still validate HMAC signatures
+4. `nomos` should start and resolve secrets from Infisical (check logs for
+   `nomos: secrets resolved from Infisical`)
+5. Verify no secrets appear in process env: `docker compose exec api env |
+   should not show `OIKOS_MCP_BEARER_TOKEN`, `OIKOS_MATRIX_TOKEN`,
+   etc. (they come from Infisical at startup, not env)

plans/index.md

@@ -21,6 +21,7 @@ went sideways, open an investigation.
 | 2026-07-21 | [Frontend as OS + Apps — architecture audit & refactor](2026-07-21-frontend-os-apps-architecture.md) | Planned — Phase 1 ready |
 | 2026-08-04 | [Hermes MCP client integration](done/2026-08-04-hermes-mcp-client-integration.md) | Done — deployed |
 | 2026-08-05 | [Agent execution safety: QEMU guest agent gate + host-mutation guard](done/2026-08-05-agent-execution-safety-qemu-guest-agent-gate.md) | Done — implemented (1b9c761) |
+| 2026-08-05 | [Backend evaluation: architecture, security, and reliability improvements](2026-08-05-backend-evaluation-improvements.md) | Planned — 6 phases, security items first |
 
 ## Done