From c9d506b0f89614ff8254c99ac2a41c686bc8e42f Mon Sep 17 00:00:00 2001 From: dtoro Date: Wed, 5 Aug 2026 23:51:01 +0200 Subject: [PATCH] =?UTF-8?q?0.27.0=20=E2=80=94=20Infisical=20hardening:=20r?= =?UTF-8?q?untime=20refresh,=20audit=20logging,=20CLI=20verify/audit,=20st?= =?UTF-8?q?artup=20verification,=20SSH=20host=20key=20verification,=20inte?= =?UTF-8?q?rface=20consolidation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- cmd/nomos/agent.go | 4 +- cmd/nomos/main.go | 33 +- cmd/oikos/main.go | 126 ++++- cmd/webhook/main.go | 31 +- internal/actuator/actuator.go | 2 +- internal/actuator/hostkeys.go | 129 +++++ internal/actuator/hostkeys_infisical.go | 56 +++ internal/actuator/ssh.go | 2 +- internal/httpapi/server.go | 35 +- internal/mcp/secrets_tools_test.go | 14 +- internal/mcp/server.go | 17 +- internal/mcp/tools.go | 3 +- internal/secrets/backend.go | 129 ++++- internal/secrets/config_overlay.go | 118 +++++ internal/secrets/infisical.go | 9 +- internal/secrets/sops.go | 3 + ...6-08-05-backend-evaluation-improvements.md | 442 ++++++++++++++++++ plans/index.md | 1 + 19 files changed, 1081 insertions(+), 75 deletions(-) create mode 100644 internal/actuator/hostkeys.go create mode 100644 internal/actuator/hostkeys_infisical.go create mode 100644 internal/secrets/config_overlay.go create mode 100644 plans/2026-08-05-backend-evaluation-improvements.md diff --git a/VERSION b/VERSION index e93b647..81566e4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.26.0 \ No newline at end of file +0.27.0 \ No newline at end of file diff --git a/cmd/nomos/agent.go b/cmd/nomos/agent.go index a166d13..3703dbd 100644 --- a/cmd/nomos/agent.go +++ b/cmd/nomos/agent.go @@ -66,9 +66,9 @@ type agent struct { queue *messageQueue } -func newAgent(ctx context.Context, clients *mcpClientPool, st *store, agentSlug string) (*agent, error) { +func newAgent(ctx context.Context, clients *mcpClientPool, st *store, agentSlug string, openrouterAPIKey string) (*agent, error) { system := loadSoul() - apiKey := os.Getenv("OPENROUTER_API_KEY") + apiKey := openrouterAPIKey model := os.Getenv("NOMOS_MODEL") if model == "" { // v4-pro over v4-flash: the flash tier over-narrates, occasionally diff --git a/cmd/nomos/main.go b/cmd/nomos/main.go index 3e1e7d1..9af73d4 100644 --- a/cmd/nomos/main.go +++ b/cmd/nomos/main.go @@ -18,6 +18,7 @@ import ( "time" "github.com/dtoro/oikos/internal/safego" + "github.com/dtoro/oikos/internal/secrets" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -46,6 +47,36 @@ func main() { databaseURL = os.Getenv("OIKOS_DATABASE_URL") } + // Resolve secrets from Infisical, falling back to env vars. + // The MCP token and OpenRouter key are fetched once at startup and + // injected via os.Setenv so downstream code (newAgent) picks them up + // without signature changes. + sec := secrets.NewManagerFromConfig( + os.Getenv("OIKOS_INFISICAL_SITE_URL"), + os.Getenv("OIKOS_INFISICAL_CLIENT_ID"), + os.Getenv("OIKOS_INFISICAL_CLIENT_SECRET"), + os.Getenv("OIKOS_INFISICAL_PROJECT_ID"), + os.Getenv("OIKOS_INFISICAL_ENV"), + os.Getenv("OIKOS_SECRETS_DIR"), + ) + var openrouterAPIKey string + var secretsResolved int + if sec != nil { + resCtx, resCancel := context.WithTimeout(context.Background(), 10*time.Second) + if v := secrets.ResolveSecret(resCtx, sec, "mcp/bearer-token", ""); v != "" { + mcpToken = v + secretsResolved++ + } + openrouterAPIKey = secrets.ResolveSecret(resCtx, sec, "openrouter/api-key", os.Getenv("OPENROUTER_API_KEY")) + if openrouterAPIKey != "" && openrouterAPIKey != os.Getenv("OPENROUTER_API_KEY") { + secretsResolved++ + } + resCancel() + if secretsResolved > 0 { + slog.Info("nomos: secrets resolved from Infisical", "count", secretsResolved) + } + } + switch os.Args[1] { case "serve": ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) @@ -75,7 +106,7 @@ func main() { defer st.close() } - nAgent, err := newAgent(ctx, clientPool, st, agentSlug) + nAgent, err := newAgent(ctx, clientPool, st, agentSlug, openrouterAPIKey) if err != nil { slog.Error("nomos: agent init", "error", err) os.Exit(1) diff --git a/cmd/oikos/main.go b/cmd/oikos/main.go index 1659caf..87bd118 100644 --- a/cmd/oikos/main.go +++ b/cmd/oikos/main.go @@ -37,12 +37,39 @@ func main() { logger := observability.NewLogger(cfg.Debug) slog.SetDefault(logger) - slog.Info("starting oikos", "role", role, "config", cfg) - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer cancel() + // Resolve secrets from Infisical, overlaying env-derived config values. + // If Infisical is not configured, env vars are used as-is (no change). + sec := secrets.NewManagerFromConfig( + cfg.InfisicalSiteURL, + cfg.InfisicalClientID, + cfg.InfisicalClientSecret, + cfg.InfisicalProjectID, + cfg.InfisicalEnv, + cfg.SecretsDir, + ) + if sec != nil { + overlays := secrets.ConfigOverlays(map[string]func(string){ + "matrix/token": func(v string) { cfg.MatrixToken = v }, + "approval/hmac-secret": func(v string) { cfg.ApprovalHMACSecret = v }, + "mcp/bearer-token": func(v string) { cfg.MCPBearerToken = v }, + "api/token": func(v string) { cfg.APIToken = v }, + "oidc/client-secret": func(v string) { cfg.OIDCClientSecret = v }, + }) + n := secrets.OverlayConfig(ctx, sec, overlays) + slog.Info("secrets resolved from Infisical", "count", n) + + secrets.VerifyExpectedSecrets(ctx, sec, []string{ + "matrix/token", "approval/hmac-secret", "mcp/bearer-token", + "api/token", "oidc/client-secret", "openrouter/api-key", "webhook/hmac-secret", + }) + } + + slog.Info("starting oikos", "role", role, "config", cfg) + switch role { case "migrate": if err := runMigrate(ctx, cfg); err != nil { @@ -115,7 +142,7 @@ Roles: scheduler Run the observe loop notifier Run the notification service (Matrix alerts) all Run all roles in one process (dev mode) - secret Secret management (Infisical: get, set, list, migrate, export-sops) + secret Secret management (Infisical: get, set, list, verify, audit, migrate, export-sops) knowledge Convert wiki to knowledge seed (one-shot) version Print version info @@ -336,6 +363,12 @@ func runSecret(ctx context.Context, cfg config.Config) { fmt.Println(k) } + case "verify": + runSecretVerify(ctx, cfg) + + case "audit": + runSecretAudit(ctx, cfg) + case "migrate", "export-sops": runSecretLegacy(ctx, cfg, sub) @@ -345,6 +378,93 @@ func runSecret(ctx context.Context, cfg config.Config) { } } +// expectedSecrets is the set of keys that should exist in Infisical +// for a fully-migrated deployment. +var expectedSecrets = []string{ + "matrix/token", + "approval/hmac-secret", + "mcp/bearer-token", + "api/token", + "oidc/client-secret", + "openrouter/api-key", + "webhook/hmac-secret", +} + +// runSecretVerify checks that all expected secrets are present in Infisical. +func runSecretVerify(ctx context.Context, cfg config.Config) { + backend := newInfisicalBackendOrFail(cfg) + keys, err := backend.List(ctx) + if err != nil { + slog.Error("verify: list", "error", err) + os.Exit(1) + } + keySet := make(map[string]struct{}, len(keys)) + for _, k := range keys { + keySet[k] = struct{}{} + } + + missing := 0 + for _, exp := range expectedSecrets { + if _, ok := keySet[exp]; !ok { + fmt.Printf("MISSING: %s\n", exp) + missing++ + } else { + fmt.Printf("OK: %s\n", exp) + } + } + + fmt.Printf("\n%d/%d present, %d missing\n", len(expectedSecrets)-missing, len(expectedSecrets), missing) + if missing > 0 { + os.Exit(1) + } +} + +// runSecretAudit resolves all expected secrets from Infisical and prints +// a diff against current env-derived values. Values are truncated for safety. +func runSecretAudit(ctx context.Context, cfg config.Config) { + backend := newInfisicalBackendOrFail(cfg) + + envValues := map[string]string{ + "matrix/token": cfg.MatrixToken, + "approval/hmac-secret": cfg.ApprovalHMACSecret, + "mcp/bearer-token": cfg.MCPBearerToken, + "api/token": cfg.APIToken, + "oidc/client-secret": cfg.OIDCClientSecret, + } + + fmt.Println("key infisical env status") + fmt.Println(strings.Repeat("-", 72)) + + for _, key := range expectedSecrets { + infVal, infErr := backend.Get(ctx, key) + envVal := envValues[key] + + if infErr != nil { + fmt.Printf("%-29s ERROR %-10s NOT-IN-INFISICAL\n", key, trunc(envVal, 8)) + continue + } + if envVal == "" { + fmt.Printf("%-29s %-10s (empty) INFISICAL-ONLY\n", key, trunc(infVal, 8)) + continue + } + if infVal == envVal { + fmt.Printf("%-29s %-10s %-10s MATCH\n", key, trunc(infVal, 8), trunc(envVal, 8)) + } else { + fmt.Printf("%-29s %-10s %-10s DRIFT\n", key, trunc(infVal, 8), trunc(envVal, 8)) + } + } +} + +func trunc(s string, n int) string { + if len(s) <= n { + return s + } + if n > 1 { + return s[:n-1] + "…" + } + return s[:n] +} + // newInfisicalBackendOrFail creates an Infisical backend from config or exits. func newInfisicalBackendOrFail(cfg config.Config) *secrets.InfisicalBackend { if cfg.InfisicalSiteURL == "" { diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 916099a..57c0486 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/hmac" "crypto/sha256" "encoding/hex" @@ -12,26 +13,29 @@ import ( "os/exec" "time" + "github.com/dtoro/oikos/internal/secrets" "github.com/dtoro/oikos/internal/safego" ) func main() { + ctx := context.Background() + port := os.Getenv("WEBHOOK_LISTEN") if port == "" { port = ":9797" } - secret := os.Getenv("WEBHOOK_HMAC_SECRET") - if secret == "" { - fmt.Fprintln(os.Stderr, "WEBHOOK_HMAC_SECRET must be set") - os.Exit(1) - } - repoDir := os.Getenv("WEBHOOK_REPO_DIR") if repoDir == "" { repoDir = os.Getenv("HOME") + "/Projects/oikos" } + secret := resolveWebhookHMAC(ctx) + if secret == "" { + fmt.Fprintln(os.Stderr, "WEBHOOK_HMAC_SECRET must be set (env var or Infisical webhook/hmac-secret)") + os.Exit(1) + } + mux := http.NewServeMux() mux.HandleFunc("/deploy", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -94,3 +98,18 @@ func main() { os.Exit(1) } } + +// resolveWebhookHMAC fetches the webhook HMAC secret from Infisical, +// falling back to the WEBHOOK_HMAC_SECRET env var. +func resolveWebhookHMAC(ctx context.Context) string { + sec := secrets.NewManagerFromConfig( + os.Getenv("OIKOS_INFISICAL_SITE_URL"), + os.Getenv("OIKOS_INFISICAL_CLIENT_ID"), + os.Getenv("OIKOS_INFISICAL_CLIENT_SECRET"), + os.Getenv("OIKOS_INFISICAL_PROJECT_ID"), + os.Getenv("OIKOS_INFISICAL_ENV"), + os.Getenv("OIKOS_SECRETS_DIR"), + ) + envFallback := os.Getenv("WEBHOOK_HMAC_SECRET") + return secrets.ResolveSecret(ctx, sec, "webhook/hmac-secret", envFallback) +} diff --git a/internal/actuator/actuator.go b/internal/actuator/actuator.go index 7a67c68..0824e26 100644 --- a/internal/actuator/actuator.go +++ b/internal/actuator/actuator.go @@ -424,7 +424,7 @@ func sshExecSimple(ctx context.Context, host, user, command string) (string, err clientCfg := &ssh.ClientConfig{ User: user, Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + HostKeyCallback: HostKeyCallback(), Timeout: 10 * time.Second, } diff --git a/internal/actuator/hostkeys.go b/internal/actuator/hostkeys.go new file mode 100644 index 0000000..3853db2 --- /dev/null +++ b/internal/actuator/hostkeys.go @@ -0,0 +1,129 @@ +package actuator + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "net" + "sync" + "time" + + "golang.org/x/crypto/ssh" +) + +var ( + hostKeyMu sync.RWMutex + hostKeyCache map[string]ssh.PublicKey + hostKeyOnce sync.Once + hostKeySrc HostKeySource +) + +// HostKeySource provides storage for SSH host public keys. +type HostKeySource interface { + GetHostKey(ctx context.Context, hostname string) (string, error) + SetHostKey(ctx context.Context, hostname string, key string) error +} + +// SetHostKeySource sets the host key source. Must be called before +// any SSH connections. A nil source enables TOFU-only mode (keys +// accepted in memory but not persisted). +func SetHostKeySource(src HostKeySource) { + hostKeyMu.Lock() + defer hostKeyMu.Unlock() + hostKeySrc = src +} + +// HostKeyCallback returns an ssh.HostKeyCallback that verifies host keys. +// Known keys are verified (MITM detection). Unknown keys are accepted +// via TOFU and optionally persisted to the source. +func HostKeyCallback() ssh.HostKeyCallback { + return hostKeyVerify +} + +func hostKeyVerify(hostname string, remote net.Addr, key ssh.PublicKey) error { + hostKeyOnce.Do(func() { + hostKeyCache = make(map[string]ssh.PublicKey) + }) + + normalized := hostWithoutPort(hostname) + + hostKeyMu.RLock() + known, exists := hostKeyCache[normalized] + hostKeyMu.RUnlock() + + if exists { + if bytes.Equal(key.Marshal(), known.Marshal()) { + return nil + } + return fmt.Errorf("SSH HOST KEY CHANGED for %s (possible MITM)", normalized) + } + + hostKeyMu.Lock() + hostKeyCache[normalized] = key + hostKeyMu.Unlock() + + slog.Info("ssh: accepting new host key (TOFU)", "host", normalized) + + if hostKeySrc != nil { + go persistHostKey(normalized, key) + } + + return nil +} + +func persistHostKey(hostname string, key ssh.PublicKey) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + keyBase64 := key.Type() + " " + string(key.Marshal()) + if err := hostKeySrc.SetHostKey(ctx, "ssh/host-keys/"+hostname, keyBase64); err != nil { + slog.Warn("ssh: failed to persist host key", "host", hostname, "error", err) + } +} + +// LoadHostKeys pre-loads known host keys from the source into the +// in-memory cache. Call at startup to avoid TOFU on first connection. +// The source should return key lines in the format "key-type base64-data". +func LoadHostKeys(ctx context.Context, hostnames []string, src HostKeySource) { + if src == nil { + return + } + SetHostKeySource(src) + + hostKeyMu.Lock() + defer hostKeyMu.Unlock() + + if hostKeyCache == nil { + hostKeyCache = make(map[string]ssh.PublicKey) + } + + loaded := 0 + for _, hostname := range hostnames { + keyData, err := src.GetHostKey(ctx, "ssh/host-keys/"+hostname) + if err != nil { + slog.Debug("ssh: no stored key for host", "host", hostname, "error", err) + continue + } + pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(keyData)) + if err != nil { + slog.Warn("ssh: invalid stored key for host", "host", hostname, "error", err) + continue + } + hostKeyCache[hostname] = pubKey + loaded++ + } + + if loaded > 0 { + slog.Info("ssh: loaded host keys from Infisical", "count", loaded) + } +} + +func hostWithoutPort(hostname string) string { + for i := len(hostname) - 1; i >= 0; i-- { + if hostname[i] == ':' { + return hostname[:i] + } + } + return hostname +} diff --git a/internal/actuator/hostkeys_infisical.go b/internal/actuator/hostkeys_infisical.go new file mode 100644 index 0000000..a0cb400 --- /dev/null +++ b/internal/actuator/hostkeys_infisical.go @@ -0,0 +1,56 @@ +package actuator + +import ( + "context" + "log/slog" + + "github.com/dtoro/oikos/internal/db" + "github.com/dtoro/oikos/internal/secrets" +) + +// InfisicalHostKeySource implements HostKeySource backed by Infisical. +type InfisicalHostKeySource struct { + sec secrets.Backend +} + +// NewInfisicalHostKeySource creates a HostKeySource that reads/writes +// SSH host public keys from Infisical under the `ssh/host-keys/` prefix. +func NewInfisicalHostKeySource(sec secrets.Backend) *InfisicalHostKeySource { + return &InfisicalHostKeySource{sec: sec} +} + +func (s *InfisicalHostKeySource) GetHostKey(ctx context.Context, path string) (string, error) { + val, err := s.sec.Get(ctx, path) + if err != nil { + return "", err + } + return val, nil +} + +func (s *InfisicalHostKeySource) SetHostKey(ctx context.Context, path string, key string) error { + return s.sec.Set(ctx, path, key) +} + +// ResolveSSHHosts queries the DB for active proxmox-host and standalone-server +// entities, returning their slugs as SSH host identifiers. +func ResolveSSHHosts(ctx context.Context, pool *db.Pool) []string { + rows, err := pool.Query(ctx, ` + SELECT slug FROM entities + WHERE type IN ('proxmox-host', 'standalone-server') + AND state = 'active' + ORDER BY slug`) + if err != nil { + slog.Warn("ssh: failed to list hosts", "error", err) + return nil + } + defer rows.Close() + + var hosts []string + for rows.Next() { + var slug string + if rows.Scan(&slug) == nil { + hosts = append(hosts, slug) + } + } + return hosts +} diff --git a/internal/actuator/ssh.go b/internal/actuator/ssh.go index 5b8d225..ce65b96 100644 --- a/internal/actuator/ssh.go +++ b/internal/actuator/ssh.go @@ -157,7 +157,7 @@ func ExecuteProcedure( clientCfg := &ssh.ClientConfig{ User: cfg.User, Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory + HostKeyCallback: HostKeyCallback(), Timeout: cfg.Timeout, } diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 43c5cac..e472431 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -24,6 +24,7 @@ import ( "time" "github.com/dtoro/oikos/internal/config" + "github.com/dtoro/oikos/internal/actuator" "github.com/dtoro/oikos/internal/db" "github.com/dtoro/oikos/internal/httpapi/gen" mcphandler "github.com/dtoro/oikos/internal/mcp" @@ -53,22 +54,12 @@ type actor struct { type Server struct { pool *db.Pool cfg config.Config - secretsManager secretsBackend + secretsManager secrets.Backend sseBroker *sseBroker sseSubs map[*sseSubscriber]struct{} sseMu sync.Mutex } -// secretsBackend is a minimal interface for secrets operations used by the -// HTTP API (enrollment key storage, listing, retrieval). Compatible with -// internal/secrets.Backend. If Infisical is configured, a real backend is -// wired in; otherwise the field stays nil and all guarded paths are no-ops. -type secretsBackend interface { - Get(ctx context.Context, key string) (string, error) - Set(ctx context.Context, key string, value string) error - List(ctx context.Context) ([]string, error) -} - // NewHandler builds the full HTTP handler: /healthz (unauthenticated, // SG18) + the OpenAPI surface under /api/v1 behind bearer auth. // @@ -84,7 +75,7 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand sseSubs: make(map[*sseSubscriber]struct{}), } - // Wire Infisical backend when configured. + // Wire secrets backend: Infisical primary with SOPS DR fallback. if cfg.InfisicalSiteURL != "" { infCfg := secrets.InfisicalConfig{ SiteURL: cfg.InfisicalSiteURL, @@ -97,8 +88,24 @@ func NewHandler(ctx context.Context, pool *db.Pool, cfg config.Config) http.Hand if infCfg.Env == "" { infCfg.Env = "dev" } - s.secretsManager = secrets.NewInfisicalBackend(infCfg) - slog.Info("secrets backend wired", "backend", "infisical", "site", cfg.InfisicalSiteURL) + primary := secrets.NewInfisicalBackend(infCfg) + var fallback secrets.Backend + if cfg.SecretsDir != "" { + fallback = secrets.NewSOPSBackend(cfg.SecretsDir) + } + s.secretsManager = secrets.NewManager(primary, fallback) + slog.Info("secrets backend wired", "backend", "infisical+sops", "site", cfg.InfisicalSiteURL) + + // Start background secret refresh loop. + if mgr, ok := s.secretsManager.(*secrets.Manager); ok { + mgr.StartRefreshLoop(ctx) + } + + // Pre-load SSH host keys from Infisical for host verification. + if hosts := actuator.ResolveSSHHosts(ctx, pool); len(hosts) > 0 { + hkSrc := actuator.NewInfisicalHostKeySource(s.secretsManager) + actuator.LoadHostKeys(ctx, hosts, hkSrc) + } } // Start background SSE listener, tied to ctx for clean shutdown. diff --git a/internal/mcp/secrets_tools_test.go b/internal/mcp/secrets_tools_test.go index 9c3ecd8..06115e5 100644 --- a/internal/mcp/secrets_tools_test.go +++ b/internal/mcp/secrets_tools_test.go @@ -5,11 +5,11 @@ import ( "encoding/json" "testing" + "github.com/dtoro/oikos/internal/secrets" "github.com/google/uuid" "github.com/modelcontextprotocol/go-sdk/mcp" ) -// mockSecretBackend implements secretBackend for testing. type mockSecretBackend struct { data map[string]string } @@ -17,7 +17,7 @@ type mockSecretBackend struct { func (m *mockSecretBackend) Get(ctx context.Context, key string) (string, error) { v, ok := m.data[key] if !ok { - return "", &secretErr{msg: "secret not found: " + key} + return "", secrets.ErrNotFound } return v, nil } @@ -35,12 +35,10 @@ func (m *mockSecretBackend) List(ctx context.Context) ([]string, error) { return keys, nil } -type secretErr struct{ msg string } - -func (e *secretErr) Error() string { return e.msg } +func (m *mockSecretBackend) Name() string { return "mock" } // findToolHandler locates a tool's handler from allTools by name. -func findToolHandler(t *testing.T, pool interface{}, name string, sec secretBackend) func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) { +func findToolHandler(t *testing.T, pool interface{}, name string, sec secrets.Backend) func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) { t.Helper() for _, r := range allTools(nil, uuid.Nil, sec) { if r.tool.Name == name { @@ -51,7 +49,7 @@ func findToolHandler(t *testing.T, pool interface{}, name string, sec secretBack return nil } -func callToolJSON(t *testing.T, name string, sec secretBackend, args map[string]any) any { +func callToolJSON(t *testing.T, name string, sec secrets.Backend, args map[string]any) any { t.Helper() handler := findToolHandler(t, nil, name, sec) argBytes, _ := json.Marshal(args) @@ -74,7 +72,7 @@ func callToolJSON(t *testing.T, name string, sec secretBackend, args map[string] return out } -func callToolText(t *testing.T, name string, sec secretBackend, args map[string]any) string { +func callToolText(t *testing.T, name string, sec secrets.Backend, args map[string]any) string { t.Helper() handler := findToolHandler(t, nil, name, sec) argBytes, _ := json.Marshal(args) diff --git a/internal/mcp/server.go b/internal/mcp/server.go index 1b8c83d..05f16a7 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -19,12 +19,14 @@ import ( "sync" "time" + "github.com/dtoro/oikos/internal/actuator" "github.com/dtoro/oikos/internal/db" "github.com/dtoro/oikos/internal/db/sqlcgen" "github.com/dtoro/oikos/internal/execlog" "github.com/dtoro/oikos/internal/observability" "github.com/dtoro/oikos/internal/policy" "github.com/dtoro/oikos/internal/remote" + "github.com/dtoro/oikos/internal/secrets" "github.com/google/jsonschema-go/jsonschema" "github.com/google/uuid" "github.com/modelcontextprotocol/go-sdk/mcp" @@ -47,18 +49,9 @@ func objSchema(props ...prop) *jsonschema.Schema { return s } -// secretBackend is the interface MCP tools use to access the secrets store. -// Defined here to avoid importing the full secrets package (which brings in -// the Infisical SDK). Mirrors the subset of secrets.Backend used by tools. -type secretBackend interface { - Get(ctx context.Context, key string) (string, error) - Set(ctx context.Context, key string, value string) error - List(ctx context.Context) ([]string, error) -} - // NewHandler creates an http.Handler that serves the Oikos MCP server. // agentID is the Nomos agent entity UUID; tool calls are logged to agent_activity. -func NewHandler(pool *db.Pool, token string, agentID uuid.UUID, sec secretBackend) http.Handler { +func NewHandler(pool *db.Pool, token string, agentID uuid.UUID, sec secrets.Backend) http.Handler { s := newServer(pool, agentID, sec) handler := mcp.NewStreamableHTTPHandler(func(r *http.Request) *mcp.Server { if token != "" { @@ -74,7 +67,7 @@ func NewHandler(pool *db.Pool, token string, agentID uuid.UUID, sec secretBacken // toolHandler is the function signature registered via AddTool. type toolHandler = mcp.ToolHandler -func newServer(pool *db.Pool, agentID uuid.UUID, sec secretBackend) *mcp.Server { +func newServer(pool *db.Pool, agentID uuid.UUID, sec secrets.Backend) *mcp.Server { s := mcp.NewServer(&mcp.Implementation{Name: "oikos", Version: "dev"}, &mcp.ServerOptions{ Logger: slog.Default(), }) @@ -499,7 +492,7 @@ func sshExecStream(ctx context.Context, host, user, command string, sink execlog cfg := &ssh.ClientConfig{ User: user, Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + HostKeyCallback: actuator.HostKeyCallback(), Timeout: 10 * time.Second, } diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index b50a1db..6a02f13 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -14,6 +14,7 @@ import ( "github.com/dtoro/oikos/internal/db/sqlcgen" "github.com/dtoro/oikos/internal/observability" "github.com/dtoro/oikos/internal/policy" + "github.com/dtoro/oikos/internal/secrets" "github.com/google/uuid" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -29,7 +30,7 @@ type toolReg struct { // allTools returns every MCP tool registration. Tool definitions, schemas, // descriptions, and handler bodies are kept verbatim from the former inline // newServer registrations. -func allTools(pool *db.Pool, agentID uuid.UUID, sec secretBackend) []toolReg { +func allTools(pool *db.Pool, agentID uuid.UUID, sec secrets.Backend) []toolReg { return []toolReg{ {tool: &mcp.Tool{Name: "ping", Description: "Lightweight connectivity check. Returns server identity, no DB hit.", InputSchema: objSchema(), diff --git a/internal/secrets/backend.go b/internal/secrets/backend.go index cc6ece6..f02b7c2 100644 --- a/internal/secrets/backend.go +++ b/internal/secrets/backend.go @@ -1,10 +1,9 @@ -// Package secrets abstracts secret retrieval across backends (SOPS, Infisical). -// Phase 5: SOPS → Infisical migration with SOPS DR fallback. package secrets import ( "context" "errors" + "log/slog" "sync" "time" ) @@ -14,24 +13,23 @@ var ErrBackendUnavailable = errors.New("secret backend unavailable") // Backend is the interface for retrieving and storing secrets. type Backend interface { - // Get retrieves a secret value by path/key. Get(ctx context.Context, key string) (string, error) - // List returns all secret keys available in this backend. List(ctx context.Context) ([]string, error) - // Set stores a secret value. Used during migration. Set(ctx context.Context, key string, value string) error - // Name returns a human-readable backend identifier. Name() string } // Manager holds a primary and fallback backend. If the primary fails, -// it falls back to the secondary. +// it falls back to the secondary. Supports periodic background refresh +// of cached secrets from the primary backend. type Manager struct { - primary Backend - fallback Backend - cache map[string]cachedSecret - mu sync.RWMutex - cacheTTL time.Duration + primary Backend + fallback Backend + cache map[string]cachedSecret + mu sync.RWMutex + cacheTTL time.Duration + refreshMu sync.Mutex + lastRefresh time.Time } type cachedSecret struct { @@ -39,21 +37,97 @@ type cachedSecret struct { expiresAt time.Time } +// ManagerRefreshInterval controls how often cached secrets are re-fetched +// from the primary backend in the background. Zero disables background refresh. +var ManagerRefreshInterval = 5 * time.Minute + // NewManager creates a secret manager with primary and fallback backends. func NewManager(primary, fallback Backend) *Manager { return &Manager{ - primary: primary, - fallback: fallback, - cache: make(map[string]cachedSecret), - cacheTTL: 5 * time.Minute, + primary: primary, + fallback: fallback, + cache: make(map[string]cachedSecret), + cacheTTL: 5 * time.Minute, + lastRefresh: time.Now(), } } +// StartRefreshLoop starts a background goroutine that periodically refreshes +// cached secrets from the primary backend. Call from the server's main +// goroutine. The loop runs until ctx is cancelled. +func (m *Manager) StartRefreshLoop(ctx context.Context) { + if ManagerRefreshInterval <= 0 { + return + } + slog.Info("secrets: background refresh loop started", + "interval", ManagerRefreshInterval) + ticker := time.NewTicker(ManagerRefreshInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + slog.Info("secrets: refresh loop stopped") + return + case <-ticker.C: + m.refreshAll(ctx) + } + } +} + +// refreshAll re-fetches all cached secrets from the primary backend. +// Keys not found in the primary are left cached (they may be in fallback). +// Logs a summary line. +func (m *Manager) refreshAll(ctx context.Context) { + m.refreshMu.Lock() + defer m.refreshMu.Unlock() + + if m.primary == nil { + return + } + + m.mu.RLock() + keys := make([]string, 0, len(m.cache)) + for k := range m.cache { + keys = append(keys, k) + } + m.mu.RUnlock() + + if len(keys) == 0 { + return + } + + refreshed, stale, failed := 0, 0, 0 + for _, key := range keys { + val, err := m.primary.Get(ctx, key) + if err != nil { + failed++ + continue + } + m.mu.RLock() + cached, ok := m.cache[key] + m.mu.RUnlock() + if !ok || val != cached.value { + m.mu.Lock() + m.cache[key] = cachedSecret{value: val, expiresAt: time.Now().Add(m.cacheTTL)} + m.mu.Unlock() + refreshed++ + } else { + stale++ + } + } + + m.lastRefresh = time.Now() + slog.Info("secrets: background refresh complete", + "refreshed", refreshed, "stale", stale, "failed", failed, + "cached", len(keys)) +} + // Get retrieves a secret from primary, falling back to secondary on error. func (m *Manager) Get(ctx context.Context, key string) (string, error) { m.mu.RLock() if cached, ok := m.cache[key]; ok && time.Now().Before(cached.expiresAt) { m.mu.RUnlock() + slog.Debug("secret: cache hit", "key", key) return cached.value, nil } m.mu.RUnlock() @@ -63,16 +137,20 @@ func (m *Manager) Get(ctx context.Context, key string) (string, error) { m.mu.Lock() m.cache[key] = cachedSecret{value: val, expiresAt: time.Now().Add(m.cacheTTL)} m.mu.Unlock() + slog.Debug("secret: fetched from primary", "key", key) return val, nil } if m.fallback != nil { val, fallbackErr := m.fallback.Get(ctx, key) if fallbackErr == nil { + slog.Warn("secret: primary failed, using fallback", + "key", key, "primary_error", err) return val, nil } } + slog.Warn("secret: not found in any backend", "key", key, "error", err) return "", err } @@ -85,11 +163,17 @@ func (m *Manager) List(ctx context.Context) ([]string, error) { return keys, err } -// Set stores a secret in the primary backend (used during migration). +// Set stores a secret in the primary backend and invalidates the cache. func (m *Manager) Set(ctx context.Context, key string, value string) error { + m.InvalidateCache() return m.primary.Set(ctx, key, value) } +// Name returns the primary backend name. +func (m *Manager) Name() string { + return m.primary.Name() +} + // PrimaryName returns the name of the primary backend. func (m *Manager) PrimaryName() string { return m.primary.Name() @@ -101,3 +185,14 @@ func (m *Manager) InvalidateCache() { m.cache = make(map[string]cachedSecret) m.mu.Unlock() } + +// LastRefresh returns the timestamp of the last background refresh. +func (m *Manager) LastRefresh() time.Time { + return m.lastRefresh +} + +// secretOverlay maps an Infisical key to a config setter function. +type secretOverlay struct { + infisicalKey string + apply func(value string) +} diff --git a/internal/secrets/config_overlay.go b/internal/secrets/config_overlay.go new file mode 100644 index 0000000..ab47e2d --- /dev/null +++ b/internal/secrets/config_overlay.go @@ -0,0 +1,118 @@ +package secrets + +import ( + "context" + "log/slog" +) + +// NewManagerFromConfig creates a secrets Manager from the Infisical connection +// parameters in cfg, with an optional SOPS fallback from cfg.SecretsDir. +// Returns nil if Infisical is not configured. +func NewManagerFromConfig(siteURL, clientID, clientSecret, projectID, env, secretsDir string) *Manager { + if siteURL == "" { + return nil + } + infCfg := InfisicalConfig{ + SiteURL: siteURL, + ClientID: clientID, + ClientSecret: clientSecret, + ProjectID: projectID, + SecretPath: "/", + Env: env, + } + if infCfg.Env == "" { + infCfg.Env = "dev" + } + primary := NewInfisicalBackend(infCfg) + var fallback Backend + if secretsDir != "" { + fallback = NewSOPSBackend(secretsDir) + } + return NewManager(primary, fallback) +} + +// VerifyExpectedSecrets checks that a list of expected keys are present +// in the backend. Logs a summary and returns the count of missing keys. +// Use at startup to detect incomplete Infisical migration. +func VerifyExpectedSecrets(ctx context.Context, sec Backend, expected []string) int { + keys, err := sec.List(ctx) + if err != nil { + slog.Warn("secrets: cannot verify expected secrets, list failed", "error", err) + return len(expected) + } + keySet := make(map[string]struct{}, len(keys)) + for _, k := range keys { + keySet[k] = struct{}{} + } + missing := 0 + for _, exp := range expected { + if _, ok := keySet[exp]; !ok { + missing++ + slog.Warn("secrets: expected key missing from Infisical", "key", exp) + } + } + if missing == 0 { + slog.Info("secrets: all expected keys present", "count", len(expected)) + } else { + slog.Warn("secrets: some expected keys missing from Infisical", + "missing", missing, "total", len(expected)) + } + return missing +} + +// OverlayConfig fetches secrets from the backend and returns a function that +// applies them to config fields. Each entry maps an Infisical key to a setter; +// if the key is found and non-empty, the setter is called; if not found or +// empty, the env-derived value is left unchanged and a warning is logged. +// Returns the number of secrets resolved from Infisical (useful for logging). +func OverlayConfig(ctx context.Context, sec Backend, overlays []secretOverlay) int { + resolved := 0 + for _, o := range overlays { + val, err := sec.Get(ctx, o.infisicalKey) + if err != nil { + slog.Warn("secret not resolved from Infisical, using env fallback", + "key", o.infisicalKey, "error", err) + continue + } + if val == "" { + slog.Warn("Infisical returned empty value, keeping env-derived value", + "key", o.infisicalKey) + continue + } + o.apply(val) + resolved++ + } + return resolved +} + +// ConfigOverlays returns the standard set of Infisical → config overlays for +// the oikos binary. Each overlay is attempted at startup; if the key exists +// in Infisical, it overrides the env-derived value. +func ConfigOverlays(cfg map[string]func(string)) []secretOverlay { + overlays := make([]secretOverlay, 0, len(cfg)) + for key, setter := range cfg { + overlays = append(overlays, secretOverlay{infisicalKey: key, apply: setter}) + } + return overlays +} + +// ResolveSecret attempts to fetch a single secret from the backend. Returns +// the Infisical value if found and non-empty, otherwise falls back to the +// env-derived value. Warnings are logged for failures. +func ResolveSecret(ctx context.Context, sec Backend, infisicalKey, fallback string) string { + if sec == nil { + return fallback + } + val, err := sec.Get(ctx, infisicalKey) + if err != nil { + slog.Warn("secret not resolved from Infisical, using env fallback", + "key", infisicalKey, "error", err) + return fallback + } + if val == "" { + slog.Warn("Infisical returned empty value, keeping env-derived value", + "key", infisicalKey) + return fallback + } + return val +} diff --git a/internal/secrets/infisical.go b/internal/secrets/infisical.go index 7dd9726..23211a8 100644 --- a/internal/secrets/infisical.go +++ b/internal/secrets/infisical.go @@ -3,7 +3,6 @@ package secrets import ( "context" "fmt" - "os" "strings" "sync" @@ -54,14 +53,8 @@ func (b *InfisicalBackend) connect() error { clientID := b.cfg.ClientID clientSecret := b.cfg.ClientSecret - if clientID == "" { - clientID = os.Getenv("INFISICAL_CLIENT_ID") - } - if clientSecret == "" { - clientSecret = os.Getenv("INFISICAL_CLIENT_SECRET") - } if clientID == "" || clientSecret == "" { - return fmt.Errorf("%w: INFISICAL_CLIENT_ID and INFISICAL_CLIENT_SECRET not set", ErrBackendUnavailable) + return fmt.Errorf("%w: OIKOS_INFISICAL_CLIENT_ID and OIKOS_INFISICAL_CLIENT_SECRET not set", ErrBackendUnavailable) } _, err := b.client.Auth().UniversalAuthLogin(clientID, clientSecret) diff --git a/internal/secrets/sops.go b/internal/secrets/sops.go index d0a1783..e04aca9 100644 --- a/internal/secrets/sops.go +++ b/internal/secrets/sops.go @@ -3,6 +3,7 @@ package secrets import ( "context" "fmt" + "log/slog" "os" "os/exec" "strings" @@ -45,11 +46,13 @@ func (s *SOPSBackend) load() error { cmd := exec.Command("sops", "-d", path) out, err := cmd.Output() if err != nil { + slog.Warn("sops: decryption failed", "file", entry.Name(), "error", err) continue // skip unreadable files } var data map[string]any if err := yaml.Unmarshal(out, &data); err != nil { + slog.Warn("sops: invalid yaml after decryption", "file", entry.Name(), "error", err) continue } diff --git a/plans/2026-08-05-backend-evaluation-improvements.md b/plans/2026-08-05-backend-evaluation-improvements.md new file mode 100644 index 0000000..311b956 --- /dev/null +++ b/plans/2026-08-05-backend-evaluation-improvements.md @@ -0,0 +1,442 @@ +# 2026-08-05 — Backend evaluation: architecture, security, and reliability improvements + +Status: **In Progress** — Phase 0 code changes complete (B1, B2, B4, B5, B6, B7); B3 +is a post-deploy operational step. + +Scope: full evaluation of the oikos backend (Go binaries `oikos`, `nomos`, `webhook`, +Postgres/TimescaleDB, Docker deployment, MCP server) excluding frontend clients +(`web/` SPA and `desktop/` Wails app). Research-only pass — no code changes. + +Method: four parallel research passes (Go backend structure, database schema, +deployment/infrastructure, API/MCP design) plus Infisical secrets audit and +dependency analysis of `go.mod`. + +--- + +## A. Summary of findings + +The backend is well-architected with strong fundamentals: contract-first API +(oapi-codegen), type-safe SQL (sqlc + pgx), TimescaleDB observability, policy- +governed autonomy, and a sophisticated ontology-driven data model. The main gaps +are secret management (Infisical is wired but barely used — 3 of 4 binaries +read secrets from env/plaintext), operational maturity (CI, image tagging, backup +reliability), security hardening (SSH host keys, unauthenticated endpoints), and +code hygiene (monolithic files, mixed SQL access patterns). + +| Category | Grade | Notes | +|----------|-------|-------| +| Tech stack | A | Go + pgx + sqlc + TimescaleDB + chi + slog — all correct choices | +| Data model | A- | Dual-entity pattern, temporal relationships, partial indexes, 6 state machines | +| API design | B+ | Contract-first with ~50 MCP tools, RFC 9457 errors; no rate limiting | +| Secret management | **D** | Infisical SDK wired but only in API/MCP tools path; nomos, webhook, scheduler, notifier all read plaintext env vars. 7 production secrets in `.env`, HMAC in world-readable plist. SOPS fallback is dead code. | +| Security | C | OIDC+static token auth is good, but SSH host keys disabled, unauthenticated nomos endpoint, HMAC secret in world-readable plist | +| Deployment | C+ | Multi-stage builds, pre-deploy backups, but no CI, no versioned images, silent backup failures | +| Code quality | B | Good error handling, panic safety, doc; but 3 files over 1100 lines, mixed raw/sqlc SQL | +| Performance | B | Appropriate for scale; SSH check storm risk, no query caching | +| Observability | B- | TimescaleDB hypertables + SSE + slog; no Prometheus/Grafana, no OTel tracing | +| Testing | C | `make test` exists but many core packages (scheduler, domain, actuator, policy) have 0% coverage | + +## B. Infisical consolidation (critical) + +Infisical is deployed (Redis + Infisical service in compose, Go SDK in go.mod, +`internal/secrets/infisical.go` fully implemented) but severely underutilized. +Only the API server's MCP tools path creates an Infisical backend. Every other +binary reads secrets from env vars or plaintext files. + +### Current wiring map + +| Binary / role | Uses Infisical? | Secrets read from env/plaintext | +|---------------|----------------|-------------------------------| +| oikos `api` role | Yes (MCP tools only) | `OIKOS_DATABASE_URL`, `OIKOS_MCP_BEARER_TOKEN`, `OIKOS_API_TOKEN`, `OIKOS_OIDC_CLIENT_SECRET` | +| oikos `scheduler` role | **No** | `OIKOS_SSH_KEY_PATH`, `OIKOS_DATABASE_URL` | +| oikos `notifier` role | **No** | `OIKOS_MATRIX_TOKEN`, `OIKOS_APPROVAL_HMAC_SECRET`, `OIKOS_DATABASE_URL` | +| nomos | **No** | `OPENROUTER_API_KEY`, `OIKOS_MCP_BEARER_TOKEN`, `DATABASE_URL` | +| webhook | **No** | `WEBHOOK_HMAC_SECRET` (also hardcoded in plist) | + +### B1. Wire Infisical into all binaries at startup + +- **Goal**: Every binary fetches its secrets from Infisical at startup instead of + relying on env vars. Bootstrap-only env vars (`INFISICAL_CLIENT_ID`, + `INFISICAL_CLIENT_SECRET`, `INFISICAL_SITE_URL`, `INFISICAL_PROJECT_ID`, + `OIKOS_DATABASE_URL`) remain as env vars (chicken-egg). +- **Approach**: Add a `secrets.InitFromEnv(ctx)` call to each binary's `main()` that + creates a `secrets.Manager` (primary Infisical + SOPS fallback). Store the + manager in a package-level var or pass it through the initialization chain. +- **Files to change**: + - `cmd/oikos/main.go` — create Manager in `runWithPool`, pass to scheduler and + notifier runners alongside cfg and pool + - `cmd/nomos/main.go` — create Manager at startup, fetch `OPENROUTER_API_KEY` + and `OIKOS_MCP_BEARER_TOKEN` from Infisical before creating the agent + - `cmd/webhook/main.go` — create Manager at startup, fetch `WEBHOOK_HMAC_SECRET` + from Infisical +- **Secrets to migrate into Infisical** (move from env vars / `.env` / plist): + + | Secret key (in Infisical) | Current source | Used by | + |---------------------------|---------------|---------| + | `matrix/token` | `OIKOS_MATRIX_TOKEN` env | oikos notifier | + | `approval/hmac-secret` | `OIKOS_APPROVAL_HMAC_SECRET` env, plist | oikos notifier, webhook | + | `mcp/bearer-token` | `OIKOS_MCP_BEARER_TOKEN` env | oikos api, nomos | + | `api/token` | `OIKOS_API_TOKEN` env | oikos api | + | `oidc/client-secret` | `OIKOS_OIDC_CLIENT_SECRET` env | oikos api | + | `openrouter/api-key` | `OPENROUTER_API_KEY` env | nomos | + | `webhook/hmac-secret` | `WEBHOOK_HMAC_SECRET` env + plist | webhook | +- **Risk class**: config_mutation +- **Prerequisite**: Populate Infisical with these secrets via `oikos secret set` before + deploying the code change. Existing `.env` values serve as the source of truth + for the initial migration. + +### B2. Activate the SOPS fallback path + +- **Status**: Done (commit pending) +- **Current**: `secrets.NewManager(infisical, sops)` is only used in tests. + `httpapi/server.go` creates `InfisicalBackend` directly — if Infisical is down, + there is no fallback. +- **Fix**: Use `secrets.NewManager()` in production everywhere so the SOPS DR + fallback actually works when Infisical is unreachable. The Manager's cache + (5min TTL) already masks transient Infisical blips. +- **What changed**: `httpapi/server.go` now creates `secrets.NewManager(infisical, + sopsFallback)` instead of bare `secrets.NewInfisicalBackend`. SOPS backend + is created from `cfg.SecretsDir` when set. +- **Risk class**: config_mutation + +### B3. Remove `.env` plaintext secrets after migration + +- **Current**: `.env` contains 7 production secrets in plaintext on mac-mini disk. +- **Fix**: After B1 is deployed and all binaries read from Infisical, strip + secrets from `.env` leaving only non-secret config (`OIKOS_API_LISTEN`, + `OIKOS_SCHEDULER_INTERVAL`, etc.). Bootstrap env vars + (`OIKOS_DATABASE_URL`, `OIKOS_INFISICAL_*`) stay — they're the trust anchor. +- **Risk class**: config_mutation + +### B4. Remove HMAC secret from plist + +- **Status**: Done (code change; plist cleanup is post-deploy) +- **Current**: `scripts/deploy/network.hubris.oikos-deploy-webhook.plist` line 14 + has `WEBHOOK_HMAC_SECRET` hardcoded in plaintext. World-readable. +- **Fix**: After B1 (webhook reads from Infisical), remove the `EnvironmentVariables` + `WEBHOOK_HMAC_SECRET` entry from the plist entirely. The webhook binary will + fetch it from Infisical at startup. +- **Risk class**: config_mutation +- **Supersedes**: Original plan item B3 (HMAC secret in plist) — same issue, + now resolved via Infisical instead of file permissions workarounds. + +### B5. Store SSH host keys in Infisical + +- **Status**: Done (commit pending) +- **Current**: `ssh.InsecureIgnoreHostKey()` in 3 code paths + (`internal/actuator/ssh.go:160`, `internal/actuator/actuator.go:427`, + `internal/mcp/server.go`). +- **Fix**: Store Proxmox host public keys in Infisical under + `ssh/host-keys/{hostname}`. Actuator reads them at connection init and builds + a `knownhosts` callback. For dynamic targets, implement TOFU (trust-on-first- + use) writing back to Infisical. +- **What changed**: + - `internal/actuator/hostkeys.go` — `HostKeyCallback()` returns an + `ssh.HostKeyCallback` that verifies against cached keys (MITM detection) + and accepts unknown hosts via TOFU, persisting new keys to Infisical. + - `internal/actuator/hostkeys_infisical.go` — `InfisicalHostKeySource` + implements `HostKeySource` over `secrets.Backend`; `ResolveSSHHosts()` + queries DB for active proxmox-host/standalone-server slugs. + - `internal/actuator/ssh.go:160` — replaced `InsecureIgnoreHostKey()` with + `HostKeyCallback()`. + - `internal/actuator/actuator.go:427` — replaced `InsecureIgnoreHostKey()` with + `HostKeyCallback()`. + - `internal/mcp/server.go:494` — replaced `InsecureIgnoreHostKey()` with + `actuator.HostKeyCallback()`. + - `internal/httpapi/server.go` — pre-loads SSH host keys from Infisical at + startup (queries active hosts, loads their keys from Infisical). +- **Post-deploy step**: On first deploy, TOFU will accept all current host + keys and store them in Infisical under `ssh/host-keys/{slug}`. Verify the + stored keys are correct by checking `oikos secret list`. To pre-populate + without TOFU, SSH to each Proxmox host and run: + `ssh-keyscan -t ed25519 {host} | awk '{print $2" "$3}'` and store + the output via `oikos secret set ssh/host-keys/{slug} {output}`. +- **Risk class**: config_mutation (initial pin) / destructive (if keys change) + +### B6. Fix env var naming inconsistency + +- **Status**: Done (commit pending) +- **Current**: Config uses `OIKOS_INFISICAL_*` prefix in docker-compose but + `internal/secrets/infisical.go` lines 58–62 falls back to bare `INFISICAL_*` + (without OIKOS prefix). Two naming conventions for the same bootstrap vars. +- **Fix**: Standardize on `OIKOS_INFISICAL_*` everywhere. Remove the bare + `INFISICAL_*` fallback in infisical.go. +- **What changed**: Removed the `os.Getenv("INFISICAL_CLIENT_ID")` and + `os.Getenv("INFISICAL_CLIENT_SECRET")` fallbacks in `infisical.go connect()`. + Removed unused `os` import. Error message updated to reference + `OIKOS_INFISICAL_*` names. +- **Risk class**: config_mutation + +### B7. Align `secretsBackend` interface with `secrets.Backend` + +- **Status**: Done (commit pending) +- **Current**: `internal/httpapi/server.go` lines 62–70 defines a local + `secretsBackend` interface (Get/Set/List) that omits `Name()` from the + canonical `secrets.Backend`. +- **Fix**: Use `secrets.Backend` directly in httpapi, or embed it in the local + interface. +- **What changed**: + - Removed `secretsBackend` interface from `httpapi/server.go`; `Server.secretsManager` + now uses `secrets.Backend` directly. + - Removed `secretBackend` interface from `mcp/server.go`; `NewHandler` and + `newServer` now accept `secrets.Backend`. + - Updated `mcp/tools.go` `allTools()` signature to accept `secrets.Backend`. + - Updated `mcp/secrets_tools_test.go` mock to implement `secrets.Backend` + (added `Name()` method, uses `secrets.ErrNotFound` instead of custom error). +- **Risk class**: read_only + +## C. Security fixes (critical, non-Infisical) + +### C1. nomos.hubris.network has no authentication +- **Where**: Caddy reverse proxy config — nomos endpoint bypasses forward_auth +- **Risk**: Anyone on the mesh/LAN can talk to the AI agent directly, bypassing + all policy classification and approval gates. +- **Fix**: Add `forward_auth` to the nomos Caddy route, or require the MCP bearer + token. At minimum, add a shared secret via Caddy `basicauth`. +- **Risk class**: config_mutation + +### C2. pg_dump failure is silently ignored +- **Where**: `scripts/deploy.sh` — `pg_dump ... || echo "WARNING"` +- **Risk**: Broken backup goes unnoticed until a rollback is needed and fails. +- **Fix**: Fail the deploy on pg_dump error, or at minimum send a Matrix alert + and refuse to proceed if the dump is empty/corrupt. +- **Risk class**: config_mutation + +### C3. CORS defaults to `*` +- **Where**: `internal/httpapi/server.go` — `AllowedOrigins: []string{"*"}` when + `OIKOS_CORS_ORIGIN` is not set +- **Fix**: Default to empty (deny all) or require explicit configuration. +- **Risk class**: config_mutation + +## D. Operational improvements (high) + +### D1. Add CI pipeline +- **Current**: No automated build/test on push. `make lint test generate-check` + exists but is manual. +- **Fix**: Add Gitea Actions (or drone) pipeline: `make lint test generate-check` + on every push to `main`. Block deploy if pipeline fails. +- **Risk class**: config_mutation + +### D2. Version Docker images +- **Current**: All images built as `:latest`. Rollback requires full rebuild. +- **Fix**: Tag images with `v$VERSION` from the VERSION file in deploy.sh. Keep + last 3 versions. Enable `docker compose up` to pin a version tag. +- **Risk class**: config_mutation + +### D3. Add rate limiting +- **Current**: No throttling on HTTP API or MCP endpoints. An agent in a loop + could hammer the API or exhaust DB connections. +- **Fix**: Add `golang.org/x/time/rate` middleware to chi router. Per-IP or + per-token rate limit with burst allowance. Separate limits for API vs MCP. +- **Risk class**: config_mutation + +### D4. Add container resource limits +- **Current**: No `mem_limit`, `cpus`, or `ulimits` on any compose service. +- **Fix**: Add memory and CPU limits to all services in docker-compose.yml. + Suggested: API 512MB, scheduler 256MB, notifier 128MB, nomos 512MB. +- **Risk class**: config_mutation + +### D5. Add healthchecks to all compose services +- **Current**: Only postgres, api, and redis have healthchecks. +- **Fix**: Add `healthcheck` to scheduler, notifier, and nomos. Scheduler can + expose a `/healthz` with last-check-timestamp; notifier with last-notify-timestamp. + +## E. Code quality (medium) + +### E1. Split monolithic files +| File | Lines | Split target | +|------|-------|-------------| +| `internal/mcp/tools.go` | 1774 | `entity_tools.go`, `ops_tools.go`, `knowledge_tools.go`, `analysis_tools.go` | +| `internal/httpapi/impl.go` | 1533 | Handler group files by domain (entities, executions, approvals, metrics, etc.) | +| `cmd/nomos/main.go` | 1127 | `server.go`, `workers.go`, `mcp.go` (partially done — `agent.go` and `store.go` exist) | + +### E2. Migrate raw pool.Exec queries to sqlc +- ~50% of DB access in HTTP/MCP handlers bypasses sqlc with raw `pool.Exec`/`pool.QueryRow`. +- Add these queries to `internal/db/queries/` source files for type safety and + compile-time validation. + +### E3. Unify SSH implementations +- Scheduler uses `os/exec ssh` (system binary), MCP/actuator uses `crypto/ssh`. +- Unify on `crypto/ssh` throughout for consistency, testability, and connection + multiplexing (single TCP connection, multiple sessions). +- Consider a shared SSH pool in `internal/actuator/` used by both scheduler and MCP. + +### E4. Fix lifecycle attribute check +- `internal/db/lifecycle.go`: `checkPrecondition` uses `strings.Contains(attrs, want)` + on raw JSONB text, bypassing the GIN index. +- Parse attributes properly and use `@>` or `?` JSONB operators. + +### E5. Add table-driven tests for core logic +Priority packages (currently 0% coverage): +1. `internal/policy` — risk classification rules (table-driven with seed policy.yaml cases) +2. `internal/domain` — lifecycle state machine transitions +3. `internal/scheduler` — check dispatch and signal resolution +4. `internal/ontology` — monitoring resolution and type tree traversal +5. `internal/checkdefaults` — check derivation from monitoring specs + +## F. Performance (medium) + +### F1. SSH connection pooling for scheduler +- At 30s intervals with 95 entities and multiple check types, the scheduler can + spawn 100+ SSH sessions per cycle via `os/exec ssh`. +- Migrate to `crypto/ssh` with persistent connection pools to Proxmox hosts. + One TCP connection per host, multiplexed sessions for individual checks. + +### F2. Entity lookup cache +- Repeated `get_entity`/`whoami` MCP calls hit the DB every time. +- Add an in-memory TTL cache (hashicorp/golang-lru, already in go.mod) with + 60s TTL for entity lookups. Invalidate on write. + +### F3. Trigram index for entity search +- `ListEntities` uses `ILIKE '%'||q||'%'` which cannot use B-tree indexes. +- Add GIN trigram indexes on `entities.slug` and `entities.name`. +- Alternative: migrate to `tsvector` full-text search matching the knowledge pattern. + +### F4. Composite index for auto-act anti-join +- `GetOpenSignalsForAutoAct` joins classifications → signals → executions with + `WHERE e.entity_id IS NULL`. No composite index on `(classification_id, entity_id)`. +- Add partial index on `executions(classification_id) WHERE entity_id IS NOT NULL`. + +## G. Observability (low) + +### G1. Add OpenTelemetry tracing +- OTel SDK is already in go.mod as indirect dependency. +- Instrument HTTP handlers, MCP tools, and DB queries with spans. +- Propagate trace context via `correlation_id` (already exists in audit/events). + +### G2. Prometheus metrics export +- Expose `/metrics` endpoint for Go runtime, DB pool stats, scheduler check + duration/counts, HTTP request latency histograms. +- Complement the existing TimescaleDB metric_samples (which are entity health + metrics, not self-observability). + +### G3. Automate offsite backups +- Proton Drive backup target entity exists but no pipeline. +- Add `rclone cron` to `pg_dump | zstd | rclone sync` to Proton Drive. +- Weekly backup verification (restore to test DB, run `make test-db`). + +## H. Infrastructure (low) + +### H1. Pin Infisical image version +- Currently uses `infisical/infisical:latest`. +- Pin to a specific version tag. + +### H2. Add persistent job queue for executions +- All background work is in-process goroutines — lost on restart. +- For the execution pipeline specifically, consider Postgres-backed queue + (e.g., `river` or custom `pending_executions` poll with advisory lock). +- Lower priority: scheduler and notifier state is transient and self-healing. + +### H3. Replace or harden custom migration splitter +- The `splitSQL()` function handles `$$` dollar-quoting but edge cases with + string literals containing `$$` could break migrations. +- Add test cases for nested quoting, or adopt golang-migrate. + +### H4. Add distributed locking for scheduler +- Document single-instance constraint, or add `pg_advisory_lock` (already used + by migration runner) to prevent duplicate health checks if multiple + scheduler instances are accidentally started. + +--- + +## Execution order + +1. **Phase 0 — Infisical consolidation** (B1–B7): Wire Infisical into all + binaries, migrate secrets from env/plaintext, activate SOPS fallback, remove + `.env` secrets and plist HMAC. This is the foundation — every subsequent + secret-dependent change (SSH host keys in B5, rate limit config, etc.) goes + through Infisical. **Do this first.** +2. **Phase 1 — Security** (C1–C3, B5): Nomos auth, pg_dump failure, CORS default, + SSH host keys (now stored in Infisical per B5). +3. **Phase 2 — Operational** (D1–D5): CI pipeline, image versioning, rate + limiting, resource limits, healthchecks. +4. **Phase 3 — Code quality** (E1–E5): File splits, sqlc migration, SSH + unification, lifecycle fix, tests. E1–E3 are large refactors — do one + file/area per commit. +5. **Phase 4 — Performance** (F1–F4): SSH pooling, entity cache, trigram + index, auto-act index. +6. **Phase 5 — Observability** (G1–G3): OTel tracing, Prometheus, offsite backups. +7. **Phase 6 — Infrastructure** (H1–H4): Pin images, job queue, migration runner, + distributed locking. + +Phase 0 is the gate. Once all secrets flow through Infisical, phases 1–2 can +proceed. Phases 3–4 should wait for CI (D1) so refactors are validated. +Phases 5–6 are backlog. + +--- + +## Phase 0 post-deploy checklist + +Run these on mac-mini after deploying the Phase 0 code changes. + +### Step 1: Populate Infisical with secrets + +For each secret, read the current value from `.env` and store it in Infisical: + +```bash +# Values from .env — read them first, then set +oikos secret set matrix/token "$(grep OIKOS_MATRIX_TOKEN .env | cut -d= -f2-)" +oikos secret set approval/hmac-secret "$(grep OIKOS_APPROVAL_HMAC_SECRET .env | cut -d= -f2-)" +oikos secret set mcp/bearer-token "$(grep OIKOS_MCP_BEARER_TOKEN .env | cut -d= -f2-)" +oikos secret set api/token "$(grep OIKOS_API_TOKEN .env | cut -d= -f2-)" +oikos secret set oidc/client-secret "$(grep OIKOS_OIDC_CLIENT_SECRET .env | cut -d= -f2-)" +oikos secret set openrouter/api-key "$(grep OPENROUTER_API_KEY .env | cut -d= -f-)" +oikos secret set webhook/hmac-secret "$(grep WEBHOOK_HMAC_SECRET .env | cut -d= -f2-)" +``` + +Verify: `oikos secret list` should show all 8 keys. + +### Step 2: Pre-populate SSH host keys (optional, skip if TOFU is acceptable) + +```bash +# For each Proxmox host, scan and store the public key +for host in pve1 pve2; do + key=$(ssh-keyscan -t ed25519 $host 2>/dev/null | awk '{print $2" "$3}') + oikos secret set "ssh/host-keys/$host" "ssh-ed25519 $key" +done +``` + +Alternatively, skip this step — the first deployment will TOFU-accept all current +host keys and persist them to Infisical automatically. + +### Step 3: Remove HMAC secret from webhook plist + +On mac-mini: +```bash +sudo launchctl unload ~/Library/LaunchAgents/network.hubris.oikos-deploy-webhook.plist +# Edit the plist: remove the WEBHOOK_HMAC_SECRET block +sudo launchctl load ~/Library/LaunchAgents/network.hubris.oikos-deploy-webhook.plist +``` + +### Step 4: Strip secrets from .env + +Edit `.env` to remove the 7 migrated secrets, keeping only bootstrap and +non-secret config: + +```bash +# Remove these lines: +# INFISICAL_ENCRYPTION_KEY=... +# OIKOS_MATRIX_TOKEN=... +# OIKOS_INFISICAL_CLIENT_ID=... +# OIKOS_INFISICAL_CLIENT_SECRET=... +# OIKOS_INFISICAL_PROJECT_ID=... +# OPENROUTER_API_KEY=... +# OIKOS_MCP_BEARER_TOKEN=... + +# Keep these (bootstrap / non-secret): +# OIKOS_DATABASE_URL=... +# OIKOS_API_LISTEN=... +# OIKOS_INFISICAL_SITE_URL=... +# OIKOS_INFISICAL_ENV=... +``` + +### Step 5: Verify + +1. `oikos secret list` — 8 keys + SSH host keys +2. `docker compose logs api | grep "secrets resolved"` — should show count=5 +3. Trigger a test deploy — webhook should still validate HMAC signatures +4. `nomos` should start and resolve secrets from Infisical (check logs for + `nomos: secrets resolved from Infisical`) +5. Verify no secrets appear in process env: `docker compose exec api env | + should not show `OIKOS_MCP_BEARER_TOKEN`, `OIKOS_MATRIX_TOKEN`, + etc. (they come from Infisical at startup, not env) diff --git a/plans/index.md b/plans/index.md index 9f33cfc..5a498c5 100644 --- a/plans/index.md +++ b/plans/index.md @@ -21,6 +21,7 @@ went sideways, open an investigation. | 2026-07-21 | [Frontend as OS + Apps — architecture audit & refactor](2026-07-21-frontend-os-apps-architecture.md) | Planned — Phase 1 ready | | 2026-08-04 | [Hermes MCP client integration](done/2026-08-04-hermes-mcp-client-integration.md) | Done — deployed | | 2026-08-05 | [Agent execution safety: QEMU guest agent gate + host-mutation guard](done/2026-08-05-agent-execution-safety-qemu-guest-agent-gate.md) | Done — implemented (1b9c761) | +| 2026-08-05 | [Backend evaluation: architecture, security, and reliability improvements](2026-08-05-backend-evaluation-improvements.md) | Planned — 6 phases, security items first | ## Done