scheduler: add ping + ssh-script check kinds, metrics refactor, 17 host check scripts

- Refactor executeCheck to return checkResult struct with metrics map
- Add ping check kind (ICMP reachability via system ping, macOS+Linux)
- Add ssh-script check kind (remote host exec via SSH, allowlisted scripts)
- Add threshold evaluation (warn/crit per metric from check config JSONB)
- Add inode tracking to disk check
- All 4 existing checks now return structured metrics
- 17 check scripts: cpu, memory, load, swap, disk_usage, disk_smart,
  updates, zfs, process, uptime, oom, journal, time, fd, docker_health,
  caddy_error_rate, backup_freshness
- Auto-deploy via tools/setup-checks.sh -> checks/install.sh on git pull
- Add ping to OpenAPI CheckKind enum and generated Go types

internal/httpapi/gen/api.gen.go

@@ -81,6 +81,7 @@ const (
 	CheckKindDisk       CheckKind = "disk"
 	CheckKindDrift      CheckKind = "drift"
 	CheckKindHttp       CheckKind = "http"
+	CheckKindPing       CheckKind = "ping"
 	CheckKindSshScript  CheckKind = "ssh-script"
 	CheckKindTcp        CheckKind = "tcp"
 )
@@ -91,6 +92,7 @@ const (
 	CheckCreateKindDisk       CheckCreateKind = "disk"
 	CheckCreateKindDrift      CheckCreateKind = "drift"
 	CheckCreateKindHttp       CheckCreateKind = "http"
+	CheckCreateKindPing       CheckCreateKind = "ping"
 	CheckCreateKindSshScript  CheckCreateKind = "ssh-script"
 	CheckCreateKindTcp        CheckCreateKind = "tcp"
 )

internal/scheduler/scheduler.go

@@ -11,6 +11,10 @@ import (
 	"log/slog"
 	"net"
 	"net/http"
+	"os/exec"
+	"regexp"
+	"runtime"
+	"strconv"
 	"time"
 
 	"github.com/dtoro/oikos/internal/config"
@@ -82,29 +86,33 @@ func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDef
 	q := sqlcgen.New(pool)
 	start := time.Now()
 
-	health, signalKind, evidence, checkErr := executeCheck(ctx, cd)
+	result := executeCheck(ctx, cd)
 
 	latency := time.Since(start).Milliseconds()
 
-	// Write metric
-	_ = q.InsertMetricSample(ctx, sqlcgen.InsertMetricSampleParams{
-		EntityID: cd.EntityID,
-		Metric:   "probe_latency_ms",
-		Value:    float64(latency),
-		Tags:     []byte(`{}`),
-	})
+	if result.metrics == nil {
+		result.metrics = make(map[string]float64)
+	}
+	result.metrics["probe_latency_ms"] = float64(latency)
 
-	if checkErr != nil {
+	for metric, value := range result.metrics {
+		_ = q.InsertMetricSample(ctx, sqlcgen.InsertMetricSampleParams{
+			EntityID: cd.EntityID,
+			Metric:   metric,
+			Value:    value,
+			Tags:     []byte(`{}`),
+		})
+	}
+
+	if result.err != nil {
 		slog.Warn("scheduler: check failed",
-			"entity", cd.EntitySlug, "kind", cd.Kind, "error", checkErr)
+			"entity", cd.EntitySlug, "kind", cd.Kind, "error", result.err)
 	}
 
 	prevHealth := currentHealth(ctx, pool, cd.EntityID)
 
-	if signalKind == "" || health == "healthy" {
-		// Recovery: resolve any open signal for this check
+	if result.signalKind == "" || result.health == "healthy" {
 		resolveSignal(ctx, pool, cd.EntityID, cd.EntitySlug)
-		// Update entity_status to healthy
 		_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
 			EntityID:    cd.EntityID,
 			Health:      "healthy",
@@ -118,45 +126,38 @@ func runCheck(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCheckDef
 		return
 	}
 
-	// Failure: upsert signal (dedup via partial unique index)
 	slog.Warn("scheduler: raising signal",
-		"entity", cd.EntitySlug, "kind", signalKind, "evidence", evidence)
+		"entity", cd.EntitySlug, "kind", result.signalKind, "evidence", result.evidence)
 
-	severity := "warning"
-	if signalKind == "down" {
-		severity = "critical"
-	}
+	severity := evaluateSeverity(cd.Kind, result.signalKind, cd.Config, result.metrics)
 
 	sig, err := q.UpsertSignal(ctx, sqlcgen.UpsertSignalParams{
 		EntityID:       cd.EntityID,
-		Kind:           signalKind,
+		Kind:           result.signalKind,
 		Severity:       severity,
 		TargetEntityID: cd.TargetID,
-		Evidence:       &evidence,
+		Evidence:       &result.evidence,
 	})
 	if err != nil {
 		slog.Error("scheduler: upsert signal", "error", err)
 		return
 	}
 
-	// Update entity_status
 	_ = q.UpsertEntityStatus(ctx, sqlcgen.UpsertEntityStatusParams{
 		EntityID:    cd.EntityID,
-		Health:      health,
+		Health:      result.health,
 		LastCheckAt: &[]time.Time{time.Now()}[0],
 		Details:     []byte(`{}`),
 	})
-	_ = sig // used for flap detection below
+	_ = sig
 
-	// Emit only on transition into failure so a persistently-down entity
-	// doesn't flood the stream every tick.
 	if prevHealth == "" || prevHealth == "healthy" {
 		emitSchedulerEvent(ctx, pool, "signal.raised", cd.EntityID, severity,
-			map[string]any{"slug": cd.EntitySlug, "kind": signalKind, "evidence": evidence})
+			map[string]any{"slug": cd.EntitySlug, "kind": result.signalKind, "evidence": result.evidence})
 	}
-	if prevHealth != health {
+	if prevHealth != result.health {
 		emitSchedulerEvent(ctx, pool, "health.changed", cd.EntityID, severity,
-			map[string]any{"slug": cd.EntitySlug, "from": prevHealth, "to": health})
+			map[string]any{"slug": cd.EntitySlug, "from": prevHealth, "to": result.health})
 	}
 }
 
@@ -197,8 +198,17 @@ func resolveSignal(ctx context.Context, pool *db.Pool, entityID uuid.UUID, slug
 	slog.Info("scheduler: signal resolved", "entity", slug)
 }
 
+// checkResult bundles the outcome of a single check execution.
+type checkResult struct {
+	health     string
+	signalKind string
+	evidence   string
+	metrics    map[string]float64
+	err        error
+}
+
 // executeCheck dispatches to the appropriate checker by kind.
-func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (health string, signalKind string, evidence string, err error) {
+func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
 	switch cd.Kind {
 	case "http":
 		return checkHTTP(ctx, cd)
@@ -208,8 +218,12 @@ func executeCheck(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (heal
 		return checkDisk(ctx, cd)
 	case "cert-expiry":
 		return checkCertExpiry(ctx, cd)
+	case "ping":
+		return checkPing(ctx, cd)
+	case "ssh-script":
+		return checkSSHScript(ctx, cd)
 	default:
-		return "unknown", "", "", nil
+		return checkResult{health: "unknown"}
 	}
 }
 
@@ -229,7 +243,7 @@ func housekeeping(ctx context.Context, pool *db.Pool) {
 }
 
 // checkHTTP performs an HTTP health check.
-func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) {
+func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
 	cfg := struct {
 		URL            string `json:"url"`
 		ExpectedStatus int    `json:"expected_status"`
@@ -241,7 +255,7 @@ func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string,
 		_ = json.Unmarshal(cd.Config, &cfg)
 	}
 	if cfg.URL == "" {
-		return "healthy", "", "", nil
+		return checkResult{health: "healthy"}
 	}
 
 	timeout := time.Duration(cd.TimeoutS) * time.Second
@@ -258,25 +272,35 @@ func checkHTTP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string,
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, cfg.URL, nil)
 	if err != nil {
-		return "down", "http", fmt.Sprintf("invalid URL %q: %v", cfg.URL, err), err
+		return checkResult{
+			health: "down", signalKind: "http",
+			evidence: fmt.Sprintf("invalid URL %q: %v", cfg.URL, err),
+			err:      err,
+		}
 	}
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return "down", "http", fmt.Sprintf("GET %s: %v", cfg.URL, err), err
+		return checkResult{
+			health: "down", signalKind: "http",
+			evidence: fmt.Sprintf("GET %s: %v", cfg.URL, err),
+			err:      err,
+		}
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != cfg.ExpectedStatus {
-		return "degraded", "http",
-			fmt.Sprintf("GET %s returned %d (expected %d)", cfg.URL, resp.StatusCode, cfg.ExpectedStatus), nil
+		return checkResult{
+			health: "degraded", signalKind: "http",
+			evidence: fmt.Sprintf("GET %s returned %d (expected %d)", cfg.URL, resp.StatusCode, cfg.ExpectedStatus),
+		}
 	}
 
-	return "healthy", "", "", nil
+	return checkResult{health: "healthy"}
 }
 
 // checkTCP performs a TCP dial check.
-func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) {
+func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
 	cfg := struct {
 		Host string `json:"host"`
 		Port int    `json:"port"`
@@ -285,7 +309,7 @@ func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string,
 		_ = json.Unmarshal(cd.Config, &cfg)
 	}
 	if cfg.Host == "" || cfg.Port == 0 {
-		return "healthy", "", "", nil
+		return checkResult{health: "healthy"}
 	}
 
 	timeout := time.Duration(cd.TimeoutS) * time.Second
@@ -296,14 +320,18 @@ func checkTCP(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string,
 	addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port))
 	conn, err := net.DialTimeout("tcp", addr, timeout)
 	if err != nil {
-		return "down", "tcp", fmt.Sprintf("dial %s: %v", addr, err), err
+		return checkResult{
+			health: "down", signalKind: "tcp",
+			evidence: fmt.Sprintf("dial %s: %v", addr, err),
+			err:      err,
+		}
 	}
 	conn.Close()
-	return "healthy", "", "", nil
+	return checkResult{health: "healthy"}
 }
 
-// checkDisk performs a disk usage check via local or SSH.
-func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) {
+// checkDisk performs a disk usage check.
+func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
 	cfg := struct {
 		Path         string `json:"path"`
 		ThresholdPct int    `json:"threshold_pct"`
@@ -315,29 +343,45 @@ func checkDisk(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string,
 		_ = json.Unmarshal(cd.Config, &cfg)
 	}
 
-	// Use unix.Statfs for disk usage.
 	var stat unix.Statfs_t
 	if err := unix.Statfs(cfg.Path, &stat); err != nil {
-		return "down", "disk", fmt.Sprintf("statfs %s: %v", cfg.Path, err), err
+		return checkResult{
+			health: "down", signalKind: "disk",
+			evidence: fmt.Sprintf("statfs %s: %v", cfg.Path, err),
+			err:      err,
+		}
 	}
 
 	total := stat.Blocks * uint64(stat.Bsize)
 	free := stat.Bfree * uint64(stat.Bsize)
 	if total == 0 {
-		return "healthy", "", "", nil
+		return checkResult{health: "healthy"}
 	}
 
 	usedPct := float64(total-free) / float64(total) * 100
-	if usedPct > float64(cfg.ThresholdPct) {
-		return "degraded", "disk",
-			fmt.Sprintf("%s %.1f%% full (threshold %d%%)", cfg.Path, usedPct, cfg.ThresholdPct), nil
+	inodePct := 0.0
+	if stat.Files > 0 {
+		inodePct = float64(stat.Files-stat.Ffree) / float64(stat.Files) * 100
 	}
 
-	return "healthy", "", "", nil
+	metrics := map[string]float64{
+		"disk_used_pct":  usedPct,
+		"disk_inode_pct": inodePct,
+	}
+
+	if usedPct > float64(cfg.ThresholdPct) {
+		return checkResult{
+			health: "degraded", signalKind: "disk",
+			evidence:   fmt.Sprintf("%s %.1f%% full (threshold %d%%)", cfg.Path, usedPct, cfg.ThresholdPct),
+			metrics:    metrics,
+		}
+	}
+
+	return checkResult{health: "healthy", metrics: metrics}
 }
 
 // checkCertExpiry checks TLS certificate expiry.
-func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (string, string, string, error) {
+func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
 	cfg := struct {
 		Host     string `json:"host"`
 		Port     int    `json:"port"`
@@ -352,7 +396,7 @@ func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (s
 		_ = json.Unmarshal(cd.Config, &cfg)
 	}
 	if cfg.Host == "" {
-		return "healthy", "", "", nil
+		return checkResult{health: "healthy"}
 	}
 
 	timeout := time.Duration(cd.TimeoutS) * time.Second
@@ -365,31 +409,250 @@ func checkCertExpiry(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) (s
 	d := tls.Dialer{Config: &tls.Config{InsecureSkipVerify: true}}
 	conn, err := d.DialContext(ctx, "tcp", addr)
 	if err != nil {
-		return "down", "cert-expiry", fmt.Sprintf("TLS dial %s: %v", addr, err), err
+		return checkResult{
+			health: "down", signalKind: "cert-expiry",
+			evidence: fmt.Sprintf("TLS dial %s: %v", addr, err),
+			err:      err,
+		}
 	}
 	defer conn.Close()
 
 	tlsConn := conn.(*tls.Conn)
-	// Use crypto/tls ConnectionState to get verified chains
 	cs := tlsConn.ConnectionState()
 	if len(cs.PeerCertificates) == 0 {
-		return "down", "cert-expiry", "no peer certificates", nil
+		return checkResult{
+			health: "down", signalKind: "cert-expiry",
+			evidence: "no peer certificates",
+		}
 	}
 
 	cert := cs.PeerCertificates[0]
 	daysLeft := int(time.Until(cert.NotAfter).Hours() / 24)
 
-	if daysLeft <= cfg.CritDays {
-		return "down", "cert-expiry",
-			fmt.Sprintf("%s expires in %d days (crit=%d)", cfg.Host, daysLeft, cfg.CritDays), nil
-	}
-	if daysLeft <= cfg.WarnDays {
-		return "degraded", "cert-expiry",
-			fmt.Sprintf("%s expires in %d days (warn=%d)", cfg.Host, daysLeft, cfg.WarnDays), nil
+	metrics := map[string]float64{
+		"cert_days_left": float64(daysLeft),
 	}
 
-	return "healthy", "", "", nil
+	if daysLeft <= cfg.CritDays {
+		return checkResult{
+			health: "down", signalKind: "cert-expiry",
+			evidence: fmt.Sprintf("%s expires in %d days (crit=%d)", cfg.Host, daysLeft, cfg.CritDays),
+			metrics:  metrics,
+		}
+	}
+	if daysLeft <= cfg.WarnDays {
+		return checkResult{
+			health: "degraded", signalKind: "cert-expiry",
+			evidence: fmt.Sprintf("%s expires in %d days (warn=%d)", cfg.Host, daysLeft, cfg.WarnDays),
+			metrics:  metrics,
+		}
+	}
+
+	return checkResult{health: "healthy", metrics: metrics}
+}
+
+// checkPing performs an ICMP ping check using the system ping command.
+func checkPing(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
+	cfg := struct {
+		Host  string `json:"host"`
+		Count int    `json:"count"`
+	}{}
+	if len(cd.Config) > 0 {
+		_ = json.Unmarshal(cd.Config, &cfg)
+	}
+	if cfg.Host == "" {
+		return checkResult{health: "healthy"}
+	}
+	if cfg.Count <= 0 {
+		cfg.Count = 1
+	}
+
+	timeout := time.Duration(cd.TimeoutS) * time.Second
+	if timeout <= 0 {
+		timeout = 10 * time.Second
+	}
+
+	deadline := time.Duration(cfg.Count+1) * timeout
+	ctx, cancel := context.WithTimeout(ctx, deadline)
+	defer cancel()
+
+	countStr := strconv.Itoa(cfg.Count)
+	timeoutSec := strconv.Itoa(int(timeout.Seconds()))
+	if timeoutSec == "0" {
+		timeoutSec = "1"
+	}
+
+	cmd := exec.CommandContext(ctx, "ping", "-c", countStr, "-W", timeoutSec, cfg.Host)
+	if runtime.GOOS == "darwin" {
+		cmd = exec.CommandContext(ctx, "ping", "-c", countStr, "-t", timeoutSec, cfg.Host)
+	}
+
+	output, err := cmd.Output()
+	if err != nil {
+		return checkResult{
+			health: "down", signalKind: "ping",
+			evidence: fmt.Sprintf("ping %s: %v", cfg.Host, err),
+			err:      err,
+		}
+	}
+
+	latency := parsePingLatency(output)
+	metrics := map[string]float64{}
+	if latency > 0 {
+		metrics["ping_latency_ms"] = latency
+	}
+
+	return checkResult{health: "healthy", metrics: metrics}
+}
+
+var pingRttRe = regexp.MustCompile(`(?:rtt\s+min\/avg\/max\/mdev|round-trip\s+min\/avg\/max\/stddev)\s*=\s*[\d.]+\/([\d.]+)\/`)
+
+func parsePingLatency(output []byte) float64 {
+	matches := pingRttRe.FindSubmatch(output)
+	if len(matches) < 2 {
+		return 0
+	}
+	val, err := strconv.ParseFloat(string(matches[1]), 64)
+	if err != nil {
+		return 0
+	}
+	return val
+}
+
+// checkSSHScript executes an allowlisted script on a remote host via SSH.
+func checkSSHScript(ctx context.Context, cd sqlcgen.ListEnabledCheckDefsRow) checkResult {
+	cfg := struct {
+		Host   string `json:"host"`
+		Port   int    `json:"port"`
+		User   string `json:"user"`
+		Script string `json:"script"`
+	}{}
+	if len(cd.Config) > 0 {
+		_ = json.Unmarshal(cd.Config, &cfg)
+	}
+	if cfg.Host == "" || cfg.Script == "" {
+		return checkResult{health: "healthy"}
+	}
+	if cfg.Port == 0 {
+		cfg.Port = 22
+	}
+	if cfg.User == "" {
+		cfg.User = "root"
+	}
+
+	if !allowlistedScript(cfg.Script) {
+		return checkResult{
+			health: "unknown", signalKind: "ssh-script",
+			evidence: fmt.Sprintf("script %q not allowlisted", cfg.Script),
+		}
+	}
+
+	timeout := time.Duration(cd.TimeoutS) * time.Second
+	if timeout <= 0 {
+		timeout = 10 * time.Second
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	scriptPath := "/opt/oikos/checks/" + cfg.Script
+	addr := net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port))
+
+	output, err := sshExec(ctx, addr, cfg.User, scriptPath, timeout)
+	if err != nil {
+		return checkResult{
+			health: "down", signalKind: "ssh-script",
+			evidence: fmt.Sprintf("ssh %s %s: %v", addr, cfg.Script, err),
+			err:      err,
+		}
+	}
+
+	type scriptOutput struct {
+		Health     string             `json:"health"`
+		SignalKind string             `json:"signalKind"`
+		Evidence   string             `json:"evidence"`
+		Metrics    map[string]float64 `json:"metrics"`
+	}
+	var so scriptOutput
+	if err := json.Unmarshal(output, &so); err != nil {
+		return checkResult{
+			health: "down", signalKind: "ssh-script",
+			evidence: fmt.Sprintf("invalid script output from %s: %v", cfg.Script, err),
+			err:      err,
+		}
+	}
+
+	health := so.Health
+	if health == "" {
+		health = "healthy"
+	}
+
+	metrics := so.Metrics
+	if metrics == nil {
+		metrics = make(map[string]float64)
+	}
+
+	return checkResult{
+		health:     health,
+		signalKind: so.SignalKind,
+		evidence:   so.Evidence,
+		metrics:    metrics,
+	}
+}
+
+var scriptNameRe = regexp.MustCompile(`^[a-z][a-z0-9_-]+\.sh$`)
+
+func allowlistedScript(name string) bool {
+	return scriptNameRe.MatchString(name)
+}
+
+func sshExec(ctx context.Context, addr, user, cmd string, timeout time.Duration) ([]byte, error) {
+	args := []string{
+		"-o", "ConnectTimeout=" + strconv.Itoa(int(timeout.Seconds())),
+		"-o", "StrictHostKeyChecking=yes",
+		"-o", "BatchMode=yes",
+		"-l", user,
+		addr,
+		cmd,
+	}
+	c := exec.CommandContext(ctx, "ssh", args...)
+	return c.Output()
 }
 
 
+// metricThreshold defines warn/crit thresholds for a single metric.
+type metricThreshold struct {
+	Warn float64 `json:"warn"`
+	Crit float64 `json:"crit"`
+}
+
+// thresholdsConfig is parsed from check_defs.config.thresholds JSONB.
+type thresholdsConfig map[string]metricThreshold
+
+// evaluateSeverity determines signal severity from check result and thresholds.
+func evaluateSeverity(kind string, signalKind string, config []byte, metrics map[string]float64) string {
+	var thresholds thresholdsConfig
+	if len(config) > 0 {
+		_ = json.Unmarshal(config, &thresholds)
+	}
+
+	for metric, value := range metrics {
+		t, ok := thresholds[metric]
+		if !ok {
+			continue
+		}
+		if t.Crit > 0 && value >= t.Crit {
+			return "critical"
+		}
+		if t.Warn > 0 && value >= t.Warn {
+			return "warning"
+		}
+	}
+
+	if signalKind == "down" {
+		return "critical"
+	}
+	return "warning"
+}
+
 var _ = uuid.UUID{} // ensure uuid import stays