package app import ( "encoding/json" "log/slog" "strings" "github.com/dtoro/oikos/internal/ontology" ) // Semantic monitoring kinds, as declared on entity types. These are not // check_defs.kind values — one semantic kind can expand to several concrete // checks (`resource` becomes four ssh-script rows). const ( KindPing = "ping" KindResource = "resource" KindUpdates = "updates" KindProcess = "process" KindHTTP = "http" KindCapacity = "capacity" KindBackup = "backup-freshness" KindCertExpiry = "cert-expiry" KindVMStatus = "vm-status" KindQuorum = "quorum" KindDNS = "dns" ) // defaultBackupMaxAge is how long a backup target may go without a new // artifact before it is stale. A day suits the nightly jobs in this lab; // override per target with `backup_max_age_s` in the entity's attributes. const defaultBackupMaxAge = 86400 // CheckTarget is the entity default checks are being derived for. type CheckTarget struct { ID string Slug string Type string // Name is the entity's name column, not an attribute. The old code read // attrs["name"], which is never populated — seeds put `name` beside // `attributes`, not inside it — so every service silently produced no // process check. Name string Attrs []byte } // CheckDef is one concrete derived check: kind, config payload, interval. type CheckDef struct { Kind string Config map[string]any IntervalS int } // DeriveResult reports what Derive produced, so callers can log a type that // declared monitoring but produced nothing instead of failing silently. type DeriveResult struct { Created int // Skipped records kinds that were declared but could not be built, with // the reason. A non-empty Skipped on an active entity is a real gap. Skipped []Skip // Undeclared is true when no ancestor of the type declared monitoring — // an ontology gap rather than a fleet gap. Undeclared bool } // Skip is one declared-but-unbuilt check kind. type Skip struct { Kind string Reason string } // HostLookup resolves the hosting entity's attributes when the entity // itself carries no address (a service lives on its container; a backup // target on whatever writes to it). It is invoked lazily — only when the // entity's own attributes lack a host — so the pure derivation below stays // separated from the graph read the caller performs. type HostLookup func() map[string]any // Derive computes the default checks for one entity from its type's // monitoring spec (with per-entity `monitoring` attribute overrides). // lookup may be nil. It performs no I/O of its own; the caller's lookup // thunk may. The postgres adapter pairs this with writeCheck upserts. func Derive(tree *ontology.TypeTree, t CheckTarget, lookup HostLookup) ([]CheckDef, DeriveResult) { var res DeriveResult mon := tree.Monitoring(t.Type) if !mon.Declared { res.Undeclared = true return nil, res } if mon.None() { return nil, res } var attrs map[string]any if len(t.Attrs) > 0 { _ = json.Unmarshal(t.Attrs, &attrs) } if attrs == nil { attrs = map[string]any{} } // Per-entity override: an explicit `monitoring` attribute wins over the // type declaration. A single entity can opt out (monitoring: none) or pick // different kinds without introducing a new type — e.g. service:haos opts // out because its VM is already covered by a vm-status check and the // service can't be SSH-probed (haos blocks SSH). if mo, ok := attrs["monitoring"]; ok { mon = resolveMonitoringAttr(mo, mon) if mon.None() { return nil, res } } // A service has no address of its own — it lives on the container that // provides it. Fall back to the graph before giving up. host := resolveHost(attrs) if host == "" && lookup != nil { hostAttrs := lookup() host = resolveHost(hostAttrs) if user := resolveSSHUser(hostAttrs); host != "" && user != "root" { attrs["ssh"] = hostAttrs["ssh"] } } user := resolveSSHUser(attrs) port := resolveSSHPort(attrs) var defs []CheckDef for _, kind := range mon.Kinds { built, reason := buildKind(kind, t, attrs, host, user, port) if len(built) == 0 { res.Skipped = append(res.Skipped, Skip{Kind: kind, Reason: reason}) continue } defs = append(defs, built...) } return defs, res } // resolveMonitoringAttr turns an entity's `monitoring` attribute into a // MonitoringResolution that overrides the type's declaration. Accepts the // scalar "none" (or empty) to opt out, or a list of kind strings to override. func resolveMonitoringAttr(v any, fallback ontology.MonitoringResolution) ontology.MonitoringResolution { switch vv := v.(type) { case string: if vv == "none" || vv == "" { return ontology.MonitoringResolution{Declared: true, Source: "attribute"} } case []any: kinds := make([]string, 0, len(vv)) for _, k := range vv { if s, ok := k.(string); ok && s != "" { kinds = append(kinds, s) } } return ontology.MonitoringResolution{Declared: true, Kinds: kinds, Source: "attribute"} } return fallback } // buildKind turns one declared semantic kind into concrete checks, or // returns the reason it could not. func buildKind(kind string, t CheckTarget, attrs map[string]any, host, user string, port int) ([]CheckDef, string) { ssh := func(script string, args ...string) CheckDef { cfg := map[string]any{"script": script, "host": host} if user != "" && user != "root" { cfg["user"] = user } if port != 0 && port != 22 { cfg["port"] = port } if len(args) > 0 && args[0] != "" { cfg["args"] = args[0] } return CheckDef{Kind: "ssh-script", Config: cfg, IntervalS: 60} } switch kind { case KindPing: if host == "" { return nil, "no address on the entity or its host" } return []CheckDef{{Kind: "ping", Config: map[string]any{"host": host}, IntervalS: 30}}, "" case KindResource: if host == "" { return nil, "no address on the entity or its host" } return []CheckDef{ ssh("cpu_check.sh"), ssh("memory_check.sh"), ssh("load_check.sh"), ssh("disk_usage_check.sh"), }, "" case KindUpdates: if host == "" { return nil, "no address on the entity or its host" } // Daily. updates_check.sh runs `apt update` against the distro // mirrors; the shared 60s ssh-script default would have meant 1,440 // mirror hits per machine per day to answer a question whose answer // changes about once a day. u := ssh("updates_check.sh") u.IntervalS = 86400 return []CheckDef{u}, "" case KindCapacity: if host == "" { return nil, "no address on the entity or its host" } return []CheckDef{ssh("disk_usage_check.sh")}, "" case KindProcess: if host == "" { return nil, "no address on the entity or its host" } // A service's name is a logical label, not usually its systemd unit // or container name (matrix = matrix-synapse.service + containers). // Prefer an explicit probe target when declared; process_check.sh also // matches a unit prefix or a docker container as a fallback. unit := "" for _, key := range []string{"probe_unit", "systemd_unit", "container"} { if v, _ := attrs[key].(string); v != "" { unit = v break } } // Ontology intent: "http when it has a url, else a process check." A // url-fronted service is already liveness-probed via http (the real // endpoint, through the TLS terminator); the process check is redundant // and fragile (needs host access + the exact unit/container name), and // under worst-of aggregation it lets a broken supplementary probe veto // a working service. Emit it only for services WITHOUT a url, or when // an explicit probe_unit opts into binary-level depth. if unit == "" { if httpURL(t, attrs) != "" { return nil, "url present and no probe_unit; http check covers liveness" } unit = t.Name } if unit == "" { return nil, "no name to check a process for" } // process_check.sh takes the unit/container name as $1 and reports // "unknown" without it. return []CheckDef{ssh("process_check.sh", unit)}, "" case KindBackup: // A backup target is checked from the machine that writes to it, so it // needs both an address (resolved via the backs-up-to edge) and the // path to look at. path, _ := attrs["path"].(string) if path == "" { return nil, "entity carries no path attribute to check for backups" } if host == "" { return nil, "no address on the entity or whatever backs up to it" } maxAge := defaultBackupMaxAge if v, ok := attrs["backup_max_age_s"].(float64); ok && v > 0 { maxAge = int(v) } cfg := map[string]any{"path": path, "host": host, "max_age_s": maxAge} if user != "" && user != "root" { cfg["user"] = user } if port != 0 && port != 22 { cfg["port"] = port } // Daily. The freshness budget itself is a day, so probing more often // cannot surface anything sooner — it just costs an SSH round trip. return []CheckDef{{Kind: "backup-freshness", Config: cfg, IntervalS: 86400}}, "" case KindHTTP: url := httpURL(t, attrs) if url == "" { return nil, "no url attribute, public_host, or hostname-shaped name" } // max_status rather than an exact expected_status: most services sit // behind Authentik and answer 302/401, which is a working service. return []CheckDef{{ Kind: "http", Config: map[string]any{"url": url, "max_status": 500}, IntervalS: 60, }}, "" case KindDNS: // Resolve the entity's name via DNS to verify the zone is reachable. // Uses the entity name (zone apex) or falls back to the slug. name := t.Name if name == "" { name = strings.TrimPrefix(t.Slug, "zone:") } if name == "" { return nil, "no name to resolve" } return []CheckDef{{ Kind: "dns", Config: map[string]any{"name": name}, IntervalS: 300, // 5 min — DNS changes are rare; the cost of a miss // is a stale IP, not a service outage. }}, "" case KindCertExpiry: // The host whose cert to read (SNI / cert CN). Prefer an explicit // `hostname` attribute, then `cn`, then a dotted name. Hourly: expiry // changes once a day, but a renewal or mis-issued cert is worth // noticing within the hour. host := certHost(t, attrs) if host == "" { return nil, "no hostname / cn / dotted name to dial for the cert" } // `dial` is the TLS terminator's address to connect to (Caddy's lab // IP), used when the hostname doesn't resolve/reach from the scheduler. // Without it the probe can't reach *.hubris.network from a container // with no mesh / split-horizon DNS. dial, _ := attrs["dial"].(string) config := map[string]any{"host": host, "warn_days": 30, "crit_days": 7} if dial != "" { config["dial"] = dial } return []CheckDef{{ Kind: "cert-expiry", Config: config, IntervalS: 3600, }}, "" case KindVMStatus: // "Is the VM powered on" via `qm status` on its Proxmox host — the // right reachability probe for a VM, since many block ICMP and lack a // guest agent. checkVMStatus re-reads pve_id + host at runtime. if _, ok := attrs["pve_id"]; !ok { return nil, "no pve_id to run qm status" } return []CheckDef{{ Kind: "vm-status", Config: map[string]any{}, IntervalS: 60, }}, "" case KindQuorum: // Proxmox cluster quorum via `pvecm status`. Only meaningful on // proxmox-host entities. Runs every 60s — corosync flaps are // transient and the probe is lightweight (local binary, no network). if host == "" { return nil, "no address on the entity or its host" } return []CheckDef{ssh("pvecm_quorum_check.sh")}, "" } return nil, "no builder for this kind yet" } // certHost works out the hostname to TLS-dial for a certificate's expiry. func certHost(t CheckTarget, attrs map[string]any) string { for _, key := range []string{"hostname", "cn", "san"} { if v, ok := attrs[key].(string); ok && v != "" { return v } } // A dotted name is a hostname (hubris.network, media.hubris.network). if strings.Contains(t.Name, ".") && !strings.Contains(t.Name, " ") { return t.Name } return "" } // httpURL works out what to GET for an http check. // // Ingress routes carry their hostname as the entity name rather than as an // attribute (`name: media.hubris.network`), and most declare no attributes // at all — so the name is the only thing to go on. Requiring a `url` attribute // left all 21 of them unmonitored, which is a shame given an ingress check is // the most end-to-end probe available: it exercises Caddy, DNS, TLS and the // upstream in one request. func httpURL(t CheckTarget, attrs map[string]any) string { if url, ok := attrs["url"].(string); ok && url != "" { return url } if h, ok := attrs["public_host"].(string); ok && h != "" { return "https://" + h } // A dotted name is a hostname; a service name like "jellyfin" is not. if strings.Contains(t.Name, ".") && !strings.Contains(t.Name, " ") { return "https://" + t.Name } return "" } func resolveHost(attrs map[string]any) string { if attrs == nil { return "" } if ip, ok := attrs["lan_ip"].(string); ok && ip != "" { return ip } // public_ipv4 before mesh: the scheduler container has no mesh interface, // so a standalone-server reachable only by mesh IP (netbird-vps) is // unprobeable even though a public IPv4 is available. if ip, ok := attrs["public_ipv4"].(string); ok && ip != "" { return ip } if mesh, ok := attrs["mesh"].(map[string]any); ok { if nb, ok := mesh["netbird"].(map[string]any); ok { if ip, ok := nb["ip"].(string); ok && ip != "" { return ip } // Seeds record the mesh name, not an address — ws:mac-mini // carries only `fqdn`, which is why it resolved to nothing. if fqdn, ok := nb["fqdn"].(string); ok && fqdn != "" { return fqdn } } } if ip, ok := attrs["mesh_ip"].(string); ok && ip != "" { return ip } for _, key := range []string{"host", "address", "public_host"} { if v, ok := attrs[key].(string); ok && v != "" { return v } } return "" } func resolveSSHUser(attrs map[string]any) string { if ssh, ok := attrs["ssh"].(map[string]any); ok { if u, ok := ssh["user"].(string); ok && u != "" { return u } } // Workstations carry their login as a top-level `user` attribute // (mac-mini: user: dtoro) rather than under ssh.user. Take it only when // no explicit ssh.user was set, so a host that genuinely wants root still // gets root. if u, ok := attrs["user"].(string); ok && u != "" { return u } return "root" } func resolveSSHPort(attrs map[string]any) int { if ssh, ok := attrs["ssh"].(map[string]any); ok { switch p := ssh["port"].(type) { case float64: return int(p) case int: return p } } return 22 } // LogDeriveResult emits the one line that was missing: a type that asked for // monitoring and did not get it. func LogDeriveResult(slug, entityType string, res DeriveResult) { switch { case res.Undeclared: slog.Info("checkdefaults: type declares no monitoring", "entity", slug, "type", entityType) case len(res.Skipped) > 0: for _, s := range res.Skipped { slog.Warn("checkdefaults: declared check not created", "entity", slug, "type", entityType, "kind", s.Kind, "reason", s.Reason) } } }