feat(remote): route service checks through their hosting compute entity

A service check used to bake its hosting LXC's lan_ip and SSH it directly as
root, which failed because the scheduler key is authorized on the Proxmox hosts
but not inside every guest — leaving all 8 service process checks 'down' even
after the guest routing and scripts were fixed.

ResolveExecTargetForCheck now, for a non-guest target, walks the
provides/runs-on/hosts edges to the compute entity that runs it and routes
through that: pct/qm exec if the host is a guest, direct SSH with the host's
correct user (workstation `user` attr) if it's a machine. The guest-resolution
path is shared via resolveGuest, and the scheduler no longer needs an
isMachine special case — one resolver handles guest, machine, and service.

internal/checkdefaults/defaults.go

@@ -299,14 +299,15 @@ func writeCheck(ctx context.Context, tx pgx.Tx, t Target, idx int, def checkDef)
 	// absent from the DO UPDATE below — a re-seed must not reset the schedule
 	// and re-herd everything.
 	tag, err := tx.Exec(ctx,
-		`INSERT INTO check_defs (entity_id, target_id, kind, config, interval_s, timeout_s, enabled, last_run_at)
+		`INSERT INTO check_defs (entity_id, target_id, target_type, kind, config, interval_s, timeout_s, enabled, last_run_at)
-		 VALUES ($1, $2, $3, $4, $5, 30, true,
+		 VALUES ($1, $2, $6, $3, $4, $5, 30, true,
 		         now() - make_interval(secs => random() * $5::int))
 		 ON CONFLICT (entity_id) DO UPDATE
-		   SET target_id = EXCLUDED.target_id, kind = EXCLUDED.kind,
+		   SET target_id = EXCLUDED.target_id, target_type = EXCLUDED.target_type,
+		       kind = EXCLUDED.kind,
 		       config = EXCLUDED.config, interval_s = EXCLUDED.interval_s,
 		       updated_at = now()`,
-		checkID, t.ID, def.kind, configJSON, def.interval)
+		checkID, t.ID, def.kind, configJSON, def.interval, t.Type)
 	if err != nil {
 		return false, fmt.Errorf("upsert check_def %s: %w", checkSlug, err)
 	}

internal/remote/remote.go

@@ -164,27 +164,28 @@ func ResolveExecTarget(ctx context.Context, pool *db.Pool, targetSlug, fallbackU
 // their hosting machine) is reached by direct SSH to the entity's own address.
 func ResolveExecTargetForCheck(ctx context.Context, pool *db.Pool, targetID uuid.UUID, targetType, fallbackUser string) (ExecTarget, error) {
 	if IsGuest(targetType) {
-		var (
+		return resolveGuest(ctx, pool, targetID, targetType, fallbackUser)
-			pveID    string
-			hostAttr string
-		)
-		if err := pool.QueryRow(ctx,
-			"SELECT attributes->>'pve_id', COALESCE(attributes->>'host','') FROM entities WHERE id = $1",
-			targetID).Scan(&pveID, &hostAttr); err != nil || pveID == "" {
-			return ExecTarget{}, fmt.Errorf("guest %s missing pve_id", targetID)
 	}
-		hostSlug := ResolveProxmoxHostSlug(ctx, pool, targetID, hostAttr)
+
-		addr, user, err := ResolveHost(ctx, pool, hostSlug, fallbackUser)
+	// A service (or other non-compute target) has no address of its own — it
+	// runs on whatever compute entity provides/hosts it. Resolve that host and
+	// route through it: pct if the host is a guest, direct SSH (with the
+	// host's correct user) if it's a machine. Previously a service check baked
+	// its hosting LXC's lan_ip and SSHed it directly as root, which fails
+	// because the scheduler key isn't in each LXC — only on the Proxmox hosts.
+	if hostID, hostType, ok := hostingCompute(ctx, pool, targetID); ok {
+		if IsGuest(hostType) {
+			return resolveGuest(ctx, pool, hostID, hostType, fallbackUser)
+		}
+		addr, user, err := resolveHostByID(ctx, pool, hostID, fallbackUser)
 		if err != nil {
 			return ExecTarget{}, err
 		}
-		return ExecTarget{Host: addr, User: user, Wrap: guestWrap(targetType, pveID)}, nil
+		return ExecTarget{Host: addr, User: user, Wrap: func(cmd string) string { return cmd }}, nil
 	}
 
-	// Host-like target: reach it directly at its own address. Services and
+	// No hosting entity found: reach the target directly at its own address
-	// other non-host entities that reach here should already have had their
+	// (a host/workstation, or a service whose host wasn't resolvable).
-	// host address baked into check config at seed time; this path covers
-	// host/workstation targets whose address is resolved live.
 	addr, user, err := resolveHostByID(ctx, pool, targetID, fallbackUser)
 	if err != nil {
 		return ExecTarget{}, err
@@ -192,6 +193,43 @@ func ResolveExecTargetForCheck(ctx context.Context, pool *db.Pool, targetID uuid
 	return ExecTarget{Host: addr, User: user, Wrap: func(cmd string) string { return cmd }}, nil
 }
 
+// hostingCompute walks the provides/runs-on/hosts edges backward from a target
+// to the compute entity that runs it (a service's LXC, an LXC's Proxmox host).
+// Returns the host's id, type, and whether one was found. Most-specific edge
+// first: provides names the runtime container directly.
+func hostingCompute(ctx context.Context, pool *db.Pool, targetID uuid.UUID) (uuid.UUID, string, bool) {
+	var hid uuid.UUID
+	var htype string
+	err := pool.QueryRow(ctx, `
+		SELECT e.id, e.type FROM relationships r
+		JOIN entities e ON e.id = r.source_id
+		WHERE r.target_id = $1 AND r.valid_to IS NULL
+		  AND r.type IN ('provides','runs-on','hosts')
+		ORDER BY CASE r.type WHEN 'provides' THEN 0 WHEN 'runs-on' THEN 1 ELSE 2 END
+		LIMIT 1`, targetID).Scan(&hid, &htype)
+	if err != nil {
+		return uuid.Nil, "", false
+	}
+	return hid, htype, true
+}
+
+// resolveGuest resolves a guest's execution endpoint: the owning Proxmox host
+// (SSH'd directly) with a pct/qm exec wrapper around the command.
+func resolveGuest(ctx context.Context, pool *db.Pool, guestID uuid.UUID, guestType, fallbackUser string) (ExecTarget, error) {
+	var pveID, hostAttr string
+	if err := pool.QueryRow(ctx,
+		"SELECT attributes->>'pve_id', COALESCE(attributes->>'host','') FROM entities WHERE id = $1",
+		guestID).Scan(&pveID, &hostAttr); err != nil || pveID == "" {
+		return ExecTarget{}, fmt.Errorf("guest %s missing pve_id", guestID)
+	}
+	hostSlug := ResolveProxmoxHostSlug(ctx, pool, guestID, hostAttr)
+	addr, user, err := ResolveHost(ctx, pool, hostSlug, fallbackUser)
+	if err != nil {
+		return ExecTarget{}, err
+	}
+	return ExecTarget{Host: addr, User: user, Wrap: guestWrap(guestType, pveID)}, nil
+}
+
 // resolveHostByID is ResolveHost keyed by entity id.
 func resolveHostByID(ctx context.Context, pool *db.Pool, id uuid.UUID, fallbackUser string) (addr, user string, err error) {
 	var raw string

internal/remote/remote_test.go

@@ -170,3 +170,34 @@ func TestResolveExecTargetForCheckHostIsDirect(t *testing.T) {
 		t.Errorf("host wrap must be identity, got %q", cmd)
 	}
 }
+
+func TestResolveExecTargetForCheckServiceRoutesViaHostingGuest(t *testing.T) {
+	// A service has no address of its own; it must route through its hosting
+	// LXC via the provides edge, host-hopping through the LXC's proxmox host.
+	pool := newRemotePool(t)
+	ctx := context.Background()
+	hostID := uuid.New()
+	guestID := uuid.New()
+	svcID := uuid.New()
+	mustExec(t, pool, ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
+		VALUES ($1,'host:hubris','proxmox-host','hubris','active','{"lan_ip":"192.168.8.77"}'::jsonb,1,now(),now())`, hostID)
+	mustExec(t, pool, ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
+		VALUES ($1,'lxc:gitea','lxc','gitea','active','{"pve_id":"104","lan_ip":"192.168.8.121"}'::jsonb,1,now(),now())`, guestID)
+	mustExec(t, pool, ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
+		VALUES ($1,'service:gitea','service','gitea','active','{}'::jsonb,1,now(),now())`, svcID)
+	// provides: lxc -> service; hosts: proxmox-host -> lxc
+	mustExec(t, pool, ctx, `INSERT INTO relationships (source_id, target_id, type, valid_from, created_at) VALUES ($1,$2,'provides',now(),now())`, guestID, svcID)
+	mustExec(t, pool, ctx, `INSERT INTO relationships (source_id, target_id, type, valid_from, created_at) VALUES ($1,$2,'hosts',now(),now())`, hostID, guestID)
+
+	et, err := ResolveExecTargetForCheck(ctx, pool, svcID, "service", DefaultUser)
+	if err != nil {
+		t.Fatalf("ResolveExecTargetForCheck for service: %v", err)
+	}
+	// Reaches the proxmox host (host-hop), wrapped as pct exec into the guest.
+	if et.Host != "192.168.8.77" {
+		t.Errorf("Host = %q, want proxmox host 192.168.8.77 (via provides->hosts)", et.Host)
+	}
+	if out := et.Wrap("p"); !strings.Contains(out, "pct exec 104") {
+		t.Errorf("service check must wrap as pct exec 104, got %q", out)
+	}
+}

internal/scheduler/scheduler.go

@@ -759,36 +759,42 @@ func checkSSHScript(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCh
 		scriptPath += " '" + strings.ReplaceAll(cfg.Args, "'", `'\''`) + "'"
 	}
 
-	// Resolve the execution endpoint. Guests host-hop; host/workstation types
+	// Resolve the execution endpoint. The resolver handles every target kind:
-	// resolve their address + user live; everything else uses the baked config.
+	// LXC/VM host-hop via pct/qm exec; hosts/workstations direct at their own
+	// address; services route through their hosting compute entity (via the
+	// provides edge) so a service check reaches the right machine with the
+	// right user instead of baking an LXC lan_ip and SSHing it as root.
 	host, port, user := cfg.Host, strconv.Itoa(oru(cfg.Port, 22)), orStr(cfg.User, sshUser)
 	wrap := func(cmd string) string { return cmd }
 	targetType := ""
 	if cd.TargetType != nil {
 		targetType = *cd.TargetType
 	}
-	if cd.TargetID != nil {
+	// target_type was omitted by older writeCheck inserts, so resolve it from
-		switch {
+	// the target entity when the column is blank — otherwise the guest routing
-		case remote.IsGuest(targetType):
+	// below (IsGuest) never triggers and a guest check falls back to its baked
+	// (often mesh-only) address.
+	if targetType == "" && cd.TargetID != nil {
+		if err := pool.QueryRow(ctx, "SELECT type FROM entities WHERE id = $1", *cd.TargetID).Scan(&targetType); err != nil {
+			targetType = ""
+		}
+	}
+	if cd.TargetID != nil && targetType != "" {
 		et, err := remote.ResolveExecTargetForCheck(ctx, pool, *cd.TargetID, targetType, sshUser)
 		if err != nil {
+			if remote.IsGuest(targetType) {
 				return checkResult{
 					health: "down", signalKind: "ssh-script",
 					evidence: fmt.Sprintf("route guest %s: %v", cd.EntitySlug, err), err: err,
 				}
 			}
-			host, port, user, wrap = et.Host, "22", et.User, et.Wrap
+			// Non-guest: log the resolution failure so an opaque ssh "down"
-		case isMachine(targetType):
+			// doesn't hide that the real cause was host/user resolution, then
-			et, err := remote.ResolveExecTargetForCheck(ctx, pool, *cd.TargetID, targetType, sshUser)
+			// fall back to the baked config below.
-			if err == nil {
+			slog.Warn("scheduler: target resolution failed, using baked config",
-				host, port, user, wrap = et.Host, "22", et.User, et.Wrap
-			} else {
-				// Log the resolution failure so an opaque ssh "down" doesn't
-				// hide that the real cause was host/user resolution (e.g. a
-				// missing attribute), then fall back to the baked config below.
-				slog.Warn("scheduler: machine target resolution failed, using baked config",
 				"entity", cd.EntitySlug, "target_type", targetType, "error", err)
-			}
+		} else {
+			host, port, user, wrap = et.Host, "22", et.User, et.Wrap
 		}
 	}
 
@@ -834,18 +840,6 @@ func checkSSHScript(ctx context.Context, pool *db.Pool, cd sqlcgen.ListEnabledCh
 	}
 }
 
-// isMachine reports whether a target type is a physical/virtual machine that
-// should be reached by direct SSH at its own resolved address (rather than the
-// baked hosting-container address a service uses). These are the machine
-// subtypes in the ontology.
-func isMachine(entityType string) bool {
-	switch entityType {
-	case "proxmox-host", "standalone-server", "workstation", "appliance":
-		return true
-	}
-	return false
-}
-
 // oru returns v when nonzero, else def. orStr returns v when non-empty, else def.
 func oru(v, def int) int {
 	if v != 0 {