homelab CLI: per-host ssh.user + LAN→mesh fallback; wiki for netbird-ssh JWT issuer fix

Three coordinated changes addressing follow-ups from the 2026-05-21 netbird vanilla
migration, plus a related off-LAN ergonomics fix:

bin/homelab:
- New ssh_target(name, force_mesh=False) helper resolves (addr, port, user)
  from inventory, honoring ssh.netbird_port (forces mesh path) and ssh.user
  (default "root"). Falls back to the netbird FQDN when LAN IP fails a
  cached 1.5s TCP probe — helps off-LAN operators on VPN/symmetric-NAT
  paths where the netbird subnet route doesn't reach 192.168.8.0/24.
- New ssh_base() builds the full `ssh ... user@addr` invocation; hubris_ssh()
  is now a back-compat shim. cmd_ssh, cmd_logs, cmd_restart, cmd_nuke
  refactored to use it — no more hardcoded "root@" anywhere.

inventory.yaml:
- New ssh.user convention (root by default, explicit per workstation).
- hubris.ssh.user=root (explicit, documents convention).
- republic-laptop, mac-mini: ssh.user=dtoro. ludo-mini left default (TODO).
- Comment block in the header explains the field + why it exists (netbird-ssh
  defaults to LOCAL username; "user not found" on LXCs is the gotcha).
- hosts/*.yaml regenerated from build_host_files.py.

infrastructure/mesh.md:
- Migration changelog entry updated: Device Code Stage is now configured
  (was "NOT yet" — landed in d41d73f); --setup-key no longer required.
- New subsection documenting the post-migration JWT-issuer cache bug:
  client/internal/engine_ssh.go's updateSSH() bails out when sshServer is
  already running, so systemctl restart and netbird down/up don't refresh
  the SSH JWT validator. Full daemon stop/start is the fix.
- Companion username gotcha (`netbird ssh` defaulting to local username).

operations/agent-enrollment.md:
- Three new troubleshooting rows: JWT-issuer cache, user-not-found, and
  LAN-unreachable-from-mesh-peer (the new homelab CLI behavior).

Verification: ssh_target resolution against the live inventory yields
- hubris → ssh -p 22022 root@proxmox-server.netbird.selfhosted (mesh-forced)
- jellyfin/gitea → ssh root@192.168.8.x (LAN reachable, probe passed)
- republic-laptop/mac-mini → ssh dtoro@<fqdn> (per ssh.user)
- ludo-mini → ssh root@<fqdn> (default)

bin/homelab

@@ -17,10 +17,12 @@ import argparse
 import json
 import os
 import shutil
+import socket
 import subprocess
 import sys
 import urllib.request
 from datetime import datetime
+from functools import lru_cache
 from pathlib import Path
 
 try:
@@ -81,25 +83,69 @@ def host_address(name: str, prefer_lan: bool = False) -> str:
     die(f"no reachable address for host {name}")
 
 
-def hubris_ssh() -> list[str]:
-    """SSH base command for hubris, honoring its non-default Netbird SSH port.
+@lru_cache(maxsize=None)
+def _can_tcp_connect(addr: str, port: int) -> bool:
+    """Cached 1.5s TCP probe. Used to detect LAN unreachability before
+    falling back to the mesh FQDN — cheaper and more predictable than
+    waiting for ssh to timeout."""
+    try:
+        with socket.create_connection((addr, port), timeout=1.5):
+            return True
+    except OSError:
+        return False
 
-    The non-default port (22022 = netbird-ssh-server) only listens on hubris's
-    netbird interface — it is NOT reachable via the LAN IP 192.168.8.77 even
-    from peers that route 192.168.8.0/24 through hubris. So when that port is
-    in use, the target address MUST be the netbird FQDN/IP, not the lan_ip
-    that `host_address()` prefers. Standard port 22 (LAN ssh) keeps the
-    `host_address()` default.
+
+def ssh_target(name: str, *, force_mesh: bool = False) -> tuple[str, int, str]:
+    """Resolve (addr, port, user) for ssh-ing into `name`.
+
+    Address rules:
+      - If `ssh.netbird_port` is set (e.g. hubris with 22022), use it +
+        force the netbird FQDN/IP since netbird-ssh-server binds only the
+        netbird interface, not the LAN IP.
+      - Otherwise: try the LAN IP first. If a 1.5s TCP probe fails (off-LAN,
+        VPN/symmetric-NAT path, etc.), fall back to the mesh FQDN.
+      - `force_mesh=True` skips the LAN probe entirely.
+
+    User: `ssh.user` from inventory if set, else `root`. Mismatched user is
+    the netbird-ssh "user not found" gotcha — defaults baked into the
+    inventory prevent it.
     """
-    h = host("hubris")
-    port = h.get("ssh", {}).get("netbird_port", 22)
-    netbird_port = h.get("ssh", {}).get("netbird_port")
-    if port == netbird_port:
-        nb = h.get("mesh", {}).get("netbird") or {}
-        addr = nb.get("fqdn") or nb.get("ip") or host_address("hubris")
-    else:
-        addr = host_address("hubris")
-    return ["ssh", "-p", str(port), f"root@{addr}"]
+    h = host(name)
+    ssh = h.get("ssh", {}) or {}
+    user = ssh.get("user", "root")
+    nb = h.get("mesh", {}).get("netbird") or {}
+    mesh_addr = nb.get("fqdn") or nb.get("ip")
+
+    nb_port = ssh.get("netbird_port")
+    if nb_port is not None:
+        addr = mesh_addr or host_address(name)
+        return addr, nb_port, user
+
+    port = ssh.get("port", 22)
+    lan = h.get("lan_ip")
+    if force_mesh or not lan:
+        return mesh_addr or host_address(name), port, user
+    if _can_tcp_connect(lan, port):
+        return lan, port, user
+    if mesh_addr:
+        return mesh_addr, port, user
+    return lan, port, user  # last resort; ssh will surface the real error
+
+
+def ssh_base(name: str, *, force_mesh: bool = False) -> list[str]:
+    """Construct an `ssh ... user@addr` invocation for `name`, honoring port
+    and per-host user from inventory."""
+    addr, port, user = ssh_target(name, force_mesh=force_mesh)
+    cmd = ["ssh"]
+    if port != 22:
+        cmd.extend(["-p", str(port)])
+    cmd.append(f"{user}@{addr}")
+    return cmd
+
+
+def hubris_ssh() -> list[str]:
+    """Back-compat shim — prefer `ssh_base("hubris")` in new code."""
+    return ssh_base("hubris")
 
 
 def confirm(question: str, default_no: bool = True) -> bool:
@@ -443,15 +489,10 @@ def cmd_list(args: argparse.Namespace) -> int:
 
 def cmd_ssh(args: argparse.Namespace) -> int:
     name = args.host
-    h = host(name)
-    addr = host_address(name)
-    cmd = ["ssh"]
-    # hubris uses a non-default Netbird SSH port for the mesh path
-    if name == "hubris":
-        port = h.get("ssh", {}).get("netbird_port", 22)
-        cmd.extend(["-p", str(port)])
-    user = args.user or "root"
-    cmd.append(f"{user}@{addr}")
+    cmd = ssh_base(name)
+    # Optional --user override replaces the resolved user in the last arg
+    if args.user:
+        cmd[-1] = f"{args.user}@{cmd[-1].split('@', 1)[1]}"
     if args.command:
         cmd.append(" ".join(args.command))
     os.execvp(cmd[0], cmd)
@@ -486,10 +527,7 @@ def cmd_logs(args: argparse.Namespace) -> int:
     svc = args.service
     host_name = service_backend_host(svc)
     unit = service(svc).get("systemd_unit", svc)
-    if host_name == "hubris":
-        base = hubris_ssh()
-    else:
-        base = ["ssh", f"root@{host_address(host_name)}"]
+    base = ssh_base(host_name)
     remote = ["journalctl", "-u", unit, "-n", str(args.lines), "--no-pager"]
     if args.follow:
         remote.append("-f")
@@ -503,10 +541,7 @@ def cmd_restart(args: argparse.Namespace) -> int:
     if not args.yes:
         if not confirm(f"restart systemd unit '{unit}' on {host_name}?"):
             return 1
-    if host_name == "hubris":
-        base = hubris_ssh()
-    else:
-        base = ["ssh", f"root@{host_address(host_name)}"]
+    base = ssh_base(host_name)
     return subprocess.call(base + ["--", "systemctl", "restart", unit])
 
 
@@ -929,14 +964,14 @@ def cmd_nuke(args: argparse.Namespace) -> int:
     if not args.yes:
         if not confirm(f"destroy /etc/age/key.txt + /opt/homelab-context on {name}?"):
             return 1
-    addr = host_address(name)
+    base = ssh_base(name)
     remote = ("set -euo pipefail; "
               "shred -u /etc/age/key.txt 2>/dev/null || true; "
               "rm -rf /opt/homelab-context; "
               "systemctl disable --now homelab-context-sync.timer 2>/dev/null || true; "
               "launchctl bootout system/network.hubris.homelab-context-sync 2>/dev/null || true; "
               "echo nuked")
-    return subprocess.call(["ssh", f"root@{addr}", remote])
+    return subprocess.call(base + [remote])
 
 
 # ---------- argparse ----------