diff --git a/bin/homelab b/bin/homelab index c049774..82eb8a3 100755 --- a/bin/homelab +++ b/bin/homelab @@ -17,10 +17,12 @@ import argparse import json import os import shutil +import socket import subprocess import sys import urllib.request from datetime import datetime +from functools import lru_cache from pathlib import Path try: @@ -81,25 +83,69 @@ def host_address(name: str, prefer_lan: bool = False) -> str: die(f"no reachable address for host {name}") -def hubris_ssh() -> list[str]: - """SSH base command for hubris, honoring its non-default Netbird SSH port. +@lru_cache(maxsize=None) +def _can_tcp_connect(addr: str, port: int) -> bool: + """Cached 1.5s TCP probe. Used to detect LAN unreachability before + falling back to the mesh FQDN — cheaper and more predictable than + waiting for ssh to timeout.""" + try: + with socket.create_connection((addr, port), timeout=1.5): + return True + except OSError: + return False - The non-default port (22022 = netbird-ssh-server) only listens on hubris's - netbird interface — it is NOT reachable via the LAN IP 192.168.8.77 even - from peers that route 192.168.8.0/24 through hubris. So when that port is - in use, the target address MUST be the netbird FQDN/IP, not the lan_ip - that `host_address()` prefers. Standard port 22 (LAN ssh) keeps the - `host_address()` default. + +def ssh_target(name: str, *, force_mesh: bool = False) -> tuple[str, int, str]: + """Resolve (addr, port, user) for ssh-ing into `name`. + + Address rules: + - If `ssh.netbird_port` is set (e.g. hubris with 22022), use it + + force the netbird FQDN/IP since netbird-ssh-server binds only the + netbird interface, not the LAN IP. + - Otherwise: try the LAN IP first. If a 1.5s TCP probe fails (off-LAN, + VPN/symmetric-NAT path, etc.), fall back to the mesh FQDN. + - `force_mesh=True` skips the LAN probe entirely. + + User: `ssh.user` from inventory if set, else `root`. Mismatched user is + the netbird-ssh "user not found" gotcha — defaults baked into the + inventory prevent it. """ - h = host("hubris") - port = h.get("ssh", {}).get("netbird_port", 22) - netbird_port = h.get("ssh", {}).get("netbird_port") - if port == netbird_port: - nb = h.get("mesh", {}).get("netbird") or {} - addr = nb.get("fqdn") or nb.get("ip") or host_address("hubris") - else: - addr = host_address("hubris") - return ["ssh", "-p", str(port), f"root@{addr}"] + h = host(name) + ssh = h.get("ssh", {}) or {} + user = ssh.get("user", "root") + nb = h.get("mesh", {}).get("netbird") or {} + mesh_addr = nb.get("fqdn") or nb.get("ip") + + nb_port = ssh.get("netbird_port") + if nb_port is not None: + addr = mesh_addr or host_address(name) + return addr, nb_port, user + + port = ssh.get("port", 22) + lan = h.get("lan_ip") + if force_mesh or not lan: + return mesh_addr or host_address(name), port, user + if _can_tcp_connect(lan, port): + return lan, port, user + if mesh_addr: + return mesh_addr, port, user + return lan, port, user # last resort; ssh will surface the real error + + +def ssh_base(name: str, *, force_mesh: bool = False) -> list[str]: + """Construct an `ssh ... user@addr` invocation for `name`, honoring port + and per-host user from inventory.""" + addr, port, user = ssh_target(name, force_mesh=force_mesh) + cmd = ["ssh"] + if port != 22: + cmd.extend(["-p", str(port)]) + cmd.append(f"{user}@{addr}") + return cmd + + +def hubris_ssh() -> list[str]: + """Back-compat shim — prefer `ssh_base("hubris")` in new code.""" + return ssh_base("hubris") def confirm(question: str, default_no: bool = True) -> bool: @@ -443,15 +489,10 @@ def cmd_list(args: argparse.Namespace) -> int: def cmd_ssh(args: argparse.Namespace) -> int: name = args.host - h = host(name) - addr = host_address(name) - cmd = ["ssh"] - # hubris uses a non-default Netbird SSH port for the mesh path - if name == "hubris": - port = h.get("ssh", {}).get("netbird_port", 22) - cmd.extend(["-p", str(port)]) - user = args.user or "root" - cmd.append(f"{user}@{addr}") + cmd = ssh_base(name) + # Optional --user override replaces the resolved user in the last arg + if args.user: + cmd[-1] = f"{args.user}@{cmd[-1].split('@', 1)[1]}" if args.command: cmd.append(" ".join(args.command)) os.execvp(cmd[0], cmd) @@ -486,10 +527,7 @@ def cmd_logs(args: argparse.Namespace) -> int: svc = args.service host_name = service_backend_host(svc) unit = service(svc).get("systemd_unit", svc) - if host_name == "hubris": - base = hubris_ssh() - else: - base = ["ssh", f"root@{host_address(host_name)}"] + base = ssh_base(host_name) remote = ["journalctl", "-u", unit, "-n", str(args.lines), "--no-pager"] if args.follow: remote.append("-f") @@ -503,10 +541,7 @@ def cmd_restart(args: argparse.Namespace) -> int: if not args.yes: if not confirm(f"restart systemd unit '{unit}' on {host_name}?"): return 1 - if host_name == "hubris": - base = hubris_ssh() - else: - base = ["ssh", f"root@{host_address(host_name)}"] + base = ssh_base(host_name) return subprocess.call(base + ["--", "systemctl", "restart", unit]) @@ -929,14 +964,14 @@ def cmd_nuke(args: argparse.Namespace) -> int: if not args.yes: if not confirm(f"destroy /etc/age/key.txt + /opt/homelab-context on {name}?"): return 1 - addr = host_address(name) + base = ssh_base(name) remote = ("set -euo pipefail; " "shred -u /etc/age/key.txt 2>/dev/null || true; " "rm -rf /opt/homelab-context; " "systemctl disable --now homelab-context-sync.timer 2>/dev/null || true; " "launchctl bootout system/network.hubris.homelab-context-sync 2>/dev/null || true; " "echo nuked") - return subprocess.call(["ssh", f"root@{addr}", remote]) + return subprocess.call(base + [remote]) # ---------- argparse ---------- diff --git a/hosts/hubris.yaml b/hosts/hubris.yaml index c82e544..89aff22 100644 --- a/hosts/hubris.yaml +++ b/hosts/hubris.yaml @@ -20,6 +20,7 @@ mounts: ssh: port: 22 netbird_port: 22022 + user: root runs: - proxmox_ui services_hosted: diff --git a/hosts/mac-mini.yaml b/hosts/mac-mini.yaml index 5bd6cf8..ddd5ad3 100644 --- a/hosts/mac-mini.yaml +++ b/hosts/mac-mini.yaml @@ -13,6 +13,8 @@ mesh_globals: accepted: - netbird - tailscale +ssh: + user: dtoro notes: - Only macOS in the fleet. Bootstrap uses launchd. mcp_endpoint: https://mcp.hubris.network/sse diff --git a/hosts/republic-laptop.yaml b/hosts/republic-laptop.yaml index ad4be76..1962ddd 100644 --- a/hosts/republic-laptop.yaml +++ b/hosts/republic-laptop.yaml @@ -13,5 +13,7 @@ mesh_globals: accepted: - netbird - tailscale +ssh: + user: dtoro mcp_endpoint: https://mcp.hubris.network/sse secrets_issuance_endpoint: https://secrets.hubris.network/issue diff --git a/infrastructure/mesh.md b/infrastructure/mesh.md index 01e6720..22416b2 100644 --- a/infrastructure/mesh.md +++ b/infrastructure/mesh.md @@ -123,7 +123,38 @@ Same migration also swapped OIDC from the combined image's embedded Dex IdP to A Also during this work: IONOS upstream was found to filter TCP 3478 in addition to UDP 3478. Added a TCP-3478 inbound exception in the IONOS firewall (see ICE/STUN section above for the verification probe). -The new Authentik provider for NetBird is `Client type: Public` (PKCE-only). Confidential would break the dashboard SPA's token exchange. Device Code Stage is NOT yet configured in Authentik → `netbird up` interactive auth flow returns an empty consent screen; new peers must use `--setup-key` until the stage is added. +The new Authentik provider for NetBird is `Client type: Public` (PKCE-only). Confidential would break the dashboard SPA's token exchange. The Device Code grant flow is wired (see [containers/124-authentik.md](../containers/124-authentik.md#device-code-grant--configured-2026-05-21)) so interactive `netbird up` works — `--setup-key` is no longer required for new peers. + +**Post-migration JWT-issuer gotcha on existing peers** (cost ~30 min to diagnose 2026-05-21): + +Existing peers — registered against the old combined image's embedded Dex IdP at `https://netbird.hubris.network/oauth2` — cache the OLD expected SSH-JWT issuer in the netbird daemon's in-memory state. After the migration, incoming `netbird ssh` connections were rejected with: + +``` +JWT authentication failed: validate token ( + expected issuer=https://netbird.hubris.network/oauth2, + audiences=[netbird-dashboard netbird-cli], + actual issuer=https://auth.hubris.network/application/o/netbird/, + audience=netbird-dashboard +) +``` + +Neither `systemctl restart netbird` nor `netbird down && netbird up` clears the cache. Root cause: in `client/internal/engine_ssh.go`, `updateSSH()` bails out with `if e.sshServer != nil { return nil }` whenever the SSH server is already running, so mgmt-pushed JWT config updates are silently ignored. Only a full daemon-process tear-down lets the SSH server re-initialize with the new validator config: + +``` +sudo systemctl stop netbird +sleep 3 +sudo systemctl start netbird +``` + +After that, `grep -iE "issuer|audience" /var/log/netbird/client.log | tail` shows the new Authentik issuer. Run this on every existing peer (PVE host + every LXC + every workstation) once after a future IdP swap. + +**Username gotcha (related):** `netbird ssh` defaults the remote username to the LOCAL one (e.g. `dtoro` from the operator's laptop). Hubris + the LXCs only have `root`, so the JWT is accepted but the session immediately fails with `user dtoro not found`. Always use the explicit `root@` prefix when invoking netbird-ssh manually: + +``` +netbird ssh -p 22022 root@proxmox-server.netbird.selfhosted +``` + +The `homelab` CLI handles this automatically via the per-host `ssh.user` field in `inventory.yaml` (defaults to `root`; set explicitly only for workstations whose login user isn't `root`). Open follow-up: TURN-over-TLS on TCP 5349 (cert via certbot or extract Traefik's acme.json) for hostile-middlebox networks; plain TCP 3478 is sufficient for current usage. diff --git a/inventory.yaml b/inventory.yaml index 899f75c..e64998d 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -15,6 +15,12 @@ # committed back via `homelab client add --finalize-pubkey `. # - When a service moves hosts, update only the `services:` section here; # never duplicate addresses elsewhere. +# - `ssh.user:` per-host login user. Default is `root` if omitted (matches +# every LXC + the PVE host). Set explicitly for workstations whose login +# user differs from `root`. Used by the `homelab` CLI to build +# `user@host` and to inform anyone running raw `netbird ssh` (which +# defaults to the LOCAL username — the gotcha that creates "user not +# found" errors when ssh'ing INTO machines that only have `root`). # # `homelab client add/remove` does surgical line-edits — comments survive. # Avoid round-tripping the file through yaml.safe_dump (it strips comments). @@ -107,6 +113,7 @@ hosts: ssh: port: 22 netbird_port: 22022 + user: root mounts: - /mnt/library age_pubkey: age1xkklkvnk5z0fsnh6cfgv70hy9ksfy8rdprwerzw4yk3p4p7cxcqs2yvpz6 @@ -329,6 +336,8 @@ hosts: mesh: netbird: fqdn: republic-laptop.netbird.selfhosted + ssh: + user: dtoro mac-mini: kind: workstation os: macos @@ -336,6 +345,8 @@ hosts: mesh: netbird: fqdn: mac-mini-234-17.netbird.selfhosted + ssh: + user: dtoro notes: - Only macOS in the fleet. Bootstrap uses launchd. age_pubkey: '' @@ -346,3 +357,6 @@ hosts: mesh: netbird: fqdn: ludo-mini.netbird.selfhosted + # ssh.user defaults to root; uncomment + set to the actual login user + # before relying on `homelab ssh ludo-mini` or netbird-ssh INTO this host. + # ssh: { user: ludo } diff --git a/operations/agent-enrollment.md b/operations/agent-enrollment.md index 9371a02..7140cc8 100644 --- a/operations/agent-enrollment.md +++ b/operations/agent-enrollment.md @@ -268,9 +268,15 @@ The CLI prints a follow-up checklist that the operator must do manually: | `homelab-context-sync.service` journal shows `fatal: could not read Username for 'https://git.hubris.network'` | Pre-fix bootstrap set the gitea credential helper via `git config --global`, which writes to `/root/.gitconfig` — invisible to the systemd timer's git process (no HOME set). | One-time migration: `sudo git config --system credential.helper "store --file=/etc/homelab-context/git-credentials"`. New bootstraps store the helper in `/etc/gitconfig` instead. | | Chat-mode `!` shell can't `sudo` (`a terminal is required to read the password`) | Claude Code's `!` invocation doesn't allocate a tty, and standard `sudo` won't read its password from stdin or a non-tty pipe. | Run the sudo'd command in a real terminal outside chat. For commands the agent issues repeatedly, configure passwordless sudo for the narrow set (e.g. `/etc/sudoers.d/homelab-self` with ` ALL=(ALL) NOPASSWD: /usr/bin/dnf upgrade -y, /usr/bin/apt-get *`). | | `netbird status -d` reports `192.168.8.180:53 ... is Unavailable` but DNS actually works | netbird's UDP-53 probe times out over the relay latency (~90ms), but actual queries still flow through systemd-resolved. Cosmetic. | Ignore unless `dig @192.168.8.180 git.hubris.network` also fails — then check dnsmasq on [LXC 124](../containers/124-authentik.md). | +| `netbird ssh` rejected with `JWT authentication failed: validate token (expected issuer=https://netbird.hubris.network/oauth2 ...)` | Peer's SSH JWT validator cached the OLD embedded-Dex issuer from before the 2026-05-21 Authentik migration. `systemctl restart netbird` and `netbird down/up` don't clear it — `client/internal/engine_ssh.go` bails out of `updateSSH()` if the SSH server is already running. | Full daemon bounce: `sudo systemctl stop netbird; sleep 3; sudo systemctl start netbird`. Verify with `grep -iE "issuer\|audience" /var/log/netbird/client.log \| tail`. Apply once per peer post-migration. | +| `netbird ssh` JWT passes but session closes with `user privilege check failed: user dtoro not found: unknown user dtoro` | netbird-ssh defaults the remote username to the LOCAL one (operator's laptop user). Hubris and LXCs only have `root`. | Always use explicit `root@` prefix manually: `netbird ssh -p 22022 root@proxmox-server.netbird.selfhosted`. `homelab ssh ` does this automatically via `inventory.yaml`'s per-host `ssh.user` field (defaults to `root`). | +| `homelab ssh hubris` (or any host on the LAN) fails with `Connection refused` or hangs, despite mesh routing being up | Off-LAN networks (operator on a VPN / coffee shop / symmetric NAT) sometimes can't reach the LAN IP even with the netbird subnet route. | Newer homelab CLIs probe the LAN with a 1.5s TCP connect and transparently fall back to the netbird FQDN. If your `/usr/local/bin/homelab` is a symlink to `/opt/homelab-context/bin/homelab` it'll pick up the fix on the next 5-min context sync. Otherwise pull the latest from gitea. | ## Changelog +### 2026-05-21 — netbird-ssh JWT issuer + username + LAN-fallback troubleshooting rows +Added three rows to the troubleshooting table covering issues surfaced during the netbird vanilla migration: (1) post-migration SSH JWT validator cache stuck on old Dex issuer (full `systemctl stop/start` required, not `restart`), (2) `user not found` from netbird-ssh's local-username default (use explicit `root@`), and (3) homelab CLI's LAN→netbird-FQDN fallback for off-LAN operators. Companion code change: per-host `ssh.user` field in `inventory.yaml` + `homelab` CLI's `ssh_target()` helper. + ### 2026-05-20 — initial page Captures the enrollment flow validated during Phase 2 of the homelab context distribution rollout. hubris + LXC 105 (apps) enrolled; first