diff --git a/bin/homelab b/bin/homelab index 48791f8..80c4069 100755 --- a/bin/homelab +++ b/bin/homelab @@ -81,10 +81,24 @@ def host_address(name: str, prefer_lan: bool = False) -> str: def hubris_ssh() -> list[str]: - """SSH base command for hubris, honoring its non-default Netbird SSH port.""" + """SSH base command for hubris, honoring its non-default Netbird SSH port. + + The non-default port (22022 = netbird-ssh-server) only listens on hubris's + netbird interface — it is NOT reachable via the LAN IP 192.168.8.77 even + from peers that route 192.168.8.0/24 through hubris. So when that port is + in use, the target address MUST be the netbird FQDN/IP, not the lan_ip + that `host_address()` prefers. Standard port 22 (LAN ssh) keeps the + `host_address()` default. + """ h = host("hubris") port = h.get("ssh", {}).get("netbird_port", 22) - return ["ssh", "-p", str(port), f"root@{host_address('hubris')}"] + netbird_port = h.get("ssh", {}).get("netbird_port") + if port == netbird_port: + nb = h.get("mesh", {}).get("netbird") or {} + addr = nb.get("fqdn") or nb.get("ip") or host_address("hubris") + else: + addr = host_address("hubris") + return ["ssh", "-p", str(port), f"root@{addr}"] def confirm(question: str, default_no: bool = True) -> bool: @@ -926,6 +940,194 @@ def cmd_nuke(args: argparse.Namespace) -> int: # ---------- argparse ---------- +# ====== apt fleet operations (audit + upgrade wrapper) ====== +# +# Both subcommands fan out to "apt targets": hubris (the PVE host) + every LXC. +# VMs (haos, zimaos) and workstations are intentionally excluded — they have +# their own update flows. +# +# Designed to survive ssh teardown: `apt-upgrade` wraps the remote apt run in +# `screen -dmS` so a flaky ssh control socket can't kill it mid-transaction +# (the failure mode from the 2026-05-21 fleet sweep). All output is tee'd to +# /var/log/homelab-apt-upgrade.log on the target; check via `apt-upgrade +# --status`. + +_APT_AUDIT_PROBE = r""" +audit=$(dpkg --audit 2>&1 | grep -c '^ ') +holds=$(apt-mark showhold 2>/dev/null | wc -l) +upgr=$(apt list --upgradable 2>/dev/null | grep -cv '^Listing') +nonapt=$(for f in /usr/bin/caddy /usr/bin/docker /usr/bin/jellyfin /usr/local/bin/*; do + [ -e "$f" ] && (dpkg -S "$f" 2>/dev/null >/dev/null || echo "$f") +done 2>/dev/null | wc -l) +if getent hosts deb.debian.org >/dev/null 2>&1; then dns=ok; else dns=fail; fi +echo "$audit $holds $upgr $nonapt $dns" +""" + +_APT_UPGRADE_WRAPPER = r""" +set -e +command -v systemd-run >/dev/null || { echo "systemd-run not available on target" >&2; exit 3; } +mkdir -p /var/log +LOG=/var/log/homelab-apt-upgrade.log +UNIT="apt-upgrade-$(hostname -s)" +if systemctl is-active --quiet "$UNIT.service" 2>/dev/null; then + echo "unit $UNIT already running on $(hostname -s); not relaunching" >&2 + exit 4 +fi +# systemd-run creates a transient unit that survives our ssh teardown. +# --collect cleans up the unit after the command finishes (no stale units). +systemd-run --unit="$UNIT" --collect --quiet bash -c ' +exec > >(tee -a '"$LOG"') 2>&1 +echo "=== START $(date -Is) host=$(hostname -s) ===" +export DEBIAN_FRONTEND=noninteractive +apt -o Acquire::Retries=3 -o Acquire::ForceIPv4=true -y update +apt -o Acquire::Retries=3 -o Dpkg::Options::=--force-confold -y upgrade +rc=$? +echo "=== END rc=$rc $(date -Is) ===" +exit $rc +' +echo "launched unit=$UNIT log=$LOG host=$(hostname -s)" +""" + + +def _apt_targets(include_hubris: bool = True, include_lxcs: bool = True) -> list[tuple[str, str | None]]: + """Standard apt targets: hubris (None pve_id) + every LXC (pve_id as str).""" + inv = inventory() + targets: list[tuple[str, str | None]] = [] + if include_hubris: + targets.append(("hubris", None)) + if include_lxcs: + for name, entry in inv.get("hosts", {}).items(): + if entry.get("kind") == "lxc": + pid = entry.get("pve_id") + if pid: + targets.append((name, str(pid))) + return targets + + +def _run_on_target(name: str, pve_id: str | None, remote_argv: list[str], + stdin_text: str | None = None) -> subprocess.CompletedProcess: + """Run `remote_argv` on the target. + + pve_id=None → on hubris directly. + pve_id= → via `pct exec` inside that LXC. + Stdin (typically a heredoc'd bash script) can be piped via stdin_text. + """ + if pve_id is None: + cmd = hubris_ssh() + ["--"] + remote_argv + else: + cmd = hubris_ssh() + ["--", "pct", "exec", pve_id, "--"] + remote_argv + return subprocess.run(cmd, input=stdin_text, capture_output=True, text=True) + + +def _audit_one(name: str, pve_id: str | None) -> dict | None: + """Run the audit probe on one target. Returns metrics dict or None if unreachable.""" + res = _run_on_target(name, pve_id, ["bash", "-s"], stdin_text=_APT_AUDIT_PROBE) + if res.returncode != 0: + return None + parts = res.stdout.strip().split() + if len(parts) != 5: + return None + try: + return { + "dpkg_dirty": int(parts[0]), + "holds": int(parts[1]), + "upgradable": int(parts[2]), + "nonapt_bins": int(parts[3]), + "dns": parts[4], + } + except ValueError: + return None + + +def cmd_apt_audit(args: argparse.Namespace) -> int: + """Per-host pre-flight: dpkg state, holds, upgradable count, non-apt binaries, DNS health. + + Exit nonzero if any host shows dpkg-interrupted state (would block apt-upgrade).""" + targets = _apt_targets() + if args.target: + targets = [(n, i) for n, i in targets if n == args.target] + if not targets: + die(f"target '{args.target}' is not a known apt target") + print(f"{'HOST':<20} {'DPKG':<7} {'HOLDS':<6} {'UPGR':<6} {'NONAPT':<7} {'DNS':<5}") + any_dirty = False + for name, pve_id in targets: + m = _audit_one(name, pve_id) + if m is None: + print(f"{name:<20} unreachable") + continue + if m["dpkg_dirty"]: + dpkg_s = f"DIRTY({m['dpkg_dirty']})" + any_dirty = True + else: + dpkg_s = "ok" + dns_s = m["dns"] if m["dns"] == "ok" else "FAIL" + print(f"{name:<20} {dpkg_s:<7} {m['holds']:<6} {m['upgradable']:<6} {m['nonapt_bins']:<7} {dns_s:<5}") + if any_dirty: + print() + print("DIRTY hosts have unconfigured packages — recover with: ssh dpkg --configure -a", file=sys.stderr) + return 1 if any_dirty else 0 + + +def cmd_apt_upgrade(args: argparse.Namespace) -> int: + """Launch apt update+upgrade inside a detached `screen` on each target. + + Survives ssh teardown. Output tee'd to /var/log/homelab-apt-upgrade.log on + the target. Refuses if pre-flight audit finds dpkg-interrupted state + (override with --force).""" + targets = _apt_targets() + if args.target: + targets = [(n, i) for n, i in targets if n == args.target] + if not targets: + die(f"target '{args.target}' is not a known apt target") + elif not args.all: + die("specify --target or --all") + + if args.status: + status_probe = r""" +u="apt-upgrade-$(hostname -s)" +if systemctl is-active --quiet "$u.service" 2>/dev/null; then + echo "$u: ACTIVE" +else + echo "$u: not running" +fi +echo --- +if [ -f /var/log/homelab-apt-upgrade.log ]; then + tail -20 /var/log/homelab-apt-upgrade.log +else + echo "(no log)" +fi +""" + for name, pve_id in targets: + print(f"=== {name} ===") + res = _run_on_target(name, pve_id, ["bash", "-s"], stdin_text=status_probe) + print(res.stdout.rstrip()) + return 0 + + # Pre-flight audit gate + if not args.force: + dirty = [] + for name, pve_id in targets: + m = _audit_one(name, pve_id) + if m and m["dpkg_dirty"]: + dirty.append(name) + if dirty: + print(f"refusing: pre-existing dpkg-interrupted state on: {', '.join(dirty)}", file=sys.stderr) + print(" recover with: homelab ssh -- dpkg --configure -a", file=sys.stderr) + print(" or rerun with --force to skip the audit gate", file=sys.stderr) + return 2 + + # Launch on each target + rc = 0 + for name, pve_id in targets: + res = _run_on_target(name, pve_id, ["bash", "-s"], stdin_text=_APT_UPGRADE_WRAPPER) + if res.returncode != 0: + print(f"{name}: launch failed (rc={res.returncode}): {res.stderr.strip()}", file=sys.stderr) + rc = 1 + continue + print(f"{name}: {res.stdout.strip()}") + return rc + + def main() -> int: p = argparse.ArgumentParser(prog="homelab", description=__doc__) sub = p.add_subparsers(dest="cmd", required=True) @@ -987,6 +1189,23 @@ def main() -> int: sp.add_argument("args", nargs=argparse.REMAINDER) sp.set_defaults(func=cmd_mcp) + sp = sub.add_parser("apt-audit", + help="per-host pre-flight: dpkg state, holds, upgradable count, non-apt binaries, DNS") + sp.add_argument("--target", default=None, + help="only audit one host (default: hubris + every LXC)") + sp.set_defaults(func=cmd_apt_audit) + + sp = sub.add_parser("apt-upgrade", + help="launch `apt update && apt upgrade` inside detached screen on each target") + grp = sp.add_mutually_exclusive_group() + grp.add_argument("--target", default=None, help="upgrade one host") + grp.add_argument("--all", action="store_true", help="upgrade hubris + every LXC") + sp.add_argument("--status", action="store_true", + help="show running screen sessions + tail the upgrade log on each target") + sp.add_argument("--force", action="store_true", + help="skip the pre-flight dpkg-audit gate") + sp.set_defaults(func=cmd_apt_upgrade) + sp = sub.add_parser("nuke", help="shred /etc/age/key.txt + /opt/homelab-context on a host") sp.add_argument("name") sp.add_argument("--yes", "-y", action="store_true")