diff --git a/bin/homelab b/bin/homelab index 48d6c9f..48791f8 100755 --- a/bin/homelab +++ b/bin/homelab @@ -558,6 +558,174 @@ def cmd_secret(args: argparse.Namespace) -> int: return subprocess.call(["sops", "-d", str(path)], env=env) +def cmd_doctor(args: argparse.Namespace) -> int: + """Run health checks for the homelab-context client setup.""" + results: list[tuple[str, str, str]] = [] # (status, label, detail) + + def ok(label, detail=""): results.append(("ok", label, detail)) + def warn(label, detail=""): results.append(("warn", label, detail)) + def fail(label, detail=""): results.append(("fail", label, detail)) + + # 1. /opt/homelab-context is a git clone. + if (CONTEXT / ".git").is_dir(): + try: + head = subprocess.run( + ["git", "-C", str(CONTEXT), "log", "--oneline", "-1"], + capture_output=True, text=True, check=True).stdout.strip() + ok("clone present", f"head: {head[:60]}") + except Exception as e: + warn("clone present but git unhappy", str(e)) + else: + fail("clone missing", f"{CONTEXT}/.git does not exist") + + # 2. Sync mechanism — systemd timer (Linux) or launchd plist (macOS). + if sys.platform == "darwin": + try: + out = subprocess.run( + ["launchctl", "list"], capture_output=True, text=True, + ).stdout + if "network.hubris.homelab-context-sync" in out: + ok("launchd job loaded") + else: + fail("launchd job missing", + "expected network.hubris.homelab-context-sync") + except FileNotFoundError: + fail("launchctl missing", "is this really macOS?") + else: + proc = subprocess.run( + ["systemctl", "list-timers", "homelab-context-sync.timer", + "--no-pager", "--no-legend"], + capture_output=True, text=True, + ) + if "homelab-context-sync.timer" in proc.stdout: + ok("sync timer active", proc.stdout.strip()[:80]) + else: + fail("sync timer not active", + "systemctl enable --now homelab-context-sync.timer") + + # 3. Age key. + if AGE_KEY.exists(): + try: + st = AGE_KEY.stat() + mode = st.st_mode & 0o777 + if mode != 0o600: + warn("age key permissions loose", + f"{AGE_KEY} mode={oct(mode)} (expected 0o600)") + else: + ok("age key present", f"{AGE_KEY} 0600 root") + except PermissionError: + ok("age key present", f"{AGE_KEY} (can't stat — expected as non-root)") + else: + warn("age key missing", + "rerun bootstrap or call /issue manually") + + # 4. /usr/local/bin/homelab is a symlink into the repo. + cli = Path("/usr/local/bin/homelab") + if cli.is_symlink() and cli.resolve() == (CONTEXT / "bin" / "homelab").resolve(): + ok("CLI is symlinked into sync target") + elif cli.exists(): + warn("CLI is a copy, not a symlink", + f"sudo ln -sfn {CONTEXT}/bin/homelab {cli}") + else: + fail("CLI not installed", f"{cli} missing") + + # 5. AGENTS.md symlink. + expected_link = Path("/root/AGENTS.md") if sys.platform != "darwin" else Path("/etc/AGENTS.md") + if expected_link.is_symlink(): + ok("AGENTS.md symlink", str(expected_link)) + else: + warn("AGENTS.md not symlinked", f"expected {expected_link}") + + # 6. hosts/.yaml present (own identity). + hname = os.uname().nodename.split(".")[0] + if (HOSTS_DIR / f"{hname}.yaml").exists(): + ok("inventory entry", f"hosts/{hname}.yaml") + else: + fail("no inventory entry for this hostname", + f"need hosts/{hname}.yaml — run 'homelab client add {hname}' on an enrolled client") + + # 7. MCP reachability. + inv = inventory() + mcp_url = inv.get("services", {}).get("homelab_mcp", {}).get("endpoint") + if mcp_url: + proc = subprocess.run( + ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-H", "Accept: text/event-stream", "--max-time", "3", mcp_url], + capture_output=True, text=True, + ) + code = proc.stdout.strip() + if code.startswith("2"): + ok("MCP reachable", f"{mcp_url} -> {code}") + else: + fail("MCP unreachable", f"{mcp_url} -> {code or 'no response'}") + + # 8. Secrets-issuance health. + issue_url = inv.get("services", {}).get("secrets_issuance", {}).get("endpoint", "") + if issue_url: + health_url = issue_url.rstrip("/").rsplit("/", 1)[0] + "/health" + proc = subprocess.run( + ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "--max-time", "3", health_url], + capture_output=True, text=True, + ) + code = proc.stdout.strip() + if code == "200": + ok("secrets-issuance healthy", health_url) + else: + warn("secrets-issuance not responding 200", f"{health_url} -> {code or 'no response'}") + + # 9. sops can decrypt the canary secret (needs age key access — root only). + if (CONTEXT / "secrets" / "hello.yaml").exists(): + if os.geteuid() != 0: + ok("sops canary skipped (run as root to test)", + "homelab secret hello (will sudo)") + elif AGE_KEY.exists(): + env = {**os.environ, "SOPS_AGE_KEY_FILE": str(AGE_KEY)} + proc = subprocess.run( + ["sops", "-d", str(CONTEXT / "secrets" / "hello.yaml")], + capture_output=True, text=True, env=env, timeout=5, + ) + if proc.returncode == 0: + ok("sops canary decrypts") + else: + warn("sops canary decrypt failed", + "this client may not be a recipient of secrets/hello.yaml") + else: + warn("sops canary not tested", "age key missing") + + # 10. Git credentials — read-only or write-scoped? + creds = Path("/etc/homelab-context/git-credentials") + if creds.exists() and os.geteuid() == 0: + try: + # The write-scoped PAT comes from secrets/gitea-pat.yaml. We can't + # check scope from the file alone, but we can at least confirm + # presence + format. + content = creds.read_text().strip() + if "@" in content and "://" in content: + ok("git credentials file present", "/etc/homelab-context/git-credentials") + else: + warn("git credentials malformed", content[:80]) + except PermissionError: + ok("git credentials present (can't read as non-root)") + elif creds.exists(): + ok("git credentials present (can't read as non-root)") + + # Print results table. + for status, label, detail in results: + icon = {"ok": "✓", "warn": "!", "fail": "✗"}[status] + line = f" {icon} {label}" + if detail: + line += f" — {detail}" + print(line) + + counts = {"ok": 0, "warn": 0, "fail": 0} + for s, *_ in results: + counts[s] += 1 + print() + print(f"{counts['ok']} ok · {counts['warn']} warn · {counts['fail']} fail") + return 1 if counts["fail"] else 0 + + def cmd_refresh_creds(args: argparse.Namespace) -> int: """Replace /etc/homelab-context/git-credentials with the write-scoped PAT from secrets/gitea-pat.yaml so push (not just pull) works from this client. @@ -800,6 +968,9 @@ def main() -> int: sp = sub.add_parser("status", help="ping every host + HTTP-check every service") sp.set_defaults(func=cmd_status) + sp = sub.add_parser("doctor", help="run health checks on this client's enrollment") + sp.set_defaults(func=cmd_doctor) + sp = sub.add_parser("secret", help="decrypt a secret (sops -d wrapper)") sp.add_argument("name") sp.set_defaults(func=cmd_secret) diff --git a/secrets-issuance/backup.service b/secrets-issuance/backup.service new file mode 100644 index 0000000..21cc475 --- /dev/null +++ b/secrets-issuance/backup.service @@ -0,0 +1,9 @@ +[Unit] +Description=Snapshot secrets-issuance state to /mnt/library +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/opt/secrets-issuance/secrets-issuance/backup.sh +Nice=10 +TimeoutStartSec=300 diff --git a/secrets-issuance/backup.sh b/secrets-issuance/backup.sh new file mode 100644 index 0000000..9e0d9dd --- /dev/null +++ b/secrets-issuance/backup.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# secrets-issuance backup — snapshot the per-client age keys + denylist + +# admin token to /mnt/library so they survive an LXC-105-only failure. +# +# Destination is under /mnt/library/.secrets-issuance-backup/ (dot-prefix to +# stay out of any indexer's path). Mode 0700 root-owned. Privileged LXCs +# that mount /mnt/library would still be able to read it as their root maps +# to host root — accept that trade today; an encrypted-tarball variant +# (separate offline age key) is a follow-up if the LAN trust model changes. +# +# Retention: 14 daily snapshots. Each snapshot is the WHOLE /var/lib/ +# secrets-issuance directory (small — ~few KiB per client), so restore is +# just `tar xf -C /`. + +set -euo pipefail + +SRC=/var/lib/secrets-issuance +DEST=/mnt/library/.secrets-issuance-backup +RETAIN=14 + +if [ ! -d "$SRC" ]; then + echo "[backup] source $SRC missing — nothing to do" >&2 + exit 0 +fi +if [ ! -d /mnt/library ]; then + echo "[backup] /mnt/library not mounted — can't write backup" >&2 + exit 1 +fi + +install -d -m 0700 "$DEST" +chown root:root "$DEST" + +stamp=$(date +%Y%m%d-%H%M%S) +out="$DEST/secrets-issuance-$stamp.tar.gz" + +# Include the admin token too (recoverable separately, but trivial size and +# saves a step when restoring after total LXC 105 loss). +tar czf "$out" \ + -C / \ + var/lib/secrets-issuance \ + $([ -f /etc/secrets-issuance/admin-token ] && echo etc/secrets-issuance/admin-token || true) +chmod 0600 "$out" + +# Drop oldest beyond retention. +ls -1t "$DEST"/secrets-issuance-*.tar.gz 2>/dev/null \ + | tail -n +$((RETAIN + 1)) \ + | xargs -r rm -f + +count=$(ls -1 "$DEST"/secrets-issuance-*.tar.gz 2>/dev/null | wc -l) +size=$(du -sh "$out" | cut -f1) +echo "[backup] wrote $out ($size); kept $count snapshots" diff --git a/secrets-issuance/backup.timer b/secrets-issuance/backup.timer new file mode 100644 index 0000000..ed96d1f --- /dev/null +++ b/secrets-issuance/backup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Daily secrets-issuance backup + +[Timer] +OnCalendar=*-*-* 03:30:00 +RandomizedDelaySec=10min +Persistent=true +Unit=secrets-issuance-backup.service + +[Install] +WantedBy=timers.target diff --git a/secrets-issuance/deploy/deploy.sh b/secrets-issuance/deploy/deploy.sh index 65a40c7..697b605 100755 --- a/secrets-issuance/deploy/deploy.sh +++ b/secrets-issuance/deploy/deploy.sh @@ -36,6 +36,11 @@ install -m 644 secrets-issuance/server.service \ /etc/systemd/system/secrets-issuance.service install -m 644 secrets-issuance/deploy/webhook/secrets-issuance-deploy.service \ /etc/systemd/system/secrets-issuance-deploy.service +install -m 755 secrets-issuance/backup.sh /usr/local/bin/secrets-issuance-backup +install -m 644 secrets-issuance/backup.service \ + /etc/systemd/system/secrets-issuance-backup.service +install -m 644 secrets-issuance/backup.timer \ + /etc/systemd/system/secrets-issuance-backup.timer if [ ! -d /opt/homelab-context/.git ]; then echo " /opt/homelab-context is not a git clone — run bootstrap.sh first." >&2 @@ -50,6 +55,10 @@ if systemctl is-active --quiet secrets-issuance-deploy.service; then systemctl restart secrets-issuance-deploy.service fi +echo "[deploy] daemon-reload + enable backup timer" +systemctl daemon-reload +systemctl enable --now secrets-issuance-backup.timer + echo "[deploy] done" echo "First-time enable:" echo " systemctl enable --now secrets-issuance.service secrets-issuance-deploy.service"