secrets-issuance backup + homelab doctor smoke-test

Two additions:

1. secrets-issuance backup: daily timer snapshots /var/lib/secrets-issuance
   to /mnt/library/.secrets-issuance-backup/ as a date-stamped tar.gz,
   keeping the last 14 days. Closes the catastrophic-fail-mode where an
   LXC 105 loss wipes every client's age key with no recovery path.
   Caveat: privileged LXCs that mount /mnt/library can read the backup
   (root-uid maps to host root); encrypted-tarball variant is a future
   refinement.

2. homelab doctor: 10 invariant checks for an enrolled client — clone
   present, sync timer/launchd job active, age key perms, CLI symlinked,
   AGENTS.md linked, inventory entry exists, MCP reachable, secrets
   /health responds, sops canary decrypts, git creds present. Returns
   nonzero on any 'fail'. Useful after enrollment or whenever something
   smells off.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

bin/homelab

@@ -558,6 +558,174 @@ def cmd_secret(args: argparse.Namespace) -> int:
     return subprocess.call(["sops", "-d", str(path)], env=env)
 
 
+def cmd_doctor(args: argparse.Namespace) -> int:
+    """Run health checks for the homelab-context client setup."""
+    results: list[tuple[str, str, str]] = []  # (status, label, detail)
+
+    def ok(label, detail=""):    results.append(("ok",   label, detail))
+    def warn(label, detail=""):  results.append(("warn", label, detail))
+    def fail(label, detail=""):  results.append(("fail", label, detail))
+
+    # 1. /opt/homelab-context is a git clone.
+    if (CONTEXT / ".git").is_dir():
+        try:
+            head = subprocess.run(
+                ["git", "-C", str(CONTEXT), "log", "--oneline", "-1"],
+                capture_output=True, text=True, check=True).stdout.strip()
+            ok("clone present", f"head: {head[:60]}")
+        except Exception as e:
+            warn("clone present but git unhappy", str(e))
+    else:
+        fail("clone missing", f"{CONTEXT}/.git does not exist")
+
+    # 2. Sync mechanism — systemd timer (Linux) or launchd plist (macOS).
+    if sys.platform == "darwin":
+        try:
+            out = subprocess.run(
+                ["launchctl", "list"], capture_output=True, text=True,
+            ).stdout
+            if "network.hubris.homelab-context-sync" in out:
+                ok("launchd job loaded")
+            else:
+                fail("launchd job missing",
+                     "expected network.hubris.homelab-context-sync")
+        except FileNotFoundError:
+            fail("launchctl missing", "is this really macOS?")
+    else:
+        proc = subprocess.run(
+            ["systemctl", "list-timers", "homelab-context-sync.timer",
+             "--no-pager", "--no-legend"],
+            capture_output=True, text=True,
+        )
+        if "homelab-context-sync.timer" in proc.stdout:
+            ok("sync timer active", proc.stdout.strip()[:80])
+        else:
+            fail("sync timer not active",
+                 "systemctl enable --now homelab-context-sync.timer")
+
+    # 3. Age key.
+    if AGE_KEY.exists():
+        try:
+            st = AGE_KEY.stat()
+            mode = st.st_mode & 0o777
+            if mode != 0o600:
+                warn("age key permissions loose",
+                     f"{AGE_KEY} mode={oct(mode)} (expected 0o600)")
+            else:
+                ok("age key present", f"{AGE_KEY} 0600 root")
+        except PermissionError:
+            ok("age key present", f"{AGE_KEY} (can't stat — expected as non-root)")
+    else:
+        warn("age key missing",
+             "rerun bootstrap or call /issue manually")
+
+    # 4. /usr/local/bin/homelab is a symlink into the repo.
+    cli = Path("/usr/local/bin/homelab")
+    if cli.is_symlink() and cli.resolve() == (CONTEXT / "bin" / "homelab").resolve():
+        ok("CLI is symlinked into sync target")
+    elif cli.exists():
+        warn("CLI is a copy, not a symlink",
+             f"sudo ln -sfn {CONTEXT}/bin/homelab {cli}")
+    else:
+        fail("CLI not installed", f"{cli} missing")
+
+    # 5. AGENTS.md symlink.
+    expected_link = Path("/root/AGENTS.md") if sys.platform != "darwin" else Path("/etc/AGENTS.md")
+    if expected_link.is_symlink():
+        ok("AGENTS.md symlink", str(expected_link))
+    else:
+        warn("AGENTS.md not symlinked", f"expected {expected_link}")
+
+    # 6. hosts/<hostname>.yaml present (own identity).
+    hname = os.uname().nodename.split(".")[0]
+    if (HOSTS_DIR / f"{hname}.yaml").exists():
+        ok("inventory entry", f"hosts/{hname}.yaml")
+    else:
+        fail("no inventory entry for this hostname",
+             f"need hosts/{hname}.yaml — run 'homelab client add {hname}' on an enrolled client")
+
+    # 7. MCP reachability.
+    inv = inventory()
+    mcp_url = inv.get("services", {}).get("homelab_mcp", {}).get("endpoint")
+    if mcp_url:
+        proc = subprocess.run(
+            ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}",
+             "-H", "Accept: text/event-stream", "--max-time", "3", mcp_url],
+            capture_output=True, text=True,
+        )
+        code = proc.stdout.strip()
+        if code.startswith("2"):
+            ok("MCP reachable", f"{mcp_url} -> {code}")
+        else:
+            fail("MCP unreachable", f"{mcp_url} -> {code or 'no response'}")
+
+    # 8. Secrets-issuance health.
+    issue_url = inv.get("services", {}).get("secrets_issuance", {}).get("endpoint", "")
+    if issue_url:
+        health_url = issue_url.rstrip("/").rsplit("/", 1)[0] + "/health"
+        proc = subprocess.run(
+            ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}",
+             "--max-time", "3", health_url],
+            capture_output=True, text=True,
+        )
+        code = proc.stdout.strip()
+        if code == "200":
+            ok("secrets-issuance healthy", health_url)
+        else:
+            warn("secrets-issuance not responding 200", f"{health_url} -> {code or 'no response'}")
+
+    # 9. sops can decrypt the canary secret (needs age key access — root only).
+    if (CONTEXT / "secrets" / "hello.yaml").exists():
+        if os.geteuid() != 0:
+            ok("sops canary skipped (run as root to test)",
+               "homelab secret hello (will sudo)")
+        elif AGE_KEY.exists():
+            env = {**os.environ, "SOPS_AGE_KEY_FILE": str(AGE_KEY)}
+            proc = subprocess.run(
+                ["sops", "-d", str(CONTEXT / "secrets" / "hello.yaml")],
+                capture_output=True, text=True, env=env, timeout=5,
+            )
+            if proc.returncode == 0:
+                ok("sops canary decrypts")
+            else:
+                warn("sops canary decrypt failed",
+                     "this client may not be a recipient of secrets/hello.yaml")
+        else:
+            warn("sops canary not tested", "age key missing")
+
+    # 10. Git credentials — read-only or write-scoped?
+    creds = Path("/etc/homelab-context/git-credentials")
+    if creds.exists() and os.geteuid() == 0:
+        try:
+            # The write-scoped PAT comes from secrets/gitea-pat.yaml. We can't
+            # check scope from the file alone, but we can at least confirm
+            # presence + format.
+            content = creds.read_text().strip()
+            if "@" in content and "://" in content:
+                ok("git credentials file present", "/etc/homelab-context/git-credentials")
+            else:
+                warn("git credentials malformed", content[:80])
+        except PermissionError:
+            ok("git credentials present (can't read as non-root)")
+    elif creds.exists():
+        ok("git credentials present (can't read as non-root)")
+
+    # Print results table.
+    for status, label, detail in results:
+        icon = {"ok": "✓", "warn": "!", "fail": "✗"}[status]
+        line = f"  {icon} {label}"
+        if detail:
+            line += f" — {detail}"
+        print(line)
+
+    counts = {"ok": 0, "warn": 0, "fail": 0}
+    for s, *_ in results:
+        counts[s] += 1
+    print()
+    print(f"{counts['ok']} ok · {counts['warn']} warn · {counts['fail']} fail")
+    return 1 if counts["fail"] else 0
+
+
 def cmd_refresh_creds(args: argparse.Namespace) -> int:
     """Replace /etc/homelab-context/git-credentials with the write-scoped PAT
     from secrets/gitea-pat.yaml so push (not just pull) works from this client.
@@ -800,6 +968,9 @@ def main() -> int:
     sp = sub.add_parser("status", help="ping every host + HTTP-check every service")
     sp.set_defaults(func=cmd_status)
 
+    sp = sub.add_parser("doctor", help="run health checks on this client's enrollment")
+    sp.set_defaults(func=cmd_doctor)
+
     sp = sub.add_parser("secret", help="decrypt a secret (sops -d wrapper)")
     sp.add_argument("name")
     sp.set_defaults(func=cmd_secret)

secrets-issuance/backup.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Snapshot secrets-issuance state to /mnt/library
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/secrets-issuance/secrets-issuance/backup.sh
+Nice=10
+TimeoutStartSec=300

secrets-issuance/backup.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# secrets-issuance backup — snapshot the per-client age keys + denylist +
+# admin token to /mnt/library so they survive an LXC-105-only failure.
+#
+# Destination is under /mnt/library/.secrets-issuance-backup/ (dot-prefix to
+# stay out of any indexer's path). Mode 0700 root-owned. Privileged LXCs
+# that mount /mnt/library would still be able to read it as their root maps
+# to host root — accept that trade today; an encrypted-tarball variant
+# (separate offline age key) is a follow-up if the LAN trust model changes.
+#
+# Retention: 14 daily snapshots. Each snapshot is the WHOLE /var/lib/
+# secrets-issuance directory (small — ~few KiB per client), so restore is
+# just `tar xf <snap> -C /`.
+
+set -euo pipefail
+
+SRC=/var/lib/secrets-issuance
+DEST=/mnt/library/.secrets-issuance-backup
+RETAIN=14
+
+if [ ! -d "$SRC" ]; then
+    echo "[backup] source $SRC missing — nothing to do" >&2
+    exit 0
+fi
+if [ ! -d /mnt/library ]; then
+    echo "[backup] /mnt/library not mounted — can't write backup" >&2
+    exit 1
+fi
+
+install -d -m 0700 "$DEST"
+chown root:root "$DEST"
+
+stamp=$(date +%Y%m%d-%H%M%S)
+out="$DEST/secrets-issuance-$stamp.tar.gz"
+
+# Include the admin token too (recoverable separately, but trivial size and
+# saves a step when restoring after total LXC 105 loss).
+tar czf "$out" \
+    -C / \
+    var/lib/secrets-issuance \
+    $([ -f /etc/secrets-issuance/admin-token ] && echo etc/secrets-issuance/admin-token || true)
+chmod 0600 "$out"
+
+# Drop oldest beyond retention.
+ls -1t "$DEST"/secrets-issuance-*.tar.gz 2>/dev/null \
+    | tail -n +$((RETAIN + 1)) \
+    | xargs -r rm -f
+
+count=$(ls -1 "$DEST"/secrets-issuance-*.tar.gz 2>/dev/null | wc -l)
+size=$(du -sh "$out" | cut -f1)
+echo "[backup] wrote $out ($size); kept $count snapshots"

secrets-issuance/backup.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Daily secrets-issuance backup
+
+[Timer]
+OnCalendar=*-*-* 03:30:00
+RandomizedDelaySec=10min
+Persistent=true
+Unit=secrets-issuance-backup.service
+
+[Install]
+WantedBy=timers.target

secrets-issuance/deploy/deploy.sh

@@ -36,6 +36,11 @@ install -m 644 secrets-issuance/server.service \
                /etc/systemd/system/secrets-issuance.service
 install -m 644 secrets-issuance/deploy/webhook/secrets-issuance-deploy.service \
                /etc/systemd/system/secrets-issuance-deploy.service
+install -m 755 secrets-issuance/backup.sh /usr/local/bin/secrets-issuance-backup
+install -m 644 secrets-issuance/backup.service \
+               /etc/systemd/system/secrets-issuance-backup.service
+install -m 644 secrets-issuance/backup.timer \
+               /etc/systemd/system/secrets-issuance-backup.timer
 
 if [ ! -d /opt/homelab-context/.git ]; then
     echo "  /opt/homelab-context is not a git clone — run bootstrap.sh first." >&2
@@ -50,6 +55,10 @@ if systemctl is-active --quiet secrets-issuance-deploy.service; then
     systemctl restart secrets-issuance-deploy.service
 fi
 
+echo "[deploy] daemon-reload + enable backup timer"
+systemctl daemon-reload
+systemctl enable --now secrets-issuance-backup.timer
+
 echo "[deploy] done"
 echo "First-time enable:"
 echo "  systemctl enable --now secrets-issuance.service secrets-issuance-deploy.service"