secrets-issuance backup + homelab doctor smoke-test

Two additions:

1. secrets-issuance backup: daily timer snapshots /var/lib/secrets-issuance
   to /mnt/library/.secrets-issuance-backup/ as a date-stamped tar.gz,
   keeping the last 14 days. Closes the catastrophic-fail-mode where an
   LXC 105 loss wipes every client's age key with no recovery path.
   Caveat: privileged LXCs that mount /mnt/library can read the backup
   (root-uid maps to host root); encrypted-tarball variant is a future
   refinement.

2. homelab doctor: 10 invariant checks for an enrolled client — clone
   present, sync timer/launchd job active, age key perms, CLI symlinked,
   AGENTS.md linked, inventory entry exists, MCP reachable, secrets
   /health responds, sops canary decrypts, git creds present. Returns
   nonzero on any 'fail'. Useful after enrollment or whenever something
   smells off.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

bin/homelab

@@ -558,6 +558,174 @@ def cmd_secret(args: argparse.Namespace) -> int:
     return subprocess.call(["sops", "-d", str(path)], env=env)
 
 
+def cmd_doctor(args: argparse.Namespace) -> int:
+    """Run health checks for the homelab-context client setup."""
+    results: list[tuple[str, str, str]] = []  # (status, label, detail)
+
+    def ok(label, detail=""):    results.append(("ok",   label, detail))
+    def warn(label, detail=""):  results.append(("warn", label, detail))
+    def fail(label, detail=""):  results.append(("fail", label, detail))
+
+    # 1. /opt/homelab-context is a git clone.
+    if (CONTEXT / ".git").is_dir():
+        try:
+            head = subprocess.run(
+                ["git", "-C", str(CONTEXT), "log", "--oneline", "-1"],
+                capture_output=True, text=True, check=True).stdout.strip()
+            ok("clone present", f"head: {head[:60]}")
+        except Exception as e:
+            warn("clone present but git unhappy", str(e))
+    else:
+        fail("clone missing", f"{CONTEXT}/.git does not exist")
+
+    # 2. Sync mechanism — systemd timer (Linux) or launchd plist (macOS).
+    if sys.platform == "darwin":
+        try:
+            out = subprocess.run(
+                ["launchctl", "list"], capture_output=True, text=True,
+            ).stdout
+            if "network.hubris.homelab-context-sync" in out:
+                ok("launchd job loaded")
+            else:
+                fail("launchd job missing",
+                     "expected network.hubris.homelab-context-sync")
+        except FileNotFoundError:
+            fail("launchctl missing", "is this really macOS?")
+    else:
+        proc = subprocess.run(
+            ["systemctl", "list-timers", "homelab-context-sync.timer",
+             "--no-pager", "--no-legend"],
+            capture_output=True, text=True,
+        )
+        if "homelab-context-sync.timer" in proc.stdout:
+            ok("sync timer active", proc.stdout.strip()[:80])
+        else:
+            fail("sync timer not active",
+                 "systemctl enable --now homelab-context-sync.timer")
+
+    # 3. Age key.
+    if AGE_KEY.exists():
+        try:
+            st = AGE_KEY.stat()
+            mode = st.st_mode & 0o777
+            if mode != 0o600:
+                warn("age key permissions loose",
+                     f"{AGE_KEY} mode={oct(mode)} (expected 0o600)")
+            else:
+                ok("age key present", f"{AGE_KEY} 0600 root")
+        except PermissionError:
+            ok("age key present", f"{AGE_KEY} (can't stat — expected as non-root)")
+    else:
+        warn("age key missing",
+             "rerun bootstrap or call /issue manually")
+
+    # 4. /usr/local/bin/homelab is a symlink into the repo.
+    cli = Path("/usr/local/bin/homelab")
+    if cli.is_symlink() and cli.resolve() == (CONTEXT / "bin" / "homelab").resolve():
+        ok("CLI is symlinked into sync target")
+    elif cli.exists():
+        warn("CLI is a copy, not a symlink",
+             f"sudo ln -sfn {CONTEXT}/bin/homelab {cli}")
+    else:
+        fail("CLI not installed", f"{cli} missing")
+
+    # 5. AGENTS.md symlink.
+    expected_link = Path("/root/AGENTS.md") if sys.platform != "darwin" else Path("/etc/AGENTS.md")
+    if expected_link.is_symlink():
+        ok("AGENTS.md symlink", str(expected_link))
+    else:
+        warn("AGENTS.md not symlinked", f"expected {expected_link}")
+
+    # 6. hosts/<hostname>.yaml present (own identity).
+    hname = os.uname().nodename.split(".")[0]
+    if (HOSTS_DIR / f"{hname}.yaml").exists():
+        ok("inventory entry", f"hosts/{hname}.yaml")
+    else:
+        fail("no inventory entry for this hostname",
+             f"need hosts/{hname}.yaml — run 'homelab client add {hname}' on an enrolled client")
+
+    # 7. MCP reachability.
+    inv = inventory()
+    mcp_url = inv.get("services", {}).get("homelab_mcp", {}).get("endpoint")
+    if mcp_url:
+        proc = subprocess.run(
+            ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}",
+             "-H", "Accept: text/event-stream", "--max-time", "3", mcp_url],
+            capture_output=True, text=True,
+        )
+        code = proc.stdout.strip()
+        if code.startswith("2"):
+            ok("MCP reachable", f"{mcp_url} -> {code}")
+        else:
+            fail("MCP unreachable", f"{mcp_url} -> {code or 'no response'}")
+
+    # 8. Secrets-issuance health.
+    issue_url = inv.get("services", {}).get("secrets_issuance", {}).get("endpoint", "")
+    if issue_url:
+        health_url = issue_url.rstrip("/").rsplit("/", 1)[0] + "/health"
+        proc = subprocess.run(
+            ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}",
+             "--max-time", "3", health_url],
+            capture_output=True, text=True,
+        )
+        code = proc.stdout.strip()
+        if code == "200":
+            ok("secrets-issuance healthy", health_url)
+        else:
+            warn("secrets-issuance not responding 200", f"{health_url} -> {code or 'no response'}")
+
+    # 9. sops can decrypt the canary secret (needs age key access — root only).
+    if (CONTEXT / "secrets" / "hello.yaml").exists():
+        if os.geteuid() != 0:
+            ok("sops canary skipped (run as root to test)",
+               "homelab secret hello (will sudo)")
+        elif AGE_KEY.exists():
+            env = {**os.environ, "SOPS_AGE_KEY_FILE": str(AGE_KEY)}
+            proc = subprocess.run(
+                ["sops", "-d", str(CONTEXT / "secrets" / "hello.yaml")],
+                capture_output=True, text=True, env=env, timeout=5,
+            )
+            if proc.returncode == 0:
+                ok("sops canary decrypts")
+            else:
+                warn("sops canary decrypt failed",
+                     "this client may not be a recipient of secrets/hello.yaml")
+        else:
+            warn("sops canary not tested", "age key missing")
+
+    # 10. Git credentials — read-only or write-scoped?
+    creds = Path("/etc/homelab-context/git-credentials")
+    if creds.exists() and os.geteuid() == 0:
+        try:
+            # The write-scoped PAT comes from secrets/gitea-pat.yaml. We can't
+            # check scope from the file alone, but we can at least confirm
+            # presence + format.
+            content = creds.read_text().strip()
+            if "@" in content and "://" in content:
+                ok("git credentials file present", "/etc/homelab-context/git-credentials")
+            else:
+                warn("git credentials malformed", content[:80])
+        except PermissionError:
+            ok("git credentials present (can't read as non-root)")
+    elif creds.exists():
+        ok("git credentials present (can't read as non-root)")
+
+    # Print results table.
+    for status, label, detail in results:
+        icon = {"ok": "✓", "warn": "!", "fail": "✗"}[status]
+        line = f"  {icon} {label}"
+        if detail:
+            line += f" — {detail}"
+        print(line)
+
+    counts = {"ok": 0, "warn": 0, "fail": 0}
+    for s, *_ in results:
+        counts[s] += 1
+    print()
+    print(f"{counts['ok']} ok · {counts['warn']} warn · {counts['fail']} fail")
+    return 1 if counts["fail"] else 0
+
+
 def cmd_refresh_creds(args: argparse.Namespace) -> int:
     """Replace /etc/homelab-context/git-credentials with the write-scoped PAT
     from secrets/gitea-pat.yaml so push (not just pull) works from this client.
@@ -800,6 +968,9 @@ def main() -> int:
     sp = sub.add_parser("status", help="ping every host + HTTP-check every service")
     sp.set_defaults(func=cmd_status)
 
+    sp = sub.add_parser("doctor", help="run health checks on this client's enrollment")
+    sp.set_defaults(func=cmd_doctor)
+
     sp = sub.add_parser("secret", help="decrypt a secret (sops -d wrapper)")
     sp.add_argument("name")
     sp.set_defaults(func=cmd_secret)