secrets-issuance backup + homelab doctor smoke-test

Two additions:

1. secrets-issuance backup: daily timer snapshots /var/lib/secrets-issuance
   to /mnt/library/.secrets-issuance-backup/ as a date-stamped tar.gz,
   keeping the last 14 days. Closes the catastrophic-fail-mode where an
   LXC 105 loss wipes every client's age key with no recovery path.
   Caveat: privileged LXCs that mount /mnt/library can read the backup
   (root-uid maps to host root); encrypted-tarball variant is a future
   refinement.

2. homelab doctor: 10 invariant checks for an enrolled client — clone
   present, sync timer/launchd job active, age key perms, CLI symlinked,
   AGENTS.md linked, inventory entry exists, MCP reachable, secrets
   /health responds, sops canary decrypts, git creds present. Returns
   nonzero on any 'fail'. Useful after enrollment or whenever something
   smells off.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

secrets-issuance/backup.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Snapshot secrets-issuance state to /mnt/library
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/secrets-issuance/secrets-issuance/backup.sh
+Nice=10
+TimeoutStartSec=300

secrets-issuance/backup.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# secrets-issuance backup — snapshot the per-client age keys + denylist +
+# admin token to /mnt/library so they survive an LXC-105-only failure.
+#
+# Destination is under /mnt/library/.secrets-issuance-backup/ (dot-prefix to
+# stay out of any indexer's path). Mode 0700 root-owned. Privileged LXCs
+# that mount /mnt/library would still be able to read it as their root maps
+# to host root — accept that trade today; an encrypted-tarball variant
+# (separate offline age key) is a follow-up if the LAN trust model changes.
+#
+# Retention: 14 daily snapshots. Each snapshot is the WHOLE /var/lib/
+# secrets-issuance directory (small — ~few KiB per client), so restore is
+# just `tar xf <snap> -C /`.
+
+set -euo pipefail
+
+SRC=/var/lib/secrets-issuance
+DEST=/mnt/library/.secrets-issuance-backup
+RETAIN=14
+
+if [ ! -d "$SRC" ]; then
+    echo "[backup] source $SRC missing — nothing to do" >&2
+    exit 0
+fi
+if [ ! -d /mnt/library ]; then
+    echo "[backup] /mnt/library not mounted — can't write backup" >&2
+    exit 1
+fi
+
+install -d -m 0700 "$DEST"
+chown root:root "$DEST"
+
+stamp=$(date +%Y%m%d-%H%M%S)
+out="$DEST/secrets-issuance-$stamp.tar.gz"
+
+# Include the admin token too (recoverable separately, but trivial size and
+# saves a step when restoring after total LXC 105 loss).
+tar czf "$out" \
+    -C / \
+    var/lib/secrets-issuance \
+    $([ -f /etc/secrets-issuance/admin-token ] && echo etc/secrets-issuance/admin-token || true)
+chmod 0600 "$out"
+
+# Drop oldest beyond retention.
+ls -1t "$DEST"/secrets-issuance-*.tar.gz 2>/dev/null \
+    | tail -n +$((RETAIN + 1)) \
+    | xargs -r rm -f
+
+count=$(ls -1 "$DEST"/secrets-issuance-*.tar.gz 2>/dev/null | wc -l)
+size=$(du -sh "$out" | cut -f1)
+echo "[backup] wrote $out ($size); kept $count snapshots"

secrets-issuance/backup.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Daily secrets-issuance backup
+
+[Timer]
+OnCalendar=*-*-* 03:30:00
+RandomizedDelaySec=10min
+Persistent=true
+Unit=secrets-issuance-backup.service
+
+[Install]
+WantedBy=timers.target

secrets-issuance/deploy/deploy.sh

@@ -36,6 +36,11 @@ install -m 644 secrets-issuance/server.service \
                /etc/systemd/system/secrets-issuance.service
 install -m 644 secrets-issuance/deploy/webhook/secrets-issuance-deploy.service \
                /etc/systemd/system/secrets-issuance-deploy.service
+install -m 755 secrets-issuance/backup.sh /usr/local/bin/secrets-issuance-backup
+install -m 644 secrets-issuance/backup.service \
+               /etc/systemd/system/secrets-issuance-backup.service
+install -m 644 secrets-issuance/backup.timer \
+               /etc/systemd/system/secrets-issuance-backup.timer
 
 if [ ! -d /opt/homelab-context/.git ]; then
     echo "  /opt/homelab-context is not a git clone — run bootstrap.sh first." >&2
@@ -50,6 +55,10 @@ if systemctl is-active --quiet secrets-issuance-deploy.service; then
     systemctl restart secrets-issuance-deploy.service
 fi
 
+echo "[deploy] daemon-reload + enable backup timer"
+systemctl daemon-reload
+systemctl enable --now secrets-issuance-backup.timer
+
 echo "[deploy] done"
 echo "First-time enable:"
 echo "  systemctl enable --now secrets-issuance.service secrets-issuance-deploy.service"