cleanup: archive stale secrets/ + secrets-issuance/, add Infisical bootstrap script, update SOPS paths to archive/

archive/secrets-issuance/backup.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Snapshot secrets-issuance state to /mnt/library
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/secrets-issuance/secrets-issuance/backup.sh
+Nice=10
+TimeoutStartSec=300

archive/secrets-issuance/backup.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# secrets-issuance backup — snapshot the per-client age keys + denylist +
+# admin token to /mnt/library so they survive an LXC-105-only failure.
+#
+# Destination is under /mnt/library/.secrets-issuance-backup/ (dot-prefix to
+# stay out of any indexer's path). Mode 0700 root-owned. Privileged LXCs
+# that mount /mnt/library would still be able to read it as their root maps
+# to host root — accept that trade today; an encrypted-tarball variant
+# (separate offline age key) is a follow-up if the LAN trust model changes.
+#
+# Retention: 14 daily snapshots. Each snapshot is the WHOLE /var/lib/
+# secrets-issuance directory (small — ~few KiB per client), so restore is
+# just `tar xf <snap> -C /`.
+
+set -euo pipefail
+
+SRC=/var/lib/secrets-issuance
+DEST=/mnt/library/.secrets-issuance-backup
+RETAIN=14
+
+if [ ! -d "$SRC" ]; then
+    echo "[backup] source $SRC missing — nothing to do" >&2
+    exit 0
+fi
+if [ ! -d /mnt/library ]; then
+    echo "[backup] /mnt/library not mounted — can't write backup" >&2
+    exit 1
+fi
+
+install -d -m 0700 "$DEST"
+chown root:root "$DEST"
+
+stamp=$(date +%Y%m%d-%H%M%S)
+out="$DEST/secrets-issuance-$stamp.tar.gz"
+
+# Include the admin token too (recoverable separately, but trivial size and
+# saves a step when restoring after total LXC 105 loss).
+tar czf "$out" \
+    -C / \
+    var/lib/secrets-issuance \
+    $([ -f /etc/secrets-issuance/admin-token ] && echo etc/secrets-issuance/admin-token || true)
+chmod 0600 "$out"
+
+# Drop oldest beyond retention.
+ls -1t "$DEST"/secrets-issuance-*.tar.gz 2>/dev/null \
+    | tail -n +$((RETAIN + 1)) \
+    | xargs -r rm -f
+
+count=$(ls -1 "$DEST"/secrets-issuance-*.tar.gz 2>/dev/null | wc -l)
+size=$(du -sh "$out" | cut -f1)
+echo "[backup] wrote $out ($size); kept $count snapshots"

archive/secrets-issuance/backup.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Daily secrets-issuance backup
+
+[Timer]
+OnCalendar=*-*-* 03:30:00
+RandomizedDelaySec=10min
+Persistent=true
+Unit=secrets-issuance-backup.service
+
+[Install]
+WantedBy=timers.target

archive/secrets-issuance/deploy/deploy.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Install/update secrets-issuance on LXC 105 (apps). Idempotent.
+set -euo pipefail
+
+REPO_DIR=${REPO_DIR:-/opt/secrets-issuance}
+
+cd "$REPO_DIR"
+echo "[deploy] git pull"
+git pull --ff-only
+
+echo "[deploy] ensure python venv + deps"
+if [ ! -d /opt/secrets-issuance/.venv ]; then
+    python3 -m venv /opt/secrets-issuance/.venv
+fi
+/opt/secrets-issuance/.venv/bin/pip install --quiet --upgrade pip
+/opt/secrets-issuance/.venv/bin/pip install --quiet pyyaml
+
+echo "[deploy] verify dependencies (age, shred)"
+command -v age-keygen >/dev/null || { echo "age-keygen not installed: apt install -y age" >&2; exit 1; }
+command -v shred >/dev/null     || { echo "shred not installed: apt install -y coreutils" >&2; exit 1; }
+
+echo "[deploy] state dir"
+install -d -m 700 /var/lib/secrets-issuance
+install -d -m 700 /var/lib/secrets-issuance/keys
+
+echo "[deploy] admin token dir"
+install -d -m 700 /etc/secrets-issuance
+if [ ! -s /etc/secrets-issuance/admin-token ]; then
+    head -c 32 /dev/urandom | base64 > /etc/secrets-issuance/admin-token
+    chmod 600 /etc/secrets-issuance/admin-token
+    echo "[deploy] generated admin token at /etc/secrets-issuance/admin-token"
+fi
+
+echo "[deploy] install systemd units"
+install -m 644 secrets-issuance/server.service \
+               /etc/systemd/system/secrets-issuance.service
+install -m 644 secrets-issuance/deploy/webhook/secrets-issuance-deploy.service \
+               /etc/systemd/system/secrets-issuance-deploy.service
+install -m 755 secrets-issuance/backup.sh /usr/local/bin/secrets-issuance-backup
+install -m 644 secrets-issuance/backup.service \
+               /etc/systemd/system/secrets-issuance-backup.service
+install -m 644 secrets-issuance/backup.timer \
+               /etc/systemd/system/secrets-issuance-backup.timer
+
+if [ ! -d /opt/homelab-context/.git ]; then
+    echo "  /opt/homelab-context is not a git clone — run bootstrap.sh first." >&2
+    exit 1
+fi
+
+systemctl daemon-reload
+if systemctl is-active --quiet secrets-issuance.service; then
+    systemctl restart secrets-issuance.service
+fi
+if systemctl is-active --quiet secrets-issuance-deploy.service; then
+    systemctl restart secrets-issuance-deploy.service
+fi
+
+echo "[deploy] daemon-reload + enable backup timer"
+systemctl daemon-reload
+systemctl enable --now secrets-issuance-backup.timer
+
+echo "[deploy] done"
+echo "First-time enable:"
+echo "  systemctl enable --now secrets-issuance.service secrets-issuance-deploy.service"
+echo "Webhook first-time setup:"
+echo "  $REPO_DIR/secrets-issuance/deploy/webhook/install.sh"

archive/secrets-issuance/deploy/webhook/install.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# First-time setup for the secrets-issuance deploy webhook. Generates a
+# secret, installs the systemd unit, and starts it.
+set -euo pipefail
+
+SECRET_DIR=/etc/secrets-issuance-deploy
+SECRET=$SECRET_DIR/secret
+UNIT=secrets-issuance-deploy.service
+
+install -d -m 700 "$SECRET_DIR"
+if [ ! -s "$SECRET" ]; then
+    head -c 32 /dev/urandom | base64 > "$SECRET"
+    chmod 600 "$SECRET"
+    echo "[install] generated webhook secret at $SECRET"
+fi
+
+systemctl daemon-reload
+systemctl enable --now "$UNIT"
+systemctl status "$UNIT" --no-pager | head -10
+
+cat <<EOF
+
+[install] webhook listening on :9821/deploy.
+Configure Gitea (dtoro/Homelab-Docs → Settings → Webhooks → Add Webhook → Gitea):
+  Target URL:   http://<lxc-105-mesh-ip>:9821/deploy
+  HTTP Method:  POST
+  Content-Type: application/json
+  Secret:       $(cat $SECRET)
+  Trigger:      Push events
+EOF

archive/secrets-issuance/deploy/webhook/secrets-issuance-deploy.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Gitea deploy webhook for dtoro/Homelab-Docs → secrets-issuance
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 /opt/secrets-issuance/secrets-issuance/deploy/webhook/webhook.py
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

archive/secrets-issuance/deploy/webhook/webhook.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Deploy webhook for dtoro/Homelab-Docs → secrets-issuance on LXC 105.
+
+Listens on 0.0.0.0:9821/deploy. Validates Gitea HMAC, runs deploy.sh.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import logging
+import os
+import subprocess
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+BIND_HOST = "0.0.0.0"
+BIND_PORT = 9821
+SECRET_PATH = "/etc/secrets-issuance-deploy/secret"
+DEPLOY_CMD = ["/opt/secrets-issuance/secrets-issuance/deploy/deploy.sh"]
+TARGET_REF = "refs/heads/main"
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger("secrets-issuance-deploy")
+_deploy_lock = threading.Lock()
+
+
+def load_secret() -> bytes:
+    with open(SECRET_PATH, "rb") as f:
+        return f.read().strip()
+
+
+class Handler(BaseHTTPRequestHandler):
+    def log_message(self, fmt, *args):
+        log.info("%s - %s", self.address_string(), fmt % args)
+
+    def _reply(self, status: int, msg: str = "") -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", "text/plain")
+        self.end_headers()
+        if msg:
+            self.wfile.write(msg.encode())
+
+    def do_POST(self) -> None:
+        if self.path != "/deploy":
+            self._reply(404, "not found")
+            return
+        length = int(self.headers.get("Content-Length", "0"))
+        body = self.rfile.read(length) if length else b""
+        sig = self.headers.get("X-Gitea-Signature", "")
+        secret = load_secret()
+        expected = hmac.new(secret, body, hashlib.sha256).hexdigest()
+        if not hmac.compare_digest(expected, sig):
+            log.warning("signature mismatch")
+            self._reply(403, "bad signature")
+            return
+        try:
+            payload = json.loads(body)
+        except json.JSONDecodeError:
+            self._reply(400, "bad json")
+            return
+        if payload.get("ref", "") != TARGET_REF:
+            self._reply(204)
+            return
+        if not _deploy_lock.acquire(blocking=False):
+            self._reply(202, "already running")
+            return
+        try:
+            result = subprocess.run(DEPLOY_CMD, capture_output=True, text=True, timeout=180)
+            if result.returncode != 0:
+                log.error("deploy failed: %s\n%s", result.stdout, result.stderr)
+                self._reply(500, "deploy failed")
+                return
+            self._reply(204)
+        finally:
+            _deploy_lock.release()
+
+
+def main() -> int:
+    if not os.path.exists(SECRET_PATH):
+        log.error("secret file missing: %s", SECRET_PATH)
+        return 1
+    server = HTTPServer((BIND_HOST, BIND_PORT), Handler)
+    log.info("listening on %s:%d", BIND_HOST, BIND_PORT)
+    server.serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

archive/secrets-issuance/server.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+"""
+Secrets-issuance HTTP service.
+
+Lives on LXC 105 alongside the MCP server. Mesh-bound (nftables restricts to
+Netbird + Tailscale subnets). Identifies callers by source mesh IP and returns
+the per-client age private key.
+
+State:
+  /var/lib/secrets-issuance/keys/<hostname>.key       (private, 0600 root)
+  /var/lib/secrets-issuance/keys/<hostname>.pub       (public)
+  /var/lib/secrets-issuance/denylist.txt              (one hostname per line)
+
+Inventory lookup:
+  /opt/homelab-context/inventory.yaml — the issuance server is itself a
+  homelab-context client, so it sees inventory updates within 5 min.
+
+Endpoints:
+  POST /issue       body: {"hostname": "..."}   -> raw age private key
+  POST /revoke      body: {"hostname": "..."}   -> shred key + add to denylist
+                                                   (called by 'homelab client remove')
+  GET  /health
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import json
+import logging
+import os
+import secrets as sysrandom
+import subprocess
+import sys
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+
+import yaml
+
+BIND_HOST = "0.0.0.0"
+BIND_PORT = 9820
+CONTEXT_DIR = Path(os.environ.get("HOMELAB_CONTEXT_DIR", "/opt/homelab-context"))
+STATE_DIR = Path(os.environ.get("SECRETS_ISSUANCE_STATE", "/var/lib/secrets-issuance"))
+KEYS_DIR = STATE_DIR / "keys"
+DENYLIST = STATE_DIR / "denylist.txt"
+ADMIN_TOKEN_PATH = Path(os.environ.get("SECRETS_ISSUANCE_ADMIN_TOKEN",
+                                       "/etc/secrets-issuance/admin-token"))
+MESH_SUBNETS = [
+    ipaddress.ip_network(s.strip())
+    for s in os.environ.get("MESH_SUBNETS", "100.64.0.0/10").split(",")
+    if s.strip()
+]
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger("secrets-issuance")
+
+
+def load_inventory() -> dict:
+    return yaml.safe_load((CONTEXT_DIR / "inventory.yaml").read_text())
+
+
+def known_hosts() -> dict[str, dict]:
+    return load_inventory().get("hosts", {})
+
+
+def host_by_mesh_ip(ip: str) -> str | None:
+    """Resolve a mesh source IP to a hostname per inventory."""
+    for name, entry in known_hosts().items():
+        mesh = entry.get("mesh", {})
+        for proto in ("netbird", "tailscale"):
+            p = mesh.get(proto, {})
+            if not isinstance(p, dict):
+                continue
+            if p.get("ip") == ip:
+                return name
+    return None
+
+
+def in_mesh(ip: str) -> bool:
+    addr = ipaddress.ip_address(ip)
+    return any(addr in subnet for subnet in MESH_SUBNETS)
+
+
+def is_denied(hostname: str) -> bool:
+    if not DENYLIST.exists():
+        return False
+    return hostname in {line.strip() for line in DENYLIST.read_text().splitlines() if line.strip()}
+
+
+def deny(hostname: str) -> None:
+    DENYLIST.parent.mkdir(parents=True, exist_ok=True)
+    existing = set()
+    if DENYLIST.exists():
+        existing = {line.strip() for line in DENYLIST.read_text().splitlines() if line.strip()}
+    existing.add(hostname)
+    DENYLIST.write_text("\n".join(sorted(existing)) + "\n")
+
+
+def generate_key(hostname: str) -> tuple[Path, str]:
+    """Generate an age keypair for hostname; return (priv_path, pubkey)."""
+    KEYS_DIR.mkdir(parents=True, exist_ok=True)
+    priv = KEYS_DIR / f"{hostname}.key"
+    pub_path = KEYS_DIR / f"{hostname}.pub"
+    proc = subprocess.run(["age-keygen"], capture_output=True, text=True, check=True)
+    priv.write_text(proc.stdout)
+    priv.chmod(0o600)
+    # Extract the public key. age-keygen writes "# public key: ageXXX" line.
+    pubkey = ""
+    for line in proc.stdout.splitlines():
+        if line.startswith("# public key:"):
+            pubkey = line.split(":", 1)[1].strip()
+    if not pubkey:
+        priv.unlink(missing_ok=True)
+        raise RuntimeError("age-keygen did not emit a public key")
+    pub_path.write_text(pubkey + "\n")
+    log.info("generated new age key for %s (pubkey: %s)", hostname, pubkey)
+    return priv, pubkey
+
+
+def existing_pubkey(hostname: str) -> str | None:
+    pub_path = KEYS_DIR / f"{hostname}.pub"
+    if pub_path.exists():
+        return pub_path.read_text().strip()
+    return None
+
+
+def read_admin_token() -> str | None:
+    if ADMIN_TOKEN_PATH.exists():
+        return ADMIN_TOKEN_PATH.read_text().strip()
+    return None
+
+
+class Handler(BaseHTTPRequestHandler):
+    def log_message(self, fmt, *args):
+        log.info("%s - %s", self.address_string(), fmt % args)
+
+    def _reply(self, status: int, body: bytes | str = b"",
+               content_type: str = "text/plain") -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", content_type)
+        if isinstance(body, str):
+            body = body.encode()
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if body:
+            self.wfile.write(body)
+
+    def _read_json(self) -> dict | None:
+        length = int(self.headers.get("Content-Length", "0"))
+        if not length:
+            return {}
+        body = self.rfile.read(length)
+        try:
+            return json.loads(body)
+        except json.JSONDecodeError:
+            return None
+
+    def do_GET(self) -> None:
+        if self.path == "/health":
+            self._reply(200, "ok")
+            return
+        self._reply(404, "not found")
+
+    def do_POST(self) -> None:
+        client_ip = self.client_address[0]
+        if not in_mesh(client_ip):
+            log.warning("rejecting non-mesh source %s", client_ip)
+            self._reply(403, "non-mesh source")
+            return
+
+        if self.path == "/issue":
+            self._handle_issue(client_ip)
+        elif self.path == "/revoke":
+            self._handle_revoke()
+        else:
+            self._reply(404, "not found")
+
+    def _handle_issue(self, client_ip: str) -> None:
+        payload = self._read_json()
+        if payload is None:
+            self._reply(400, "bad json")
+            return
+        hostname = (payload.get("hostname") or "").strip()
+        if not hostname:
+            self._reply(400, "hostname required")
+            return
+
+        # Cross-check: does inventory list this hostname, and does its mesh IP
+        # match the caller (if known)?
+        hosts = known_hosts()
+        if hostname not in hosts:
+            log.warning("unknown hostname in /issue: %s (from %s)", hostname, client_ip)
+            self._reply(403, f"unknown hostname: {hostname} (run 'homelab client add {hostname}' first)")
+            return
+
+        if is_denied(hostname):
+            log.warning("denied hostname %s tried to issue (from %s)", hostname, client_ip)
+            self._reply(403, "hostname on denylist (was removed); operator must clear before re-enrolling")
+            return
+
+        inv_ip = host_by_mesh_ip(client_ip)
+        if inv_ip is not None and inv_ip != hostname:
+            log.warning("source %s maps to inventory host %s but body claims %s",
+                        client_ip, inv_ip, hostname)
+            self._reply(403, "source IP / hostname mismatch")
+            return
+        if inv_ip is None:
+            # Inventory has no IP for this hostname yet (first-bootstrap state).
+            # Accept the call but log it loudly so the operator backfills.
+            log.info("issuing for %s (source %s) — inventory has no mesh IP yet",
+                     hostname, client_ip)
+
+        priv = KEYS_DIR / f"{hostname}.key"
+        existing_pub = existing_pubkey(hostname)
+        if priv.exists() and existing_pub:
+            # Re-issue: client lost its key but is still authorized. Return
+            # the existing key.
+            log.info("returning existing key for %s", hostname)
+            self._reply(200, priv.read_text())
+            return
+
+        # Fresh provisioning.
+        priv, pubkey = generate_key(hostname)
+        # Tell the operator (via response body comment) which pubkey to commit.
+        body = priv.read_text()
+        body += f"\n# operator: add this pubkey to inventory.yaml under hosts.{hostname}.age_pubkey:\n"
+        body += f"# {pubkey}\n"
+        self._reply(200, body)
+
+    def _handle_revoke(self) -> None:
+        token = self.headers.get("X-Admin-Token", "")
+        expected = read_admin_token()
+        if not expected or token != expected:
+            self._reply(403, "admin token required")
+            return
+        payload = self._read_json()
+        if payload is None:
+            self._reply(400, "bad json")
+            return
+        hostname = (payload.get("hostname") or "").strip()
+        if not hostname:
+            self._reply(400, "hostname required")
+            return
+        priv = KEYS_DIR / f"{hostname}.key"
+        pub = KEYS_DIR / f"{hostname}.pub"
+        for p in (priv, pub):
+            if p.exists():
+                # shred-then-unlink
+                subprocess.run(["shred", "-u", str(p)], check=False)
+                if p.exists():
+                    p.unlink(missing_ok=True)
+        deny(hostname)
+        log.warning("revoked %s (shredded keys, added to denylist)", hostname)
+        self._reply(200, f"revoked {hostname}\n")
+
+
+def main() -> int:
+    KEYS_DIR.mkdir(parents=True, exist_ok=True)
+    server = HTTPServer((BIND_HOST, BIND_PORT), Handler)
+    log.info("listening on %s:%d (context=%s, state=%s)",
+             BIND_HOST, BIND_PORT, CONTEXT_DIR, STATE_DIR)
+    server.serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

archive/secrets-issuance/server.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=Homelab secrets-issuance (per-client age key provisioning)
+After=network-online.target homelab-context-sync.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/secrets-issuance
+Environment=HOMELAB_CONTEXT_DIR=/opt/homelab-context
+Environment=SECRETS_ISSUANCE_STATE=/var/lib/secrets-issuance
+Environment=MESH_SUBNETS=100.122.0.0/16,100.64.0.0/10,192.168.8.0/24
+ExecStart=/opt/secrets-issuance/.venv/bin/python /opt/secrets-issuance/secrets-issuance/server.py
+Restart=on-failure
+RestartSec=5
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+NoNewPrivileges=true
+ReadOnlyPaths=/opt/homelab-context /opt/secrets-issuance
+ReadWritePaths=/var/lib/secrets-issuance
+
+[Install]
+WantedBy=multi-user.target