dtoro/oikos
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Phase 1: cross-client homelab context + MCP scaffolding

Browse Source 
Add the foundation for distributing homelab context to every client
(LXCs, VMs, workstations including republic-laptop, mac-mini, ludo-mini)
with a single source of truth, structured query layer (MCP), and per-client
age-key issuance for secrets:

- inventory.yaml — canonical topology (hosts, services, mesh addresses)
- hosts/*.yaml — per-host identity files generated from inventory by
  mcp/build_host_files.py; do not edit by hand
- AGENTS.md — orientation doc symlinked to /root/AGENTS.md on every client
- bootstrap.sh — one-shot enroll (Linux + macOS), clones repo, fetches age
  key from issuance, installs sync timer/launchd job, drops the homelab CLI
- bin/homelab — single-binary Python CLI: whoami, list, ssh, pct, logs,
  restart, open, status, secret, sync, mcp, client add/remove, nuke
- mcp/server.py — FastMCP server: context tools + read-only management
  tools (no mutations exposed); shell-outs use mcp-reader restricted ssh key
- mcp/deploy/ — claudio-monitor-style gitea webhook deploy scaffold for the
  MCP service on LXC 105 (ports 9810 mcp, 9811 webhook)
- secrets-issuance/ — per-client age key auto-provisioning over the mesh;
  source-IP gated against inventory, with denylist for revoked clients
  (ports 9820 issue, 9821 webhook)
- secrets/, .sops.yaml — SOPS recipient scaffolding; the operator fills in
  age public keys after Phase 3a generates them
- scripts/sync/ — systemd timer (Linux) + launchd plist (macOS) pulling
  /opt/homelab-context every 5 min

Mesh: both Netbird (preferred, 100.122.0.0/16) and Tailscale accepted
during the in-flight migration; no client is gated on completing the move.

Plan reference: /root/.claude/plans/lets-make-a-plan-fluttering-trinket.md

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-05-20 15:47:48 +02:00
parent 8f598a0e7a
commit 3c25f936d3
44 changed files with 3180 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
mcp/build_host_files.py Executable file
View File

@@ -0,0 +1,158 @@
#!/usr/bin/env python3
"""
Generate hosts/<name>.yaml from inventory.yaml.
Run from the repo root:
python3 mcp/build_host_files.py # writes files, exits non-zero on diff
python3 mcp/build_host_files.py --check # exits non-zero if any output differs
Designed to be wired into a pre-commit hook or Gitea Action so generated
hosts/*.yaml never drift from inventory.yaml.
"""
from __future__ import annotations
import argparse
import difflib
import os
import sys
from pathlib import Path
try:
import yaml
except ImportError: # pragma: no cover
print("PyYAML is required: pip install pyyaml", file=sys.stderr)
sys.exit(2)
REPO = Path(__file__).resolve().parent.parent
INVENTORY = REPO / "inventory.yaml"
HOSTS_DIR = REPO / "hosts"
GENERATED_BANNER = (
"# Generated by mcp/build_host_files.py from inventory.yaml.\n"
"# Do NOT edit by hand — your changes will be overwritten.\n"
"# Source of truth: ../inventory.yaml\n"
)
def narrative_page(name: str, kind: str, pve_id: int | None) -> str | None:
"""Best-guess path to the human-authored narrative page for this host."""
if kind == "proxmox-host":
candidate = REPO / "hosts" / f"{name}.md"
elif kind == "lxc":
candidate = REPO / "containers" / f"{pve_id}-{name}.md"
elif kind == "vm":
candidate = REPO / "vms" / f"{pve_id}-{name}.md"
else:
return None
if candidate.exists():
return str(candidate.relative_to(REPO))
return None
def build_one(name: str, entry: dict, inventory: dict) -> dict:
"""Project the entry for a single host into a per-host yaml record."""
services = inventory.get("services", {})
mesh = inventory.get("mesh", {})
pve_id = entry.get("pve_id")
# Services this host runs: scan inventory.services for matching backend.
runs_services = sorted(
svc for svc, sentry in services.items()
if isinstance(sentry, dict) and sentry.get("backend") == name
)
record = {
"name": name,
"kind": entry.get("kind"),
"os": entry.get("os"),
"role": entry.get("role"),
"host": entry.get("host"),
"pve_id": pve_id,
"lan_ip": entry.get("lan_ip"),
"mesh": entry.get("mesh", {}),
"mesh_globals": {
"primary": mesh.get("primary"),
"accepted": mesh.get("accepted"),
},
"peers": entry.get("peers", []),
"mounts": entry.get("mounts", []),
"public_host": entry.get("public_host"),
"public_hosts": entry.get("public_hosts", []),
"ssh": entry.get("ssh", {}),
"runs": entry.get("runs", []) + runs_services,
"services_hosted": [
{"name": svc, **services[svc]} for svc in runs_services
],
"notes": entry.get("notes", []),
"age_pubkey": entry.get("age_pubkey", ""),
"see_also": [
page for page in [narrative_page(name, entry.get("kind", ""), pve_id)]
if page
],
"mcp_endpoint": services.get("homelab_mcp", {}).get("endpoint"),
"secrets_issuance_endpoint": (
services.get("secrets_issuance", {}).get("endpoint")
),
}
# Strip None and empty containers so the file stays readable.
return {k: v for k, v in record.items() if v not in (None, {}, [], "")}
def serialize(record: dict) -> str:
return GENERATED_BANNER + yaml.safe_dump(
record, sort_keys=False, default_flow_style=False, width=100
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--check", action="store_true",
help="exit 1 if any output would change (don't write)")
args = parser.parse_args()
inventory = yaml.safe_load(INVENTORY.read_text())
hosts = inventory.get("hosts", {})
HOSTS_DIR.mkdir(exist_ok=True)
desired: dict[Path, str] = {}
for name, entry in hosts.items():
desired[HOSTS_DIR / f"{name}.yaml"] = serialize(build_one(name, entry, inventory))
diff_count = 0
for path, content in desired.items():
existing = path.read_text() if path.exists() else ""
if existing != content:
diff_count += 1
if args.check:
diff = difflib.unified_diff(
existing.splitlines(keepends=True),
content.splitlines(keepends=True),
fromfile=str(path),
tofile=str(path) + " (generated)",
)
sys.stdout.writelines(diff)
else:
path.write_text(content)
print(f"wrote {path.relative_to(REPO)}")
# Clean up orphans (file exists but host removed from inventory).
for existing_path in HOSTS_DIR.glob("*.yaml"):
if existing_path not in desired:
diff_count += 1
if args.check:
print(f"orphan: {existing_path.relative_to(REPO)} (would delete)")
else:
existing_path.unlink()
print(f"deleted orphan {existing_path.relative_to(REPO)}")
if args.check and diff_count > 0:
print(f"\n{diff_count} file(s) would change. Run without --check to write.",
file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

46
mcp/deploy/deploy.sh Executable file
View File

@@ -0,0 +1,46 @@
#!/bin/bash
# Install/update homelab-mcp on LXC 105 (apps). Idempotent.
# Triggered by the gitea webhook or run by hand.
set -euo pipefail
REPO_DIR=${REPO_DIR:-/opt/homelab-mcp}
cd "$REPO_DIR"
echo "[deploy] git pull"
git pull --ff-only
echo "[deploy] ensure python venv + deps"
if [ ! -d /opt/homelab-mcp/.venv ]; then
python3 -m venv /opt/homelab-mcp/.venv
fi
/opt/homelab-mcp/.venv/bin/pip install --quiet --upgrade pip
/opt/homelab-mcp/.venv/bin/pip install --quiet "mcp[cli]" pyyaml
echo "[deploy] install systemd units"
install -m 644 mcp/deploy/homelab-mcp.service \
/etc/systemd/system/homelab-mcp.service
install -m 644 mcp/deploy/webhook/homelab-mcp-deploy.service \
/etc/systemd/system/homelab-mcp-deploy.service
echo "[deploy] context clone for the MCP server"
# The MCP server reads from a local clone of Homelab-Docs at /opt/homelab-context
# (the same path every client uses). Bootstrap should already have created this;
# if not, fail loudly.
if [ ! -d /opt/homelab-context/.git ]; then
echo " /opt/homelab-context is not a git clone — run bootstrap.sh first." >&2
exit 1
fi
systemctl daemon-reload
if systemctl is-active --quiet homelab-mcp.service; then
systemctl restart homelab-mcp.service
fi
if systemctl is-active --quiet homelab-mcp-deploy.service; then
systemctl restart homelab-mcp-deploy.service
fi
echo "[deploy] done"
echo "First-time enable:"
echo " systemctl enable --now homelab-mcp.service homelab-mcp-deploy.service"
echo "Webhook first-time setup (generates secret):"
echo " $REPO_DIR/mcp/deploy/webhook/install.sh"

24
mcp/deploy/homelab-mcp.service Normal file
View File

@@ -0,0 +1,24 @@
[Unit]
Description=Homelab MCP server (read-only context + management)
After=network-online.target homelab-context-sync.service
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/opt/homelab-context
Environment=HOMELAB_CONTEXT_DIR=/opt/homelab-context
Environment=HOMELAB_MCP_SSH_KEY=/etc/homelab-mcp/mcp-reader.key
Environment=HOMELAB_MCP_SSH_USER=mcp-reader
ExecStart=/opt/homelab-mcp/.venv/bin/python /opt/homelab-mcp/mcp/server.py
Restart=on-failure
RestartSec=5
# Stay confined.
ProtectSystem=strict
ProtectHome=true
PrivateTmp=true
NoNewPrivileges=true
ReadOnlyPaths=/opt/homelab-context /opt/homelab-mcp
ReadWritePaths=/var/log/homelab-mcp
[Install]
WantedBy=multi-user.target

13
mcp/deploy/webhook/homelab-mcp-deploy.service Normal file
View File

@@ -0,0 +1,13 @@
[Unit]
Description=Gitea deploy webhook for dtoro/Homelab-Docs → homelab-mcp
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart=/usr/bin/python3 /opt/homelab-mcp/mcp/deploy/webhook/webhook.py
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target

31
mcp/deploy/webhook/install.sh Executable file
View File

@@ -0,0 +1,31 @@
#!/bin/bash
# First-time setup for the homelab-mcp deploy webhook. Generates a secret,
# installs the systemd unit, and starts it. Re-run is safe (won't regenerate
# the secret if one exists).
set -euo pipefail
SECRET_DIR=/etc/homelab-mcp-deploy
SECRET=$SECRET_DIR/secret
UNIT=homelab-mcp-deploy.service
install -d -m 700 "$SECRET_DIR"
if [ ! -s "$SECRET" ]; then
head -c 32 /dev/urandom | base64 > "$SECRET"
chmod 600 "$SECRET"
echo "[install] generated webhook secret at $SECRET"
fi
systemctl daemon-reload
systemctl enable --now "$UNIT"
systemctl status "$UNIT" --no-pager | head -10
cat <<EOF
[install] webhook listening on :9811/deploy.
Configure Gitea (dtoro/Homelab-Docs → Settings → Webhooks → Add Webhook → Gitea):
Target URL: http://<lxc-105-mesh-ip>:9811/deploy
HTTP Method: POST
Content-Type: application/json
Secret: $(cat $SECRET)
Trigger: Push events
EOF

95
mcp/deploy/webhook/webhook.py Executable file
View File

@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""Deploy webhook for dtoro/Homelab-Docs on LXC 105 (apps).
Listens on 0.0.0.0:9811/deploy. Validates Gitea HMAC, runs deploy.sh.
Port 9811: claudio-bot-deploy=9797, backup-library-deploy=9798,
claudio-monitor=9799, homelab-mcp=9811, secrets-issuance=9821.
"""
from __future__ import annotations
import hashlib
import hmac
import json
import logging
import os
import subprocess
import sys
import threading
from http.server import BaseHTTPRequestHandler, HTTPServer
BIND_HOST = "0.0.0.0"
BIND_PORT = 9811
SECRET_PATH = "/etc/homelab-mcp-deploy/secret"
DEPLOY_CMD = ["/opt/homelab-mcp/mcp/deploy/deploy.sh"]
TARGET_REF = "refs/heads/main"
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("homelab-mcp-deploy")
_deploy_lock = threading.Lock()
def load_secret() -> bytes:
with open(SECRET_PATH, "rb") as f:
return f.read().strip()
class Handler(BaseHTTPRequestHandler):
def log_message(self, fmt, *args):
log.info("%s - %s", self.address_string(), fmt % args)
def _reply(self, status: int, msg: str = "") -> None:
self.send_response(status)
self.send_header("Content-Type", "text/plain")
self.end_headers()
if msg:
self.wfile.write(msg.encode())
def do_POST(self) -> None:
if self.path != "/deploy":
self._reply(404, "not found")
return
length = int(self.headers.get("Content-Length", "0"))
body = self.rfile.read(length) if length else b""
sig_header = self.headers.get("X-Gitea-Signature", "")
secret = load_secret()
expected = hmac.new(secret, body, hashlib.sha256).hexdigest()
if not hmac.compare_digest(expected, sig_header):
log.warning("signature mismatch")
self._reply(403, "bad signature")
return
try:
payload = json.loads(body)
except json.JSONDecodeError:
self._reply(400, "bad json")
return
if payload.get("ref", "") != TARGET_REF:
self._reply(204)
return
if not _deploy_lock.acquire(blocking=False):
self._reply(202, "already running")
return
try:
log.info("running deploy: %s", DEPLOY_CMD)
result = subprocess.run(DEPLOY_CMD, capture_output=True, text=True, timeout=180)
if result.returncode != 0:
log.error("deploy failed: %s\n%s", result.stdout, result.stderr)
self._reply(500, "deploy failed")
return
self._reply(204)
finally:
_deploy_lock.release()
def main() -> int:
if not os.path.exists(SECRET_PATH):
log.error("secret file missing: %s", SECRET_PATH)
return 1
server = HTTPServer((BIND_HOST, BIND_PORT), Handler)
log.info("listening on %s:%d", BIND_HOST, BIND_PORT)
server.serve_forever()
return 0
if __name__ == "__main__":
sys.exit(main())

310
mcp/server.py Executable file
View File

@@ -0,0 +1,310 @@
#!/usr/bin/env python3
"""
Homelab MCP server.
Reads the canonical state from /opt/homelab-context/ (a git clone of
dtoro/Homelab-Docs) and exposes structured tools to any MCP-capable agent.
Two tool groups:
- Context (pure read of the clone, no shell-outs)
- Management (read-only ssh/pct/systemctl/curl, NO mutations)
Mutations live in the `homelab` CLI on each client, behind operator
confirmation. The MCP server never restarts, edits, or executes arbitrary
commands.
"""
from __future__ import annotations
import os
import re
import shlex
import subprocess
from functools import lru_cache
from pathlib import Path
from typing import Any
import yaml
from mcp.server.fastmcp import FastMCP
CONTEXT_DIR = Path(os.environ.get("HOMELAB_CONTEXT_DIR", "/opt/homelab-context"))
INVENTORY = CONTEXT_DIR / "inventory.yaml"
HOSTS_DIR = CONTEXT_DIR / "hosts"
SSH_IDENTITY = os.environ.get("HOMELAB_MCP_SSH_KEY", "/etc/homelab-mcp/mcp-reader.key")
SSH_USER = os.environ.get("HOMELAB_MCP_SSH_USER", "mcp-reader")
SSH_TIMEOUT = int(os.environ.get("HOMELAB_MCP_SSH_TIMEOUT", "10"))
mcp = FastMCP("homelab")
def _load_inventory() -> dict:
if not INVENTORY.exists():
raise RuntimeError(f"inventory not found: {INVENTORY}")
return yaml.safe_load(INVENTORY.read_text())
@lru_cache(maxsize=1)
def _inventory_cache_key() -> int:
"""Cache buster keyed on mtime so we re-read after each sync pull."""
return INVENTORY.stat().st_mtime_ns
def inventory() -> dict:
_inventory_cache_key.cache_clear()
_ = _inventory_cache_key() # warm
return _load_inventory()
def _host_ssh_target(name: str) -> str:
"""Return the ssh target (user@host) the MCP reader uses for `name`."""
inv = inventory()
h = inv.get("hosts", {}).get(name)
if h is None:
raise ValueError(f"unknown host: {name}")
mesh = h.get("mesh", {})
# Netbird FQDN if available, then lan_ip, then tailscale.
target = (
mesh.get("netbird", {}).get("fqdn")
or mesh.get("netbird", {}).get("ip")
or h.get("lan_ip")
or mesh.get("tailscale", {}).get("fqdn")
)
if not target:
raise ValueError(f"no reachable address for host {name}")
return f"{SSH_USER}@{target}"
def _ssh(host: str, *cmd: str, timeout: int | None = None) -> subprocess.CompletedProcess:
"""Run a command on a remote host as the restricted mcp-reader user."""
target = _host_ssh_target(host)
full = [
"ssh", "-i", SSH_IDENTITY, "-o", "BatchMode=yes",
"-o", "StrictHostKeyChecking=accept-new",
"-o", f"ConnectTimeout={SSH_TIMEOUT}",
target, *cmd,
]
return subprocess.run(full, capture_output=True, text=True,
timeout=timeout or SSH_TIMEOUT * 3)
def _service_to_host(service: str) -> str:
"""Resolve a service name to its backend host name."""
svc = inventory().get("services", {}).get(service)
if not svc:
raise ValueError(f"unknown service: {service}")
return svc["backend"]
# ---------------------------------------------------------------------------
# Context tools
# ---------------------------------------------------------------------------
@mcp.tool()
def get_host(name: str) -> dict:
"""Return the structured record for a host (LXC, VM, workstation, or hubris)."""
inv = inventory()
entry = inv.get("hosts", {}).get(name)
if entry is None:
raise ValueError(f"unknown host: {name}")
out = {"name": name, **entry}
yaml_path = HOSTS_DIR / f"{name}.yaml"
if yaml_path.exists():
out["host_yaml_path"] = str(yaml_path)
return out
@mcp.tool()
def list_services() -> dict:
"""List every service registered in inventory.yaml."""
return inventory().get("services", {})
@mcp.tool()
def find_service(query: str) -> list[dict]:
"""Find services by name substring, role, or backend host name."""
q = query.lower()
inv = inventory()
out = []
for name, entry in inv.get("services", {}).items():
haystack = " ".join([
name,
str(entry.get("backend", "")),
str(entry.get("role", "")),
str(entry.get("note", "")),
]).lower()
if q in haystack:
out.append({"service": name, **entry})
return out
@mcp.tool()
def get_topology() -> dict:
"""Return the full inventory (hosts + services + mesh)."""
return inventory()
@mcp.tool()
def search_docs(query: str, max_results: int = 10) -> list[dict]:
"""Ripgrep the markdown wiki for query, return file:line hits."""
rg = subprocess.run(
["rg", "--no-heading", "-n", "-i", "--type", "md", "-m", "5", query, str(CONTEXT_DIR)],
capture_output=True, text=True, timeout=10,
)
results = []
for line in rg.stdout.splitlines()[: max_results * 5]:
m = re.match(r"^(.+?):(\d+):(.*)$", line)
if m:
results.append({
"path": str(Path(m.group(1)).relative_to(CONTEXT_DIR)),
"line": int(m.group(2)),
"text": m.group(3).strip(),
})
if len(results) >= max_results:
break
return results
@mcp.tool()
def get_page(path: str) -> str:
"""Return a markdown page verbatim. Path is relative to the repo root."""
full = (CONTEXT_DIR / path).resolve()
if not str(full).startswith(str(CONTEXT_DIR.resolve())):
raise ValueError("path escapes repo")
if not full.exists():
raise ValueError(f"no such page: {path}")
return full.read_text()
@mcp.tool()
def get_changelog(path: str, since: str | None = None) -> str:
"""Extract the trailing ## Changelog section from a page; optionally filter."""
text = get_page(path)
m = re.search(r"^##\s+Changelog\s*\n(.*)", text, re.MULTILINE | re.DOTALL)
if not m:
return ""
body = m.group(1).strip()
if since:
kept = []
for entry in re.split(r"^###\s+", body, flags=re.MULTILINE):
if not entry.strip():
continue
head = entry.split("\n", 1)[0]
date_m = re.match(r"(\d{4}-\d{2}-\d{2})", head)
if not date_m or date_m.group(1) >= since:
kept.append("### " + entry.rstrip())
return "\n\n".join(kept).strip()
return body
@mcp.tool()
def whoami(hostname: str) -> dict:
"""Given a hostname, return that host's full yaml record from hosts/<name>.yaml."""
candidate = HOSTS_DIR / f"{hostname}.yaml"
if not candidate.exists():
raise ValueError(f"no host record for {hostname}")
return yaml.safe_load(candidate.read_text())
@mcp.tool()
def list_my_secrets(caller_pubkey: str) -> list[str]:
"""Return the names of secrets the caller (identified by age pubkey) can decrypt.
Metadata only — the server never returns plaintext. The client decrypts
locally with its own /etc/age/key.txt.
"""
if not caller_pubkey:
return []
secrets_dir = CONTEXT_DIR / "secrets"
if not secrets_dir.exists():
return []
out = []
for path in secrets_dir.glob("*.yaml"):
try:
data = yaml.safe_load(path.read_text()) or {}
except yaml.YAMLError:
continue
recipients = (
data.get("sops", {})
.get("age", [])
)
for r in recipients:
if r.get("recipient") == caller_pubkey:
out.append(path.stem)
break
return sorted(out)
# ---------------------------------------------------------------------------
# Management tools (read-only)
# ---------------------------------------------------------------------------
@mcp.tool()
def get_service_status(service: str) -> dict:
"""systemctl is-active + is-enabled for the named service on its backend host."""
host = _service_to_host(service)
inv_svc = inventory()["services"][service]
unit = inv_svc.get("systemd_unit", service)
active = _ssh(host, "systemctl", "is-active", unit)
enabled = _ssh(host, "systemctl", "is-enabled", unit)
return {
"service": service,
"host": host,
"unit": unit,
"active": active.stdout.strip(),
"enabled": enabled.stdout.strip(),
}
@mcp.tool()
def tail_log(service: str, lines: int = 200) -> str:
"""Last N journalctl lines for the named service on its backend host."""
host = _service_to_host(service)
inv_svc = inventory()["services"][service]
unit = inv_svc.get("systemd_unit", service)
proc = _ssh(host, "journalctl", "-u", unit, "-n", str(lines), "--no-pager")
return proc.stdout
@mcp.tool()
def list_lxcs() -> str:
"""Run `pct list` on hubris."""
return _ssh("hubris", "pct", "list").stdout
@mcp.tool()
def get_lxc_state(lxc: str) -> dict:
"""`pct status` and a quick `pct config` snapshot for the named LXC."""
inv = inventory().get("hosts", {})
if lxc not in inv:
raise ValueError(f"unknown lxc: {lxc}")
pve_id = inv[lxc].get("pve_id")
if pve_id is None:
raise ValueError(f"{lxc} has no pve_id (is it actually an LXC?)")
status = _ssh("hubris", "pct", "status", str(pve_id))
cfg = _ssh("hubris", "pct", "config", str(pve_id))
return {"lxc": lxc, "pve_id": pve_id,
"status": status.stdout.strip(),
"config": cfg.stdout}
@mcp.tool()
def ping_service(service: str) -> dict:
"""HTTP check against the service's URL (from inventory)."""
inv_svc = inventory().get("services", {}).get(service)
if not inv_svc:
raise ValueError(f"unknown service: {service}")
url = inv_svc.get("url") or inv_svc.get("endpoint") or inv_svc.get("backend_url")
if not url:
return {"service": service, "ok": False, "reason": "no URL in inventory"}
proc = subprocess.run(
["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code} %{time_total}",
"--max-time", "5", url],
capture_output=True, text=True,
)
code, t = (proc.stdout.strip().split() + ["", ""])[:2]
return {"service": service, "url": url, "http_code": code, "time_s": t,
"ok": code.startswith("2") or code.startswith("3")}
if __name__ == "__main__":
mcp.run(transport="sse")