docs: add client lifecycle plan, cleanup stale files, document repo for 3 audiences

Problem: Repo had no developer guide, no client onboarding doc, no agent
dev instructions. Stale files (675KB SQL dump, one-off convert script,
legacy MCP builder) cluttered the tree. Client enrollment was a documented
intention with no Go implementation.

Changes:
- New docs: CONTRIBUTING.md (dev setup), CLIENTS.md (client onboarding),
  .agents/dev/CONTRIBUTING.md (agent codebase map)
- New plan: plans/2026-07-07-client-lifecycle-in-go.md — full client
  lifecycle (planned→provisioning→active→deprecated→destroyed) in Go,
  replacing archived Python secrets-issuance, adding client API endpoints
  and 6 missing MCP tools
- Cleanup: deleted archive/convert-wiki.py (one-off), archive/mcp/
  build_host_files.py (legacy), backups/pre-deploy-7f7d039.sql (local)
- Fixes: plans/index.md duplicate row removed, README.md repo layout
  updated for current state, AGENTS.md header points to new guides

Risk: low. Docs only + stale file deletion. No code changes. New plan is
proposal, not implementation.
Verification: git diff reviewed, all changes are prose/docs/plans.
This commit is contained in:
2026-07-07 23:45:32 +02:00
parent 85b541a1cc
commit 638e313c66
9 changed files with 921 additions and 563 deletions

View File

@@ -1,392 +0,0 @@
#!/usr/bin/env python3
"""One-shot: convert knowledge/wiki/ to seeds/knowledge.yaml."""
import os, re, yaml
from pathlib import Path
from hashlib import sha256
REPO = Path("/Users/dtoro/Projects/oikos")
WIKI = REPO / "archive" / "knowledge"
SOURCES = REPO / "archive" / "knowledge"
GLOSSARY = REPO / "archive" / "knowledge" / "GLOSSARY.md"
# Maps wiki path components to entity slugs
# Format: (path_pattern, entity_slug)
PATH_TO_ENTITY = {
# Containers
"containers/101-jellyfin": "lxc:jellyfin",
"containers/102-nfs-export": "lxc:nfs-export",
"containers/103-paperless": "lxc:paperless",
"containers/104-gitea": "lxc:gitea",
"containers/105-apps": "lxc:apps",
"containers/106-auth-outpost": "lxc:auth-outpost",
"containers/107-dns": "lxc:dns",
"containers/114-nextcloud": "lxc:nextcloud",
"containers/118-elementsynapse": "lxc:elementsynapse",
"containers/119-sophia": "lxc:sophia",
"containers/120-mule-images": "lxc:mule-images",
"containers/121-caddy": "lxc:caddy",
"containers/122-arriman": "lxc:arriman",
"containers/128-trmnl": "lxc:trmnl",
"containers/129-house": "lxc:house",
"containers/130-grimmory": "lxc:grimmory",
"containers/131-teddycloud": "lxc:teddycloud",
"containers/132-rclone": "lxc:rclone",
"containers/133-seanime": "lxc:seanime",
"containers/134-romm": "lxc:romm",
# Hosts
"hosts/hubris": "host:hubris",
"hosts/strong": "host:strong",
# VMs
"vms/100-zimaos": "vm:zimaos",
"vms/108-haos": "vm:haos",
# Infrastructure → services
"infrastructure/auto-deploy": None,
"infrastructure/backups": None,
"infrastructure/dns": "service:dns",
"infrastructure/homelab-context": "service:homelab-mcp",
"infrastructure/ingress": "service:caddy",
"infrastructure/media-permissions": "service:jellyfin",
"infrastructure/mesh": None,
"infrastructure/monitoring": None,
"infrastructure/network": None,
"infrastructure/ssh-access": None,
"infrastructure/topology": None,
"infrastructure/vps-hardening": "host:netbird-vps",
}
def parse_page(path):
"""Parse a wiki page into structured sections."""
if not path.exists():
return None
text = path.read_text()
lines = text.split('\n')
# Title is first H1
title = ""
for line in lines:
if line.startswith('# ') and not line.startswith('## '):
title = line[2:].strip()
break
# Find sections by H2 headings
sections = {}
current_heading = "_preamble"
current_content = []
for line in lines:
if line.startswith('## ') and not line.startswith('### '):
if current_content:
sections[current_heading] = '\n'.join(current_content).strip()
current_heading = line[3:].strip().lower()
current_content = []
else:
current_content.append(line)
if current_content:
sections[current_heading] = '\n'.join(current_content).strip()
# Parse at-a-glance
at_glance = {}
ag_text = sections.get('at a glance', '')
for line in ag_text.split('\n'):
line = line.strip()
# Strip leading bullet
line = re.sub(r'^[-*]\s+', '', line)
# Match **Key:** value or **Key Word:** value
m = re.match(r'\*\*([^*]+?):?\*\*\s+(.+)', line)
if not m:
m = re.match(r'([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*):\s+(.+)', line)
if m:
key = m.group(1).lower().strip().replace(' ', '_').replace('/', '_')
val = m.group(2).strip()
# Strip trailing parenthetical notes
val = re.sub(r'\s*\([^)]*\)$', '', val)
# Strip markdown formatting from value
val = re.sub(r'\*\*([^*]+)\*\*', r'\1', val)
val = re.sub(r'`([^`]+)`', r'\1', val)
# Simplify link text
val = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', val)
val = re.sub(r'', '', val).strip()
# Normalize keys
key_map = {
'cores': 'cores', 'core': 'cores',
'ram': 'ram', 'memory': 'ram',
'mounts': 'mounts', 'mount': 'mounts',
'host': 'host', 'ip': 'ip',
'public_host': 'public_host', 'public_hostname': 'public_host',
'lan_ip': 'lan_ip',
'os': 'os', 'kind': 'kind',
'runtime': 'runtime', 'role': 'role',
'pve_id': 'pve_id', 'privilege': 'privileged',
'resources': 'resources', 'gpu': 'gpu',
'swap': 'swap', 'rootfs': 'rootfs',
'version': 'version', 'hardware': 'hardware',
}
key = key_map.get(key, key)
at_glance[key] = val
# Parse changelog
changelog = []
cl_text = sections.get('changelog', '')
current_entry = None
for line in cl_text.split('\n'):
m = re.match(r'###\s+(\d{4}-\d{2}-\d{2})\s+[—–-]\s+(.+)', line)
if m:
if current_entry:
changelog.append(current_entry)
current_entry = {'date': m.group(1), 'title': m.group(2).strip(), 'body': ''}
elif current_entry is not None:
stripped = line.strip()
if stripped and not stripped.startswith('#'):
if current_entry['body']:
current_entry['body'] += ' '
current_entry['body'] += stripped
if current_entry:
changelog.append(current_entry)
# Tags from path
parts = path.relative_to(REPO).parts
tags = []
if 'containers' in parts:
tags.append('container')
elif 'hosts' in parts:
tags.append('host')
elif 'vms' in parts:
tags.append('vm')
elif 'infrastructure' in parts:
tags.append('infrastructure')
# Determine slug from relative path
rel = str(path.relative_to(WIKI))
slug = rel.replace('.md', '')
# Entity mapping
entity_slug = PATH_TO_ENTITY.get(slug, None)
return {
'slug': slug,
'title': title,
'content': text,
'entity_slug': entity_slug,
'tags': tags,
'at_glance': at_glance,
'changelog': changelog,
'is_investigation': 'investigations' in rel,
}
def parse_investigation(path):
"""Parse an investigation page."""
if not path.exists():
return None
text = path.read_text()
lines = text.split('\n')
title = ""
for line in lines:
if line.startswith('# '):
title = line[2:].strip()
break
# Extract date from title or filename
date = ""
status = "resolved"
duration = ""
for line in lines[:30]:
m = re.search(r'(\d{4}-\d{2}-\d{2})', line)
if m:
date = m.group(1)
break
for line in lines:
if '**Status:**' in line:
status = line.split('**Status:**')[-1].strip().lower()
if '**Duration:**' in line:
duration = line.split('**Duration:**')[-1].strip()
# Extract entity references for about_slugs
about_slugs = []
entity_patterns = [
(r'\bcaddy\b', 'service:caddy'),
(r'\bauthentik\b', 'service:authentik'),
(r'\bdns\b', 'service:dns'),
(r'\bgitea\b', 'service:gitea'),
(r'\bjellyfin\b', 'service:jellyfin'),
(r'\bmatrix\b', 'service:matrix'),
(r'\bpaperless\b', 'service:paperless'),
(r'\bnextcloud\b', 'service:nextcloud'),
(r'\bartifacto\b', 'service:artifacto'),
(r'\barriman\b', 'lxc:arriman'),
(r'\btrmnl\b', 'service:trmnl'),
(r'\bmac-mini\b', 'ws:mac-mini'),
(r'\bhubris\b', 'host:hubris'),
(r'\bstrong\b', 'host:strong'),
]
for pattern, slug in entity_patterns:
if re.search(pattern, text, re.IGNORECASE):
about_slugs.append(slug)
rel = str(path.relative_to(WIKI))
slug = rel.replace('.md', '')
return {
'slug': slug,
'title': title,
'date': date,
'status': status,
'duration': duration,
'content': text,
'about_slugs': about_slugs,
'tags': ['investigation'],
}
def main():
documents = []
investigations = []
runbooks = []
# Container pages
containers_dir = WIKI / "containers"
for f in sorted(containers_dir.glob("*.md")):
if 'index' in f.name:
continue
if f.parent.name == 'archive':
continue
result = parse_page(f)
if result and result['title']:
documents.append(result)
print(f" document: {result['slug']}{result['entity_slug']}")
# Host pages
hosts_dir = WIKI / "hosts"
for f in sorted(hosts_dir.glob("*.md")):
if 'index' in f.name:
continue
result = parse_page(f)
if result and result['title']:
documents.append(result)
print(f" document: {result['slug']}{result['entity_slug']}")
# VM pages
vms_dir = WIKI / "vms"
for f in sorted(vms_dir.glob("*.md")):
if 'index' in f.name:
continue
result = parse_page(f)
if result and result['title']:
documents.append(result)
print(f" document: {result['slug']}{result['entity_slug']}")
# Infrastructure pages
infra_dir = WIKI / "infrastructure"
for f in sorted(infra_dir.glob("*.md")):
if 'index' in f.name:
continue
result = parse_page(f)
if result and result['title']:
documents.append(result)
print(f" document: {result['slug']}{result['entity_slug']}")
# Investigation pages
inv_dir = SOURCES / "investigations"
for f in sorted(inv_dir.glob("*.md")):
if 'index' in f.name:
continue
result = parse_investigation(f)
if result and result['title']:
investigations.append(result)
print(f" investigation: {result['slug']}{result['about_slugs']}")
# Archive investigations too
inv_archive = inv_dir / "archive"
if inv_archive.exists():
for f in sorted(inv_archive.glob("*.md")):
result = parse_investigation(f)
if result and result['title']:
investigations.append(result)
print(f" investigation: {result['slug']}{result['about_slugs']}")
# Runbooks from .agents/skills/
skills_dir = REPO / ".agents" / "skills"
for skill_dir in sorted(skills_dir.iterdir()):
if not skill_dir.is_dir():
continue
skill_file = skill_dir / "SKILL.md"
if not skill_file.exists():
continue
text = skill_file.read_text()
lines = text.split('\n')
title = ""
for line in lines:
if line.startswith('# '):
title = line[2:].strip()
break
# Extract risk_class and entity_type from frontmatter
risk_class = "read_only"
entity_type = "service"
for line in lines[:30]:
m = re.match(r'\*\*risk_class:\*\*\s*(\w+)', line, re.IGNORECASE)
if m:
risk_class = m.group(1)
m = re.match(r'\*\*applies_to:\*\*\s*(\w[\w-]*)', line, re.IGNORECASE)
if m:
entity_type = m.group(1)
name = skill_dir.name
runbooks.append({
'slug': name,
'name': title or name,
'risk_class': risk_class,
'entity_type': entity_type,
'procedure': {}, # SKILL.md is narrative, not structured yet
'content': text,
'tags': ['skill', 'runbook'],
})
print(f" runbook: {name}")
# Build seed YAML
seed = {
'version': 1,
'documents': [{
'slug': d['slug'],
'title': d['title'],
'content': d['content'],
'entity_slug': d['entity_slug'],
'tags': d['tags'],
'at_glance': d['at_glance'],
'changelog': d['changelog'],
} for d in documents],
'investigations': [{
'slug': i['slug'],
'title': i['title'],
'date': i['date'],
'status': i['status'],
'duration': i['duration'],
'content': i['content'],
'about_slugs': i['about_slugs'],
'tags': i['tags'],
} for i in investigations],
'runbooks': [{
'slug': r['slug'],
'name': r['name'],
'risk_class': r['risk_class'],
'entity_type': r['entity_type'],
'procedure': r['procedure'],
'content': r['content'],
'tags': r['tags'],
} for r in runbooks],
}
out_path = REPO / "seeds" / "knowledge.yaml"
out_path.write_text(yaml.dump(seed, allow_unicode=True, width=120, sort_keys=False))
print(f"\nWrote {out_path}")
print(f" {len(documents)} documents")
print(f" {len(investigations)} investigations")
print(f" {len(runbooks)} runbooks")
if __name__ == "__main__":
main()

View File

@@ -1,162 +0,0 @@
#!/usr/bin/env python3
"""
Generate hosts/<name>.yaml from inventory.yaml.
Run from the repo root:
python3 mcp/build_host_files.py # writes files, exits non-zero on diff
python3 mcp/build_host_files.py --check # exits non-zero if any output differs
Designed to be wired into a pre-commit hook or Gitea Action so generated
hosts/*.yaml never drift from inventory.yaml.
"""
from __future__ import annotations
import argparse
import difflib
import os
import sys
from pathlib import Path
try:
import yaml
except ImportError: # pragma: no cover
print("PyYAML is required: pip install pyyaml", file=sys.stderr)
sys.exit(2)
REPO = Path(__file__).resolve().parent.parent
INVENTORY = REPO / "inventory.yaml"
HOSTS_DIR = REPO / "hosts"
GENERATED_BANNER = (
"# Generated by mcp/build_host_files.py from inventory.yaml.\n"
"# Do NOT edit by hand — your changes will be overwritten.\n"
"# Source of truth: ../inventory.yaml\n"
)
def narrative_page(name: str, kind: str, pve_id: int | None) -> str | None:
"""Best-guess path to the human-authored narrative page for this host."""
if kind == "proxmox-host":
candidate = REPO / "hosts" / f"{name}.md"
elif kind == "lxc":
candidate = REPO / "containers" / f"{pve_id}-{name}.md"
elif kind == "vm":
candidate = REPO / "vms" / f"{pve_id}-{name}.md"
else:
return None
if candidate.exists():
return str(candidate.relative_to(REPO))
return None
def build_one(name: str, entry: dict, inventory: dict) -> dict:
"""Project the entry for a single host into a per-host yaml record."""
services = inventory.get("services", {})
mesh = inventory.get("mesh", {})
pve_id = entry.get("pve_id")
# Services this host runs: scan inventory.services for matching backend.
runs_services = sorted(
svc for svc, sentry in services.items()
if isinstance(sentry, dict) and sentry.get("backend") == name
)
record = {
"name": name,
"kind": entry.get("kind"),
"os": entry.get("os"),
"role": entry.get("role"),
# Oikos lifecycle (oikos/ontology.yaml); absent in inventory = active
"state": entry.get("state", "active"),
"host": entry.get("host"),
"pve_id": pve_id,
"storage": entry.get("storage"),
"depends_on": entry.get("depends_on", []),
"lan_ip": entry.get("lan_ip"),
"mesh": entry.get("mesh", {}),
"mesh_globals": {
"primary": mesh.get("primary"),
"accepted": mesh.get("accepted"),
},
"peers": entry.get("peers", []),
"mounts": entry.get("mounts", []),
"public_host": entry.get("public_host"),
"public_hosts": entry.get("public_hosts", []),
"ssh": entry.get("ssh", {}),
"runs": entry.get("runs", []) + runs_services,
"services_hosted": [
{"name": svc, **services[svc]} for svc in runs_services
],
"notes": entry.get("notes", []),
"age_pubkey": entry.get("age_pubkey", ""),
"see_also": [
page for page in [narrative_page(name, entry.get("kind", ""), pve_id)]
if page
],
"mcp_endpoint": services.get("homelab_mcp", {}).get("endpoint"),
"secrets_issuance_endpoint": (
services.get("secrets_issuance", {}).get("endpoint")
),
}
# Strip None and empty containers so the file stays readable.
return {k: v for k, v in record.items() if v not in (None, {}, [], "")}
def serialize(record: dict) -> str:
return GENERATED_BANNER + yaml.safe_dump(
record, sort_keys=False, default_flow_style=False, width=100
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--check", action="store_true",
help="exit 1 if any output would change (don't write)")
args = parser.parse_args()
inventory = yaml.safe_load(INVENTORY.read_text())
hosts = inventory.get("hosts", {})
HOSTS_DIR.mkdir(exist_ok=True)
desired: dict[Path, str] = {}
for name, entry in hosts.items():
desired[HOSTS_DIR / f"{name}.yaml"] = serialize(build_one(name, entry, inventory))
diff_count = 0
for path, content in desired.items():
existing = path.read_text() if path.exists() else ""
if existing != content:
diff_count += 1
if args.check:
diff = difflib.unified_diff(
existing.splitlines(keepends=True),
content.splitlines(keepends=True),
fromfile=str(path),
tofile=str(path) + " (generated)",
)
sys.stdout.writelines(diff)
else:
path.write_text(content)
print(f"wrote {path.relative_to(REPO)}")
# Clean up orphans (file exists but host removed from inventory).
for existing_path in HOSTS_DIR.glob("*.yaml"):
if existing_path not in desired:
diff_count += 1
if args.check:
print(f"orphan: {existing_path.relative_to(REPO)} (would delete)")
else:
existing_path.unlink()
print(f"deleted orphan {existing_path.relative_to(REPO)}")
if args.check and diff_count > 0:
print(f"\n{diff_count} file(s) would change. Run without --check to write.",
file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())