dtoro
5e3b946ded
- Rewrite AGENTS.md: DB as source of truth, MCP knowledge tools, archive refs - Fix OIKOS.md: seeds/ paths, remove Python-era notes, update deployment status - Fix commands.md, agent-enrollment.md: archive/knowledge/ links - Fix all SKILL.md files: remove hosts/*.yaml refs, point to inventory.yaml - Fix HERMES.md, schema.md, page-templates.md, llm-wiki.md: update paths - Fix bootstrap.sh: identity check reads inventory.yaml - Fix README.md, cutover-checklist.md: stale wiki references - Move convert-wiki.py to archive/ (one-shot done)
392 lines
13 KiB
Python
392 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""One-shot: convert knowledge/wiki/ to seeds/knowledge.yaml."""
|
|
|
|
import os, re, yaml
|
|
from pathlib import Path
|
|
from hashlib import sha256
|
|
|
|
REPO = Path("/Users/dtoro/Projects/oikos")
|
|
WIKI = REPO / "archive" / "knowledge"
|
|
SOURCES = REPO / "archive" / "knowledge"
|
|
GLOSSARY = REPO / "archive" / "knowledge" / "GLOSSARY.md"
|
|
|
|
# Maps wiki path components to entity slugs
|
|
# Format: (path_pattern, entity_slug)
|
|
PATH_TO_ENTITY = {
|
|
# Containers
|
|
"containers/101-jellyfin": "lxc:jellyfin",
|
|
"containers/102-nfs-export": "lxc:nfs-export",
|
|
"containers/103-paperless": "lxc:paperless",
|
|
"containers/104-gitea": "lxc:gitea",
|
|
"containers/105-apps": "lxc:apps",
|
|
"containers/106-auth-outpost": "lxc:auth-outpost",
|
|
"containers/107-dns": "lxc:dns",
|
|
"containers/114-nextcloud": "lxc:nextcloud",
|
|
"containers/118-elementsynapse": "lxc:elementsynapse",
|
|
"containers/119-sophia": "lxc:sophia",
|
|
"containers/120-mule-images": "lxc:mule-images",
|
|
"containers/121-caddy": "lxc:caddy",
|
|
"containers/122-arriman": "lxc:arriman",
|
|
"containers/128-trmnl": "lxc:trmnl",
|
|
"containers/129-house": "lxc:house",
|
|
"containers/130-grimmory": "lxc:grimmory",
|
|
"containers/131-teddycloud": "lxc:teddycloud",
|
|
"containers/132-rclone": "lxc:rclone",
|
|
"containers/133-seanime": "lxc:seanime",
|
|
"containers/134-romm": "lxc:romm",
|
|
# Hosts
|
|
"hosts/hubris": "host:hubris",
|
|
"hosts/strong": "host:strong",
|
|
# VMs
|
|
"vms/100-zimaos": "vm:zimaos",
|
|
"vms/108-haos": "vm:haos",
|
|
# Infrastructure → services
|
|
"infrastructure/auto-deploy": None,
|
|
"infrastructure/backups": None,
|
|
"infrastructure/dns": "service:dns",
|
|
"infrastructure/homelab-context": "service:homelab-mcp",
|
|
"infrastructure/ingress": "service:caddy",
|
|
"infrastructure/media-permissions": "service:jellyfin",
|
|
"infrastructure/mesh": None,
|
|
"infrastructure/monitoring": None,
|
|
"infrastructure/network": None,
|
|
"infrastructure/ssh-access": None,
|
|
"infrastructure/topology": None,
|
|
"infrastructure/vps-hardening": "host:netbird-vps",
|
|
}
|
|
|
|
def parse_page(path):
|
|
"""Parse a wiki page into structured sections."""
|
|
if not path.exists():
|
|
return None
|
|
text = path.read_text()
|
|
lines = text.split('\n')
|
|
|
|
# Title is first H1
|
|
title = ""
|
|
for line in lines:
|
|
if line.startswith('# ') and not line.startswith('## '):
|
|
title = line[2:].strip()
|
|
break
|
|
|
|
# Find sections by H2 headings
|
|
sections = {}
|
|
current_heading = "_preamble"
|
|
current_content = []
|
|
|
|
for line in lines:
|
|
if line.startswith('## ') and not line.startswith('### '):
|
|
if current_content:
|
|
sections[current_heading] = '\n'.join(current_content).strip()
|
|
current_heading = line[3:].strip().lower()
|
|
current_content = []
|
|
else:
|
|
current_content.append(line)
|
|
if current_content:
|
|
sections[current_heading] = '\n'.join(current_content).strip()
|
|
|
|
# Parse at-a-glance
|
|
at_glance = {}
|
|
ag_text = sections.get('at a glance', '')
|
|
for line in ag_text.split('\n'):
|
|
line = line.strip()
|
|
# Strip leading bullet
|
|
line = re.sub(r'^[-*]\s+', '', line)
|
|
# Match **Key:** value or **Key Word:** value
|
|
m = re.match(r'\*\*([^*]+?):?\*\*\s+(.+)', line)
|
|
if not m:
|
|
m = re.match(r'([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*):\s+(.+)', line)
|
|
if m:
|
|
key = m.group(1).lower().strip().replace(' ', '_').replace('/', '_')
|
|
val = m.group(2).strip()
|
|
# Strip trailing parenthetical notes
|
|
val = re.sub(r'\s*\([^)]*\)$', '', val)
|
|
# Strip markdown formatting from value
|
|
val = re.sub(r'\*\*([^*]+)\*\*', r'\1', val)
|
|
val = re.sub(r'`([^`]+)`', r'\1', val)
|
|
# Simplify link text
|
|
val = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', val)
|
|
val = re.sub(r'↔', '', val).strip()
|
|
# Normalize keys
|
|
key_map = {
|
|
'cores': 'cores', 'core': 'cores',
|
|
'ram': 'ram', 'memory': 'ram',
|
|
'mounts': 'mounts', 'mount': 'mounts',
|
|
'host': 'host', 'ip': 'ip',
|
|
'public_host': 'public_host', 'public_hostname': 'public_host',
|
|
'lan_ip': 'lan_ip',
|
|
'os': 'os', 'kind': 'kind',
|
|
'runtime': 'runtime', 'role': 'role',
|
|
'pve_id': 'pve_id', 'privilege': 'privileged',
|
|
'resources': 'resources', 'gpu': 'gpu',
|
|
'swap': 'swap', 'rootfs': 'rootfs',
|
|
'version': 'version', 'hardware': 'hardware',
|
|
}
|
|
key = key_map.get(key, key)
|
|
at_glance[key] = val
|
|
|
|
# Parse changelog
|
|
changelog = []
|
|
cl_text = sections.get('changelog', '')
|
|
current_entry = None
|
|
for line in cl_text.split('\n'):
|
|
m = re.match(r'###\s+(\d{4}-\d{2}-\d{2})\s+[—–-]\s+(.+)', line)
|
|
if m:
|
|
if current_entry:
|
|
changelog.append(current_entry)
|
|
current_entry = {'date': m.group(1), 'title': m.group(2).strip(), 'body': ''}
|
|
elif current_entry is not None:
|
|
stripped = line.strip()
|
|
if stripped and not stripped.startswith('#'):
|
|
if current_entry['body']:
|
|
current_entry['body'] += ' '
|
|
current_entry['body'] += stripped
|
|
if current_entry:
|
|
changelog.append(current_entry)
|
|
|
|
# Tags from path
|
|
parts = path.relative_to(REPO).parts
|
|
tags = []
|
|
if 'containers' in parts:
|
|
tags.append('container')
|
|
elif 'hosts' in parts:
|
|
tags.append('host')
|
|
elif 'vms' in parts:
|
|
tags.append('vm')
|
|
elif 'infrastructure' in parts:
|
|
tags.append('infrastructure')
|
|
|
|
# Determine slug from relative path
|
|
rel = str(path.relative_to(WIKI))
|
|
slug = rel.replace('.md', '')
|
|
|
|
# Entity mapping
|
|
entity_slug = PATH_TO_ENTITY.get(slug, None)
|
|
|
|
return {
|
|
'slug': slug,
|
|
'title': title,
|
|
'content': text,
|
|
'entity_slug': entity_slug,
|
|
'tags': tags,
|
|
'at_glance': at_glance,
|
|
'changelog': changelog,
|
|
'is_investigation': 'investigations' in rel,
|
|
}
|
|
|
|
|
|
def parse_investigation(path):
|
|
"""Parse an investigation page."""
|
|
if not path.exists():
|
|
return None
|
|
text = path.read_text()
|
|
lines = text.split('\n')
|
|
|
|
title = ""
|
|
for line in lines:
|
|
if line.startswith('# '):
|
|
title = line[2:].strip()
|
|
break
|
|
|
|
# Extract date from title or filename
|
|
date = ""
|
|
status = "resolved"
|
|
duration = ""
|
|
|
|
for line in lines[:30]:
|
|
m = re.search(r'(\d{4}-\d{2}-\d{2})', line)
|
|
if m:
|
|
date = m.group(1)
|
|
break
|
|
|
|
for line in lines:
|
|
if '**Status:**' in line:
|
|
status = line.split('**Status:**')[-1].strip().lower()
|
|
if '**Duration:**' in line:
|
|
duration = line.split('**Duration:**')[-1].strip()
|
|
|
|
# Extract entity references for about_slugs
|
|
about_slugs = []
|
|
entity_patterns = [
|
|
(r'\bcaddy\b', 'service:caddy'),
|
|
(r'\bauthentik\b', 'service:authentik'),
|
|
(r'\bdns\b', 'service:dns'),
|
|
(r'\bgitea\b', 'service:gitea'),
|
|
(r'\bjellyfin\b', 'service:jellyfin'),
|
|
(r'\bmatrix\b', 'service:matrix'),
|
|
(r'\bpaperless\b', 'service:paperless'),
|
|
(r'\bnextcloud\b', 'service:nextcloud'),
|
|
(r'\bartifacto\b', 'service:artifacto'),
|
|
(r'\barriman\b', 'lxc:arriman'),
|
|
(r'\btrmnl\b', 'service:trmnl'),
|
|
(r'\bmac-mini\b', 'ws:mac-mini'),
|
|
(r'\bhubris\b', 'host:hubris'),
|
|
(r'\bstrong\b', 'host:strong'),
|
|
]
|
|
for pattern, slug in entity_patterns:
|
|
if re.search(pattern, text, re.IGNORECASE):
|
|
about_slugs.append(slug)
|
|
|
|
rel = str(path.relative_to(WIKI))
|
|
slug = rel.replace('.md', '')
|
|
|
|
return {
|
|
'slug': slug,
|
|
'title': title,
|
|
'date': date,
|
|
'status': status,
|
|
'duration': duration,
|
|
'content': text,
|
|
'about_slugs': about_slugs,
|
|
'tags': ['investigation'],
|
|
}
|
|
|
|
|
|
def main():
|
|
documents = []
|
|
investigations = []
|
|
runbooks = []
|
|
|
|
# Container pages
|
|
containers_dir = WIKI / "containers"
|
|
for f in sorted(containers_dir.glob("*.md")):
|
|
if 'index' in f.name:
|
|
continue
|
|
if f.parent.name == 'archive':
|
|
continue
|
|
result = parse_page(f)
|
|
if result and result['title']:
|
|
documents.append(result)
|
|
print(f" document: {result['slug']} → {result['entity_slug']}")
|
|
|
|
# Host pages
|
|
hosts_dir = WIKI / "hosts"
|
|
for f in sorted(hosts_dir.glob("*.md")):
|
|
if 'index' in f.name:
|
|
continue
|
|
result = parse_page(f)
|
|
if result and result['title']:
|
|
documents.append(result)
|
|
print(f" document: {result['slug']} → {result['entity_slug']}")
|
|
|
|
# VM pages
|
|
vms_dir = WIKI / "vms"
|
|
for f in sorted(vms_dir.glob("*.md")):
|
|
if 'index' in f.name:
|
|
continue
|
|
result = parse_page(f)
|
|
if result and result['title']:
|
|
documents.append(result)
|
|
print(f" document: {result['slug']} → {result['entity_slug']}")
|
|
|
|
# Infrastructure pages
|
|
infra_dir = WIKI / "infrastructure"
|
|
for f in sorted(infra_dir.glob("*.md")):
|
|
if 'index' in f.name:
|
|
continue
|
|
result = parse_page(f)
|
|
if result and result['title']:
|
|
documents.append(result)
|
|
print(f" document: {result['slug']} → {result['entity_slug']}")
|
|
|
|
# Investigation pages
|
|
inv_dir = SOURCES / "investigations"
|
|
for f in sorted(inv_dir.glob("*.md")):
|
|
if 'index' in f.name:
|
|
continue
|
|
result = parse_investigation(f)
|
|
if result and result['title']:
|
|
investigations.append(result)
|
|
print(f" investigation: {result['slug']} → {result['about_slugs']}")
|
|
|
|
# Archive investigations too
|
|
inv_archive = inv_dir / "archive"
|
|
if inv_archive.exists():
|
|
for f in sorted(inv_archive.glob("*.md")):
|
|
result = parse_investigation(f)
|
|
if result and result['title']:
|
|
investigations.append(result)
|
|
print(f" investigation: {result['slug']} → {result['about_slugs']}")
|
|
|
|
# Runbooks from .agents/skills/
|
|
skills_dir = REPO / ".agents" / "skills"
|
|
for skill_dir in sorted(skills_dir.iterdir()):
|
|
if not skill_dir.is_dir():
|
|
continue
|
|
skill_file = skill_dir / "SKILL.md"
|
|
if not skill_file.exists():
|
|
continue
|
|
text = skill_file.read_text()
|
|
lines = text.split('\n')
|
|
title = ""
|
|
for line in lines:
|
|
if line.startswith('# '):
|
|
title = line[2:].strip()
|
|
break
|
|
|
|
# Extract risk_class and entity_type from frontmatter
|
|
risk_class = "read_only"
|
|
entity_type = "service"
|
|
for line in lines[:30]:
|
|
m = re.match(r'\*\*risk_class:\*\*\s*(\w+)', line, re.IGNORECASE)
|
|
if m:
|
|
risk_class = m.group(1)
|
|
m = re.match(r'\*\*applies_to:\*\*\s*(\w[\w-]*)', line, re.IGNORECASE)
|
|
if m:
|
|
entity_type = m.group(1)
|
|
|
|
name = skill_dir.name
|
|
runbooks.append({
|
|
'slug': name,
|
|
'name': title or name,
|
|
'risk_class': risk_class,
|
|
'entity_type': entity_type,
|
|
'procedure': {}, # SKILL.md is narrative, not structured yet
|
|
'content': text,
|
|
'tags': ['skill', 'runbook'],
|
|
})
|
|
print(f" runbook: {name}")
|
|
|
|
# Build seed YAML
|
|
seed = {
|
|
'version': 1,
|
|
'documents': [{
|
|
'slug': d['slug'],
|
|
'title': d['title'],
|
|
'content': d['content'],
|
|
'entity_slug': d['entity_slug'],
|
|
'tags': d['tags'],
|
|
'at_glance': d['at_glance'],
|
|
'changelog': d['changelog'],
|
|
} for d in documents],
|
|
'investigations': [{
|
|
'slug': i['slug'],
|
|
'title': i['title'],
|
|
'date': i['date'],
|
|
'status': i['status'],
|
|
'duration': i['duration'],
|
|
'content': i['content'],
|
|
'about_slugs': i['about_slugs'],
|
|
'tags': i['tags'],
|
|
} for i in investigations],
|
|
'runbooks': [{
|
|
'slug': r['slug'],
|
|
'name': r['name'],
|
|
'risk_class': r['risk_class'],
|
|
'entity_type': r['entity_type'],
|
|
'procedure': r['procedure'],
|
|
'content': r['content'],
|
|
'tags': r['tags'],
|
|
} for r in runbooks],
|
|
}
|
|
|
|
out_path = REPO / "seeds" / "knowledge.yaml"
|
|
out_path.write_text(yaml.dump(seed, allow_unicode=True, width=120, sort_keys=False))
|
|
print(f"\nWrote {out_path}")
|
|
print(f" {len(documents)} documents")
|
|
print(f" {len(investigations)} investigations")
|
|
print(f" {len(runbooks)} runbooks")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |