db as source of truth: wiki→seeds, archive old artifacts, knowledge ingestion
- Migrations 010 (content_hash) + 011 (search tsvector column) - new: internal/knowledge/seed.go — knowledge seed ingest engine - new: internal/httpapi/knowledge.go — SearchKnowledge + GetEntityKnowledge - wire knowledge ingest into oikos seed pipeline - convert all 36 wiki docs + 6 investigations + 12 runbooks → seeds/knowledge.yaml - archive: knowledge/wiki/→archive/, oikos/cards/→archive/, .hermes/plans/→archive/ - delete: 9 superseded Python kernel files, ledger/, mcp/build_host_files.py - remove empty knowledge/ directory tree
This commit is contained in:
397
scripts/convert-wiki.py
Normal file
397
scripts/convert-wiki.py
Normal file
@@ -0,0 +1,397 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-shot: convert knowledge/wiki/ to seeds/knowledge.yaml."""
|
||||
|
||||
import os, re, yaml
|
||||
from pathlib import Path
|
||||
from hashlib import sha256
|
||||
|
||||
REPO = Path("/Users/dtoro/Projects/oikos")
|
||||
WIKI = REPO / "knowledge/wiki"
|
||||
SOURCES = REPO / "knowledge/sources"
|
||||
GLOSSARY = REPO / "knowledge/GLOSSARY.md"
|
||||
|
||||
# Maps wiki path components to entity slugs
|
||||
# Format: (path_pattern, entity_slug)
|
||||
PATH_TO_ENTITY = {
|
||||
# Containers
|
||||
"containers/101-jellyfin": "lxc:jellyfin",
|
||||
"containers/102-nfs-export": "lxc:nfs-export",
|
||||
"containers/103-paperless": "lxc:paperless",
|
||||
"containers/104-gitea": "lxc:gitea",
|
||||
"containers/105-apps": "lxc:apps",
|
||||
"containers/106-auth-outpost": "lxc:auth-outpost",
|
||||
"containers/107-dns": "lxc:dns",
|
||||
"containers/114-nextcloud": "lxc:nextcloud",
|
||||
"containers/118-elementsynapse": "lxc:elementsynapse",
|
||||
"containers/119-sophia": "lxc:sophia",
|
||||
"containers/120-mule-images": "lxc:mule-images",
|
||||
"containers/121-caddy": "lxc:caddy",
|
||||
"containers/122-arriman": "lxc:arriman",
|
||||
"containers/128-trmnl": "lxc:trmnl",
|
||||
"containers/129-house": "lxc:house",
|
||||
"containers/130-grimmory": "lxc:grimmory",
|
||||
"containers/131-teddycloud": "lxc:teddycloud",
|
||||
"containers/132-rclone": "lxc:rclone",
|
||||
"containers/133-seanime": "lxc:seanime",
|
||||
"containers/134-romm": "lxc:romm",
|
||||
# Hosts
|
||||
"hosts/hubris": "proxmox-host:hubris",
|
||||
"hosts/strong": "proxmox-host:strong",
|
||||
# VMs
|
||||
"vms/100-zimaos": "vm:zimaos",
|
||||
"vms/108-haos": "vm:haos",
|
||||
# Infrastructure → services
|
||||
"infrastructure/auto-deploy": None,
|
||||
"infrastructure/backups": None,
|
||||
"infrastructure/dns": "service:dns",
|
||||
"infrastructure/homelab-context": "service:homelab_mcp",
|
||||
"infrastructure/ingress": "service:caddy",
|
||||
"infrastructure/media-permissions": "service:jellyfin",
|
||||
"infrastructure/mesh": None,
|
||||
"infrastructure/monitoring": None,
|
||||
"infrastructure/network": None,
|
||||
"infrastructure/ssh-access": None,
|
||||
"infrastructure/topology": None,
|
||||
"infrastructure/vps-hardening": "standalone-server:netbird-vps",
|
||||
}
|
||||
|
||||
def parse_page(path):
|
||||
"""Parse a wiki page into structured sections."""
|
||||
if not path.exists():
|
||||
return None
|
||||
text = path.read_text()
|
||||
lines = text.split('\n')
|
||||
|
||||
# Title is first H1
|
||||
title = ""
|
||||
for line in lines:
|
||||
if line.startswith('# ') and not line.startswith('## '):
|
||||
title = line[2:].strip()
|
||||
break
|
||||
|
||||
# Find sections by H2 headings
|
||||
sections = {}
|
||||
current_heading = "_preamble"
|
||||
current_content = []
|
||||
|
||||
for line in lines:
|
||||
if line.startswith('## ') and not line.startswith('### '):
|
||||
if current_content:
|
||||
sections[current_heading] = '\n'.join(current_content).strip()
|
||||
current_heading = line[3:].strip().lower()
|
||||
current_content = []
|
||||
else:
|
||||
current_content.append(line)
|
||||
if current_content:
|
||||
sections[current_heading] = '\n'.join(current_content).strip()
|
||||
|
||||
# Parse at-a-glance
|
||||
at_glance = {}
|
||||
ag_text = sections.get('at a glance', '')
|
||||
for line in ag_text.split('\n'):
|
||||
line = line.strip()
|
||||
# Strip leading bullet
|
||||
line = re.sub(r'^[-*]\s+', '', line)
|
||||
# Match **Key:** value or **Key Word:** value
|
||||
m = re.match(r'\*\*([^*]+?):?\*\*\s+(.+)', line)
|
||||
if not m:
|
||||
m = re.match(r'([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*):\s+(.+)', line)
|
||||
if m:
|
||||
key = m.group(1).lower().strip().replace(' ', '_').replace('/', '_')
|
||||
val = m.group(2).strip()
|
||||
# Strip trailing parenthetical notes
|
||||
val = re.sub(r'\s*\([^)]*\)$', '', val)
|
||||
# Strip markdown formatting from value
|
||||
val = re.sub(r'\*\*([^*]+)\*\*', r'\1', val)
|
||||
val = re.sub(r'`([^`]+)`', r'\1', val)
|
||||
# Simplify link text
|
||||
val = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', val)
|
||||
val = re.sub(r'↔', '', val).strip()
|
||||
# Normalize keys
|
||||
key_map = {
|
||||
'cores': 'cores', 'core': 'cores',
|
||||
'ram': 'ram', 'memory': 'ram',
|
||||
'mounts': 'mounts', 'mount': 'mounts',
|
||||
'host': 'host', 'ip': 'ip',
|
||||
'public_host': 'public_host', 'public_hostname': 'public_host',
|
||||
'lan_ip': 'lan_ip',
|
||||
'os': 'os', 'kind': 'kind',
|
||||
'runtime': 'runtime', 'role': 'role',
|
||||
'pve_id': 'pve_id', 'privilege': 'privileged',
|
||||
'resources': 'resources', 'gpu': 'gpu',
|
||||
'swap': 'swap', 'rootfs': 'rootfs',
|
||||
'version': 'version', 'hardware': 'hardware',
|
||||
}
|
||||
key = key_map.get(key, key)
|
||||
at_glance[key] = val
|
||||
|
||||
# Parse changelog
|
||||
changelog = []
|
||||
cl_text = sections.get('changelog', '')
|
||||
current_entry = None
|
||||
for line in cl_text.split('\n'):
|
||||
m = re.match(r'###\s+(\d{4}-\d{2}-\d{2})\s+[—–-]\s+(.+)', line)
|
||||
if m:
|
||||
if current_entry:
|
||||
changelog.append(current_entry)
|
||||
current_entry = {'date': m.group(1), 'title': m.group(2).strip(), 'body': ''}
|
||||
elif current_entry is not None:
|
||||
stripped = line.strip()
|
||||
if stripped and not stripped.startswith('#'):
|
||||
if current_entry['body']:
|
||||
current_entry['body'] += ' '
|
||||
current_entry['body'] += stripped
|
||||
if current_entry:
|
||||
changelog.append(current_entry)
|
||||
|
||||
# Tags from path
|
||||
parts = path.relative_to(REPO).parts
|
||||
tags = []
|
||||
if 'containers' in parts:
|
||||
tags.append('container')
|
||||
elif 'hosts' in parts:
|
||||
tags.append('host')
|
||||
elif 'vms' in parts:
|
||||
tags.append('vm')
|
||||
elif 'infrastructure' in parts:
|
||||
tags.append('infrastructure')
|
||||
|
||||
# Determine slug from relative path
|
||||
rel = str(path.relative_to(REPO))
|
||||
if rel.startswith('knowledge/wiki/'):
|
||||
slug_rel = rel[len('knowledge/wiki/'):]
|
||||
elif rel.startswith('knowledge/sources/investigations/'):
|
||||
slug_rel = rel[len('knowledge/sources/'):]
|
||||
else:
|
||||
slug_rel = rel
|
||||
|
||||
slug = slug_rel.replace('.md', '')
|
||||
|
||||
# Entity mapping
|
||||
entity_slug = PATH_TO_ENTITY.get(slug, None)
|
||||
|
||||
return {
|
||||
'slug': slug,
|
||||
'title': title,
|
||||
'content': text,
|
||||
'entity_slug': entity_slug,
|
||||
'tags': tags,
|
||||
'at_glance': at_glance,
|
||||
'changelog': changelog,
|
||||
'is_investigation': 'investigations' in rel,
|
||||
}
|
||||
|
||||
|
||||
def parse_investigation(path):
|
||||
"""Parse an investigation page."""
|
||||
if not path.exists():
|
||||
return None
|
||||
text = path.read_text()
|
||||
lines = text.split('\n')
|
||||
|
||||
title = ""
|
||||
for line in lines:
|
||||
if line.startswith('# '):
|
||||
title = line[2:].strip()
|
||||
break
|
||||
|
||||
# Extract date from title or filename
|
||||
date = ""
|
||||
status = "resolved"
|
||||
duration = ""
|
||||
|
||||
for line in lines[:30]:
|
||||
m = re.search(r'(\d{4}-\d{2}-\d{2})', line)
|
||||
if m:
|
||||
date = m.group(1)
|
||||
break
|
||||
|
||||
for line in lines:
|
||||
if '**Status:**' in line:
|
||||
status = line.split('**Status:**')[-1].strip().lower()
|
||||
if '**Duration:**' in line:
|
||||
duration = line.split('**Duration:**')[-1].strip()
|
||||
|
||||
# Extract entity references for about_slugs
|
||||
about_slugs = []
|
||||
entity_patterns = [
|
||||
(r'\bcaddy\b', 'service:caddy'),
|
||||
(r'\bauthentik\b', 'service:authentik'),
|
||||
(r'\bdns\b', 'service:dns'),
|
||||
(r'\bgitea\b', 'service:gitea'),
|
||||
(r'\bjellyfin\b', 'service:jellyfin'),
|
||||
(r'\bmatrix\b', 'service:matrix'),
|
||||
(r'\bpaperless\b', 'service:paperless'),
|
||||
(r'\bnextcloud\b', 'service:nextcloud'),
|
||||
(r'\bartifacto\b', 'service:artifacto'),
|
||||
(r'\barriman\b', 'lxc:arriman'),
|
||||
(r'\btrmnl\b', 'service:trmnl'),
|
||||
(r'\bmac-mini\b', 'workstation:mac-mini'),
|
||||
(r'\bhubris\b', 'proxmox-host:hubris'),
|
||||
(r'\bstrong\b', 'proxmox-host:strong'),
|
||||
]
|
||||
for pattern, slug in entity_patterns:
|
||||
if re.search(pattern, text, re.IGNORECASE):
|
||||
about_slugs.append(slug)
|
||||
|
||||
rel = str(path.relative_to(SOURCES))
|
||||
slug = rel.replace('.md', '')
|
||||
|
||||
return {
|
||||
'slug': slug,
|
||||
'title': title,
|
||||
'date': date,
|
||||
'status': status,
|
||||
'duration': duration,
|
||||
'content': text,
|
||||
'about_slugs': about_slugs,
|
||||
'tags': ['investigation'],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
documents = []
|
||||
investigations = []
|
||||
runbooks = []
|
||||
|
||||
# Container pages
|
||||
containers_dir = WIKI / "containers"
|
||||
for f in sorted(containers_dir.glob("*.md")):
|
||||
if 'index' in f.name or 'archive' in str(f):
|
||||
continue
|
||||
result = parse_page(f)
|
||||
if result and result['title']:
|
||||
documents.append(result)
|
||||
print(f" document: {result['slug']} → {result['entity_slug']}")
|
||||
|
||||
# Host pages
|
||||
hosts_dir = WIKI / "hosts"
|
||||
for f in sorted(hosts_dir.glob("*.md")):
|
||||
if 'index' in f.name:
|
||||
continue
|
||||
result = parse_page(f)
|
||||
if result and result['title']:
|
||||
documents.append(result)
|
||||
print(f" document: {result['slug']} → {result['entity_slug']}")
|
||||
|
||||
# VM pages
|
||||
vms_dir = WIKI / "vms"
|
||||
for f in sorted(vms_dir.glob("*.md")):
|
||||
if 'index' in f.name:
|
||||
continue
|
||||
result = parse_page(f)
|
||||
if result and result['title']:
|
||||
documents.append(result)
|
||||
print(f" document: {result['slug']} → {result['entity_slug']}")
|
||||
|
||||
# Infrastructure pages
|
||||
infra_dir = WIKI / "infrastructure"
|
||||
for f in sorted(infra_dir.glob("*.md")):
|
||||
if 'index' in f.name:
|
||||
continue
|
||||
result = parse_page(f)
|
||||
if result and result['title']:
|
||||
documents.append(result)
|
||||
print(f" document: {result['slug']} → {result['entity_slug']}")
|
||||
|
||||
# Investigation pages
|
||||
inv_dir = SOURCES / "investigations"
|
||||
for f in sorted(inv_dir.glob("*.md")):
|
||||
if 'index' in f.name or 'archive' in str(f):
|
||||
continue
|
||||
result = parse_investigation(f)
|
||||
if result and result['title']:
|
||||
investigations.append(result)
|
||||
print(f" investigation: {result['slug']} → {result['about_slugs']}")
|
||||
|
||||
# Archive investigations too
|
||||
inv_archive = inv_dir / "archive"
|
||||
if inv_archive.exists():
|
||||
for f in sorted(inv_archive.glob("*.md")):
|
||||
result = parse_investigation(f)
|
||||
if result and result['title']:
|
||||
investigations.append(result)
|
||||
print(f" investigation: {result['slug']} → {result['about_slugs']}")
|
||||
|
||||
# Runbooks from .agents/skills/
|
||||
skills_dir = REPO / ".agents" / "skills"
|
||||
for skill_dir in sorted(skills_dir.iterdir()):
|
||||
if not skill_dir.is_dir():
|
||||
continue
|
||||
skill_file = skill_dir / "SKILL.md"
|
||||
if not skill_file.exists():
|
||||
continue
|
||||
text = skill_file.read_text()
|
||||
lines = text.split('\n')
|
||||
title = ""
|
||||
for line in lines:
|
||||
if line.startswith('# '):
|
||||
title = line[2:].strip()
|
||||
break
|
||||
|
||||
# Extract risk_class and entity_type from frontmatter
|
||||
risk_class = "read_only"
|
||||
entity_type = "service"
|
||||
for line in lines[:30]:
|
||||
m = re.match(r'\*\*risk_class:\*\*\s*(\w+)', line, re.IGNORECASE)
|
||||
if m:
|
||||
risk_class = m.group(1)
|
||||
m = re.match(r'\*\*applies_to:\*\*\s*(\w[\w-]*)', line, re.IGNORECASE)
|
||||
if m:
|
||||
entity_type = m.group(1)
|
||||
|
||||
name = skill_dir.name
|
||||
runbooks.append({
|
||||
'slug': name,
|
||||
'name': title or name,
|
||||
'risk_class': risk_class,
|
||||
'entity_type': entity_type,
|
||||
'procedure': {}, # SKILL.md is narrative, not structured yet
|
||||
'content': text,
|
||||
'tags': ['skill', 'runbook'],
|
||||
})
|
||||
print(f" runbook: {name}")
|
||||
|
||||
# Build seed YAML
|
||||
seed = {
|
||||
'version': 1,
|
||||
'documents': [{
|
||||
'slug': d['slug'],
|
||||
'title': d['title'],
|
||||
'content': d['content'],
|
||||
'entity_slug': d['entity_slug'],
|
||||
'tags': d['tags'],
|
||||
'at_glance': d['at_glance'],
|
||||
'changelog': d['changelog'],
|
||||
} for d in documents],
|
||||
'investigations': [{
|
||||
'slug': i['slug'],
|
||||
'title': i['title'],
|
||||
'date': i['date'],
|
||||
'status': i['status'],
|
||||
'duration': i['duration'],
|
||||
'content': i['content'],
|
||||
'about_slugs': i['about_slugs'],
|
||||
'tags': i['tags'],
|
||||
} for i in investigations],
|
||||
'runbooks': [{
|
||||
'slug': r['slug'],
|
||||
'name': r['name'],
|
||||
'risk_class': r['risk_class'],
|
||||
'entity_type': r['entity_type'],
|
||||
'procedure': r['procedure'],
|
||||
'content': r['content'],
|
||||
'tags': r['tags'],
|
||||
} for r in runbooks],
|
||||
}
|
||||
|
||||
out_path = REPO / "seeds" / "knowledge.yaml"
|
||||
out_path.write_text(yaml.dump(seed, allow_unicode=True, width=120, sort_keys=False))
|
||||
print(f"\nWrote {out_path}")
|
||||
print(f" {len(documents)} documents")
|
||||
print(f" {len(investigations)} investigations")
|
||||
print(f" {len(runbooks)} runbooks")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user