dtoro/oikos
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Oikos Week 3: scheduler, drift detectors, signals, classifier, approvals

Browse Source 
New kernel modules, all wired into `homelab` CLI + tested against live
production where reachable:

- oikos/scheduler.py — Observe stage: HTTP health probes for every
  service, disk-usage probes on hubris/strong, writes oikos/state.json
  (gitignored — regenerates every run). `homelab service <name> health`
  is now cache-first; `--live` forces a fresh probe. Deploys via
  oikos/systemd/oikos-scheduler.{timer,service} on LXC 105.

- oikos/drift.py — SOPS-recipient-vs-inventory and lifecycle-consistency
  detectors (fully local, no SSH) plus pct-list and Caddy-backend
  detectors (best-effort SSH, degrade to an info finding when
  unreachable rather than a false drift alarm). Found real, currently-
  true drift on first run: republic-laptop's age key granted on every
  secret but missing from inventory.yaml, grimmory missing from
  hello.yaml's recipients, and an undocumented pve_id 131 on hubris —
  recorded in OIKOS.md for the operator, not auto-fixed (each is a
  config_mutation/destructive decision).

- oikos/signal.py — the attention layer: raised -> acknowledged ->
  acting -> resolved|muted lifecycle, severity-based routing, dedup via
  open_signal_for(). `homelab signal list|raise|ack|resolve|mute`.

- oikos/decide.py — the Decide-stage classifier: risk class x blast
  radius x ledger-history confidence -> auto-act/escalate. Adds an
  action-alias layer (oikos/policy.py ACTION_ALIASES) and auto-infers
  service_name from the entity for per-service policy overrides.
  `homelab decide <action> <entity>`.

- oikos/approve.py — the escalate route. No dedicated Matrix bot exists
  in this homelab, so this is the repo-side half only: request/reply/
  grant lifecycle with short-TTL HMAC-signed tokens (new secret
  secrets/oikos-approval-hmac.yaml, recipients apps+hubris). Matrix
  delivery is Hermes's existing @dtoro:avispero send path (documented
  integration contract in the module docstring), not a new bot.
  `homelab restart` now mechanically refuses config_mutation/destructive
  services without a valid --approval-id, regardless of -y/interactivity.

- oikos/report.py — daily brief + weekly report from signal/approval/
  ledger state (no Prometheus yet, so point-in-time counts only).

- plans/2026-07-05-oikos-prometheus-lxc.md — Prometheus is `planned`,
  not provisioned: no pve_id is guessed here since Proxmox assigns real
  IDs at creation time, and drift already found an unclaimed ID (131) to
  investigate first.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
authentik Default Admin
2026-07-05 23:29:39 +02:00
parent 48debc0911
commit 2084a1583e
17 changed files with 1791 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
bin/homelab
View File

@@ -42,11 +42,15 @@ AGE_KEY = Path(os.environ.get("SOPS_AGE_KEY_FILE", "/etc/age/key.txt"))
# "feature unavailable" instead of crashing every subcommand.
sys.path.insert(0, str(CONTEXT))
try:
from oikos import approve as oikos_approve
from oikos import decide as oikos_decide
from oikos import ledger as oikos_ledger
from oikos import policy as oikos_policy
from oikos import relations as oikos_relations
from oikos import signal as oikos_signal
except ImportError:
oikos_ledger = oikos_policy = oikos_relations = None
oikos_approve = oikos_decide = oikos_ledger = oikos_policy = None
oikos_relations = oikos_signal = None
# ---------- helpers ----------
@@ -184,12 +188,14 @@ def service_backend_host(name: str) -> str:
def _record_change(entity: str, action: str, risk: str, *,
result: str | None = None, verification: str | None = None) -> None:
result: str | None = None, verification: str | None = None,
approval_ref: str | None = None) -> None:
"""Append a ledger entry and push it standalone (used by mutations that
don't already go through push_inventory, e.g. restart)."""
if oikos_ledger is None:
return
oikos_ledger.append(entity, action, risk, result=result, verification=verification)
oikos_ledger.append(entity, action, risk, result=result, verification=verification,
approval_ref=approval_ref)
try:
subprocess.run(["git", "add", "ledger/"], check=True, cwd=CONTEXT)
if subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=CONTEXT).returncode != 0:
@@ -598,15 +604,30 @@ def cmd_restart(args: argparse.Namespace) -> int:
svc = args.service
host_name = service_backend_host(svc)
unit = service(svc).get("systemd_unit", svc)
risk = (oikos_policy.classify_action("service-restart", svc)
if oikos_policy else "reversible_low") or "reversible_low"
approval = oikos_policy.approval_for(risk) if oikos_policy else "none"
# Mechanical gate: config_mutation/destructive risk classes require a
# live grant regardless of -y/interactivity — an agent (or a human
# bypassing the confirm() prompt with -y) cannot mutate a gated service
# without a real oikos/approve.py approval. See oikos/policy.yaml.
if approval != "none":
if not args.approval_id:
die(f"restarting '{svc}' is risk class '{risk}' (approval: {approval}) — "
f"pass --approval-id <id> from an approved 'homelab approval request'")
ok, reason = oikos_approve.check_grant(args.approval_id, f"service:{svc}", "service-restart")
if not ok:
die(f"approval {args.approval_id} not valid for this action: {reason}")
if not args.yes:
if not confirm(f"restart systemd unit '{unit}' on {host_name}?"):
return 1
base = ssh_base(host_name)
rc = subprocess.call(base + ["--", "systemctl", "restart", unit])
risk = (oikos_policy.classify_action("service-restart", svc)
if oikos_policy else "reversible_low") or "reversible_low"
_record_change(f"service:{svc}", "restart", risk,
result=("ok" if rc == 0 else f"failed rc={rc}"))
result=("ok" if rc == 0 else f"failed rc={rc}"),
approval_ref=args.approval_id)
return rc
@@ -1241,12 +1262,24 @@ def cmd_service(args: argparse.Namespace) -> int:
url = svc.get("url") or svc.get("endpoint")
if not url:
die(f"service {name} has no url/endpoint in inventory")
if not args.live:
try:
from oikos import scheduler as oikos_scheduler
cached = oikos_scheduler.cached_service_health(name)
except ImportError:
cached = None
if cached is not None and cached.get("checked"):
status = "ok" if cached.get("ok") else "unhealthy"
print(f"{name}: {cached.get('checked_url', url)} -> "
f"{cached.get('http_code') or 'no response'} ({status}, "
f"as of {cached['as_of']} — pass --live to force a fresh probe)")
return 0 if cached.get("ok") else 1
proc = subprocess.run(
["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", url],
capture_output=True, text=True,
)
code = proc.stdout.strip() or "no response"
print(f"{name}: {url} -> {code} (live probe — cache-first reads land Week 3)")
print(f"{name}: {url} -> {code} (live probe)")
return 0 if code.startswith(("2", "3")) else 1
if args.action == "docs":
@@ -1325,6 +1358,93 @@ def cmd_node_relations(args: argparse.Namespace) -> int:
return 0
def cmd_decide(args: argparse.Namespace) -> int:
"""Route a proposed action: auto-act or escalate. See oikos/decide.py."""
_require_oikos()
result = oikos_decide.classify(args.action, args.entity, service_name=args.service_name,
record=not args.no_record)
print(json.dumps(result, indent=2))
return 0 if result["route"] == "auto-act" else 1
def cmd_approval_request(args: argparse.Namespace) -> int:
_require_oikos()
entry = oikos_approve.request(
args.entity, args.action, args.risk, args.evidence,
verification=args.verification, requires_phrase=args.requires_phrase,
ttl_hours=args.ttl_hours,
)
print(json.dumps({k: v for k, v in entry.items() if k != "matrix_message"}, indent=2))
print()
print("--- post this to Matrix ---")
print(entry["matrix_message"])
return 0
def cmd_approval_list(args: argparse.Namespace) -> int:
_require_oikos()
for e in oikos_approve.list_approvals(state=args.state):
print(json.dumps(e))
return 0
def cmd_approval_reply(args: argparse.Namespace) -> int:
_require_oikos()
try:
entry = oikos_approve.reply(args.id, args.decision, phrase=args.phrase,
decided_by=args.decided_by)
except (ValueError, RuntimeError) as e:
die(str(e))
print(json.dumps(entry, indent=2))
return 0
def cmd_approval_check(args: argparse.Namespace) -> int:
_require_oikos()
ok, reason = oikos_approve.check_grant(args.id, args.entity, args.action)
print(f"{'GRANTED' if ok else 'DENIED'}: {reason}")
return 0 if ok else 1
def cmd_signal_raise(args: argparse.Namespace) -> int:
_require_oikos()
action = None
if args.action_runbook or args.action_risk:
action = {"runbook": args.action_runbook, "risk": args.action_risk}
entry = oikos_signal.raise_signal(args.kind, args.severity, args.entity, args.evidence,
likely_cause=args.likely_cause,
recommended_action=action,
verification=args.verification)
print(json.dumps(entry, indent=2))
return 0
def cmd_signal_list(args: argparse.Namespace) -> int:
_require_oikos()
for e in oikos_signal.list_signals(state=args.state, entity=args.entity,
severity=args.severity, kind=args.kind):
print(json.dumps(e))
return 0
def cmd_signal_ack(args: argparse.Namespace) -> int:
_require_oikos()
print(json.dumps(oikos_signal.acknowledge(args.id, args.note), indent=2))
return 0
def cmd_signal_resolve(args: argparse.Namespace) -> int:
_require_oikos()
print(json.dumps(oikos_signal.resolve(args.id, args.note), indent=2))
return 0
def cmd_signal_mute(args: argparse.Namespace) -> int:
_require_oikos()
print(json.dumps(oikos_signal.mute(args.id, args.ttl_hours, args.note), indent=2))
return 0
def cmd_nuke(args: argparse.Namespace) -> int:
name = args.name
if not args.yes:
@@ -1642,6 +1762,9 @@ def main() -> int:
sp = sub.add_parser("restart", help="restart a service")
sp.add_argument("service")
sp.add_argument("--yes", "-y", action="store_true")
sp.add_argument("--approval-id", default=None,
help="required if the service's risk class needs approval "
"(see 'homelab approval request')")
sp.set_defaults(func=cmd_restart)
sp = sub.add_parser("open", help="open a service's URL in browser")
@@ -1702,6 +1825,8 @@ def main() -> int:
sp.add_argument("action", choices=["explain", "health", "docs", "log", "actions", "history"])
sp.add_argument("--lines", "-n", type=int, default=200, help="for 'log'")
sp.add_argument("--limit", type=int, default=20, help="for 'history'")
sp.add_argument("--live", action="store_true",
help="for 'health': force a fresh probe instead of the scheduler's cache")
sp.set_defaults(func=cmd_service)
change = sub.add_parser("change", help="change/mutation workflow")
@@ -1715,6 +1840,81 @@ def main() -> int:
sp.add_argument("action", choices=["relations"])
sp.set_defaults(func=cmd_node_relations)
sp = sub.add_parser("decide", help="classify a proposed action: auto-act or escalate")
sp.add_argument("action")
sp.add_argument("entity")
sp.add_argument("--service-name", default=None)
sp.add_argument("--no-record", action="store_true",
help="skip writing this classification to the change ledger")
sp.set_defaults(func=cmd_decide)
approval = sub.add_parser("approval", help="approval-engine requests (escalate route)")
apsub = approval.add_subparsers(dest="action", required=True)
ap_req = apsub.add_parser("request")
ap_req.add_argument("entity")
ap_req.add_argument("action")
ap_req.add_argument("risk")
ap_req.add_argument("evidence")
ap_req.add_argument("--verification")
ap_req.add_argument("--requires-phrase", action="store_true")
ap_req.add_argument("--ttl-hours", type=int, default=24)
ap_req.set_defaults(func=cmd_approval_request)
ap_list = apsub.add_parser("list")
ap_list.add_argument("--state", choices=["pending", "approved", "denied", "expired"])
ap_list.set_defaults(func=cmd_approval_list)
ap_reply = apsub.add_parser("reply")
ap_reply.add_argument("id")
ap_reply.add_argument("decision", choices=["approve", "deny"])
ap_reply.add_argument("--phrase")
ap_reply.add_argument("--decided-by")
ap_reply.set_defaults(func=cmd_approval_reply)
ap_check = apsub.add_parser("check")
ap_check.add_argument("id")
ap_check.add_argument("entity")
ap_check.add_argument("action")
ap_check.set_defaults(func=cmd_approval_check)
signal = sub.add_parser("signal", help="the attention layer (oikos/signal.py)")
sigsub = signal.add_subparsers(dest="action", required=True)
sig_raise = sigsub.add_parser("raise")
sig_raise.add_argument("kind")
sig_raise.add_argument("severity", choices=["info", "warning", "critical"])
sig_raise.add_argument("entity")
sig_raise.add_argument("evidence")
sig_raise.add_argument("--likely-cause")
sig_raise.add_argument("--action-runbook")
sig_raise.add_argument("--action-risk")
sig_raise.add_argument("--verification")
sig_raise.set_defaults(func=cmd_signal_raise)
sig_list = sigsub.add_parser("list")
sig_list.add_argument("--state", choices=["raised", "acknowledged", "acting", "resolved", "muted"])
sig_list.add_argument("--entity")
sig_list.add_argument("--severity", choices=["info", "warning", "critical"])
sig_list.add_argument("--kind")
sig_list.set_defaults(func=cmd_signal_list)
sig_ack = sigsub.add_parser("ack")
sig_ack.add_argument("id")
sig_ack.add_argument("--note")
sig_ack.set_defaults(func=cmd_signal_ack)
sig_resolve = sigsub.add_parser("resolve")
sig_resolve.add_argument("id")
sig_resolve.add_argument("--note")
sig_resolve.set_defaults(func=cmd_signal_resolve)
sig_mute = sigsub.add_parser("mute")
sig_mute.add_argument("id")
sig_mute.add_argument("--ttl-hours", type=int, default=24)
sig_mute.add_argument("--note")
sig_mute.set_defaults(func=cmd_signal_mute)
sp = sub.add_parser("nuke", help="shred /etc/age/key.txt + /opt/homelab-context on a host")
sp.add_argument("name")
sp.add_argument("--yes", "-y", action="store_true")