Compare commits
96 Commits
ccbf6a8aac
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 7160eee1e1 | |||
| 30ecdc16c2 | |||
| d79b0862bd | |||
| 53823595de | |||
| 7ecf720166 | |||
| febc153b7f | |||
| 7d6a3320d4 | |||
| 60bc9d555d | |||
| ebe1b95acf | |||
| 5e10437fe3 | |||
| 7236c46e5c | |||
| 75c0848a6f | |||
| 712b66422b | |||
| a30c024ef8 | |||
| 137a2afb8d | |||
| c8b1ec5af2 | |||
| 8ff382a50d | |||
| fa79c1ea25 | |||
| ef762794e7 | |||
| 0b6c546aae | |||
| cecfd8b0e4 | |||
| 653ea3a116 | |||
| 89a94c24c9 | |||
| 7a34d8c8a0 | |||
| c9d506b0f8 | |||
| e3449b24c1 | |||
| 38c472a118 | |||
| 1d0197da69 | |||
| 0920c4cb6d | |||
| 2254a07baf | |||
| 1b9c761274 | |||
| a126cfa710 | |||
| 86fa57b5cd | |||
| 8e97d589af | |||
| 0dd8c28815 | |||
| 4e294b3630 | |||
| 85f0bb67fa | |||
| c3f478b8f8 | |||
| 1aaedf498a | |||
| 20adb89650 | |||
| 058f1afcdc | |||
| 2b73290994 | |||
| 428f4fe945 | |||
| 195d45a0e9 | |||
| 5b68bdc16c | |||
| 757ef2f34b | |||
| b27e1bf3ec | |||
| 39e9227fdb | |||
| bb05f215c6 | |||
| 467589d78a | |||
| e25e979757 | |||
| bc0ccb4cdc | |||
| c9a00a9532 | |||
| eb16796bf0 | |||
| a3914a1d41 | |||
| 8eb1ca2bac | |||
| e4104eb344 | |||
| 0929c17cbb | |||
| 6487032461 | |||
| fb6b6f9160 | |||
| 62c9fc5c86 | |||
| 6007e922b4 | |||
| 2d8eb91b25 | |||
| 3d88f52988 | |||
| 9016c3a43b | |||
| 04775192c1 | |||
| a104cb4bb4 | |||
| 72f0f46528 | |||
| b87735a111 | |||
| 1540f74342 | |||
| c7729b2ef6 | |||
| b8b4aa2aee | |||
| c10f6920cd | |||
| ad29295c93 | |||
| 6ca6d5b352 | |||
| af450dac2a | |||
| 6ed9dc39e8 | |||
| cc8eae4979 | |||
| 4f706fa65f | |||
| 50e899e5ee | |||
| 42751623ea | |||
| 98e19bb14a | |||
| d7b526a112 | |||
| 1dca2cfd7a | |||
| 7e1ccad5f4 | |||
| 89312a9ce4 | |||
| ce0e4142ff | |||
| 873b00ac42 | |||
| b345783eef | |||
| 29d5cb8b85 | |||
| 8f440c5ad5 | |||
| 4e4e2c169c | |||
| c151a66627 | |||
| 1c12d40712 | |||
| 482c7f3448 | |||
| 50aed11cc4 |
@@ -30,9 +30,9 @@ one pass through **Observe → Orient → Decide → Act**:
|
||||
- **queue**: informational — console + reports
|
||||
The classifier can only *lower* autonomy relative to policy, never raise
|
||||
it. When in doubt, escalate.
|
||||
4. **Act** — execute through `homelab` commands or runbooks (never ad-hoc
|
||||
SSH), then **verify** with the action's verification command, write a
|
||||
**ledger** entry, resolve the Signal, and update docs in the same session.
|
||||
4. **Act** — execute through MCP `run` or runbooks (never ad-hoc
|
||||
SSH), then **verify** with the action's verification command, write a
|
||||
**ledger** entry, resolve the Signal, and update docs in the same session.
|
||||
|
||||
## Primitives
|
||||
|
||||
@@ -88,7 +88,7 @@ via the API's `/api/v1/graph` endpoint, and the Mermaid export at
|
||||
## Conventions carried forward
|
||||
|
||||
- Inventory is the truth; live state wins over narrative docs.
|
||||
- Prefer `homelab` CLI and MCP over ad-hoc SSH.
|
||||
- Prefer MCP tools over ad-hoc SSH.
|
||||
- Meaningful changes update docs in the same session.
|
||||
- Secrets are decrypted locally via per-client keys; never into docs/comments.
|
||||
- Tracked configs change by commit + push, not local edits.
|
||||
@@ -141,7 +141,7 @@ in the Go binary.
|
||||
- Standalone Nomos MCP client binary (`cmd/nomos`) with gateway mode
|
||||
(:8092). Structured queries + natural-language routing to the MCP tool
|
||||
list (see AGENTS.md §3). Agent activity logging on every tool call. No SSH keys.
|
||||
- `nomos/` directory with config, SOUL.md, homelab-ops skill.
|
||||
- `nomos/` directory with config, SOUL.md, `homelab-ops` skill at `nomos/skills/homelab-ops/`.
|
||||
- Nomos Docker service in `docker-compose.yml` (profile: full).
|
||||
- Go packages: `cmd/nomos/`, `compose/nomos/`.
|
||||
|
||||
|
||||
@@ -20,6 +20,6 @@ Run from the repo root:
|
||||
Exit code is non-zero when any violation is found, so it can gate a commit. The banned-vocabulary
|
||||
list mirrors `writing-style.md`; update both together if the standard changes.
|
||||
|
||||
> **Known baseline.** `archive/knowledge/archive/knowledge/containers/101-jellyfin.md` links into a sibling repo
|
||||
> **Known baseline.** `archive/knowledge/containers/101-jellyfin.md` links into a sibling repo
|
||||
> (`devops/homelab-authentik-admin`) that this checkout does not contain — expected, not a bug.
|
||||
> Any other broken link is a real regression; investigate before dismissing it as baseline noise.
|
||||
|
||||
87
.agents/skills/knowledge-graph-audit/SKILL.md
Normal file
87
.agents/skills/knowledge-graph-audit/SKILL.md
Normal file
@@ -0,0 +1,87 @@
|
||||
---
|
||||
name: knowledge-graph-audit
|
||||
risk_class: read_only
|
||||
inputs: []
|
||||
verification: "audit_knowledge_graph returns a report with summary.total_findings"
|
||||
docs_update_checklist: []
|
||||
---
|
||||
|
||||
# Knowledge-graph audit
|
||||
|
||||
Goal: validate that the knowledge graph (entities, relationships, checks) and
|
||||
the monitoring built on it reflect live reality — without mutating anything.
|
||||
Read-only. Run this before trusting health, blast-radius, or coverage answers,
|
||||
and whenever something feels off (a healthy host reports `down`, a retired
|
||||
service still alarms, the graph looks thin).
|
||||
|
||||
## 1. Run the drift report
|
||||
|
||||
Call MCP `audit_knowledge_graph` (or `GET /api/v1/audit/drift`). It returns a
|
||||
ranked list of findings, each with `{category, severity, count, entities,
|
||||
evidence, suggested_runbook}`, plus a `summary` with totals by category.
|
||||
|
||||
The DB-side categories:
|
||||
|
||||
- **orphan_checks** — check entities with truncated/random slugs left by the
|
||||
old `shortSlug()` collision bug. Remediation: `scripts/cleanup-orphan-checks.sh`.
|
||||
- **dead_checks** — enabled `check_defs` whose target entity is `deprecated`/
|
||||
`destroyed`. Remediation: `lifecycle-deprecate-node` / `lifecycle-destroy-node`
|
||||
(the scheduler already skips these, but the rows should be retired).
|
||||
- **down_checks** — enabled probes reporting `down`. Remediation:
|
||||
`service-health-check` (then check whether the failure is real or a
|
||||
probe-config/routing problem — see step 3).
|
||||
- **unknown_checks** — probes that ran but reported `unknown` (usually a
|
||||
misconfigured or not-yet-deployed probe script).
|
||||
- **unmonitored** — active entities whose type declares monitoring but have no
|
||||
enabled `check_def`.
|
||||
- **dangling_edges** — live `hosts`/`provides`/`mounts` edges still pointing at
|
||||
destroyed/deprecated targets. Remediation: `lifecycle-destroy-node`.
|
||||
|
||||
## 2. Triage
|
||||
|
||||
`severity: critical` (down_checks) first. For each finding, read `evidence` and
|
||||
open the entities with `get_entity` / `get_relations` to confirm the diagnosis
|
||||
before acting — the report is a pointer, not a verdict.
|
||||
|
||||
## 3. Common probe-failure causes
|
||||
|
||||
A `down_checks` finding that is NOT a real outage is usually one of:
|
||||
|
||||
- **Guest reached wrong** — an LXC/VM check SSHed the guest directly instead of
|
||||
routing through its Proxmox host. Confirm with `get_relations` that a `hosts`
|
||||
edge exists and the guest has `pve_id`; checks route via `pct exec`/`qm guest
|
||||
exec` automatically when both are present.
|
||||
- **Script not deployed** — the probe script is absent at `/opt/oikos/checks/`
|
||||
inside the target. Remediation: redeploy via `tools/deploy-checks.sh`.
|
||||
- **macOS host** — a workstation check used the wrong SSH user or a Linux-only
|
||||
script flag. The scheduler resolves `user: dtoro` from the entity attribute.
|
||||
|
||||
## 4. What this audit does NOT cover (follow-ups)
|
||||
|
||||
Live-infrastructure discovery has its own tool — run **`discover_infra_drift`**
|
||||
alongside this one. It compares running Proxmox guests (`pct`/`qm list` on every
|
||||
proxmox host) against the DB graph and returns:
|
||||
|
||||
- **missing entities** — a guest running in Proxmox with no DB entity.
|
||||
- **ghost entities** — a DB lxc/vm whose `pve_id` is no longer live.
|
||||
|
||||
Still manual until that machinery lands:
|
||||
|
||||
- **Misplaced parent** — compare each guest's actual Proxmox host against its
|
||||
`hosts` edge (migrations leave these stale).
|
||||
- **Undeployed scripts** — per-guest `/opt/oikos/checks/` presence.
|
||||
- **Unmodeled certs** — now modeled; verify with `audit_knowledge_graph` /
|
||||
the cert-expiry checks.
|
||||
- **Seed drift** — run `oikos export` and `git diff seeds/` to find
|
||||
runtime-created entities not in version control.
|
||||
|
||||
## 5. Acting on findings
|
||||
|
||||
This skill is read-only — make no changes here. Route each confirmed finding to
|
||||
its `suggested_runbook`, classify the action against `seeds/policy.yaml`, and
|
||||
proceed through the normal lifecycle/approval flow. Re-run the audit afterward
|
||||
to confirm the finding cleared.
|
||||
|
||||
Docs-update checklist: none — the audit reads state; it changes nothing. If a
|
||||
finding reveals stale `risk_notes` or a wrong `doc_page`, fix `inventory.yaml`
|
||||
in that remediation session.
|
||||
@@ -36,8 +36,9 @@ ledger entry.
|
||||
6. Update the entity's `state` to `destroyed` in `seeds/inventory.yaml`
|
||||
(or move it to an `archaeology:`-style section if the schema still has
|
||||
one) — `pve_id`, `destroyed` date, `reason` — then `oikos seed` to
|
||||
ingest. Add a row to `containers/index.md` "Recently destroyed" table
|
||||
(kept for human-readable browsing alongside the structured data).
|
||||
ingest. Add a row to the legacy `archive/knowledge/containers/index.md`
|
||||
"Recently destroyed" table (kept for human-readable browsing
|
||||
alongside the structured data in the DB).
|
||||
7. No manual ledger step — mutations through the API are recorded
|
||||
automatically in the `audit_log` table (MCP `get_audit_trail`,
|
||||
`get_change_history`). The old `oikos/ledger.py append` was retired
|
||||
|
||||
@@ -27,9 +27,10 @@ chosen, doc page stub.
|
||||
will self-enroll as a client afterward (see
|
||||
[CLIENTS.md](../../../CLIENTS.md#enrollment)), the entity must exist in
|
||||
`planned`/`provisioning` state before `bootstrap.sh` runs there.
|
||||
3. Stub the doc page (`containers/<pve_id>-<name>.md` or
|
||||
`vms/<pve_id>-<name>.md`) — even a one-line "provisioning, see plan X"
|
||||
is enough to satisfy the transition requirement.
|
||||
3. Stub a document entity via MCP `upsert_knowledge` with
|
||||
`kind: document` and about set to the new entity slug — even a
|
||||
one-line "provisioning, see plan X" is enough to satisfy the
|
||||
transition requirement.
|
||||
4. Reserve the IP in DNS/DHCP notes if it's a fixed LAN address.
|
||||
|
||||
Next: [lifecycle-activate-node.md](../lifecycle-activate-node/SKILL.md).
|
||||
|
||||
@@ -78,5 +78,5 @@ Session: {id[:8]} — "{title[:60]}"
|
||||
- `internal/mcp/server.go` — all tool implementations (`run`, `list_lxcs`, …)
|
||||
- `web/src/lib/components/ToolCallGroup.svelte` — tool result display
|
||||
- `nomos/SOUL.md` — agent persona and tool selection rules
|
||||
- `plans/2026-07-09-chat-sessions-improvements.md` — prior session findings
|
||||
- `plans/2026-07-09-session-execution-and-ux-fixes.md` — latest plan
|
||||
- `plans/done/2026-07-09-chat-sessions-improvements.md` — prior session findings
|
||||
- `plans/done/2026-07-09-session-execution-and-ux-fixes.md` — latest plan
|
||||
|
||||
12
.dockerignore
Normal file
12
.dockerignore
Normal file
@@ -0,0 +1,12 @@
|
||||
# Every docker build in this repo previously sent the whole directory as
|
||||
# build context — including every OTHER git worktree under .claude/worktrees/
|
||||
# (each with its own web/node_modules, ~200-300MB apiece). That's what
|
||||
# starved the mac-mini's disk mid-build on 2026-07-27 (SHA 873b00a): the
|
||||
# context alone crossed 390MB of pure worktree cruft before the host ran out
|
||||
# of space. None of this ever belonged in an image.
|
||||
.claude/worktrees/
|
||||
.git/
|
||||
**/node_modules/
|
||||
**/dist/
|
||||
**/build/
|
||||
*.log
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -24,3 +24,7 @@ cmd/desktop/build/
|
||||
cmd/desktop/Oikos
|
||||
desktop
|
||||
/eval
|
||||
|
||||
# Local tooling artifacts (Playwright MCP session logs, stray screenshots)
|
||||
.playwright-mcp/
|
||||
config-screen.png
|
||||
|
||||
120
AGENTS.md
120
AGENTS.md
@@ -56,67 +56,79 @@ Endpoint: `https://mcp.hubris.network/mcp`. Every call needs
|
||||
enrollment and `/healthz` (see "Authentication" below for where the token
|
||||
comes from).
|
||||
|
||||
Available tools (the authoritative list — count them below if a number is
|
||||
needed; do not hardcode the count elsewhere):
|
||||
Available tools (63 total — the authoritative list; do not hardcode the count
|
||||
elsewhere; regenerate from `internal/mcp/` when tools change):
|
||||
|
||||
Context — observe + orient:
|
||||
get_entity(slug), list_entities(type, limit, cursor),
|
||||
get_relations(entity), get_blast_radius(entity),
|
||||
search_knowledge(query) — ILIKE search over documents, investigations,
|
||||
runbooks in the knowledge_entities table
|
||||
get_entity_knowledge(entity_slug) — every document, investigation, and
|
||||
runbook linked to one entity, in one call
|
||||
get_patterns(status, entity_type, action) — learned action patterns
|
||||
get_skills(status) — available automation skills
|
||||
http_get(url) — fetch a public page/raw file (e.g. researching how to
|
||||
deploy something before provisioning it); HTTP/HTTPS only, ~16KB cap
|
||||
|
||||
Management — live state:
|
||||
get_service_status(service_slug) — systemctl is-active on target host
|
||||
tail_log(service_slug, lines=200) — journalctl
|
||||
list_lxcs() — all LXC containers with ID, host, IP, health
|
||||
get_lxc_state(lxc_slug) — pct status from Proxmox host
|
||||
ping_service(service_slug) — HTTP reachability from entity_status
|
||||
list_my_secrets(caller_pubkey) — secrets accessible to this client by
|
||||
age public key
|
||||
|
||||
Oikos — decisions:
|
||||
Entity Tools — knowledge graph, discovery, and lifecycle:
|
||||
ping — lightweight connectivity check
|
||||
get_entity(slug_or_id) — get an entity by slug or UUID
|
||||
list_entities(type, state, q, limit) — entities filtered by type, state, or search
|
||||
get_relations(entity_id, types) — list inbound/outbound edges for one entity
|
||||
get_blast_radius(entity_id, depth=3) — entities affected if this one goes down
|
||||
create_entity(type, name, slug, attributes, state) — create a new entity in the graph
|
||||
update_entity_attributes(slug, attributes) — merge discovered facts into an entity
|
||||
set_entity_state(slug, state) — transition entity to a new lifecycle state
|
||||
create_relationship(source, target, type) — record a discovered edge
|
||||
end_relationship(source, target, type) — soft-delete an active edge
|
||||
whoami(hostname) — entity record, peers, and health for a host
|
||||
explain(service_slug) — compact context card (type, state, health, relations)
|
||||
preflight(service_slug, action) — risk class + approval requirement
|
||||
whoami(hostname) — entity record, peers, health for a client
|
||||
get_change_history(entity_slug, limit=20) — last audit-log entries per entity
|
||||
get_state_snapshot() — fleet health, disk, drift count
|
||||
get_state_snapshot() — last scheduler Observe-pass: fleet health, disk, drift
|
||||
audit_knowledge_graph() — read-only drift report over the graph and checks
|
||||
discover_infra_drift() — running guests vs DB: missing/ghost entities
|
||||
find_entities_by(key, value, limit=25) — search entities by attribute values
|
||||
|
||||
Operations — observe + act:
|
||||
get_health_summary() — fleet health counts (healthy/degraded/down/unknown)
|
||||
get_signal_history(entity_slug, state, limit) — open + recent signals
|
||||
get_audit_trail(entity_id) — audit log filter + browse
|
||||
get_agent_activity(limit) — agent self-inspection
|
||||
query_metrics(hours=24) — time-series metric bucketed averages
|
||||
get_trend(entity_id, days=7) — metric slope over time
|
||||
get_event_timeline(severity, entity_slug, limit) — recent events
|
||||
Ops Tools — live state, signals, checks, and execution:
|
||||
run(target, command, purpose, declared_risk) — general execution primitive; read-only auto-acts, mutations queue for approval, destructive always needs explicit confirmation
|
||||
inspect_path(path, targets) — bulk mount/df/ls/stat across multiple hosts/LXCs
|
||||
get_execution_status(execution_id) — poll execution progress
|
||||
tail_log(service_slug, lines=50) — journalctl for a service
|
||||
get_service_status(service_slug) — systemctl is-active/is-enabled
|
||||
get_lxc_state(lxc_slug) — pct status from Proxmox host
|
||||
ping_service(service_slug) — HTTP reachability + scheduler health state
|
||||
list_lxcs(state) — all LXC containers with ID, host, IP, last-audited hint
|
||||
ack_signal(signal_id) — acknowledge an open signal
|
||||
resolve_signal(signal_id, resolution) — resolve a signal with optional note
|
||||
mute_signal(signal_id, duration_s=3600) — temporarily mute a signal
|
||||
cancel_execution(execution_id, reason) — cancel a queued/running execution
|
||||
update_check(check_id, enabled) — enable or disable a health check
|
||||
list_checks(entity_slug, enabled) — list health checks with verdict, probe kind
|
||||
list_executions(entity_slug, status, limit=25) — cursor-paginated execution history
|
||||
list_entity_sessions(entity_slug) — active Nomos sessions linked to an entity
|
||||
get_dashboard_summary() — fleet overview: counts, health, signals, approvals
|
||||
get_secret(key, path, environment) — retrieve a secret from the Infisical vault
|
||||
list_secrets(path_prefix) — list secret keys in the Infisical vault
|
||||
set_secret(key, value, path, environment) — store/update a secret (requires approval)
|
||||
|
||||
Knowledge — keep the graph current (none require approval; this updates
|
||||
the knowledge graph, not live infrastructure):
|
||||
upsert_knowledge(title, content) — record what you learned after solving
|
||||
a non-obvious problem; the only way anything persists past a session
|
||||
update_entity_attributes(slug, attributes) — merge a discovered fact
|
||||
(IP, version, port, ...) into an entity so a future task doesn't
|
||||
rediscover it from scratch
|
||||
create_relationship(source, target, type) — record a discovered edge
|
||||
(depends-on, hosts, routes-to, ...) between two entities
|
||||
Knowledge Tools — search, read, and maintain the knowledge base:
|
||||
search_knowledge(query) — full-text search across docs (snippets, not full body)
|
||||
get_entity_knowledge(entity_slug) — all docs/investigations/runbooks linked to a slug
|
||||
get_knowledge_content(slug) — full markdown body of one knowledge entry
|
||||
upsert_knowledge(title, content, about, tags, kind) — write what you learned
|
||||
delete_knowledge(knowledge_slug) — soft-delete a knowledge entry
|
||||
restore_knowledge(knowledge_slug) — restore a soft-deleted entry
|
||||
merge_knowledge(target_slug, source_slugs) — fold entries into a target
|
||||
rename_knowledge_tag(from, to) — bulk-rename tags across all entries
|
||||
get_knowledge_revisions(knowledge_slug) — version history for a knowledge entry
|
||||
get_knowledge_duplicates(threshold=0.6) — near-duplicate detection via trigram similarity
|
||||
get_knowledge_orphans(stale_days=90) — unlinked, untagged, or stale entries
|
||||
list_knowledge_tags() — all tags with usage counts and casing variants
|
||||
list_my_secrets(caller_pubkey) — secrets accessible to a client by age public key
|
||||
|
||||
Execution — mutating the live infrastructure:
|
||||
run(target, command) — the general execution primitive. Run any shell
|
||||
command against a host or LXC; every command is auto-classified —
|
||||
read-only inspection runs immediately, anything state-changing needs
|
||||
operator approval, and destructive patterns (rm -rf, dd, mkfs,
|
||||
pct/qm destroy, DROP TABLE, reboot, curl-pipe-to-shell, ...) always
|
||||
need approval regardless of what you declare. This is the ONLY
|
||||
mutation tool — `request_execution` was retired 2026-07-14; the
|
||||
former enum actions (restart, systemctl, pct_exec, apt_upgrade,
|
||||
pct_create) are all expressed as `run(target, command)` now.
|
||||
get_execution_status(execution_id) — poll progress
|
||||
Analysis Tools — fleet health, metrics, and introspection:
|
||||
get_health_summary(health) — fleet health per entity, optionally filtered
|
||||
get_audit_trail(entity_id) — query the audit log
|
||||
query_metrics(hours=24) — time-series with bucketed avg/min/max
|
||||
get_signal_history(entity_slug, state, limit=50) — open and recent signals
|
||||
get_patterns(status, entity_type, action) — learned action patterns
|
||||
get_skills(status) — available automation skills
|
||||
get_trend(entity_id, days=7) — metric slope, variance, and averages
|
||||
get_event_timeline(severity, entity_slug, limit=50) — recent events
|
||||
get_agent_activity(limit=50) — agent self-inspection log
|
||||
classify_command(command, declared_risk) — pre-flight risk classification before `run`
|
||||
get_ontology() — entity types, relationship types, and lifecycle definitions
|
||||
http_get(url) — fetch a public web page/raw file; ~16KB cap
|
||||
|
||||
**When to prefer MCP over grepping the clone:** always for knowledge queries.
|
||||
`search_knowledge("jellyfin hardware acceleration")` returns ranked results from
|
||||
|
||||
@@ -2284,6 +2284,23 @@ components:
|
||||
type: boolean
|
||||
version:
|
||||
type: integer
|
||||
last_health:
|
||||
type: string
|
||||
description: >-
|
||||
This check's own most recent verdict. An entity's health is the
|
||||
worst of these across its enabled checks, so this is what explains
|
||||
*why* an entity is degraded. Null until the check first runs.
|
||||
nullable: true
|
||||
enum:
|
||||
- healthy
|
||||
- degraded
|
||||
- down
|
||||
- unknown
|
||||
last_run_at:
|
||||
type: string
|
||||
format: date-time
|
||||
description: When this check last executed. Null = never run.
|
||||
nullable: true
|
||||
CheckCreate:
|
||||
type: object
|
||||
required:
|
||||
@@ -3168,8 +3185,6 @@ components:
|
||||
required:
|
||||
- age_public_key
|
||||
- age_private_key
|
||||
- infisical_client_id
|
||||
- infisical_client_secret
|
||||
properties:
|
||||
age_public_key:
|
||||
type: string
|
||||
@@ -3183,9 +3198,6 @@ components:
|
||||
infisical_client_secret:
|
||||
type: string
|
||||
description: Infisical UniversalAuth client secret
|
||||
machine_identity_token:
|
||||
type: string
|
||||
description: Infisical machine identity access token
|
||||
ClientContext:
|
||||
type: object
|
||||
required:
|
||||
|
||||
104
archive/knowledge/infrastructure/oikos-check-lifecycle.md
Normal file
104
archive/knowledge/infrastructure/oikos-check-lifecycle.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# Oikos check lifecycle — how monitoring works
|
||||
|
||||
This runbook covers how Oikos health checks are derived, created, and wired so
|
||||
an agent (Nomos) doesn't reverse-engineer source when asked to add monitoring to
|
||||
an entity — the problem that stranded session `23da10db` (2026-08-03).
|
||||
|
||||
## Concepts
|
||||
|
||||
- **`check_defs`** (scheduler config, table `check_defs`): the row the scheduler
|
||||
reads to know *what* to probe and *when*. One per check instance.
|
||||
- **`check` entity** (type `check`, slug `check:<kind>:<target>:<n>`): the
|
||||
knowledge-graph entity for that check. It carries attributes
|
||||
(`check_type`, `target`, `port`, …) and `checks` edges to the probed target.
|
||||
- **`monitoring` spec** on an entity type (`entity_types.monitoring_spec`): the
|
||||
default list of check kinds (e.g. `[http, process]` for `service`).
|
||||
- Per-entity override: set `monitoring` in the entity's attributes —
|
||||
`"none"` for zero checks, `["http"]` to replace the type defaults.
|
||||
- **`checkdefaults.Ensure`** (`internal/checkdefaults/defaults.go`): the
|
||||
function that reads the monitoring spec, resolves host/port/URL from
|
||||
attributes + relationships, and writes `check_defs` rows. Idempotent.
|
||||
|
||||
## When checks are derived
|
||||
|
||||
`checkdefaults.Ensure` runs in three situations (as of v0.17.1+):
|
||||
|
||||
1. **Seed/deploy ingest** — `internal/db/seed.go:231`. Every entity gets its
|
||||
default checks once on initial ingest.
|
||||
2. **HTTP `POST /api/v1/entities` (create)** — `ensureDefaultChecks` at
|
||||
`internal/httpapi/impl.go:1012`. Creating an entity via the REST API derives
|
||||
its checks in the same transaction.
|
||||
3. **HTTP `PATCH /api/v1/entities` (patch)** — `ensureDefaultChecks` at
|
||||
`internal/httpapi/impl.go:1280`. Changing an entity's attributes (especially
|
||||
`monitoring`) via the REST API regenerates its checks.
|
||||
4. **MCP `create_entity`** — SAME hook. Creating an entity via the MCP tool
|
||||
derives checks. (Added 2026-08-03; previously MCP had no create.)
|
||||
5. **MCP `update_entity_attributes`** — SAME hook. Changing an entity's
|
||||
`monitoring` attribute via MCP now regenerates checks. (Added 2026-08-03;
|
||||
previously MCP updates silently skipped check derivation — the exact bug
|
||||
that stranded the haos session.)
|
||||
|
||||
## Check slug grammar
|
||||
|
||||
```
|
||||
check:<kind>:<target-type>:<target-name>:<n>
|
||||
```
|
||||
|
||||
Examples: `check:http:service:jellyfin:0`, `check:vm-status:vm:haos:0`,
|
||||
`check:cert-expiry:cert:house.hubris.network:0`.
|
||||
|
||||
## Adding monitoring to an entity
|
||||
|
||||
**If the entity already exists:**
|
||||
|
||||
```
|
||||
update_entity_attributes(slug="service:haos", attributes={"monitoring":["http"]})
|
||||
```
|
||||
|
||||
This regenerates checks via `checkdefaults.Ensure`. The result message tells you
|
||||
how many checks were derived and whether any kinds were skipped (and why).
|
||||
|
||||
**If the entity does not exist yet (a new check, ingress, cert, etc.):**
|
||||
|
||||
```
|
||||
create_entity(type="check", name="HAOS http check",
|
||||
slug="check:http:service:haos:0",
|
||||
attributes={"check_type":"http:service","target":"service:haos","port":"8123"})
|
||||
```
|
||||
|
||||
This creates the entity AND derives its `check_defs`. Same for a new `ingress`
|
||||
(`type=ingress`, monitoring `[http]`) or `cert` (`type=cert`,
|
||||
monitoring `[cert-expiry]`).
|
||||
|
||||
**To remove monitoring:** set `monitoring:["none"]` or transition the entity
|
||||
to a terminal lifecycle state (`set_entity_state` → `deprecated`/`destroyed`).
|
||||
|
||||
## Caveats
|
||||
|
||||
- **A service without a `url` attribute AND without a `probe_unit` gets no
|
||||
process check** (the http check covers liveness; the process check would
|
||||
be redundant without an opt-in `probe_unit`). The skip is logged.
|
||||
- **A service whose address comes from a `hosts` edge** may produce no checks on
|
||||
initial create because the edge doesn't exist yet — the next inventory ingest
|
||||
(or a later `update_entity_attributes` after the edge is created) fills it in.
|
||||
- **A `not found` error from `update_entity_attributes`** means the entity
|
||||
doesn't exist — use `create_entity` instead.
|
||||
- **`check_defs` has target columns** (`target_id`, `target_type`). A check
|
||||
entity needs a `checks` relationship (`create_relationship(source=check:…,
|
||||
target=service:…, type="checks")`) so the scheduler can resolve what to
|
||||
probe. `create_entity` derives the check_def; `create_relationship` links
|
||||
the check entity to its target in the graph.
|
||||
|
||||
## Related files
|
||||
|
||||
- `internal/checkdefaults/defaults.go` — `Ensure`, `Target`, `LogResult`
|
||||
- `internal/httpapi/default_checks.go` — `ensureDefaultChecks` (HTTP hook)
|
||||
- `internal/db/checks.go` — `db.EnsureEntityChecks` (shared hook)
|
||||
- `internal/db/seed.go` — seed-time check derivation
|
||||
- `internal/mcp/tools.go` — `create_entity`, `update_entity_attributes`
|
||||
|
||||
## Revision history
|
||||
|
||||
- **2026-08-03:** Created after session `23da10db` stranded for lack of entity-
|
||||
creation tool and unawareness of check-derivation triggers. Covers the MCP
|
||||
create_entity + update_entity_attributes regen paths added same day.
|
||||
@@ -2,12 +2,27 @@
|
||||
# cpu_check.sh — CPU usage % and thermal temperature.
|
||||
set -euo pipefail
|
||||
|
||||
USAGE=$(top -bn1 2>/dev/null | awk '/^%Cpu/ {print 100 - $8}' || true)
|
||||
if [ -z "$USAGE" ]; then
|
||||
CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
|
||||
USAGE=$(awk -v cores="$CORES" '{print ($1+$2+$3)*100/cores}' /proc/loadavg 2>/dev/null || echo "0")
|
||||
os=$(uname -s)
|
||||
|
||||
if [ "$os" = "Darwin" ]; then
|
||||
# `top -l 1 -n 0` prints "CPU usage: X% user, Y% sys, Z% idle".
|
||||
# Usage is 100 minus the idle figure that precedes the literal `idle`.
|
||||
USAGE=$(top -l 1 -n 0 -s 0 2>/dev/null | awk '
|
||||
/^CPU usage/ {
|
||||
for (i = 1; i <= NF; i++) {
|
||||
if ($i == "idle") { gsub(/%/, "", $(i - 1)); printf "%.1f", 100 - $(i - 1) }
|
||||
}
|
||||
}' || true)
|
||||
else
|
||||
USAGE=$(top -bn1 2>/dev/null | awk '/^%Cpu/ {print 100 - $8}' || true)
|
||||
if [ -z "$USAGE" ]; then
|
||||
CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
|
||||
USAGE=$(awk -v cores="$CORES" '{print ($1+$2+$3)*100/cores}' /proc/loadavg 2>/dev/null || echo "0")
|
||||
fi
|
||||
fi
|
||||
|
||||
[ -z "$USAGE" ] && USAGE=0
|
||||
|
||||
TEMP=""
|
||||
if [ -f /sys/class/thermal/thermal_zone0/temp ]; then
|
||||
TEMP=$(awk '{printf "%.1f", $1/1000}' /sys/class/thermal/thermal_zone0/temp 2>/dev/null || true)
|
||||
|
||||
22
checks/disk_usage_check.sh
Normal file → Executable file
22
checks/disk_usage_check.sh
Normal file → Executable file
@@ -2,18 +2,34 @@
|
||||
# disk_usage_check.sh — disk usage and inode usage per mountpoint.
|
||||
set -euo pipefail
|
||||
|
||||
MOUNTS=$(df -k 2>/dev/null | awk 'NR>1 && $1 ~ /^\// && $NF !~ /^\/(snap|dev|proc|sys|run|private)/ {print $NF}' || true)
|
||||
# `timeout` caps each df so a single hung/stale mountpoint (a stale NFS
|
||||
# export, a wedged ZFS pool) can't stall the whole check — that hung the
|
||||
# scheduler's 30s budget on hubris. Available on Linux (coreutils); absent on
|
||||
# Darwin, whose local mounts don't hang, so it degrades to an empty prefix.
|
||||
TO=""
|
||||
if command -v timeout >/dev/null 2>&1; then TO="timeout 8"; fi
|
||||
|
||||
# Build the mount list WITHOUT statting anything: reading /proc/mounts never
|
||||
# blocks the way `df` does on a stuck filesystem, so the enumeration itself
|
||||
# can't hang. Fall back to `df` on hosts without /proc/mounts (macOS).
|
||||
if [ -r /proc/mounts ]; then
|
||||
MOUNTS=$(awk '$1 ~ /^\// && $2 !~ /^\/(snap|dev|proc|sys|run|private)/ {print $2}' /proc/mounts || true)
|
||||
else
|
||||
MOUNTS=$($TO df -k 2>/dev/null | awk 'NR>1 && $1 ~ /^\// && $NF !~ /^\/(snap|dev|proc|sys|run|private)/ {print $NF}' || true)
|
||||
fi
|
||||
FIRST=1
|
||||
|
||||
echo -n '{"health":"healthy","metrics":{'
|
||||
for m in $MOUNTS; do
|
||||
LINE=$(df -k "$m" 2>/dev/null | awk 'NR==2 {print $3, $4, $5, $7}' | tr -d '%' || true)
|
||||
# Each df is bounded: a stuck mount times out and is skipped (LINE empty)
|
||||
# rather than hanging the probe.
|
||||
LINE=$($TO df -k "$m" 2>/dev/null | awk 'NR==2 {print $3, $4, $5, $7}' | tr -d '%' || true)
|
||||
if [ -z "$LINE" ]; then continue; fi
|
||||
USED=$(echo "$LINE" | awk '{print $1}')
|
||||
FREE=$(echo "$LINE" | awk '{print $2}')
|
||||
PCT=$(echo "$LINE" | awk '{print $3}')
|
||||
|
||||
INODE_LINE=$(df -i "$m" 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%' || echo "0")
|
||||
INODE_LINE=$($TO df -i "$m" 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%' || echo "0")
|
||||
INODE_PCT=$(echo "${INODE_LINE:-0}" | sed 's/-/0/')
|
||||
|
||||
KEY=$(echo "$m" | sed 's|/|_|g' | sed 's|^_||')
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
# process_check.sh — systemd service liveness.
|
||||
# process_check.sh — service liveness.
|
||||
#
|
||||
# A service entity's name is a logical label, rarely the literal systemd unit
|
||||
# or container name. matrix = matrix-synapse.service + element-web/mautrix-*
|
||||
# containers; authentik = authentik-server/-worker containers. So checking
|
||||
# `systemctl is-active matrix` reports "inactive" for a healthy service.
|
||||
#
|
||||
# Resolution order, any hit = healthy:
|
||||
# 1. exact systemd unit `systemctl is-active <name>`
|
||||
# 2. a systemd unit with the name as prefix `<name>*.service`
|
||||
# 3. a running docker container whose name contains <name>
|
||||
# An explicit probe target overrides the label — see checkdefaults, which
|
||||
# passes a `probe_unit`/`container`/`systemd_unit` attribute as $1 when set.
|
||||
set -euo pipefail
|
||||
|
||||
SERVICE="${1:-}"
|
||||
@@ -8,15 +20,31 @@ if [ -z "$SERVICE" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v systemctl >/dev/null 2>&1; then
|
||||
echo '{"health":"unknown","signalKind":"process-check","evidence":"systemctl not found"}'
|
||||
exit 0
|
||||
ok() { echo "{\"health\":\"healthy\"}"; exit 0; }
|
||||
|
||||
# 1. exact systemd unit
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null | head -1 || true)
|
||||
[ "$STATE" = "active" ] && ok
|
||||
|
||||
# 2. prefix match: matrix -> matrix-synapse.service, house -> house.service, etc.
|
||||
# --no-legend strips the header/footer so grep can see the unit rows; the
|
||||
# pattern is a systemd unit glob.
|
||||
if systemctl list-units --type=service --state=active --no-legend "$SERVICE*.service" 2>/dev/null \
|
||||
| grep -q '\.service'; then
|
||||
ok
|
||||
fi
|
||||
fi
|
||||
|
||||
STATE=$(systemctl is-active "$SERVICE" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [ "$STATE" = "active" ]; then
|
||||
echo "{\"health\":\"healthy\"}"
|
||||
else
|
||||
echo "{\"health\":\"degraded\",\"signalKind\":\"$SERVICE\",\"evidence\":\"$SERVICE is $STATE\"}"
|
||||
# 3. a running docker container whose name contains the label.
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
if docker ps --filter "status=running" --filter "name=$SERVICE" --format '{{.Names}}' 2>/dev/null \
|
||||
| grep -q .; then
|
||||
ok
|
||||
fi
|
||||
fi
|
||||
|
||||
STATE=${STATE:-inactive}
|
||||
STATE=${STATE//\"/}
|
||||
SAFE_SERVICE=${SERVICE//\"/}
|
||||
echo "{\"health\":\"degraded\",\"signalKind\":\"process\",\"evidence\":\"$SAFE_SERVICE is $STATE (no active unit/container matched)\"}"
|
||||
|
||||
12
checks/pvecm_quorum_check.sh
Normal file
12
checks/pvecm_quorum_check.sh
Normal file
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# pvecm_quorum_check.sh — Proxmox cluster quorum status.
|
||||
# Runs on a PVE host. Fails if the node is not quorate.
|
||||
set -euo pipefail
|
||||
|
||||
# pvecm status exit code is non-zero on non-quorate nodes
|
||||
# (e.g. "Quorate: No — Activity blocked")
|
||||
if pvecm status 2>/dev/null | grep -q 'Quorate.*Yes'; then
|
||||
echo '{"health":"healthy","metrics":{"quorate":1}}'
|
||||
else
|
||||
echo '{"health":"unhealthy","metrics":{"quorate":0}}'
|
||||
fi
|
||||
@@ -57,11 +57,18 @@ type agent struct {
|
||||
apiBase string // oikos HTTP API base, derived from NOMOS_MCP_URL, for chat-assent approvals
|
||||
apiToken string // OIKOS_MCP_BEARER_TOKEN — api's combinedAuth requires it (no dev-open bypass)
|
||||
httpClient *http.Client
|
||||
// gate serializes turns per session (at most one in-flight turn per
|
||||
// sessionID). See turngate.go and plan 2026-08-03 F1.
|
||||
gate *turnGate
|
||||
// queue holds operator messages that arrived while a turn was already
|
||||
// running; they are auto-run when the gate frees (plan 2026-08-03 F2).
|
||||
// See messagequeue.go.
|
||||
queue *messageQueue
|
||||
}
|
||||
|
||||
func newAgent(ctx context.Context, clients *mcpClientPool, st *store, agentSlug string) (*agent, error) {
|
||||
func newAgent(ctx context.Context, clients *mcpClientPool, st *store, agentSlug string, openrouterAPIKey string) (*agent, error) {
|
||||
system := loadSoul()
|
||||
apiKey := os.Getenv("OPENROUTER_API_KEY")
|
||||
apiKey := openrouterAPIKey
|
||||
model := os.Getenv("NOMOS_MODEL")
|
||||
if model == "" {
|
||||
// v4-pro over v4-flash: the flash tier over-narrates, occasionally
|
||||
@@ -117,6 +124,8 @@ func newAgent(ctx context.Context, clients *mcpClientPool, st *store, agentSlug
|
||||
apiBase: apiBase,
|
||||
apiToken: os.Getenv("OIKOS_MCP_BEARER_TOKEN"),
|
||||
httpClient: &http.Client{Timeout: 15 * time.Second},
|
||||
gate: newTurnGate(),
|
||||
queue: newMessageQueue(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -172,6 +181,11 @@ type agentEvent struct {
|
||||
Data any `json:"data,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Iteration int `json:"iteration,omitempty"`
|
||||
// IsThinking marks text/text_delta events that carry the model's internal
|
||||
// reasoning (text produced before tool calls in the same iteration), as
|
||||
// distinct from the final response text. The frontend renders these as
|
||||
// collapsible thinking blocks separated from the response.
|
||||
IsThinking bool `json:"is_thinking,omitempty"`
|
||||
}
|
||||
|
||||
func (a *agent) chat(ctx context.Context, sessionID, message string, emit func(agentEvent)) {
|
||||
@@ -368,7 +382,12 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
|
||||
var msg openai.ChatCompletionMessage
|
||||
var acc openai.ChatCompletionAccumulator
|
||||
|
||||
for attempt := 0; attempt <= maxLLMRetries; attempt++ {
|
||||
// Capture token usage from this LLM response for activity logging.
|
||||
// Previously always NULL — every agent_activity row had no token
|
||||
// count. Now each tool call in this iteration gets the same total.
|
||||
totalTokens := 0
|
||||
|
||||
for attempt := 0; attempt <= maxLLMRetries; attempt++ {
|
||||
acc = openai.ChatCompletionAccumulator{}
|
||||
stream := a.provider.Chat.Completions.NewStreaming(ctx, params, a.reqOpts...)
|
||||
for stream.Next() {
|
||||
@@ -400,14 +419,19 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
|
||||
msg = acc.Choices[0].Message
|
||||
finishReason := acc.Choices[0].FinishReason
|
||||
|
||||
// Capture token usage from this iteration.
|
||||
if acc.Usage.TotalTokens > 0 {
|
||||
totalTokens = int(acc.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
if len(msg.ToolCalls) == 0 {
|
||||
if isRefusalOrEmpty(msg.Content) {
|
||||
if attempt < maxLLMRetries {
|
||||
slog.Warn("nomos: empty or refusal response, retrying",
|
||||
"session", sessionID, "iter", i+1, "attempt", attempt+1,
|
||||
"content_len", len(msg.Content), "finish_reason", finishReason)
|
||||
continue
|
||||
}
|
||||
if attempt < maxLLMRetries {
|
||||
slog.Warn("nomos: empty or refusal response, retrying",
|
||||
"session", sessionID, "iter", i+1, "attempt", attempt+1,
|
||||
"content_len", len(msg.Content), "finish_reason", finishReason)
|
||||
continue
|
||||
}
|
||||
// B.4: surface the real error context (finish_reason +
|
||||
// refusal text) instead of a generic "empty response" —
|
||||
// the operator can tell "content_filter — rephrase" from
|
||||
@@ -456,7 +480,7 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
|
||||
// led to each step. Emitting it lets the persist layer accumulate
|
||||
// per-iteration reasoning into the row's text field.
|
||||
if strings.TrimSpace(msg.Content) != "" {
|
||||
emit(agentEvent{Type: "text", Data: msg.Content, SessionID: sessionID})
|
||||
emit(agentEvent{Type: "text", Data: msg.Content, SessionID: sessionID, IsThinking: true})
|
||||
}
|
||||
|
||||
slog.Info("nomos: tool calls", "count", len(msg.ToolCalls), "iter", i+1, "correlation", correlationID)
|
||||
@@ -491,7 +515,7 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
|
||||
slog.Warn("nomos: run retry cap hit — refusing dispatch",
|
||||
"target", t, "failures", n, "session", sessionID)
|
||||
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args,
|
||||
tc.Function.Arguments, directive, 0, false, correlationID)
|
||||
tc.Function.Arguments, directive, 0, false, correlationID, totalTokens)
|
||||
emit(agentEvent{
|
||||
Type: "tool_result",
|
||||
Data: map[string]any{"name": tc.Function.Name, "result": directive, "id": tc.ID, "retry_capped": true},
|
||||
@@ -542,7 +566,7 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
|
||||
inputStr := string(inputJSON)
|
||||
|
||||
if callErr != nil {
|
||||
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args, inputStr, callErr.Error(), elapsed, false, correlationID)
|
||||
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args, inputStr, callErr.Error(), elapsed, false, correlationID, totalTokens)
|
||||
|
||||
// Retry cap: dispatch errors (e.g. MCP client timeout)
|
||||
// count toward the cap too. A command that keeps timing
|
||||
@@ -572,7 +596,7 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
|
||||
}
|
||||
|
||||
resultJSON, _ := json.Marshal(result)
|
||||
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args, inputStr, string(resultJSON), elapsed, true, correlationID)
|
||||
a.store.logActivity(ctx, a.agentID, sessionID, tc.Function.Name, args, inputStr, string(resultJSON), elapsed, true, correlationID, totalTokens)
|
||||
|
||||
// Link any execution this tool queued/started back to this
|
||||
// session, so the auto-continuation worker can feed its result
|
||||
|
||||
@@ -76,16 +76,20 @@ func (a *agent) processIdleSweep(ctx context.Context) {
|
||||
s := s
|
||||
if s.CompletionNudges == 0 {
|
||||
safego.Go("nomos:idle-nudge:"+s.ID, func() {
|
||||
if err := a.store.bumpCompletionNudge(ctx, s.ID); err != nil {
|
||||
slog.Error("nomos: idle nudge bump failed", "session", s.ID, "error", err)
|
||||
return
|
||||
note := fmt.Sprintf("[System: this task ('%s') has been idle for %s with no complete_task call. "+
|
||||
"If the goal is done (or can't be completed), call complete_task now with the outcome and a "+
|
||||
"one-line summary. If you're still genuinely working through the plan, ignore this and continue.]",
|
||||
s.Goal, idleTaskThreshold)
|
||||
note = a.store.enrichResumeNote(ctx, s.ID, note)
|
||||
// P1: only count the nudge if it actually delivered. resumeSession
|
||||
// skips (returns false) when a turn is already active; bumping the
|
||||
// counter anyway would make the next sweep auto-close a merely-busy
|
||||
// session as "unanswered."
|
||||
if a.resumeSession(ctx, s.ID, note) {
|
||||
if err := a.store.bumpCompletionNudge(ctx, s.ID); err != nil {
|
||||
slog.Error("nomos: idle nudge bump failed", "session", s.ID, "error", err)
|
||||
}
|
||||
}
|
||||
note := fmt.Sprintf("[System: this task ('%s') has been idle for %s with no complete_task call. "+
|
||||
"If the goal is done (or can't be completed), call complete_task now with the outcome and a "+
|
||||
"one-line summary. If you're still genuinely working through the plan, ignore this and continue.]",
|
||||
s.Goal, idleTaskThreshold)
|
||||
note = a.store.enrichResumeNote(ctx, s.ID, note)
|
||||
a.resumeSession(ctx, s.ID, note)
|
||||
})
|
||||
continue
|
||||
}
|
||||
@@ -163,7 +167,9 @@ func (a *agent) processContinuations(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
a.store.markContinued(ctx, p.ExecID) // stamp first: a failure here must not cause a re-continue loop
|
||||
// markContinued now happens inside continueSession, AFTER resumeSession
|
||||
// actually runs (P0). Pre-marking here consumed the item even when
|
||||
// resumeSession skipped on a busy session, losing the result.
|
||||
safego.Go("nomos:continue-session:"+p.SessionID, func() { a.continueSession(ctx, p) })
|
||||
}
|
||||
}
|
||||
@@ -179,7 +185,18 @@ func (a *agent) processContinuations(ctx context.Context) {
|
||||
// something new to poll for.
|
||||
func (a *agent) continueSession(ctx context.Context, p pendingContinuation) {
|
||||
slog.Info("nomos: auto-continuing session", "session", p.SessionID, "execution", p.ExecID, "status", p.Status)
|
||||
a.resumeSession(ctx, p.SessionID, buildContinuationNote(p))
|
||||
// P0 (plans/2026-08-03-nomos-chat-changes-review.md): mark the execution
|
||||
// continued ONLY after the turn actually ran. resumeSession skips (returns
|
||||
// false) when another turn is already active for this session; marking
|
||||
// before that — as the old code did — consumed the item (continued_at set,
|
||||
// never re-queued by pendingContinuations) and silently lost the result.
|
||||
// On a skip, leave it pending so the next worker tick retries once the
|
||||
// active turn frees the permit.
|
||||
if !a.resumeSession(ctx, p.SessionID, buildContinuationNote(p)) {
|
||||
slog.Info("nomos: continuation deferred — a turn is active; will retry next tick", "session", p.SessionID, "execution", p.ExecID)
|
||||
return
|
||||
}
|
||||
a.store.markContinued(ctx, p.ExecID)
|
||||
}
|
||||
|
||||
// resumeSession re-invokes the agent for a session with a system-injected note —
|
||||
@@ -187,7 +204,32 @@ func (a *agent) continueSession(ctx context.Context, p pendingContinuation) {
|
||||
// (handleAnswerQuestion) — persisting progress LIVE (a placeholder row updated
|
||||
// in place as each tool call lands) so the frontend poller sees each step,
|
||||
// instead of total silence until the whole resume concludes.
|
||||
func (a *agent) resumeSession(ctx context.Context, sessionID, note string) {
|
||||
//
|
||||
// F1 (plan 2026-08-03): this is the single entry point for EVERY background
|
||||
// turn — the continuation worker, idle sweep, answer-question, /resume, and the
|
||||
// empty-message reconnect all funnel through here. It acquires the session's
|
||||
// turn permit non-blocking and SKIPS if a turn is already running. A duplicate
|
||||
// resume while a turn (live or background) is active is exactly the
|
||||
// interleaving that corrupted the activity panel and made tasks feel stuck.
|
||||
//
|
||||
// Returns whether the turn actually ran. Callers that mutate state before
|
||||
// resuming (the continuation worker's markContinued, the idle sweep's nudge
|
||||
// bump) MUST gate that mutation on a true return — otherwise a busy-skip leaves
|
||||
// the state changed but the work undone (lost continuation / false auto-close).
|
||||
// See plans/2026-08-03-nomos-chat-changes-review.md P0/P1.
|
||||
func (a *agent) resumeSession(ctx context.Context, sessionID, note string) bool {
|
||||
if !a.gate.acquire(sessionID, 0) {
|
||||
slog.Info("nomos: turn already active, skipping background resume", "session", sessionID)
|
||||
return false
|
||||
}
|
||||
// Release the gate, then drain any operator message that was queued while
|
||||
// this background turn ran (plan 2026-08-03 F2). Queued messages are run as
|
||||
// real user turns server-side; resumeSession itself never enqueues.
|
||||
defer func() {
|
||||
a.gate.release(sessionID)
|
||||
safego.Go("nomos:drain:"+sessionID, func() { a.drainQueued(context.Background(), sessionID) })
|
||||
}()
|
||||
|
||||
placeholder, _ := json.Marshal(map[string]any{
|
||||
"role": "assistant",
|
||||
"text": "",
|
||||
@@ -200,6 +242,7 @@ func (a *agent) resumeSession(ctx context.Context, sessionID, note string) {
|
||||
|
||||
var toolCalls []map[string]any
|
||||
var finalText, errText string
|
||||
var finalThinking string
|
||||
|
||||
persist := func() {
|
||||
if msgID == uuid.Nil {
|
||||
@@ -212,6 +255,7 @@ func (a *agent) resumeSession(ctx context.Context, sessionID, note string) {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"role": "assistant",
|
||||
"text": text,
|
||||
"thinking": finalThinking,
|
||||
"tool_calls": toolCalls,
|
||||
"auto": true, // marks this as an autonomous continuation, not an operator turn
|
||||
})
|
||||
@@ -242,15 +286,19 @@ func (a *agent) resumeSession(ctx context.Context, sessionID, note string) {
|
||||
if attempt > 0 {
|
||||
select {
|
||||
case <-cctx.Done():
|
||||
return
|
||||
return true // a turn ran on an earlier attempt; consume, don't re-loop
|
||||
case <-time.After(time.Duration(2<<attempt) * time.Second): // 4s, 8s
|
||||
}
|
||||
}
|
||||
toolCalls, finalText, errText = nil, "", ""
|
||||
finalThinking = ""
|
||||
// P3: accumulate per-iteration reasoning instead of overwriting
|
||||
// (same fix as main.go's chat handler). Without this, a resumed
|
||||
// turn's intermediate thinking is lost on reload.
|
||||
// (same fix as main.go's chat handler). Without this, a resumed
|
||||
// turn's intermediate thinking is lost on reload.
|
||||
var textParts []string
|
||||
var thinkingParts []string
|
||||
emit := func(ev agentEvent) {
|
||||
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
||||
if m, ok := ev.Data.(map[string]any); ok {
|
||||
@@ -277,8 +325,13 @@ func (a *agent) resumeSession(ctx context.Context, sessionID, note string) {
|
||||
}
|
||||
if ev.Type == "text" {
|
||||
if t, ok := ev.Data.(string); ok && t != "" {
|
||||
textParts = append(textParts, t)
|
||||
finalText = strings.Join(textParts, "\n\n")
|
||||
if ev.IsThinking {
|
||||
thinkingParts = append(thinkingParts, t)
|
||||
finalThinking = strings.Join(thinkingParts, "\n\n")
|
||||
} else {
|
||||
textParts = append(textParts, t)
|
||||
finalText = strings.Join(textParts, "\n\n")
|
||||
}
|
||||
persist()
|
||||
}
|
||||
}
|
||||
@@ -314,9 +367,10 @@ func (a *agent) resumeSession(ctx context.Context, sessionID, note string) {
|
||||
// No placeholder was inserted (rare), save directly.
|
||||
a.store.saveMessage(context.Background(), sessionID, "assistant", body)
|
||||
}
|
||||
return // do not call persist() again — already persisted above
|
||||
return true // do not call persist() again — already persisted above
|
||||
}
|
||||
persist() // final state — same row, updated one last time with the concluding text
|
||||
return true
|
||||
}
|
||||
|
||||
// buildContinuationNote frames the finished execution for the model: what
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestExtractExecutionIDs(t *testing.T) {
|
||||
// Real tool-result phrasings that should yield an execution id.
|
||||
@@ -37,3 +42,37 @@ func TestExtractExecutionIDs(t *testing.T) {
|
||||
t.Errorf("expected de-dup to 1 id, got %v", ids)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeSession_SkipsWhenBusy guards the P0 fix
|
||||
// (plans/2026-08-03-nomos-chat-changes-review.md): resumeSession must skip —
|
||||
// return false, body never executed — when a turn is already active for the
|
||||
// session. continueSession relies on this so it only marks a continuation
|
||||
// "continued" after a turn really ran (otherwise the result is lost: marked
|
||||
// continued, never re-queued by pendingContinuations).
|
||||
//
|
||||
// A minimal agent with only a gate is enough: if the body ever ran, chatWith
|
||||
// would dereference the nil provider and panic. Returning false cleanly proves
|
||||
// the body was skipped.
|
||||
func TestResumeSession_SkipsWhenBusy(t *testing.T) {
|
||||
a := &agent{gate: newTurnGate()}
|
||||
if !a.gate.acquire("sess", 0) {
|
||||
t.Fatal("precondition: initial acquire should succeed on a free session")
|
||||
}
|
||||
ran := a.resumeSession(context.Background(), "sess", "note")
|
||||
if ran {
|
||||
t.Fatal("resumeSession must return false (skip) while a turn is active for the session")
|
||||
}
|
||||
}
|
||||
|
||||
// TestContinueSession_DefersWhenBusy guards the other half of P0: when the
|
||||
// session is busy, continueSession defers (leaves the execution pending for the
|
||||
// next worker tick) instead of running or marking it. It must return cleanly
|
||||
// without reaching resumeSession's body (nil provider → panic) or markContinued.
|
||||
func TestContinueSession_DefersWhenBusy(t *testing.T) {
|
||||
a := &agent{gate: newTurnGate()}
|
||||
if !a.gate.acquire("sess", 0) {
|
||||
t.Fatal("precondition: initial acquire should succeed on a free session")
|
||||
}
|
||||
p := pendingContinuation{ExecID: uuid.New(), SessionID: "sess", Status: "completed"}
|
||||
a.continueSession(context.Background(), p) // must not panic; must not run/mark
|
||||
}
|
||||
|
||||
@@ -319,8 +319,10 @@ func fetchTranscript(ctx context.Context, gateway, sid string) (transcript, sess
|
||||
}
|
||||
// Fetch the plan (steps with generation numbers) for the
|
||||
// plan_generations assertion. A 404 or empty response is fine — a
|
||||
// pure-DB Q&A with no propose_plan has no plan.
|
||||
if planResp, perr := http.Get(gateway + "/sessions/" + sid + "/plan"); perr == nil {
|
||||
// pure-DB Q&A with no propose_plan has no plan. ?all=true returns every
|
||||
// generation so the assertion can count them (the default view returns
|
||||
// only the current generation).
|
||||
if planResp, perr := http.Get(gateway + "/sessions/" + sid + "/plan?all=true"); perr == nil {
|
||||
if planResp.StatusCode == 200 {
|
||||
pb, _ := io.ReadAll(planResp.Body)
|
||||
_ = json.Unmarshal(pb, &t) // fills t.PlanSteps via "steps" field
|
||||
|
||||
348
cmd/nomos/mcp.go
Normal file
348
cmd/nomos/mcp.go
Normal file
@@ -0,0 +1,348 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── MCP Streamable HTTP client ────────────────────────────────────────
|
||||
|
||||
type mcpClient struct {
|
||||
baseURL string
|
||||
token string // OIKOS_MCP_BEARER_TOKEN — api's combinedAuth requires it on every request (no dev-open bypass)
|
||||
sessionID string
|
||||
http *http.Client
|
||||
nextID int
|
||||
mu sync.Mutex // one client serializes its own MCP calls (the pool gives each session its own client, so this never blocks another session)
|
||||
// toolsCache holds the last tools/list result. The tool list is static
|
||||
// for the lifetime of one MCP connection — it only changes when the api
|
||||
// process (re)registers tools, i.e. on a restart, which this client
|
||||
// already detects and reacts to via reconnectLocked. Without this,
|
||||
// buildTools (called at the start of EVERY chat turn, including every
|
||||
// auto-continuation resume) paid a full tools/list round-trip every
|
||||
// single time for a list that's almost always identical to the last one.
|
||||
// Guarded separately from mu (not reused) so a cache check never
|
||||
// contends with an in-flight doRequest call for a different method.
|
||||
toolsMu sync.Mutex
|
||||
toolsCache []toolDef
|
||||
}
|
||||
|
||||
func newMCPClient(baseURL, token string) (*mcpClient, error) {
|
||||
c := &mcpClient{
|
||||
baseURL: baseURL,
|
||||
token: token,
|
||||
http: &http.Client{Timeout: 120 * time.Second},
|
||||
}
|
||||
|
||||
resp, err := c.doRequest("initialize", map[string]any{
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": map[string]any{},
|
||||
"clientInfo": map[string]any{"name": "nomos", "version": "2.0"},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize: %w", err)
|
||||
}
|
||||
if resp.sessionID == "" {
|
||||
return nil, fmt.Errorf("no session ID in initialize response")
|
||||
}
|
||||
c.sessionID = resp.sessionID
|
||||
|
||||
c.doRequest("notifications/initialized", map[string]any{})
|
||||
|
||||
slog.Info("nomos: mcp connected", "session", c.sessionID[:16]+"...")
|
||||
return c, nil
|
||||
}
|
||||
|
||||
type mcpJSONRPCResponse struct {
|
||||
sessionID string
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error json.RawMessage `json:"error"`
|
||||
}
|
||||
|
||||
// errStaleSession signals that the MCP server rejected our session id (e.g.
|
||||
// after an api/MCP restart), so the client should re-initialize and retry.
|
||||
var errStaleSession = fmt.Errorf("mcp session stale")
|
||||
|
||||
// doRequest serializes MCP calls and transparently re-initializes the session
|
||||
// if the server has forgotten it (common after an api redeploy), retrying the
|
||||
// original call once. Without this, an api restart permanently breaks nomos
|
||||
// until it is itself restarted.
|
||||
func (c *mcpClient) doRequest(method string, params map[string]any) (*mcpJSONRPCResponse, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
resp, err := c.send(method, params)
|
||||
if err != nil && method != "initialize" && errors.Is(err, errStaleSession) {
|
||||
slog.Warn("nomos: mcp session stale, reconnecting")
|
||||
if rerr := c.reconnectLocked(); rerr != nil {
|
||||
return nil, fmt.Errorf("mcp reconnect: %w (original: %v)", rerr, err)
|
||||
}
|
||||
return c.send(method, params)
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// reconnectLocked re-initializes the MCP session. The caller must hold c.mu.
|
||||
func (c *mcpClient) reconnectLocked() error {
|
||||
c.sessionID = ""
|
||||
// A reconnect means the api process was restarted (or forgot us) — its
|
||||
// tool registration may have changed, so the cached list is no longer
|
||||
// trustworthy.
|
||||
c.toolsMu.Lock()
|
||||
c.toolsCache = nil
|
||||
c.toolsMu.Unlock()
|
||||
resp, err := c.send("initialize", map[string]any{
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": map[string]any{},
|
||||
"clientInfo": map[string]any{"name": "nomos", "version": "2.0"},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.sessionID == "" {
|
||||
return fmt.Errorf("no session ID on re-initialize")
|
||||
}
|
||||
c.sessionID = resp.sessionID
|
||||
_, _ = c.send("notifications/initialized", map[string]any{})
|
||||
slog.Info("nomos: mcp reconnected", "session", c.sessionID[:16]+"...")
|
||||
return nil
|
||||
}
|
||||
|
||||
// send performs one MCP round-trip. It does not lock; callers hold c.mu.
|
||||
func (c *mcpClient) send(method string, params map[string]any) (*mcpJSONRPCResponse, error) {
|
||||
c.nextID++
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"method": method,
|
||||
"params": params,
|
||||
"id": c.nextID,
|
||||
})
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, c.baseURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json, text/event-stream")
|
||||
if c.sessionID != "" {
|
||||
req.Header.Set("Mcp-Session-Id", c.sessionID)
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// A rejected/unknown session comes back as 4xx (commonly 400/404).
|
||||
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusBadRequest {
|
||||
return nil, errStaleSession
|
||||
}
|
||||
|
||||
result := &mcpJSONRPCResponse{}
|
||||
result.sessionID = resp.Header.Get("Mcp-Session-Id")
|
||||
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
gotData := false
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
gotData = true
|
||||
data := line[6:]
|
||||
if err := json.Unmarshal([]byte(data), result); err != nil {
|
||||
return nil, fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if result.Error != nil {
|
||||
return nil, fmt.Errorf("rpc error: %s", string(result.Error))
|
||||
}
|
||||
|
||||
// Empty body with no result and a session set: the server likely dropped
|
||||
// our session. Notifications legitimately return no data, so exempt them.
|
||||
if !gotData && result.Result == nil && method != "notifications/initialized" {
|
||||
return nil, errStaleSession
|
||||
}
|
||||
|
||||
if result.sessionID != "" {
|
||||
c.sessionID = result.sessionID
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *mcpClient) callTool(name string, args map[string]any) (any, error) {
|
||||
resp, err := c.doRequest("tools/call", map[string]any{
|
||||
"name": name,
|
||||
"arguments": args,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var toolResult struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if err := json.Unmarshal(resp.Result, &toolResult); err != nil {
|
||||
return string(resp.Result), nil
|
||||
}
|
||||
|
||||
var texts []string
|
||||
for _, c := range toolResult.Content {
|
||||
if c.Type == "text" {
|
||||
var parsed any
|
||||
if json.Unmarshal([]byte(c.Text), &parsed) == nil {
|
||||
return parsed, nil
|
||||
}
|
||||
texts = append(texts, c.Text)
|
||||
}
|
||||
}
|
||||
if len(texts) == 1 {
|
||||
return texts[0], nil
|
||||
}
|
||||
return texts, nil
|
||||
}
|
||||
|
||||
func (c *mcpClient) listTools() ([]string, error) {
|
||||
resp, err := c.doRequest("tools/list", map[string]any{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var tr struct {
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
} `json:"tools"`
|
||||
}
|
||||
if err := json.Unmarshal(resp.Result, &tr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var names []string
|
||||
for _, t := range tr.Tools {
|
||||
names = append(names, t.Name)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (c *mcpClient) close() {
|
||||
}
|
||||
|
||||
// ─── Per-session MCP client pool ────────────────────────────────────────
|
||||
//
|
||||
// A single shared mcpClient serializes EVERY tool call across EVERY
|
||||
// concurrently-running task through one mutex (see mcpClient.mu) — `run`
|
||||
// executes its SSH command synchronously inside that lock and is capped at
|
||||
// up to 10 minutes, so one task mid-`run` stalled every other task's tool
|
||||
// calls, even trivial reads, behind it. The MCP *server* has no per-
|
||||
// connection state to protect (newServer in internal/mcp/server.go returns
|
||||
// one shared *mcp.Server instance whose tool handlers close only over the DB
|
||||
// pool, which is already safe for concurrent use) — the mutex existed purely
|
||||
// because the *client* reused one stateful transport session, not because
|
||||
// the server needed it. Giving each task's own session its own client
|
||||
// removes the cross-task serialization entirely: a task's own tool calls
|
||||
// stay sequential (which they already are — the agent loop calls tools one
|
||||
// at a time within a turn), but no longer block anyone else's.
|
||||
type mcpClientPool struct {
|
||||
baseURL string
|
||||
token string
|
||||
mu sync.Mutex
|
||||
clients map[string]*pooledMCPClient
|
||||
}
|
||||
|
||||
type pooledMCPClient struct {
|
||||
client *mcpClient
|
||||
lastUsed time.Time
|
||||
}
|
||||
|
||||
func newMCPClientPool(baseURL, token string) *mcpClientPool {
|
||||
return &mcpClientPool{baseURL: baseURL, token: token, clients: make(map[string]*pooledMCPClient)}
|
||||
}
|
||||
|
||||
// get returns the client for sessionID, creating and initializing one (a
|
||||
// real MCP handshake) on first use. Session ids that don't identify a real
|
||||
// persisted conversation ("" / "ephemeral", the no-DB-store path; "query",
|
||||
// the structured /query endpoint) still get exactly one dedicated,
|
||||
// reused client each via the same map — just keyed on a fixed string instead
|
||||
// of a real session id — so that traffic doesn't pay a fresh handshake per
|
||||
// request while still never sharing a connection with an actual task.
|
||||
func (p *mcpClientPool) get(sessionID string) (*mcpClient, error) {
|
||||
key := sessionID
|
||||
if key == "" {
|
||||
key = "ephemeral"
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
if pc, ok := p.clients[key]; ok {
|
||||
pc.lastUsed = time.Now()
|
||||
p.mu.Unlock()
|
||||
return pc.client, nil
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
// Initialize outside the lock — it's a network round-trip, and holding
|
||||
// the pool mutex for it would serialize unrelated sessions' first calls
|
||||
// behind each other, undermining the whole point of this pool.
|
||||
c, err := newMCPClient(p.baseURL, p.token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
// Another goroutine may have created one for the same key while we were
|
||||
// initializing (two of this session's tool calls racing on a cold
|
||||
// start); keep whichever won, close out the loser's connection (a no-op
|
||||
// today, but future-proof if mcpClient.close ever does real teardown).
|
||||
if existing, ok := p.clients[key]; ok {
|
||||
p.mu.Unlock()
|
||||
c.close()
|
||||
return existing.client, nil
|
||||
}
|
||||
p.clients[key] = &pooledMCPClient{client: c, lastUsed: time.Now()}
|
||||
p.mu.Unlock()
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// mcpClientIdleTimeout is how long an idle session's MCP client is kept
|
||||
// before eviction — long enough to outlive a single slow `run` (capped at 10
|
||||
// minutes server-side) plus normal think-time between a task's tool calls,
|
||||
// short enough not to accumulate one abandoned connection per finished task
|
||||
// forever.
|
||||
const mcpClientIdleTimeout = 20 * time.Minute
|
||||
|
||||
// sweep evicts clients idle past mcpClientIdleTimeout. Call on a ticker.
|
||||
func (p *mcpClientPool) sweep() {
|
||||
cutoff := time.Now().Add(-mcpClientIdleTimeout)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for key, pc := range p.clients {
|
||||
if pc.lastUsed.Before(cutoff) {
|
||||
pc.client.close()
|
||||
delete(p.clients, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *mcpClientPool) closeAll() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for key, pc := range p.clients {
|
||||
pc.client.close()
|
||||
delete(p.clients, key)
|
||||
}
|
||||
}
|
||||
82
cmd/nomos/messagequeue.go
Normal file
82
cmd/nomos/messagequeue.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// maxQueuedPerSession caps a session's queue. A held turn plus unbounded
|
||||
// enqueues would grow memory without limit; an operator nudging a long
|
||||
// autonomous turn realistically queues only a handful, so a generous cap is
|
||||
// pure insurance. Overflow drops the newest enqueue and logs (the message is
|
||||
// already persisted in the DB by handleChat before enqueue, so it isn't lost
|
||||
// from the transcript — it just won't auto-run).
|
||||
const maxQueuedPerSession = 20
|
||||
|
||||
// messageQueue holds operator messages that arrived while a turn was already
|
||||
// running for a session. Plan 2026-08-03 (F2): instead of rejecting the
|
||||
// operator's message with "Nomos is still finishing a previous step… send it
|
||||
// again", the message is queued and auto-run when the in-flight turn releases
|
||||
// the session's turn-gate permit.
|
||||
//
|
||||
// The queue only schedules WHEN a turn runs, not WHETHER the message is stored
|
||||
// — handleChat persists the user message before acquiring the gate, so a queued
|
||||
// message is already in the transcript; this just makes sure a turn eventually
|
||||
// acts on it.
|
||||
//
|
||||
// Draining is strictly one-at-a-time under the turn gate (see drainQueued in
|
||||
// main.go), so this cannot stack concurrent turns — the exact hazard the gate
|
||||
// itself exists to prevent. Background resumeSession callers never touch this
|
||||
// queue; they keep their non-blocking skip.
|
||||
type messageQueue struct {
|
||||
mu sync.Mutex
|
||||
queue map[string][]string
|
||||
}
|
||||
|
||||
func newMessageQueue() *messageQueue {
|
||||
return &messageQueue{queue: map[string][]string{}}
|
||||
}
|
||||
|
||||
// enqueue appends a message to the back of the session's FIFO. Returns false
|
||||
// (and logs) if the session is already at maxQueuedPerSession — the caller's
|
||||
// message is already persisted in the DB, so this only skips auto-running it.
|
||||
func (q *messageQueue) enqueue(sessionID, msg string) bool {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if len(q.queue[sessionID]) >= maxQueuedPerSession {
|
||||
slog.Warn("nomos: message queue full; dropping auto-run for operator message", "session", sessionID, "cap", maxQueuedPerSession)
|
||||
return false
|
||||
}
|
||||
q.queue[sessionID] = append(q.queue[sessionID], msg)
|
||||
return true
|
||||
}
|
||||
|
||||
// dequeue pops the next message from the front of the session's FIFO. Returns
|
||||
// ok=false when empty.
|
||||
func (q *messageQueue) dequeue(sessionID string) (string, bool) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
xs := q.queue[sessionID]
|
||||
if len(xs) == 0 {
|
||||
return "", false
|
||||
}
|
||||
m := xs[0]
|
||||
q.queue[sessionID] = xs[1:]
|
||||
return m, true
|
||||
}
|
||||
|
||||
// requeueFront pushes a message back to the front — used when a drainer popped
|
||||
// a message but lost the race for the gate to a live turn; that turn's own
|
||||
// release will drain it again.
|
||||
func (q *messageQueue) requeueFront(sessionID, msg string) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
q.queue[sessionID] = append([]string{msg}, q.queue[sessionID]...)
|
||||
}
|
||||
|
||||
// peek reports the queued depth for a session (test/diagnostic helper).
|
||||
func (q *messageQueue) peek(sessionID string) int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return len(q.queue[sessionID])
|
||||
}
|
||||
142
cmd/nomos/messagequeue_test.go
Normal file
142
cmd/nomos/messagequeue_test.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMessageQueue_FIFO(t *testing.T) {
|
||||
q := newMessageQueue()
|
||||
q.enqueue("s", "first")
|
||||
q.enqueue("s", "second")
|
||||
q.enqueue("s", "third")
|
||||
|
||||
want := []string{"first", "second", "third"}
|
||||
for _, w := range want {
|
||||
got, ok := q.dequeue("s")
|
||||
if !ok || got != w {
|
||||
t.Fatalf("dequeue = %q,%v want %q,true", got, ok, w)
|
||||
}
|
||||
}
|
||||
if _, ok := q.dequeue("s"); ok {
|
||||
t.Fatal("dequeue on drained queue should return ok=false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageQueue_RequeueFront(t *testing.T) {
|
||||
q := newMessageQueue()
|
||||
q.enqueue("s", "a")
|
||||
q.enqueue("s", "b")
|
||||
// Pop "a", then push it back to the front; "a" must come out before "b".
|
||||
a, _ := q.dequeue("s")
|
||||
q.requeueFront("s", a)
|
||||
got, _ := q.dequeue("s")
|
||||
if got != "a" {
|
||||
t.Fatalf("after requeueFront, dequeue = %q want %q", got, "a")
|
||||
}
|
||||
got2, _ := q.dequeue("s")
|
||||
if got2 != "b" {
|
||||
t.Fatalf("next dequeue = %q want %q", got2, "b")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageQueue_IsolatedPerSession(t *testing.T) {
|
||||
q := newMessageQueue()
|
||||
q.enqueue("s1", "one")
|
||||
q.enqueue("s2", "two")
|
||||
if got, _ := q.dequeue("s1"); got != "one" {
|
||||
t.Fatalf("s1 = %q want one", got)
|
||||
}
|
||||
if got, _ := q.dequeue("s2"); got != "two" {
|
||||
t.Fatalf("s2 = %q want two", got)
|
||||
}
|
||||
if q.peek("s1") != 0 || q.peek("s2") != 0 {
|
||||
t.Fatal("both sessions should be drained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageQueue_Concurrent(t *testing.T) {
|
||||
q := newMessageQueue()
|
||||
const n = maxQueuedPerSession // stay under the cap so every enqueue lands
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < n; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
q.enqueue("s", "m")
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
if q.peek("s") != n {
|
||||
t.Fatalf("peek = %d want %d (all enqueues must be counted)", q.peek("s"), n)
|
||||
}
|
||||
seen := 0
|
||||
for {
|
||||
if _, ok := q.dequeue("s"); !ok {
|
||||
break
|
||||
}
|
||||
seen++
|
||||
}
|
||||
if seen != n {
|
||||
t.Fatalf("drained %d want %d", seen, n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageQueue_CapsOverflow(t *testing.T) {
|
||||
q := newMessageQueue()
|
||||
for i := 0; i < maxQueuedPerSession; i++ {
|
||||
if !q.enqueue("s", "m") {
|
||||
t.Fatalf("enqueue #%d within cap should succeed", i)
|
||||
}
|
||||
}
|
||||
if q.enqueue("s", "overflow") {
|
||||
t.Fatal("enqueue past the cap should return false (dropped)")
|
||||
}
|
||||
if got := q.peek("s"); got != maxQueuedPerSession {
|
||||
t.Fatalf("peek = %d want %d (overflow must not append)", got, maxQueuedPerSession)
|
||||
}
|
||||
}
|
||||
|
||||
// drainQueued on an empty queue must be a no-op: it returns immediately and
|
||||
// never touches the gate (so the session stays free for the next turn).
|
||||
func TestDrainQueued_NoOpOnEmpty(t *testing.T) {
|
||||
a := &agent{gate: newTurnGate(), queue: newMessageQueue()}
|
||||
a.drainQueued(context.Background(), "s")
|
||||
if !a.gate.acquire("s", 0) {
|
||||
t.Fatal("gate should be free after a no-op drain (drain must not hold it)")
|
||||
}
|
||||
a.gate.release("s")
|
||||
}
|
||||
|
||||
// With a queued message but the gate held by another turn, drainQueued must
|
||||
// re-queue the message and return WITHOUT running a turn (no store/provider → a
|
||||
// real run would panic). This is the "never stack" property: a busy gate
|
||||
// defers to the holder's own release-drain.
|
||||
func TestDrainQueued_RequeuesWhenBusy(t *testing.T) {
|
||||
prev := drainAcquireWait
|
||||
drainAcquireWait = 10 * time.Millisecond
|
||||
t.Cleanup(func() { drainAcquireWait = prev })
|
||||
|
||||
a := &agent{gate: newTurnGate(), queue: newMessageQueue()}
|
||||
if !a.gate.acquire("s", 0) {
|
||||
t.Fatal("precondition: hold the gate")
|
||||
}
|
||||
a.queue.enqueue("s", "queued-msg")
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
a.drainQueued(context.Background(), "s") // must not panic; must requeue
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("drainQueued did not return promptly while the gate was busy")
|
||||
}
|
||||
if got := a.queue.peek("s"); got != 1 {
|
||||
t.Fatalf("message should be re-queued while busy; peek = %d want 1", got)
|
||||
}
|
||||
a.gate.release("s")
|
||||
}
|
||||
@@ -1,11 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -18,7 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/safego"
|
||||
"github.com/google/uuid"
|
||||
"github.com/dtoro/oikos/internal/secrets"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
@@ -27,13 +24,14 @@ func main() {
|
||||
fmt.Fprintln(os.Stderr, "usage: nomos serve")
|
||||
os.Exit(1)
|
||||
}
|
||||
if os.Args[1] == "healthcheck" {
|
||||
runHealthcheck()
|
||||
return
|
||||
}
|
||||
mcpURL := os.Getenv("NOMOS_MCP_URL")
|
||||
if mcpURL == "" {
|
||||
mcpURL = "http://localhost:8090/mcp"
|
||||
}
|
||||
// api's combinedAuth requires a bearer token on every request (no
|
||||
// dev-open bypass — plans/2026-07-12-wails-desktop-app.md 0.4); this is
|
||||
// the same shared secret api validates against (OIKOS_MCP_BEARER_TOKEN).
|
||||
mcpToken := os.Getenv("OIKOS_MCP_BEARER_TOKEN")
|
||||
|
||||
agentSlug := os.Getenv("NOMOS_AGENT_SLUG")
|
||||
@@ -46,6 +44,32 @@ func main() {
|
||||
databaseURL = os.Getenv("OIKOS_DATABASE_URL")
|
||||
}
|
||||
|
||||
sec := secrets.NewManagerFromConfig(
|
||||
os.Getenv("OIKOS_INFISICAL_SITE_URL"),
|
||||
os.Getenv("OIKOS_INFISICAL_CLIENT_ID"),
|
||||
os.Getenv("OIKOS_INFISICAL_CLIENT_SECRET"),
|
||||
os.Getenv("OIKOS_INFISICAL_PROJECT_ID"),
|
||||
os.Getenv("OIKOS_INFISICAL_ENV"),
|
||||
os.Getenv("OIKOS_SECRETS_DIR"),
|
||||
)
|
||||
var openrouterAPIKey string
|
||||
var secretsResolved int
|
||||
if sec != nil {
|
||||
resCtx, resCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if v := secrets.ResolveSecret(resCtx, sec, "mcp_bearer-token", ""); v != "" {
|
||||
mcpToken = v
|
||||
secretsResolved++
|
||||
}
|
||||
openrouterAPIKey = secrets.ResolveSecret(resCtx, sec, "openrouter_api-key", os.Getenv("OPENROUTER_API_KEY"))
|
||||
if openrouterAPIKey != "" && openrouterAPIKey != os.Getenv("OPENROUTER_API_KEY") {
|
||||
secretsResolved++
|
||||
}
|
||||
resCancel()
|
||||
if secretsResolved > 0 {
|
||||
slog.Info("nomos: secrets resolved from Infisical", "count", secretsResolved)
|
||||
}
|
||||
}
|
||||
|
||||
switch os.Args[1] {
|
||||
case "serve":
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
@@ -75,7 +99,7 @@ func main() {
|
||||
defer st.close()
|
||||
}
|
||||
|
||||
nAgent, err := newAgent(ctx, clientPool, st, agentSlug)
|
||||
nAgent, err := newAgent(ctx, clientPool, st, agentSlug, openrouterAPIKey)
|
||||
if err != nil {
|
||||
slog.Error("nomos: agent init", "error", err)
|
||||
os.Exit(1)
|
||||
@@ -161,6 +185,26 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
func runHealthcheck() {
|
||||
addr := os.Getenv("NOMOS_LISTEN")
|
||||
if addr == "" {
|
||||
addr = ":8092"
|
||||
}
|
||||
host := addr
|
||||
if strings.HasPrefix(host, ":") {
|
||||
host = "127.0.0.1" + host
|
||||
}
|
||||
client := &http.Client{Timeout: 3 * time.Second}
|
||||
resp, err := client.Get("http://" + host + "/healthz")
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func sseEvent(w http.ResponseWriter, flusher http.Flusher, event agentEvent) {
|
||||
data, _ := json.Marshal(event)
|
||||
fmt.Fprintf(w, "data: %s\n\n", data)
|
||||
@@ -186,12 +230,22 @@ func handleChat(w http.ResponseWriter, r *http.Request, a *agent, st *store) {
|
||||
return
|
||||
}
|
||||
|
||||
// Empty message with an existing session = reconnect/resume. The
|
||||
// frontend sends this after a dropped SSE stream to re-establish the
|
||||
// connection and catch up on any auto-continuation work that happened
|
||||
// while disconnected. Route into resumeSession so the agent sees a
|
||||
// system note and reports current state.
|
||||
// Empty message with an existing session = reconnect/resume. This path is
|
||||
// defensive now — the frontend (post F2) recovers a dropped SSE via the
|
||||
// poller + terminal task.status clearing, and no longer POSTs empty
|
||||
// messages. If a client ever does, route into resumeSession so the agent
|
||||
// reports current state — but SKIP a terminal session (done/failed/
|
||||
// abandoned): there's nothing to resume, and running a "report state"
|
||||
// turn there is just a spare turn the operator never asked for (P2.1).
|
||||
if req.Message == "" && req.SessionID != "" {
|
||||
if sess, err := st.getSession(context.Background(), req.SessionID); err == nil {
|
||||
switch sess.Status {
|
||||
case "done", "failed", "abandoned":
|
||||
slog.Info("nomos: reconnect skipped — session already terminal", "session", req.SessionID, "status", sess.Status)
|
||||
w.WriteHeader(202)
|
||||
return
|
||||
}
|
||||
}
|
||||
slog.Info("nomos: reconnect", "session", req.SessionID)
|
||||
safego.Go("nomos:reconnect:"+req.SessionID, func() {
|
||||
base := "[System: the operator's connection was re-established. The task may have progressed in the background.]"
|
||||
@@ -200,7 +254,7 @@ func handleChat(w http.ResponseWriter, r *http.Request, a *agent, st *store) {
|
||||
})
|
||||
// Return 202 so the frontend doesn't try to consume an SSE stream
|
||||
// from this POST — resumeSession writes to the DB directly and
|
||||
// the poller (already running from handleDisconnect) picks it up.
|
||||
// the poller picks it up.
|
||||
w.WriteHeader(202)
|
||||
return
|
||||
}
|
||||
@@ -217,6 +271,17 @@ func handleChat(w http.ResponseWriter, r *http.Request, a *agent, st *store) {
|
||||
w.Header().Set("X-Accel-Buffering", "no") // disable proxy buffering
|
||||
w.WriteHeader(200)
|
||||
|
||||
// All writes to w (events + the keepalive comment below) go through one
|
||||
// mutex: http.ResponseWriter is NOT safe for concurrent use, and the
|
||||
// keepalive ticker runs alongside the turn's event sink (plan 2026-08-03
|
||||
// F3). Without this, interleaved writes corrupt the SSE stream.
|
||||
var writeMu sync.Mutex
|
||||
writeEvent := func(ev agentEvent) {
|
||||
writeMu.Lock()
|
||||
defer writeMu.Unlock()
|
||||
sseEvent(w, flusher, ev)
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
sessionID := req.SessionID
|
||||
|
||||
@@ -268,111 +333,65 @@ func handleChat(w http.ResponseWriter, r *http.Request, a *agent, st *store) {
|
||||
st.answerQuestion(pctx, sessionID, qid, req.Message)
|
||||
}
|
||||
|
||||
sseEvent(w, flusher, agentEvent{Type: "session", Data: sessionID, SessionID: sessionID})
|
||||
writeEvent(agentEvent{Type: "session", Data: sessionID, SessionID: sessionID})
|
||||
|
||||
toolCalls := []map[string]any{}
|
||||
// P3: accumulate per-iteration reasoning instead of overwriting with
|
||||
// the final `text` event. The agent loop emits a `text` event for each
|
||||
// LLM iteration that produced text (intermediate reasoning before tool
|
||||
// calls + the final answer). Without accumulation, only the last `text`
|
||||
// survives in the persisted row — a reload shows the final summary but
|
||||
// not the thinking that led to each tool call.
|
||||
var textParts []string
|
||||
var finalText string
|
||||
|
||||
// Incremental persistence, mirroring resumeSession's existing
|
||||
// placeholder+update pattern (continue.go): insert a placeholder now,
|
||||
// update the SAME row after every tool call, so whatever happened before
|
||||
// an abort is never lost — only what hadn't happened yet is.
|
||||
placeholder, _ := json.Marshal(map[string]any{"role": "assistant", "text": ""})
|
||||
msgID, err := st.insertMessageReturningID(pctx, sessionID, "assistant", placeholder)
|
||||
if err != nil {
|
||||
slog.Error("nomos: chat placeholder insert failed", "session", sessionID, "error", err)
|
||||
}
|
||||
persist := func() {
|
||||
if msgID == uuid.Nil {
|
||||
return
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"role": "assistant",
|
||||
"text": finalText,
|
||||
"tool_calls": toolCalls,
|
||||
})
|
||||
st.updateMessage(pctx, msgID, body)
|
||||
// F1/F2 (plan 2026-08-03): serialize turns per session. The user message is
|
||||
// already persisted above, so it is never lost. Wait briefly for a finishing
|
||||
// background turn; if one is still running after that, QUEUE this message
|
||||
// (don't reject it) and tell the client so it shows a "queued" state. The
|
||||
// in-flight turn's release drains the queue (drainQueued) and runs it as a
|
||||
// real turn server-side. This never stacks concurrent turns — the gate still
|
||||
// guarantees one in-flight turn per session.
|
||||
const turnWait = 5 * time.Second
|
||||
if !a.gate.acquire(sessionID, turnWait) {
|
||||
a.queue.enqueue(sessionID, req.Message)
|
||||
slog.Info("nomos: turn already active, queued operator message", "session", sessionID)
|
||||
writeEvent(agentEvent{Type: "queued", Data: sessionID, SessionID: sessionID})
|
||||
writeEvent(agentEvent{Type: "done", Data: map[string]any{
|
||||
"session_id": sessionID,
|
||||
"queued": true,
|
||||
}, SessionID: sessionID})
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
a.gate.release(sessionID)
|
||||
// Run any message that was queued while this turn held the gate. In a
|
||||
// goroutine so the HTTP response finishes without waiting on the next
|
||||
// turn; the queued turn has no SSE client of its own.
|
||||
safego.Go("nomos:drain:"+sessionID, func() { a.drainQueued(context.Background(), sessionID) })
|
||||
}()
|
||||
|
||||
a.chat(ctx, sessionID, req.Message, func(ev agentEvent) {
|
||||
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
||||
if m, ok := ev.Data.(map[string]any); ok {
|
||||
m["type"] = ev.Type
|
||||
// One entry per tool call: tool_use creates it, tool_result
|
||||
// merges the result into the same entry (matched by id).
|
||||
// Before this fix, both events appended separate entries,
|
||||
// doubling every tool call in the persisted transcript
|
||||
// (confirmed pre-existing in d9cdcee1, v0.3.x era).
|
||||
id, _ := m["id"].(string)
|
||||
if id != "" && ev.Type == "tool_result" {
|
||||
for _, existing := range toolCalls {
|
||||
if eID, _ := existing["id"].(string); eID == id {
|
||||
for k, v := range m {
|
||||
existing[k] = v
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
toolCalls = append(toolCalls, m)
|
||||
}
|
||||
}
|
||||
persist() // live: survives even if the client disconnects right after
|
||||
}
|
||||
if ev.Type == "text" {
|
||||
// P3: accumulate. Each `text` event is one iteration's reasoning
|
||||
// (or the final answer). Join with newlines so the persisted row
|
||||
// reads as the full transcript of what the agent said, not just
|
||||
// the last thing.
|
||||
if t, ok := ev.Data.(string); ok && t != "" {
|
||||
textParts = append(textParts, t)
|
||||
finalText = strings.Join(textParts, "\n\n")
|
||||
persist()
|
||||
// F3 (plan 2026-08-03): keep the SSE alive during long turns. A turn can
|
||||
// run for many minutes (provisioning chains, deep research); the model
|
||||
// often takes 20-40s between tool iterations, and with nothing flushed in
|
||||
// that gap a proxy/browser idle timeout silently closes the stream. The
|
||||
// client then sees streaming=false while the server keeps working — the
|
||||
// "I can't tell it's working" desync. An SSE comment line (":keepalive") is
|
||||
// ignored by EventSource but resets idle timers.
|
||||
keepDone := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(12 * time.Second)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-keepDone:
|
||||
return
|
||||
case <-t.C:
|
||||
writeMu.Lock()
|
||||
fmt.Fprintf(w, ":keepalive\n\n")
|
||||
flusher.Flush()
|
||||
writeMu.Unlock()
|
||||
}
|
||||
}
|
||||
sseEvent(w, flusher, ev)
|
||||
}()
|
||||
// Defer the close (not a statement after runChatTurn) so the goroutine
|
||||
// exits even if runChatTurn panics — net/http recovers handler panics, so
|
||||
// a non-deferred close would be skipped and the ticker would keep writing
|
||||
// to a dead ResponseWriter forever.
|
||||
defer close(keepDone)
|
||||
a.runChatTurn(pctx, ctx, sessionID, req.Message, func(ev agentEvent) {
|
||||
writeEvent(ev)
|
||||
})
|
||||
|
||||
// B.6: if the turn ended with no text and no tool calls (the model
|
||||
// empty-response'd and all retries failed), delete the placeholder row
|
||||
// instead of persisting an empty bubble. The error event was already
|
||||
// streamed to the frontend via the 'done with error=true' event, so the
|
||||
// operator sees the error inline — an empty assistant bubble in the
|
||||
// transcript adds nothing and looks like the agent is broken.
|
||||
if finalText == "" && len(toolCalls) == 0 && msgID != uuid.Nil {
|
||||
st.deleteMessage(pctx, msgID)
|
||||
} else {
|
||||
persist() // final state — same row, updated one last time with the concluding text
|
||||
}
|
||||
|
||||
// Generate a meaningful title from the assistant's first answer
|
||||
// instead of reusing the raw user message for every session.
|
||||
// P2.9 (2026-07-20): prefer the goal as the title when one is set —
|
||||
// the first assistant text is often a greeting or narrative that
|
||||
// doesn't describe the task ("Hey! 👋 Nomos here, running on
|
||||
// mac-mini:8092..."). The goal is the operator's actual intent.
|
||||
// Sessions that never call set_goal (pure Q&A) fall back to the
|
||||
// assistant text, which is still better than the raw user message.
|
||||
if finalText != "" && sessionID != "ephemeral" {
|
||||
var goalTitle string
|
||||
if sess, gerr := st.getSession(pctx, sessionID); gerr == nil && sess.Goal != "" {
|
||||
goalTitle = truncate(sess.Goal, 120)
|
||||
}
|
||||
title := goalTitle
|
||||
if title == "" {
|
||||
title = truncate(finalText, 80)
|
||||
}
|
||||
if title != "" {
|
||||
st.updateSessionTitle(pctx, sessionID, title)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleSessionsList(w http.ResponseWriter, r *http.Request, st *store) {
|
||||
@@ -483,7 +502,8 @@ func handleSessionDetail(w http.ResponseWriter, r *http.Request, st *store, a *a
|
||||
if len(parts) == 2 && r.Method == http.MethodGet {
|
||||
switch parts[1] {
|
||||
case "plan":
|
||||
steps, err := st.getPlanSteps(r.Context(), id)
|
||||
all := r.URL.Query().Has("all") && r.URL.Query().Get("all") != "0" && r.URL.Query().Get("all") != "false"
|
||||
steps, err := st.getPlanSteps(r.Context(), id, all)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
@@ -672,339 +692,4 @@ func truncate(s string, n int) string {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "..."
|
||||
}
|
||||
|
||||
// ─── MCP Streamable HTTP client ────────────────────────────────────────
|
||||
|
||||
type mcpClient struct {
|
||||
baseURL string
|
||||
token string // OIKOS_MCP_BEARER_TOKEN — api's combinedAuth requires it on every request (no dev-open bypass)
|
||||
sessionID string
|
||||
http *http.Client
|
||||
nextID int
|
||||
mu sync.Mutex // this client is one stateful MCP session; serialize ITS OWN calls
|
||||
|
||||
// toolsCache holds the last tools/list result. The tool list is static
|
||||
// for the lifetime of one MCP connection — it only changes when the api
|
||||
// process (re)registers tools, i.e. on a restart, which this client
|
||||
// already detects and reacts to via reconnectLocked. Without this,
|
||||
// buildTools (called at the start of EVERY chat turn, including every
|
||||
// auto-continuation resume) paid a full tools/list round-trip every
|
||||
// single time for a list that's almost always identical to the last one.
|
||||
// Guarded separately from mu (not reused) so a cache check never
|
||||
// contends with an in-flight doRequest call for a different method.
|
||||
toolsMu sync.Mutex
|
||||
toolsCache []toolDef
|
||||
}
|
||||
|
||||
func newMCPClient(baseURL, token string) (*mcpClient, error) {
|
||||
c := &mcpClient{
|
||||
baseURL: baseURL,
|
||||
token: token,
|
||||
http: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
|
||||
resp, err := c.doRequest("initialize", map[string]any{
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": map[string]any{},
|
||||
"clientInfo": map[string]any{"name": "nomos", "version": "2.0"},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize: %w", err)
|
||||
}
|
||||
if resp.sessionID == "" {
|
||||
return nil, fmt.Errorf("no session ID in initialize response")
|
||||
}
|
||||
c.sessionID = resp.sessionID
|
||||
|
||||
c.doRequest("notifications/initialized", map[string]any{})
|
||||
|
||||
slog.Info("nomos: mcp connected", "session", c.sessionID[:16]+"...")
|
||||
return c, nil
|
||||
}
|
||||
|
||||
type mcpJSONRPCResponse struct {
|
||||
sessionID string
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error json.RawMessage `json:"error"`
|
||||
}
|
||||
|
||||
// errStaleSession signals that the MCP server rejected our session id (e.g.
|
||||
// after an api/MCP restart), so the client should re-initialize and retry.
|
||||
var errStaleSession = fmt.Errorf("mcp session stale")
|
||||
|
||||
// doRequest serializes MCP calls and transparently re-initializes the session
|
||||
// if the server has forgotten it (common after an api redeploy), retrying the
|
||||
// original call once. Without this, an api restart permanently breaks nomos
|
||||
// until it is itself restarted.
|
||||
func (c *mcpClient) doRequest(method string, params map[string]any) (*mcpJSONRPCResponse, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
resp, err := c.send(method, params)
|
||||
if err != nil && method != "initialize" && errors.Is(err, errStaleSession) {
|
||||
slog.Warn("nomos: mcp session stale, reconnecting")
|
||||
if rerr := c.reconnectLocked(); rerr != nil {
|
||||
return nil, fmt.Errorf("mcp reconnect: %w (original: %v)", rerr, err)
|
||||
}
|
||||
return c.send(method, params)
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// reconnectLocked re-initializes the MCP session. The caller must hold c.mu.
|
||||
func (c *mcpClient) reconnectLocked() error {
|
||||
c.sessionID = ""
|
||||
// A reconnect means the api process was restarted (or forgot us) — its
|
||||
// tool registration may have changed, so the cached list is no longer
|
||||
// trustworthy.
|
||||
c.toolsMu.Lock()
|
||||
c.toolsCache = nil
|
||||
c.toolsMu.Unlock()
|
||||
resp, err := c.send("initialize", map[string]any{
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": map[string]any{},
|
||||
"clientInfo": map[string]any{"name": "nomos", "version": "2.0"},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.sessionID == "" {
|
||||
return fmt.Errorf("no session ID on re-initialize")
|
||||
}
|
||||
c.sessionID = resp.sessionID
|
||||
_, _ = c.send("notifications/initialized", map[string]any{})
|
||||
slog.Info("nomos: mcp reconnected", "session", c.sessionID[:16]+"...")
|
||||
return nil
|
||||
}
|
||||
|
||||
// send performs one MCP round-trip. It does not lock; callers hold c.mu.
|
||||
func (c *mcpClient) send(method string, params map[string]any) (*mcpJSONRPCResponse, error) {
|
||||
c.nextID++
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"method": method,
|
||||
"params": params,
|
||||
"id": c.nextID,
|
||||
})
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, c.baseURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json, text/event-stream")
|
||||
if c.sessionID != "" {
|
||||
req.Header.Set("Mcp-Session-Id", c.sessionID)
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// A rejected/unknown session comes back as 4xx (commonly 400/404).
|
||||
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusBadRequest {
|
||||
return nil, errStaleSession
|
||||
}
|
||||
|
||||
result := &mcpJSONRPCResponse{}
|
||||
result.sessionID = resp.Header.Get("Mcp-Session-Id")
|
||||
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
gotData := false
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
gotData = true
|
||||
data := line[6:]
|
||||
if err := json.Unmarshal([]byte(data), result); err != nil {
|
||||
return nil, fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if result.Error != nil {
|
||||
return nil, fmt.Errorf("rpc error: %s", string(result.Error))
|
||||
}
|
||||
|
||||
// Empty body with no result and a session set: the server likely dropped
|
||||
// our session. Notifications legitimately return no data, so exempt them.
|
||||
if !gotData && result.Result == nil && method != "notifications/initialized" {
|
||||
return nil, errStaleSession
|
||||
}
|
||||
|
||||
if result.sessionID != "" {
|
||||
c.sessionID = result.sessionID
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *mcpClient) callTool(name string, args map[string]any) (any, error) {
|
||||
resp, err := c.doRequest("tools/call", map[string]any{
|
||||
"name": name,
|
||||
"arguments": args,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var toolResult struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if err := json.Unmarshal(resp.Result, &toolResult); err != nil {
|
||||
return string(resp.Result), nil
|
||||
}
|
||||
|
||||
var texts []string
|
||||
for _, c := range toolResult.Content {
|
||||
if c.Type == "text" {
|
||||
var parsed any
|
||||
if json.Unmarshal([]byte(c.Text), &parsed) == nil {
|
||||
return parsed, nil
|
||||
}
|
||||
texts = append(texts, c.Text)
|
||||
}
|
||||
}
|
||||
if len(texts) == 1 {
|
||||
return texts[0], nil
|
||||
}
|
||||
return texts, nil
|
||||
}
|
||||
|
||||
func (c *mcpClient) listTools() ([]string, error) {
|
||||
resp, err := c.doRequest("tools/list", map[string]any{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var tr struct {
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
} `json:"tools"`
|
||||
}
|
||||
if err := json.Unmarshal(resp.Result, &tr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var names []string
|
||||
for _, t := range tr.Tools {
|
||||
names = append(names, t.Name)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (c *mcpClient) close() {
|
||||
}
|
||||
|
||||
// ─── Per-session MCP client pool ────────────────────────────────────────
|
||||
//
|
||||
// A single shared mcpClient serializes EVERY tool call across EVERY
|
||||
// concurrently-running task through one mutex (see mcpClient.mu) — `run`
|
||||
// executes its SSH command synchronously inside that lock and is capped at
|
||||
// up to 10 minutes, so one task mid-`run` stalled every other task's tool
|
||||
// calls, even trivial reads, behind it. The MCP *server* has no per-
|
||||
// connection state to protect (newServer in internal/mcp/server.go returns
|
||||
// one shared *mcp.Server instance whose tool handlers close only over the DB
|
||||
// pool, which is already safe for concurrent use) — the mutex existed purely
|
||||
// because the *client* reused one stateful transport session, not because
|
||||
// the server needed it. Giving each task's own session its own client
|
||||
// removes the cross-task serialization entirely: a task's own tool calls
|
||||
// stay sequential (which they already are — the agent loop calls tools one
|
||||
// at a time within a turn), but no longer block anyone else's.
|
||||
type mcpClientPool struct {
|
||||
baseURL string
|
||||
token string
|
||||
mu sync.Mutex
|
||||
clients map[string]*pooledMCPClient
|
||||
}
|
||||
|
||||
type pooledMCPClient struct {
|
||||
client *mcpClient
|
||||
lastUsed time.Time
|
||||
}
|
||||
|
||||
func newMCPClientPool(baseURL, token string) *mcpClientPool {
|
||||
return &mcpClientPool{baseURL: baseURL, token: token, clients: make(map[string]*pooledMCPClient)}
|
||||
}
|
||||
|
||||
// get returns the client for sessionID, creating and initializing one (a
|
||||
// real MCP handshake) on first use. Session ids that don't identify a real
|
||||
// persisted conversation ("" / "ephemeral", the no-DB-store path; "query",
|
||||
// the structured /query endpoint) still get exactly one dedicated,
|
||||
// reused client each via the same map — just keyed on a fixed string instead
|
||||
// of a real session id — so that traffic doesn't pay a fresh handshake per
|
||||
// request while still never sharing a connection with an actual task.
|
||||
func (p *mcpClientPool) get(sessionID string) (*mcpClient, error) {
|
||||
key := sessionID
|
||||
if key == "" {
|
||||
key = "ephemeral"
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
if pc, ok := p.clients[key]; ok {
|
||||
pc.lastUsed = time.Now()
|
||||
p.mu.Unlock()
|
||||
return pc.client, nil
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
// Initialize outside the lock — it's a network round-trip, and holding
|
||||
// the pool mutex for it would serialize unrelated sessions' first calls
|
||||
// behind each other, undermining the whole point of this pool.
|
||||
c, err := newMCPClient(p.baseURL, p.token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
// Another goroutine may have created one for the same key while we were
|
||||
// initializing (two of this session's tool calls racing on a cold
|
||||
// start); keep whichever won, close out the loser's connection (a no-op
|
||||
// today, but future-proof if mcpClient.close ever does real teardown).
|
||||
if existing, ok := p.clients[key]; ok {
|
||||
p.mu.Unlock()
|
||||
c.close()
|
||||
return existing.client, nil
|
||||
}
|
||||
p.clients[key] = &pooledMCPClient{client: c, lastUsed: time.Now()}
|
||||
p.mu.Unlock()
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// mcpClientIdleTimeout is how long an idle session's MCP client is kept
|
||||
// before eviction — long enough to outlive a single slow `run` (capped at 10
|
||||
// minutes server-side) plus normal think-time between a task's tool calls,
|
||||
// short enough not to accumulate one abandoned connection per finished task
|
||||
// forever.
|
||||
const mcpClientIdleTimeout = 20 * time.Minute
|
||||
|
||||
// sweep evicts clients idle past mcpClientIdleTimeout. Call on a ticker.
|
||||
func (p *mcpClientPool) sweep() {
|
||||
cutoff := time.Now().Add(-mcpClientIdleTimeout)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for key, pc := range p.clients {
|
||||
if pc.lastUsed.Before(cutoff) {
|
||||
pc.client.close()
|
||||
delete(p.clients, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *mcpClientPool) closeAll() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for key, pc := range p.clients {
|
||||
pc.client.close()
|
||||
delete(p.clients, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/observability"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
@@ -25,6 +26,13 @@ const maxToolResultSize = 4096
|
||||
// The caller translates this into a directive tool result.
|
||||
var errPlanInFlight = errors.New("plan already in flight")
|
||||
|
||||
// errPlanStepNotFound is returned by updatePlanStep when no step matches the
|
||||
// given seq in the CURRENT (MAX) generation — either the seq is out of range,
|
||||
// or (after a re-plan) the model addressed a stale 1-based number. seq is
|
||||
// generation-relative, so this never resurrects a superseded generation's row.
|
||||
// The caller translates it into a directive tool result (P0.1).
|
||||
var errPlanStepNotFound = errors.New("plan step not found in current generation")
|
||||
|
||||
type store struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
@@ -171,6 +179,15 @@ func (s *store) createTaskEntity(ctx context.Context, sessionID, title string) s
|
||||
`UPDATE agent_sessions SET entity_id = $1 WHERE id = $2`, entityID, sessionID); err != nil {
|
||||
slog.Warn("nomos: could not link task entity", "session", sessionID, "error", err)
|
||||
}
|
||||
// Graph edge: task —involves→ agent:nomos (gives every task at least one
|
||||
// edge from creation, even if no run calls are ever made).
|
||||
s.pool.Exec(ctx, `INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT $1, id, 'involves', '{"by":"nomos"}'::jsonb, now()
|
||||
FROM entities WHERE slug = 'agent:nomos'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM relationships r
|
||||
WHERE r.source_id = $1 AND r.target_id = entities.id AND r.type = 'involves' AND r.valid_to IS NULL)`,
|
||||
entityID)
|
||||
return entityID.String()
|
||||
}
|
||||
|
||||
@@ -799,10 +816,11 @@ func (s *store) setGoal(ctx context.Context, sessionID, goal string) error {
|
||||
// Replace any prior plan steps (done/running/pending/...) as `replaced`.
|
||||
// The rows are kept for the generation counter + audit trail; proposePlan
|
||||
// excludes `replaced` from its in-flight check, so the next propose_plan
|
||||
// takes the fresh-generation path.
|
||||
// takes the fresh-generation path. replaced_reason records the cause
|
||||
// (2026-08-04 plan-step integrity audit).
|
||||
s.pool.Exec(ctx,
|
||||
`UPDATE session_plan_steps SET status = 'replaced', finished_at = COALESCE(finished_at, now()) WHERE session_id = $1 AND status <> 'replaced'`,
|
||||
sessionID)
|
||||
`UPDATE session_plan_steps SET status = 'replaced', replaced_reason = $2, finished_at = COALESCE(finished_at, now()) WHERE session_id = $1 AND status <> 'replaced'`,
|
||||
sessionID, "goal superseded")
|
||||
if _, err := s.pool.Exec(ctx,
|
||||
`UPDATE agent_sessions SET goal = $2, status = 'executing', title = $2, last_active_at = now() WHERE id = $1`,
|
||||
sessionID, goal); err != nil {
|
||||
@@ -842,6 +860,14 @@ func (s *store) reopenSession(ctx context.Context, sessionID string) bool {
|
||||
s.pool.Exec(ctx,
|
||||
`UPDATE agent_sessions SET status = 'executing', outcome = NULL, summary = NULL, last_active_at = now() WHERE id = $1`,
|
||||
sessionID)
|
||||
// Mark the prior plan's steps as replaced so the P1 plan-first gate in
|
||||
// classifyAndGate forces a fresh propose_plan before any run. Without
|
||||
// this, the agent could resume a session and call run against the old
|
||||
// (completed) plan — exactly what caused the ZimaOS continuation to
|
||||
// have 81 ad-hoc tool calls with zero plan structure (2026-08-04).
|
||||
s.pool.Exec(ctx,
|
||||
`UPDATE session_plan_steps SET status = 'replaced', replaced_reason = $2, finished_at = COALESCE(finished_at, now()) WHERE session_id = $1 AND status <> 'replaced'`,
|
||||
sessionID, "session reopened — awaiting new plan")
|
||||
_ = observability.Event(ctx, sqlcgen.New(s.pool), "task.reopened", s.taskEntityPtr(ctx, sessionID),
|
||||
"info", "nomos", sessionID, map[string]any{"prior_status": currentStatus})
|
||||
return true
|
||||
@@ -879,16 +905,15 @@ func (s *store) proposePlan(ctx context.Context, sessionID string, steps []planS
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var startSeq int
|
||||
var anyStarted bool
|
||||
// `replaced` steps (from a prior plan generation superseded by a
|
||||
// follow-up sub-task — see reopenSession) are excluded: they prove a
|
||||
// prior plan was completed and superseded, not that a plan is in flight.
|
||||
// Without this exclusion, reopenSession's `replaced` marking would be
|
||||
// follow-up sub-task — see setGoal/reopenSession) are excluded: they
|
||||
// prove a prior plan was completed and superseded, not that a plan is in
|
||||
// flight. Without this exclusion, setGoal's `replaced` marking would be
|
||||
// useless — propose_plan would still refuse on the follow-up.
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(max(seq), 0), COALESCE(bool_or(status NOT IN ('pending', 'replaced')), false)
|
||||
FROM session_plan_steps WHERE session_id = $1`, sessionID).Scan(&startSeq, &anyStarted); err != nil {
|
||||
SELECT COALESCE(bool_or(status NOT IN ('pending', 'replaced')), false)
|
||||
FROM session_plan_steps WHERE session_id = $1`, sessionID).Scan(&anyStarted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if anyStarted {
|
||||
@@ -898,35 +923,29 @@ func (s *store) proposePlan(ctx context.Context, sessionID string, steps []planS
|
||||
return nil, errPlanInFlight
|
||||
}
|
||||
// Fresh/revise: mark any prior PENDING steps as `replaced` (not DELETE).
|
||||
// This preserves the rows for the generation counter (MAX(generation)+1
|
||||
// below) and the plan_generations eval assertion. Without this, a first
|
||||
// plan that was proposed but never executed (all pending) would be
|
||||
// wiped, resetting the counter to 1 — making a follow-up's plan look
|
||||
// like generation 1 instead of 2. `replaced` steps are excluded from
|
||||
// the anyStarted check above, so they don't block the fresh proposal.
|
||||
// The rows are kept for the generation counter (MAX(generation)+1 below)
|
||||
// and the plan_generations eval assertion. `replaced` steps are excluded
|
||||
// from the anyStarted check above, so they don't block this proposal.
|
||||
// replaced_reason records the cause — required by the plan-step integrity
|
||||
// gate (2026-08-04 session audit).
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE session_plan_steps SET status = 'replaced', finished_at = COALESCE(finished_at, now()) WHERE session_id = $1 AND status = 'pending'`,
|
||||
sessionID); err != nil {
|
||||
`UPDATE session_plan_steps SET status = 'replaced', replaced_reason = $2, finished_at = COALESCE(finished_at, now()) WHERE session_id = $1 AND status = 'pending'`,
|
||||
sessionID, "superseded by new plan generation"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// startSeq keeps the max(seq) from the query above: if prior steps
|
||||
// exist (replaced or done), the new generation's steps start after them
|
||||
// (no seq collisions across generations). If no rows exist (first plan),
|
||||
// startSeq is 0 and the first step is seq 1.
|
||||
|
||||
// Resolve the generation number for this plan. Generation 1 is the
|
||||
// initial plan; a genuine revise (which currently goes through the same
|
||||
// fresh-start path above because all steps were pending) resets to 1
|
||||
// since the DELETE wiped the prior rows. The column is wired here so a
|
||||
// future explicit mid-flight revise path can increment it.
|
||||
// nextGen: generation 1 for the first plan, MAX(generation)+1 for every
|
||||
// revise/follow-up (prior rows were marked `replaced` above, not deleted,
|
||||
// so the counter survives). seq is generation-relative — it resets to
|
||||
// 1..N for this generation, so (session_id, generation, seq) is the
|
||||
// addressing key and the model's 1-based update_plan_step always maps to
|
||||
// the CURRENT plan after a re-plan (P0.1).
|
||||
var nextGen int
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(MAX(generation), 0) + 1
|
||||
FROM session_plan_steps WHERE session_id = $1`, sessionID).Scan(&nextGen); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// After the DELETE above, no rows remain, so MAX(generation) is NULL →
|
||||
// nextGen = 1. (Keep the query for the future revise path; it's cheap.)
|
||||
|
||||
out := make([]map[string]any, 0, len(steps))
|
||||
for i, st := range steps {
|
||||
@@ -934,7 +953,7 @@ func (s *store) proposePlan(ctx context.Context, sessionID string, steps []planS
|
||||
if st.TargetSlug != "" {
|
||||
targetSlug = &st.TargetSlug
|
||||
}
|
||||
seq := startSeq + i + 1
|
||||
seq := i + 1
|
||||
var id uuid.UUID
|
||||
if err := tx.QueryRow(ctx, `
|
||||
INSERT INTO session_plan_steps (session_id, seq, title, detail, target_slug, generation)
|
||||
@@ -973,10 +992,27 @@ func (s *store) proposePlan(ctx context.Context, sessionID string, steps []planS
|
||||
// be marked complete while an earlier step is still pending, preventing the
|
||||
// agent from marking step 5 done before step 4 (observed in production: the
|
||||
// agent rushed to close all steps in a final turn, in reverse order).
|
||||
func (s *store) updatePlanStep(ctx context.Context, sessionID string, seq int, status, execID string) error {
|
||||
func (s *store) updatePlanStep(ctx context.Context, sessionID string, seq int, status, execID, replacedReason string) error {
|
||||
if s == nil || sessionID == "" || sessionID == "ephemeral" {
|
||||
return nil
|
||||
}
|
||||
// Resolve the CURRENT generation: seq is generation-relative (1-based
|
||||
// within the plan the model is working), so (session_id, MAX(generation),
|
||||
// seq) is the addressing key. A re-plan's superseded generations have
|
||||
// their own seq space and must never be touched by a follow-up's
|
||||
// update_plan_step — that was the root cause of "the plan was off"
|
||||
// (gen-1 `replaced` rows resurrected as `done` while gen-2 work went
|
||||
// unrecorded). The MAX(generation) step is by construction the active
|
||||
// plan, never `replaced`, so this can't resurrect a superseded row (P0.1).
|
||||
var curGen int
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
`SELECT COALESCE(MAX(generation), 0) FROM session_plan_steps WHERE session_id = $1`,
|
||||
sessionID).Scan(&curGen); err != nil {
|
||||
return err
|
||||
}
|
||||
if curGen == 0 {
|
||||
return errPlanStepNotFound
|
||||
}
|
||||
stamp := ""
|
||||
switch status {
|
||||
case "running":
|
||||
@@ -984,16 +1020,18 @@ func (s *store) updatePlanStep(ctx context.Context, sessionID string, seq int, s
|
||||
case "done", "failed", "skipped", "blocked", "replaced":
|
||||
stamp = ", finished_at = now()"
|
||||
}
|
||||
// Completion ordering: for terminal states, check that no earlier step
|
||||
// is still pending. Running steps can start out of order (the agent
|
||||
// may dispatch parallel work), but completion must be sequential.
|
||||
// Completion ordering, scoped to the CURRENT generation: for terminal
|
||||
// states, no earlier step in THIS plan may still be pending. Running
|
||||
// steps can start out of order (the agent may dispatch parallel work),
|
||||
// but completion must be sequential. Earlier generations are superseded
|
||||
// and irrelevant.
|
||||
if status == "done" || status == "failed" || status == "skipped" || status == "blocked" {
|
||||
var blockedBy int
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(MIN(seq), 0)
|
||||
FROM session_plan_steps
|
||||
WHERE session_id = $1 AND seq < $2 AND status = 'pending'`,
|
||||
sessionID, seq).Scan(&blockedBy); err == nil && blockedBy > 0 {
|
||||
WHERE session_id = $1 AND generation = $2 AND seq < $3 AND status = 'pending'`,
|
||||
sessionID, curGen, seq).Scan(&blockedBy); err == nil && blockedBy > 0 {
|
||||
return fmt.Errorf("cannot complete step %d — step %d is still pending", seq, blockedBy)
|
||||
}
|
||||
}
|
||||
@@ -1004,12 +1042,33 @@ func (s *store) updatePlanStep(ctx context.Context, sessionID string, seq int, s
|
||||
var stepID uuid.UUID
|
||||
var targetSlug *string
|
||||
// stamp is a fixed literal from the switch above — never user input.
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
UPDATE session_plan_steps
|
||||
SET status = $3, execution_id = COALESCE($4, execution_id)`+stamp+`
|
||||
WHERE session_id = $1 AND seq = $2
|
||||
RETURNING id, target_slug`, sessionID, seq, status, execPtr).Scan(&stepID, &targetSlug); err != nil {
|
||||
return err
|
||||
// status <> 'replaced' is defense-in-depth: MAX(generation) can't hold a
|
||||
// replaced row, but if it ever could, this refuses the write instead of
|
||||
// resurrecting it. No matching row → errPlanStepNotFound (stale/out-of-range seq).
|
||||
if status == "replaced" && replacedReason != "" {
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
UPDATE session_plan_steps
|
||||
SET status = $4, execution_id = COALESCE($5, execution_id), replaced_reason = $6`+stamp+`
|
||||
WHERE session_id = $1 AND generation = $2 AND seq = $3 AND status <> 'replaced'
|
||||
RETURNING id, target_slug`, sessionID, curGen, seq, status, execPtr, replacedReason).Scan(&stepID, &targetSlug)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return errPlanStepNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
UPDATE session_plan_steps
|
||||
SET status = $4, execution_id = COALESCE($5, execution_id)`+stamp+`
|
||||
WHERE session_id = $1 AND generation = $2 AND seq = $3 AND status <> 'replaced'
|
||||
RETURNING id, target_slug`, sessionID, curGen, seq, status, execPtr).Scan(&stepID, &targetSlug)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return errPlanStepNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Anchor the event to the step's target entity when it has one, else the task.
|
||||
entPtr := s.taskEntityPtr(ctx, sessionID)
|
||||
@@ -1098,13 +1157,58 @@ func (s *store) completeTask(ctx context.Context, sessionID, outcome, summary st
|
||||
if outcome != "success" {
|
||||
closeStatus = "skipped"
|
||||
}
|
||||
// Auto-close only the CURRENT generation's in-flight steps — superseded
|
||||
// generations were already resolved when their plan was replaced. Stamp
|
||||
// started_at so no `done` step is left with a NULL start time (P0.1 fix
|
||||
// 5), and emit a plan.step.finished event per closed step so the panel
|
||||
// converges instead of freezing on "running" after the task completes
|
||||
// (P1.1: no bulk plan-step status write without a corresponding event).
|
||||
type closingStep struct {
|
||||
id uuid.UUID
|
||||
seq int
|
||||
targetSlug *string
|
||||
}
|
||||
var toClose []closingStep
|
||||
if rows, qerr := s.pool.Query(ctx, `
|
||||
SELECT id, seq, target_slug FROM session_plan_steps
|
||||
WHERE session_id = $1
|
||||
AND generation = (SELECT MAX(generation) FROM session_plan_steps WHERE session_id = $1)
|
||||
AND status IN ('pending', 'running')`, sessionID); qerr == nil {
|
||||
for rows.Next() {
|
||||
var cs closingStep
|
||||
if err := rows.Scan(&cs.id, &cs.seq, &cs.targetSlug); err == nil {
|
||||
toClose = append(toClose, cs)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
UPDATE session_plan_steps
|
||||
SET status = $2, finished_at = COALESCE(finished_at, now())
|
||||
WHERE session_id = $1 AND status IN ('pending', 'running')`,
|
||||
SET status = $2,
|
||||
started_at = COALESCE(started_at, now()),
|
||||
finished_at = COALESCE(finished_at, now())
|
||||
WHERE session_id = $1
|
||||
AND generation = (SELECT MAX(generation) FROM session_plan_steps WHERE session_id = $1)
|
||||
AND status IN ('pending', 'running')`,
|
||||
sessionID, closeStatus); err != nil {
|
||||
slog.Warn("nomos: completeTask failed to auto-close in-flight steps", "session", sessionID, "error", err)
|
||||
}
|
||||
// Emit one plan.step.finished per closed step so the live panel advances
|
||||
// (mirrors updatePlanStep's event). A bulk UPDATE that skips the event
|
||||
// bus guarantees a stale panel — the rule is: no plan-step status change
|
||||
// without a corresponding event.
|
||||
taskEnt := s.taskEntityPtr(ctx, sessionID)
|
||||
for _, cs := range toClose {
|
||||
evEnt := taskEnt
|
||||
if cs.targetSlug != nil && *cs.targetSlug != "" {
|
||||
var tid uuid.UUID
|
||||
if s.pool.QueryRow(ctx, `SELECT id FROM entities WHERE slug = $1`, *cs.targetSlug).Scan(&tid) == nil {
|
||||
evEnt = &tid
|
||||
}
|
||||
}
|
||||
_ = observability.Event(ctx, sqlcgen.New(s.pool), "plan.step.finished", evEnt, "info", "nomos", sessionID,
|
||||
map[string]any{"step_id": cs.id.String(), "seq": cs.seq, "status": closeStatus})
|
||||
}
|
||||
|
||||
// Clean up assent and destructive window keys from autonomy_settings.
|
||||
s.pool.Exec(ctx, `DELETE FROM autonomy_settings
|
||||
@@ -1148,9 +1252,141 @@ func (s *store) completeTask(ctx context.Context, sessionID, outcome, summary st
|
||||
_ = observability.Event(ctx, sqlcgen.New(s.pool), "task.status", entPtr, severity, "nomos", sessionID,
|
||||
map[string]any{"status": status, "outcome": outcome, "summary": summary,
|
||||
"cancelled_executions": cancelledCount, "blocker": blocker})
|
||||
// Auto-persist knowledge so the graph learns from this session regardless
|
||||
// of whether the agent remembered to call upsert_knowledge (2026-08-04
|
||||
// session audit: only 2.4% of sessions called upsert_knowledge manually).
|
||||
if outcome == "success" || outcome == "partial" {
|
||||
autoUpsertKnowledge(ctx, s, sessionID, outcome, summary)
|
||||
}
|
||||
// Plan quality metric: compute step completion rate for the session's
|
||||
// current plan generation. Tracked as a task attribute so the trend
|
||||
// can be monitored over time (2026-08-04 session audit: 38% baseline).
|
||||
writePlanCompletionRate(ctx, s, sessionID)
|
||||
// Auto-feedback: create a feedback entry linking the session's outcome
|
||||
// to its last execution, feeding the pattern-extraction pipeline that
|
||||
// has been empty since launch (2026-08-04 session audit: 0 feedback rows).
|
||||
if outcome == "success" || outcome == "partial" {
|
||||
autoFeedback(ctx, s, sessionID, outcome, summary)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// autoUpsertKnowledge creates a knowledge entry for a completed session,
|
||||
// capturing what was done and linking it to the entities involved. Called
|
||||
// automatically from completeTask so every session leaves a trace, even if
|
||||
// the agent forgot to call upsert_knowledge. Only fired for success/partial
|
||||
// outcomes (failures don't have actionable discoveries).
|
||||
func autoUpsertKnowledge(ctx context.Context, s *store, sessionID, outcome, summary string) {
|
||||
var goal string
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
`SELECT COALESCE(goal, '') FROM agent_sessions WHERE id = $1`,
|
||||
sessionID).Scan(&goal); err != nil || goal == "" {
|
||||
return
|
||||
}
|
||||
title := "Session " + sessionID[:8] + ": " + goal
|
||||
if len(title) > 200 {
|
||||
title = title[:200]
|
||||
}
|
||||
content := "## Outcome\n" + outcome + "\n\n## Summary\n" + summary
|
||||
kind := "investigation"
|
||||
slug := "investigation:nomos/" + sessionID
|
||||
tags := []string{"nomos-session", "auto-generated"}
|
||||
|
||||
// Upsert the knowledge entity.
|
||||
docID, _ := uuid.NewV7()
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
INSERT INTO entities (id, slug, type, name, attributes)
|
||||
VALUES ($1, $2, $3, $4, '{}')
|
||||
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, updated_at = now()
|
||||
RETURNING id`, docID, slug, kind, title).Scan(&docID); err != nil {
|
||||
slog.Warn("nomos: autoUpsertKnowledge entity insert", "session", sessionID, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Upsert the knowledge content.
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO knowledge_entities (entity_id, title, content, source, tags, updated_at)
|
||||
VALUES ($1, $2, $3, 'nomos-agent', $4, now())
|
||||
ON CONFLICT (entity_id) DO UPDATE
|
||||
SET title = EXCLUDED.title, content = EXCLUDED.content,
|
||||
tags = EXCLUDED.tags, updated_at = now()`,
|
||||
docID, title, content, tags); err != nil {
|
||||
slog.Warn("nomos: autoUpsertKnowledge content insert", "session", sessionID, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Link to the task entity.
|
||||
var taskEntID uuid.UUID
|
||||
if s.pool.QueryRow(ctx, `SELECT entity_id FROM agent_sessions WHERE id = $1`,
|
||||
sessionID).Scan(&taskEntID) == nil && taskEntID != uuid.Nil {
|
||||
s.pool.Exec(ctx, `
|
||||
INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT $1, $2, 'involves', '{"by":"nomos","auto":true}'::jsonb, now()
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM relationships
|
||||
WHERE source_id = $1 AND target_id = $2 AND type = 'involves' AND valid_to IS NULL)`,
|
||||
taskEntID, docID)
|
||||
}
|
||||
|
||||
slog.Info("nomos: auto-upserted knowledge for session",
|
||||
"session", sessionID, "outcome", outcome, "slug", slug)
|
||||
}
|
||||
|
||||
// writePlanCompletionRate computes the step completion rate for the current
|
||||
// plan generation and writes it as a task entity attribute so the trend can
|
||||
// be tracked. Baseline from 2026-08-04 audit: 38% (15/39 steps reached done).
|
||||
func writePlanCompletionRate(ctx context.Context, s *store, sessionID string) {
|
||||
var total, completed int
|
||||
s.pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*), COALESCE(SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END), 0)
|
||||
FROM session_plan_steps
|
||||
WHERE session_id = $1
|
||||
AND generation = (SELECT MAX(generation) FROM session_plan_steps WHERE session_id = $1)
|
||||
AND status <> 'replaced'`, sessionID).Scan(&total, &completed)
|
||||
if total > 0 {
|
||||
rate := float64(completed) / float64(total)
|
||||
attrs, _ := json.Marshal(map[string]any{"plan_completion_rate": rate, "plan_steps_total": total, "plan_steps_completed": completed})
|
||||
s.pool.Exec(ctx, `
|
||||
UPDATE entities SET attributes = attributes || $2::jsonb, updated_at = now()
|
||||
WHERE id = (SELECT entity_id FROM agent_sessions WHERE id = $1)`,
|
||||
sessionID, string(attrs))
|
||||
slog.Info("nomos: plan completion rate", "session", sessionID, "rate", fmt.Sprintf("%.0f%%", rate*100),
|
||||
"completed", completed, "total", total)
|
||||
}
|
||||
}
|
||||
|
||||
// autoFeedback creates a feedback entry linking the session's outcome to its
|
||||
// last execution, feeding the pattern-extraction pipeline that has been empty
|
||||
// since launch. Only created for success/partial outcomes (failures don't
|
||||
// have a specific execution to tie to).
|
||||
func autoFeedback(ctx context.Context, s *store, sessionID, outcome, summary string) {
|
||||
// Find the last execution linked to this session.
|
||||
var execID uuid.UUID
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT pe.execution_id FROM nomos_plan_executions pe
|
||||
WHERE pe.session_id = $1::uuid
|
||||
ORDER BY pe.created_at DESC LIMIT 1`, sessionID).Scan(&execID); err != nil || execID == uuid.Nil {
|
||||
return
|
||||
}
|
||||
fbID, _ := uuid.NewV7()
|
||||
slug := "feedback:" + fbID.String()
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO entities (id, slug, type, name, attributes) VALUES ($1, $2, 'feedback', $3, '{}')`,
|
||||
fbID, slug, "feedback for "+sessionID[:8]); err != nil {
|
||||
slog.Warn("nomos: autoFeedback entity insert", "session", sessionID, "error", err)
|
||||
return
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO feedback (entity_id, execution_id, outcome, observation, lesson, tags, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, now())`,
|
||||
fbID, execID, outcome, summary, summary, []string{"nomos-session", "auto-generated", "session:" + sessionID[:8]})
|
||||
if err != nil {
|
||||
slog.Warn("nomos: autoFeedback insert", "session", sessionID, "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("nomos: auto-feedback created for session", "session", sessionID, "outcome", outcome)
|
||||
}
|
||||
|
||||
// blockerPatterns maps a substring (case-insensitive) to a structured blocker
|
||||
// reason. Order matters — earlier patterns take precedence. These are the
|
||||
// recurring failure signatures from the 2026-07-20 session audit. A
|
||||
@@ -1240,6 +1476,53 @@ func (s *store) hadDiscovery(ctx context.Context, sessionID string) bool {
|
||||
return count > 0
|
||||
}
|
||||
|
||||
// sessionGoal returns the session's goal text, empty string if not found.
|
||||
// Used by complete_task to check whether the goal involved a reachability
|
||||
// verification before marking success.
|
||||
func (s *store) sessionGoal(ctx context.Context, sessionID string) string {
|
||||
if s == nil || sessionID == "" {
|
||||
return ""
|
||||
}
|
||||
var goal string
|
||||
s.pool.QueryRow(ctx,
|
||||
`SELECT COALESCE(goal, '') FROM agent_sessions WHERE id = $1`,
|
||||
sessionID).Scan(&goal)
|
||||
return goal
|
||||
}
|
||||
|
||||
// hadRecentVerification checks whether the session successfully verified
|
||||
// reachability in recent turns — ping_service, or a run with curl/wget that
|
||||
// returned successfully. Used by complete_task as a soft warning when the
|
||||
// goal involved a reachability check but no recent verification occurred.
|
||||
func (s *store) hadRecentVerification(ctx context.Context, sessionID string) bool {
|
||||
if s == nil || sessionID == "" {
|
||||
return true // fail safe: don't warn when we can't check
|
||||
}
|
||||
// Check for ping_service calls in the last 5 activity entries for this session.
|
||||
var pingCount int
|
||||
s.pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM (
|
||||
SELECT 1 FROM agent_activity
|
||||
WHERE session_id = $1 AND tool_name = 'ping_service' AND success = true
|
||||
ORDER BY ts DESC LIMIT 5
|
||||
) sub`, sessionID).Scan(&pingCount)
|
||||
if pingCount > 0 {
|
||||
return true
|
||||
}
|
||||
// Check for run calls with curl/wget that returned successfully.
|
||||
var curlCount int
|
||||
s.pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM (
|
||||
SELECT 1 FROM agent_activity
|
||||
WHERE session_id = $1
|
||||
AND tool_name = 'run'
|
||||
AND success = true
|
||||
AND (input_summary LIKE '%curl%' OR input_summary LIKE '%wget%')
|
||||
ORDER BY ts DESC LIMIT 10
|
||||
) sub`, sessionID).Scan(&curlCount)
|
||||
return curlCount > 0
|
||||
}
|
||||
|
||||
// staleGoalSession is a goal-bearing task that's gone idle without reaching
|
||||
// a terminal state — the idle-sweep worker's work list (fix 2+3 of
|
||||
// plans/2026-07-11-task-completion-safety-net.md).
|
||||
@@ -1347,15 +1630,23 @@ type planStep struct {
|
||||
|
||||
// getPlanSteps returns a task's plan in order — REST hydration for the context
|
||||
// panel when it first opens a task (live events only carry deltas from then on).
|
||||
func (s *store) getPlanSteps(ctx context.Context, sessionID string) ([]planStep, error) {
|
||||
// By default only the CURRENT (MAX) generation is returned — the panel shows the
|
||||
// live plan, not an archaeological record of every superseded generation. Pass
|
||||
// all=true for the audit/eval view that needs every generation (the
|
||||
// plan_generations assertion counts distinct generations across the full set).
|
||||
func (s *store) getPlanSteps(ctx context.Context, sessionID string, all bool) ([]planStep, error) {
|
||||
if s == nil {
|
||||
return nil, nil
|
||||
}
|
||||
genFilter := ""
|
||||
if !all {
|
||||
genFilter = "AND generation = (SELECT MAX(generation) FROM session_plan_steps WHERE session_id = $1)"
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id::text, seq, title, detail, status,
|
||||
execution_id::text, target_slug,
|
||||
started_at::text, finished_at::text, generation
|
||||
FROM session_plan_steps WHERE session_id = $1 ORDER BY seq`, sessionID)
|
||||
FROM session_plan_steps WHERE session_id = $1 `+genFilter+` ORDER BY generation, seq`, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1835,7 +2126,7 @@ func (s *store) resolveArgEntityID(ctx context.Context, args map[string]any) uui
|
||||
// The (nullable) session_id column carries the conversation id. args is the
|
||||
// tool call's own arguments, used to best-effort tag the row with the
|
||||
// entity it acted on (see resolveArgEntityID).
|
||||
func (s *store) logActivity(ctx context.Context, agentID uuid.UUID, sessionID, toolName string, args map[string]any, inputSummary, outputSummary string, durationMs int, success bool, correlationID string) {
|
||||
func (s *store) logActivity(ctx context.Context, agentID uuid.UUID, sessionID, toolName string, args map[string]any, inputSummary, outputSummary string, durationMs int, success bool, correlationID string, tokenCount int) {
|
||||
if s == nil || agentID == uuid.Nil {
|
||||
return
|
||||
}
|
||||
@@ -1847,8 +2138,8 @@ func (s *store) logActivity(ctx context.Context, agentID uuid.UUID, sessionID, t
|
||||
s.pool.Exec(ctx, `
|
||||
INSERT INTO agent_activity
|
||||
(agent_id, session_id, activity_type, tool_name, entity_id, input_summary, output_summary,
|
||||
duration_ms, success, correlation_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
|
||||
duration_ms, success, correlation_id, token_count)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
|
||||
agentID, sessionID, "tool_call", toolName, entityIDArg, inputSummary, outputSummary,
|
||||
durationMs, success, correlationID)
|
||||
durationMs, success, correlationID, tokenCount)
|
||||
}
|
||||
|
||||
@@ -192,7 +192,7 @@ func TestProposePlan_RefuseInFlight(t *testing.T) {
|
||||
}
|
||||
|
||||
// Mark step 1 as started.
|
||||
if err := s.updatePlanStep(ctx, sess.ID, 1, "running", ""); err != nil {
|
||||
if err := s.updatePlanStep(ctx, sess.ID, 1, "running", "", ""); err != nil {
|
||||
t.Fatalf("updatePlanStep: %v", err)
|
||||
}
|
||||
|
||||
@@ -205,7 +205,7 @@ func TestProposePlan_RefuseInFlight(t *testing.T) {
|
||||
}
|
||||
|
||||
// The original step 1 must be untouched — not erased, not appended to.
|
||||
steps, err := s.getPlanSteps(ctx, sess.ID)
|
||||
steps, err := s.getPlanSteps(ctx, sess.ID, false)
|
||||
if err != nil {
|
||||
t.Fatalf("getPlanSteps: %v", err)
|
||||
}
|
||||
@@ -217,7 +217,8 @@ func TestProposePlan_RefuseInFlight(t *testing.T) {
|
||||
}
|
||||
|
||||
// Third call BEFORE anything runs on a fresh session: every step is
|
||||
// still pending, so this must REPLACE, not refuse.
|
||||
// still pending, so this must REPLACE (mark the prior plan `replaced`),
|
||||
// not refuse. The new plan becomes generation 2.
|
||||
sess2, err := s.createSession(ctx, "plan replace test")
|
||||
if err != nil {
|
||||
t.Fatalf("createSession: %v", err)
|
||||
@@ -228,15 +229,146 @@ func TestProposePlan_RefuseInFlight(t *testing.T) {
|
||||
if _, err := s.proposePlan(ctx, sess2.ID, []planStepInput{{Title: "Revised"}}); err != nil {
|
||||
t.Fatalf("proposePlan (revise before execution): %v", err)
|
||||
}
|
||||
revisedSteps, err := s.getPlanSteps(ctx, sess2.ID)
|
||||
// Default (current generation) view: only the revised step.
|
||||
revisedSteps, err := s.getPlanSteps(ctx, sess2.ID, false)
|
||||
if err != nil {
|
||||
t.Fatalf("getPlanSteps: %v", err)
|
||||
}
|
||||
if len(revisedSteps) != 1 || revisedSteps[0].Title != "Revised" {
|
||||
t.Fatalf("got %+v, want a single 'Revised' step (pre-execution revise must replace, not refuse)", revisedSteps)
|
||||
t.Fatalf("got %+v, want a single 'Revised' step (current-generation view)", revisedSteps)
|
||||
}
|
||||
if revisedSteps[0].Generation != 1 {
|
||||
t.Fatalf("revised step generation = %d, want 1 (fresh-start after DELETE resets generation)", revisedSteps[0].Generation)
|
||||
if revisedSteps[0].Seq != 1 {
|
||||
t.Fatalf("revised step seq = %d, want 1 (seq is generation-relative, resets to 1..N)", revisedSteps[0].Seq)
|
||||
}
|
||||
if revisedSteps[0].Generation != 2 {
|
||||
t.Fatalf("revised step generation = %d, want 2 (prior pending plan is replaced, not deleted, so the counter increments)", revisedSteps[0].Generation)
|
||||
}
|
||||
// all=true audit view: both generations, the original marked `replaced`.
|
||||
allSteps, err := s.getPlanSteps(ctx, sess2.ID, true)
|
||||
if err != nil {
|
||||
t.Fatalf("getPlanSteps(all): %v", err)
|
||||
}
|
||||
if len(allSteps) != 2 {
|
||||
t.Fatalf("all=true got %d steps, want 2 (Original replaced gen1 + Revised gen2)", len(allSteps))
|
||||
}
|
||||
if allSteps[0].Title != "Original" || allSteps[0].Status != "replaced" || allSteps[0].Generation != 1 {
|
||||
t.Errorf("gen1 step = %+v, want Original/replaced/gen1", allSteps[0])
|
||||
}
|
||||
if allSteps[1].Title != "Revised" || allSteps[1].Generation != 2 || allSteps[1].Seq != 1 {
|
||||
t.Errorf("gen2 step = %+v, want Revised/gen2/seq1", allSteps[1])
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdatePlanStep_GenerationRelative is the P0.1 regression proof: after a
|
||||
// re-plan, update_plan_step(seq=N) — using the 1-based number the model
|
||||
// naturally carries — must address the CURRENT generation and never resurrect
|
||||
// a superseded generation's `replaced` row. Before the fix, seq was globally
|
||||
// increasing across generations, so seq=1 after a re-plan flipped the gen-1
|
||||
// `replaced` step back to `running`/`done` while the real gen-2 work went
|
||||
// unrecorded.
|
||||
func TestUpdatePlanStep_GenerationRelative(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
sess, err := s.createSession(ctx, "gen-relative seq test")
|
||||
if err != nil {
|
||||
t.Fatalf("createSession: %v", err)
|
||||
}
|
||||
// Generation 1: two steps.
|
||||
if _, err := s.proposePlan(ctx, sess.ID, []planStepInput{{Title: "A"}, {Title: "B"}}); err != nil {
|
||||
t.Fatalf("proposePlan #1: %v", err)
|
||||
}
|
||||
// Re-plan: setGoal marks the gen-1 plan `replaced`, proposePlan starts gen 2.
|
||||
if err := s.setGoal(ctx, sess.ID, "follow-up sub-task"); err != nil {
|
||||
t.Fatalf("setGoal: %v", err)
|
||||
}
|
||||
if _, err := s.proposePlan(ctx, sess.ID, []planStepInput{{Title: "C"}, {Title: "D"}}); err != nil {
|
||||
t.Fatalf("proposePlan #2: %v", err)
|
||||
}
|
||||
|
||||
// The model addresses the new plan with 1-based seq. seq=1 must hit
|
||||
// gen-2 "C", leaving gen-1 "A" (replaced) untouched.
|
||||
if err := s.updatePlanStep(ctx, sess.ID, 1, "running", "", ""); err != nil {
|
||||
t.Fatalf("updatePlanStep(seq=1, running): %v", err)
|
||||
}
|
||||
if err := s.updatePlanStep(ctx, sess.ID, 1, "done", "", ""); err != nil {
|
||||
t.Fatalf("updatePlanStep(seq=1, done): %v", err)
|
||||
}
|
||||
|
||||
all, err := s.getPlanSteps(ctx, sess.ID, true)
|
||||
if err != nil {
|
||||
t.Fatalf("getPlanSteps(all): %v", err)
|
||||
}
|
||||
byTitle := map[string]planStep{}
|
||||
for _, st := range all {
|
||||
byTitle[st.Title] = st
|
||||
}
|
||||
// gen-1 steps stay `replaced` — NOT resurrected to running/done.
|
||||
if byTitle["A"].Status != "replaced" || byTitle["A"].Generation != 1 {
|
||||
t.Errorf("A = %+v, want replaced/gen1 (a superseded row must never be touched)", byTitle["A"])
|
||||
}
|
||||
if byTitle["B"].Status != "replaced" || byTitle["B"].Generation != 1 {
|
||||
t.Errorf("B = %+v, want replaced/gen1", byTitle["B"])
|
||||
}
|
||||
// gen-2 seq=1 advanced; seq=2 untouched.
|
||||
if byTitle["C"].Status != "done" || byTitle["C"].Generation != 2 || byTitle["C"].Seq != 1 {
|
||||
t.Errorf("C = %+v, want done/gen2/seq1 (the 1-based update must address the current generation)", byTitle["C"])
|
||||
}
|
||||
if byTitle["D"].Status != "pending" || byTitle["D"].Seq != 2 {
|
||||
t.Errorf("D = %+v, want pending/seq2", byTitle["D"])
|
||||
}
|
||||
|
||||
// Out-of-range seq must be refused (no current-gen step there).
|
||||
if err := s.updatePlanStep(ctx, sess.ID, 99, "running", "", ""); !errors.Is(err, errPlanStepNotFound) {
|
||||
t.Fatalf("updatePlanStep(seq=99) err = %v, want errPlanStepNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompleteTask_AutoCloseEmitsEvents is the P1.1 regression proof:
|
||||
// completeTask's bulk auto-close of in-flight steps must emit one
|
||||
// plan.step.finished event per closed step (so the live panel converges
|
||||
// instead of freezing on "running" after the task completes) and must stamp
|
||||
// started_at so no closed step is left un-timestamped (P0.1 fix 5).
|
||||
func TestCompleteTask_AutoCloseEmitsEvents(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
sess, err := s.createSession(ctx, "auto-close events test")
|
||||
if err != nil {
|
||||
t.Fatalf("createSession: %v", err)
|
||||
}
|
||||
if _, err := s.proposePlan(ctx, sess.ID, []planStepInput{{Title: "A"}, {Title: "B"}}); err != nil {
|
||||
t.Fatalf("proposePlan: %v", err)
|
||||
}
|
||||
// A is running, B still pending at completion time.
|
||||
if err := s.updatePlanStep(ctx, sess.ID, 1, "running", "", ""); err != nil {
|
||||
t.Fatalf("updatePlanStep(1, running): %v", err)
|
||||
}
|
||||
if err := s.completeTask(ctx, sess.ID, "success", "done"); err != nil {
|
||||
t.Fatalf("completeTask: %v", err)
|
||||
}
|
||||
|
||||
// Every auto-closed step should now carry both a started_at and a
|
||||
// finished_at (no NULL-started `done` step).
|
||||
steps, err := s.getPlanSteps(ctx, sess.ID, true)
|
||||
if err != nil {
|
||||
t.Fatalf("getPlanSteps: %v", err)
|
||||
}
|
||||
for _, st := range steps {
|
||||
if st.Status == "done" && st.StartedAt == nil {
|
||||
t.Errorf("step %q done but started_at is NULL (P0.1 fix 5: stamp it)", st.Title)
|
||||
}
|
||||
}
|
||||
|
||||
// Exactly two plan.step.finished events — one per closed step (A and B).
|
||||
var finished int
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
`SELECT COUNT(*) FROM events WHERE type = 'plan.step.finished' AND correlation_id = $1`,
|
||||
sess.ID).Scan(&finished); err != nil {
|
||||
t.Fatalf("count events: %v", err)
|
||||
}
|
||||
if finished != 2 {
|
||||
t.Fatalf("plan.step.finished events = %d, want 2 (one per auto-closed step)", finished)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,7 +397,7 @@ func TestHadDiscoveryAndWriteback(t *testing.T) {
|
||||
|
||||
// A `run` call (discovery) — should set hadDiscovery, not hadEntityWriteback.
|
||||
agentID := uuid.New()
|
||||
s.logActivity(ctx, agentID, sess.ID, "run", nil, "", "uptime output", 100, true, "corr-1")
|
||||
s.logActivity(ctx, agentID, sess.ID, "run", nil, "", "uptime output", 100, true, "corr-1", 0)
|
||||
if !s.hadDiscovery(ctx, sess.ID) {
|
||||
t.Fatal("hadDiscovery = false after a successful run call, want true")
|
||||
}
|
||||
@@ -278,7 +410,7 @@ func TestHadDiscoveryAndWriteback(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("createSession: %v", err)
|
||||
}
|
||||
s.logActivity(ctx, agentID, sess2.ID, "run", nil, "", "ssh timeout", 100, false, "corr-2")
|
||||
s.logActivity(ctx, agentID, sess2.ID, "run", nil, "", "ssh timeout", 100, false, "corr-2", 0)
|
||||
if s.hadDiscovery(ctx, sess2.ID) {
|
||||
t.Fatal("hadDiscovery = true after a failed run call, want false (no facts learned)")
|
||||
}
|
||||
@@ -288,7 +420,7 @@ func TestHadDiscoveryAndWriteback(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("createSession: %v", err)
|
||||
}
|
||||
s.logActivity(ctx, agentID, sess3.ID, "get_entity", nil, "", "entity row", 10, true, "corr-3")
|
||||
s.logActivity(ctx, agentID, sess3.ID, "get_entity", nil, "", "entity row", 10, true, "corr-3", 0)
|
||||
if s.hadDiscovery(ctx, sess3.ID) {
|
||||
t.Fatal("hadDiscovery = true after get_entity, want false (DB lookups are not discovery)")
|
||||
}
|
||||
@@ -298,12 +430,12 @@ func TestHadDiscoveryAndWriteback(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("createSession: %v", err)
|
||||
}
|
||||
s.logActivity(ctx, agentID, sess4.ID, "update_entity_attributes", nil, "", "ok", 10, true, "corr-4")
|
||||
s.logActivity(ctx, agentID, sess4.ID, "update_entity_attributes", nil, "", "ok", 10, true, "corr-4", 0)
|
||||
if !s.hadEntityWriteback(ctx, sess4.ID) {
|
||||
t.Fatal("hadEntityWriteback = false after update_entity_attributes, want true")
|
||||
}
|
||||
// And the discovery+writeback combination (the conv3 scenario).
|
||||
s.logActivity(ctx, agentID, sess4.ID, "run", nil, "", "apt-get update output", 100, true, "corr-5")
|
||||
s.logActivity(ctx, agentID, sess4.ID, "run", nil, "", "apt-get update output", 100, true, "corr-5", 0)
|
||||
if !s.hadDiscovery(ctx, sess4.ID) {
|
||||
t.Fatal("hadDiscovery = false after run+writeback, want true")
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -245,13 +246,17 @@ func (a *agent) handleTaskTool(ctx context.Context, sessionID, name string, args
|
||||
// the seq-order enforcement (5.6) require it to be completed last,
|
||||
// and D.1's complete_task gate enforces the actual calls. Together
|
||||
// they close the loop structurally — neither relies on the agent
|
||||
// reading SOUL.md.
|
||||
// reading SOUL.md. The match is broadened past the literal tool
|
||||
// names so a natural-language step ("Write back: update entity
|
||||
// attributes…") isn't doubled by an auto-appended duplicate (P1.2).
|
||||
hasWritebackStep := false
|
||||
for _, st := range steps {
|
||||
if strings.Contains(st.Title, "update_entity_attributes") ||
|
||||
strings.Contains(st.Title, "create_relationship") ||
|
||||
strings.Contains(st.Detail, "update_entity_attributes") ||
|
||||
strings.Contains(st.Detail, "create_relationship") {
|
||||
t := strings.ToLower(st.Title + " " + st.Detail)
|
||||
if strings.Contains(t, "update_entity_attributes") ||
|
||||
strings.Contains(t, "create_relationship") ||
|
||||
strings.Contains(t, "upsert_knowledge") ||
|
||||
strings.Contains(t, "write back") ||
|
||||
strings.Contains(t, "writeback") {
|
||||
hasWritebackStep = true
|
||||
break
|
||||
}
|
||||
@@ -280,8 +285,19 @@ func (a *agent) handleTaskTool(ctx context.Context, sessionID, name string, args
|
||||
// The writeback step is now always present (D.2 auto-appends it if
|
||||
// the agent forgot), so the old advisory nudge is replaced by the
|
||||
// structural gate: D.1 refuses complete_task without the actual
|
||||
// update_entity_attributes/create_relationship calls.
|
||||
result := fmt.Sprintf("Plan set (%d steps)%s. If all steps are read-only, execute now — call update_plan_step(running) + run for each step, no approval needed. If any step is config_mutation/destructive, STOP and wait for operator approval (\"approved\", \"yes\", \"go\", \"proceed\", \"continue\", \"ok\", \"go ahead\"). Do not call propose_plan again.", len(persisted), appendedNote)
|
||||
// update_entity_attributes/create_relationship calls. Enumerate the
|
||||
// step seqs so the model knows exactly which numbers to address with
|
||||
// update_plan_step (seq is 1-based within this plan — the addressing
|
||||
// key, not a global counter).
|
||||
var seqs strings.Builder
|
||||
for i, p := range persisted {
|
||||
if i > 0 {
|
||||
seqs.WriteString("; ")
|
||||
}
|
||||
title := fmt.Sprint(p["title"])
|
||||
fmt.Fprintf(&seqs, "%v=%s", p["seq"], title)
|
||||
}
|
||||
result := fmt.Sprintf("Plan set (%d steps): %s.%s Address them with update_plan_step(seq=N). If all steps are read-only, execute now — call update_plan_step(running) + run for each step, no approval needed. If any step is config_mutation/destructive, STOP and wait for operator approval (\"approved\", \"yes\", \"go\", \"proceed\", \"continue\", \"ok\", \"go ahead\"). Do not call propose_plan again.", len(persisted), seqs.String(), appendedNote)
|
||||
return result, true
|
||||
|
||||
case "update_plan_step":
|
||||
@@ -291,7 +307,17 @@ func (a *agent) handleTaskTool(ctx context.Context, sessionID, name string, args
|
||||
if seq <= 0 || status == "" {
|
||||
return "error: update_plan_step needs seq (>=1) and status", true
|
||||
}
|
||||
if err := a.store.updatePlanStep(ctx, sessionID, seq, status, execID); err != nil {
|
||||
reason, _ := args["replaced_reason"].(string)
|
||||
if err := a.store.updatePlanStep(ctx, sessionID, seq, status, execID, reason); err != nil {
|
||||
if errors.Is(err, errPlanStepNotFound) {
|
||||
// The seq doesn't address a step in the CURRENT plan — most
|
||||
// often a stale 1-based number the model carried across a
|
||||
// re-plan, or an out-of-range seq. seq is generation-relative
|
||||
// (1..N within the latest propose_plan), so a superseded
|
||||
// generation's row is never touched (P0.1 fix 3). Direct the
|
||||
// model instead of silently no-op'ing.
|
||||
return fmt.Sprintf("Step %d is not in the current plan. seq is 1-based within your latest propose_plan (a re-plan resets it to 1..N, so an old step number no longer applies). The plan was not changed. Re-address with the correct 1-based seq, or if you've lost track, re-read the plan.", seq), true
|
||||
}
|
||||
return fmt.Sprintf("error updating step %d: %v", seq, err), true
|
||||
}
|
||||
return fmt.Sprintf("Step %d → %s. (Advance with update_plan_step + run; do not re-propose.)", seq, status), true
|
||||
@@ -349,6 +375,18 @@ func (a *agent) handleTaskTool(ctx context.Context, sessionID, name string, args
|
||||
if outcome == "success" && a.store.hadDiscovery(ctx, sessionID) && !a.store.hadEntityWriteback(ctx, sessionID) {
|
||||
return "Refused: this session ran `run` against live targets (discovery) but did not call update_entity_attributes or create_relationship to persist what you learned. The knowledge graph will drift if you complete without writeback. Call update_entity_attributes for each entity you ran against (versions, states, counts, timestamps), and create_relationship for any edge you discovered, then call complete_task again. Outcome is held at 'executing' until you do.", true
|
||||
}
|
||||
// D.2: refuse success when the goal mentions a reachability/uptime
|
||||
// check but no verification was done. The agent can't claim "X is
|
||||
// reachable" based on a shell command alone — the proxy (Caddy) can
|
||||
// return 200 for a terminal page (ttyd) or fallback while the actual
|
||||
// dashboard is still down. Must call ping_service or run a successful
|
||||
// curl before claiming success.
|
||||
if outcome == "success" && a.store.hadDiscovery(ctx, sessionID) {
|
||||
goal := a.store.sessionGoal(ctx, sessionID)
|
||||
if mentionsReachability(goal) && !a.store.hadRecentVerification(ctx, sessionID) {
|
||||
return "Refused: the goal involves a reachability or uptime check (\"make X reachable\", \"get X up\", etc.), but no ping_service call or successful curl/HTTP request against the target was detected. Caddy can return 200 for a terminal or fallback page while the actual service is still down — you must verify the service itself, not just the proxy. Call ping_service(target) or run a curl against the actual service URL, then call complete_task again. Outcome held until verified.", true
|
||||
}
|
||||
}
|
||||
if err := a.store.completeTask(ctx, sessionID, outcome, summary); err != nil {
|
||||
if errors.Is(err, errTaskAlreadyComplete) {
|
||||
return "Task is already complete. Do not call complete_task again. If the operator pointed out a UI/sidebar inconsistency, fix it with update_plan_step (reconcile step states) or summarize the panel in your reply — do not re-execute the work.", true
|
||||
@@ -365,6 +403,28 @@ func (a *agent) handleTaskTool(ctx context.Context, sessionID, name string, args
|
||||
}
|
||||
}
|
||||
|
||||
// reachabilityPatterns matches goal text that involves making something
|
||||
// reachable/accessible/working. Used by complete_task to surface a soft
|
||||
// warning when the session goal was about reachability but no verification
|
||||
// occurred before marking success.
|
||||
var reachabilityPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)https?://[^\s]+`),
|
||||
regexp.MustCompile(`(?i)\.hubris\.net\w+`),
|
||||
regexp.MustCompile(`(?i)(un)?reachable`),
|
||||
regexp.MustCompile(`(?i)(not?\s+)?(accessible|reachable|responding|resolving)`),
|
||||
regexp.MustCompile(`(?i)diagnose\s+why`),
|
||||
regexp.MustCompile(`(?i)(fix|restore|bring\s+back).*(accessible|reachable|online)`),
|
||||
}
|
||||
|
||||
func mentionsReachability(goal string) bool {
|
||||
for _, p := range reachabilityPatterns {
|
||||
if p.MatchString(goal) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// autoCompleteTrivialTask is the case-1 fix from
|
||||
// plans/2026-07-11-task-completion-safety-net.md: a session that never
|
||||
// called set_goal never framed itself as a structured task, so a turn that
|
||||
|
||||
90
cmd/nomos/turngate.go
Normal file
90
cmd/nomos/turngate.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// turnGate enforces at most one in-flight agent turn per session.
|
||||
//
|
||||
// Why this exists (plan 2026-08-03, F1): handleChat runs a turn in the HTTP
|
||||
// request goroutine, and every "resume" path (the empty-message reconnect,
|
||||
// the auto-continuation worker, the idle sweep, answer-question, the /resume
|
||||
// endpoint) launches ANOTHER goroutine running a full turn. Nothing prevented
|
||||
// two turns for the SAME session at once, so a network blip that triggered a
|
||||
// reconnect would spawn a duplicate resumeSession while the original turn was
|
||||
// still alive — their tool calls interleaved on the wire and in the persisted
|
||||
// transcript, which is the root cause behind the "parallel/nesting/sequence
|
||||
// is off" and "task didn't end / flaky" reports.
|
||||
//
|
||||
// Model: one permit (buffered-1 channel seeded with a single token) per
|
||||
// session id. Acquiring consumes the token; releasing puts it back.
|
||||
// - Background/best-effort callers (resumeSession and everything it backs)
|
||||
// use a non-blocking acquire and SKIP when busy — a duplicate nudge while a
|
||||
// turn is already running adds nothing, and the continuation/idle tickers
|
||||
// will retry on their own.
|
||||
// - The live chat path (an operator message) waits briefly for a finishing
|
||||
// background turn, then bails with an actionable error if still busy — see
|
||||
// handleChat.
|
||||
//
|
||||
// The permits map grows one entry per session id seen. For this single-agent
|
||||
// homelab process that set is small and bounded by real sessions; cleanup is
|
||||
// intentionally omitted (a sweep would race with acquire/release and the
|
||||
// memory is negligible).
|
||||
type turnGate struct {
|
||||
mu sync.Mutex
|
||||
permits map[string]chan struct{}
|
||||
}
|
||||
|
||||
func newTurnGate() *turnGate {
|
||||
return &turnGate{permits: make(map[string]chan struct{})}
|
||||
}
|
||||
|
||||
// permit returns the single token-channel for sessionID, creating and seeding
|
||||
// it on first use. Creation is guarded so two concurrent first-callers for the
|
||||
// same id share one channel.
|
||||
func (g *turnGate) permit(sessionID string) chan struct{} {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
ch, ok := g.permits[sessionID]
|
||||
if !ok {
|
||||
ch = make(chan struct{}, 1)
|
||||
ch <- struct{}{}
|
||||
g.permits[sessionID] = ch
|
||||
}
|
||||
return ch
|
||||
}
|
||||
|
||||
// acquire takes the session's permit. With wait <= 0 it is non-blocking
|
||||
// (returns false immediately if a turn is active). With wait > 0 it blocks up
|
||||
// to wait for the permit, returning false on timeout. Every true return MUST
|
||||
// be paired with exactly one release.
|
||||
func (g *turnGate) acquire(sessionID string, wait time.Duration) bool {
|
||||
ch := g.permit(sessionID)
|
||||
if wait <= 0 {
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
t := time.NewTimer(wait)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
case <-t.C:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// release returns the session's permit. Idempotent: a release with no matching
|
||||
// acquire (or a double release) is a no-op rather than a blocking send.
|
||||
func (g *turnGate) release(sessionID string) {
|
||||
ch := g.permit(sessionID)
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
114
cmd/nomos/turngate_test.go
Normal file
114
cmd/nomos/turngate_test.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTurnGate_NonBlockingSkipsWhenBusy(t *testing.T) {
|
||||
g := newTurnGate()
|
||||
if !g.acquire("s1", 0) {
|
||||
t.Fatal("first non-blocking acquire should succeed on a free session")
|
||||
}
|
||||
// A second non-blocking acquire (a background resume) must skip, not queue.
|
||||
if g.acquire("s1", 0) {
|
||||
t.Fatal("second non-blocking acquire should fail while a turn is active")
|
||||
}
|
||||
// A different session is independent.
|
||||
if !g.acquire("s2", 0) {
|
||||
t.Fatal("acquire on a different session should succeed")
|
||||
}
|
||||
g.release("s2")
|
||||
g.release("s1")
|
||||
// After release, the session is free again.
|
||||
if !g.acquire("s1", 0) {
|
||||
t.Fatal("acquire should succeed again after release")
|
||||
}
|
||||
g.release("s1")
|
||||
}
|
||||
|
||||
func TestTurnGate_BlockingAcquireWaitsForRelease(t *testing.T) {
|
||||
g := newTurnGate()
|
||||
if !g.acquire("s1", 0) {
|
||||
t.Fatal("first acquire should succeed")
|
||||
}
|
||||
|
||||
got := make(chan bool, 1)
|
||||
go func() { got <- g.acquire("s1", 2*time.Second) }()
|
||||
|
||||
select {
|
||||
case <-got:
|
||||
t.Fatal("blocking acquire should wait, not return before release")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
// expected: still waiting
|
||||
}
|
||||
|
||||
g.release("s1")
|
||||
select {
|
||||
case ok := <-got:
|
||||
if !ok {
|
||||
t.Fatal("blocking acquire should succeed after release")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("blocking acquire did not return after release")
|
||||
}
|
||||
g.release("s1")
|
||||
}
|
||||
|
||||
func TestTurnGate_BlockingAcquireTimesOut(t *testing.T) {
|
||||
g := newTurnGate()
|
||||
g.acquire("s1", 0) // hold the permit
|
||||
|
||||
start := time.Now()
|
||||
if g.acquire("s1", 60*time.Millisecond) {
|
||||
t.Fatal("acquire should time out while permit is held")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 50*time.Millisecond {
|
||||
t.Fatalf("acquire returned too fast (%v); expected to wait ~60ms", elapsed)
|
||||
}
|
||||
g.release("s1")
|
||||
}
|
||||
|
||||
// TestTurnGate_SingleFlightConcurrent is the core F1 guarantee: many concurrent
|
||||
// background acquirers on the SAME session, exactly one runs at a time. This is
|
||||
// the property that prevents two turns interleaving tool calls.
|
||||
func TestTurnGate_SingleFlightConcurrent(t *testing.T) {
|
||||
g := newTurnGate()
|
||||
const n = 50
|
||||
var inFlight, maxInFlight int64
|
||||
var runs int64
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(n)
|
||||
start := make(chan struct{})
|
||||
for i := 0; i < n; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
if !g.acquire("shared", 0) { // background-style: skip if busy
|
||||
return
|
||||
}
|
||||
defer g.release("shared")
|
||||
cur := atomic.AddInt64(&inFlight, 1)
|
||||
for {
|
||||
m := atomic.LoadInt64(&maxInFlight)
|
||||
if cur <= m || atomic.CompareAndSwapInt64(&maxInFlight, m, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
atomic.AddInt64(&runs, 1)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
atomic.AddInt64(&inFlight, -1)
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
|
||||
if maxInFlight != 1 {
|
||||
t.Fatalf("max in-flight turns = %d, want 1 (turns must not overlap)", maxInFlight)
|
||||
}
|
||||
if runs == 0 {
|
||||
t.Fatal("expected at least one turn to run")
|
||||
}
|
||||
}
|
||||
152
cmd/nomos/workers.go
Normal file
152
cmd/nomos/workers.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// runChatTurn is the shared core of an operator-initiated turn: insert an
|
||||
// assistant placeholder, run a.chat with incremental persistence (so whatever
|
||||
// happened before an abort is never lost), finalize the row, and derive a
|
||||
// title. It is agnostic to the transport: `sink` receives every agent event
|
||||
// for delivery (SSE for a live handleChat, a no-op for a queued turn that has
|
||||
// no client attached — the frontend learns about those via the poller + the
|
||||
// status-driven "working" signal). The caller MUST already hold the session's
|
||||
// turn-gate permit.
|
||||
func (a *agent) runChatTurn(pctx, ctx context.Context, sessionID, message string, sink func(agentEvent)) {
|
||||
toolCalls := []map[string]any{}
|
||||
// P3: accumulate per-iteration reasoning instead of overwriting with the
|
||||
// final `text` event (see the original inline comment in handleChat).
|
||||
var textParts []string
|
||||
var thinkingParts []string
|
||||
var finalText string
|
||||
var finalThinking string
|
||||
|
||||
placeholder, _ := json.Marshal(map[string]any{"role": "assistant", "text": ""})
|
||||
msgID, err := a.store.insertMessageReturningID(pctx, sessionID, "assistant", placeholder)
|
||||
if err != nil {
|
||||
slog.Error("nomos: chat placeholder insert failed", "session", sessionID, "error", err)
|
||||
}
|
||||
persist := func() {
|
||||
if msgID == uuid.Nil {
|
||||
return
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"role": "assistant",
|
||||
"text": finalText,
|
||||
"thinking": finalThinking,
|
||||
"tool_calls": toolCalls,
|
||||
})
|
||||
a.store.updateMessage(pctx, msgID, body)
|
||||
}
|
||||
|
||||
a.chat(ctx, sessionID, message, func(ev agentEvent) {
|
||||
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
||||
if m, ok := ev.Data.(map[string]any); ok {
|
||||
m["type"] = ev.Type
|
||||
// One entry per tool call: tool_use creates it, tool_result
|
||||
// merges the result into the same entry (matched by id).
|
||||
id, _ := m["id"].(string)
|
||||
if id != "" && ev.Type == "tool_result" {
|
||||
for _, existing := range toolCalls {
|
||||
if eID, _ := existing["id"].(string); eID == id {
|
||||
for k, v := range m {
|
||||
existing[k] = v
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
toolCalls = append(toolCalls, m)
|
||||
}
|
||||
}
|
||||
persist() // live: survives even if the client disconnects right after
|
||||
}
|
||||
if ev.Type == "text" {
|
||||
if t, ok := ev.Data.(string); ok && t != "" {
|
||||
if ev.IsThinking {
|
||||
thinkingParts = append(thinkingParts, t)
|
||||
finalThinking = strings.Join(thinkingParts, "\n\n")
|
||||
} else {
|
||||
textParts = append(textParts, t)
|
||||
finalText = strings.Join(textParts, "\n\n")
|
||||
}
|
||||
persist()
|
||||
}
|
||||
}
|
||||
sink(ev)
|
||||
})
|
||||
|
||||
// B.6: if the turn ended with no text and no tool calls (the model
|
||||
// empty-response'd and all retries failed), delete the placeholder row
|
||||
// instead of persisting an empty bubble.
|
||||
if finalText == "" && len(toolCalls) == 0 && msgID != uuid.Nil {
|
||||
a.store.deleteMessage(pctx, msgID)
|
||||
} else {
|
||||
persist() // final state — same row, updated one last time
|
||||
}
|
||||
|
||||
// Title: prefer the goal once set; else the first assistant answer.
|
||||
if finalText != "" && sessionID != "ephemeral" {
|
||||
var goalTitle string
|
||||
if sess, gerr := a.store.getSession(pctx, sessionID); gerr == nil && sess.Goal != "" {
|
||||
goalTitle = truncate(sess.Goal, 120)
|
||||
}
|
||||
title := goalTitle
|
||||
if title == "" {
|
||||
title = truncate(finalText, 80)
|
||||
}
|
||||
if title != "" {
|
||||
a.store.updateSessionTitle(pctx, sessionID, title)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// drainAcquireWait is how long drainQueued blocks for a busy gate before
|
||||
// re-queuing and deferring to the holder's own release-drain. A package var so
|
||||
// tests can shorten it; in production it just needs to outlast the brief
|
||||
// release→drain handoff window.
|
||||
var drainAcquireWait = 5 * time.Second
|
||||
|
||||
// drainQueued runs every queued operator message for a session as its own turn,
|
||||
// one at a time, under the turn gate. Called (in a goroutine) whenever a turn
|
||||
// releases the gate — from handleChat (live) and resumeSession (background) —
|
||||
// so a message queued while the agent was busy is acted on as soon as it's
|
||||
// free, without the operator re-sending. See messagequeue.go (plan 2026-08-03
|
||||
// F2).
|
||||
//
|
||||
// Each queued turn is persisted incrementally and has no SSE client (the
|
||||
// browser detached after receiving the `queued` event); the frontend sees the
|
||||
// result via the 3s poller and the status-driven "working" indicator.
|
||||
func (a *agent) drainQueued(ctx context.Context, sessionID string) {
|
||||
for {
|
||||
msg, ok := a.queue.dequeue(sessionID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// Block briefly for the gate. If a live turn grabbed it first, put the
|
||||
// message back — that turn's release will drain it again. Never stack.
|
||||
if !a.gate.acquire(sessionID, drainAcquireWait) {
|
||||
a.queue.requeueFront(sessionID, msg)
|
||||
return
|
||||
}
|
||||
slog.Info("nomos: running queued operator message", "session", sessionID)
|
||||
pctx := context.Background()
|
||||
// Run the turn inside a per-iteration closure so the gate release is
|
||||
// deferred to the end of THIS turn (and runs even if runChatTurn
|
||||
// panics — safego recovers the panic at the goroutine boundary, so a
|
||||
// non-deferred release would be skipped and the session's permit held
|
||||
// forever, deadlocking all future turns). A bare `defer release` in
|
||||
// the loop would be wrong too: Go defers run at function exit, not
|
||||
// iteration exit, so the gate would stay held across iterations.
|
||||
func() {
|
||||
defer a.gate.release(sessionID)
|
||||
a.runChatTurn(pctx, ctx, sessionID, msg, func(agentEvent) {})
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/execworker"
|
||||
"github.com/dtoro/oikos/internal/httpapi"
|
||||
"github.com/dtoro/oikos/internal/knowledge"
|
||||
"github.com/dtoro/oikos/internal/notifier"
|
||||
@@ -23,6 +24,7 @@ import (
|
||||
|
||||
var schedulerRunner = scheduler.RunnerForMain()
|
||||
var notifierRunner = notifier.RunnerForMain()
|
||||
var execWorkerRunner = execworker.RunnerForMain()
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
@@ -37,12 +39,39 @@ func main() {
|
||||
logger := observability.NewLogger(cfg.Debug)
|
||||
slog.SetDefault(logger)
|
||||
|
||||
slog.Info("starting oikos", "role", role, "config", cfg)
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(),
|
||||
syscall.SIGTERM, syscall.SIGINT)
|
||||
defer cancel()
|
||||
|
||||
// Resolve secrets from Infisical, overlaying env-derived config values.
|
||||
// If Infisical is not configured, env vars are used as-is (no change).
|
||||
sec := secrets.NewManagerFromConfig(
|
||||
cfg.InfisicalSiteURL,
|
||||
cfg.InfisicalClientID,
|
||||
cfg.InfisicalClientSecret,
|
||||
cfg.InfisicalProjectID,
|
||||
cfg.InfisicalEnv,
|
||||
cfg.SecretsDir,
|
||||
)
|
||||
if sec != nil {
|
||||
overlays := secrets.ConfigOverlays(map[string]func(string){
|
||||
"matrix_token": func(v string) { cfg.MatrixToken = v },
|
||||
"approval_hmac-secret": func(v string) { cfg.ApprovalHMACSecret = v },
|
||||
"mcp_bearer-token": func(v string) { cfg.MCPBearerToken = v },
|
||||
"api_token": func(v string) { cfg.APIToken = v },
|
||||
"oidc_client-secret": func(v string) { cfg.OIDCClientSecret = v },
|
||||
})
|
||||
n := secrets.OverlayConfig(ctx, sec, overlays)
|
||||
slog.Info("secrets resolved from Infisical", "count", n)
|
||||
|
||||
secrets.VerifyExpectedSecrets(ctx, sec, []string{
|
||||
"matrix_token", "approval_hmac-secret", "mcp_bearer-token",
|
||||
"api_token", "openrouter_api-key", "webhook_hmac-secret",
|
||||
})
|
||||
}
|
||||
|
||||
slog.Info("starting oikos", "role", role, "config", cfg)
|
||||
|
||||
switch role {
|
||||
case "migrate":
|
||||
if err := runMigrate(ctx, cfg); err != nil {
|
||||
@@ -68,6 +97,8 @@ func main() {
|
||||
runWithPool(ctx, cfg, "scheduler", schedulerRunner)
|
||||
case "notifier":
|
||||
runWithPool(ctx, cfg, "notifier", notifierRunner)
|
||||
case "execution-worker":
|
||||
runWithPool(ctx, cfg, "execution-worker", execWorkerRunner)
|
||||
case "all":
|
||||
pool, err := db.New(ctx, cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
@@ -83,8 +114,9 @@ func main() {
|
||||
|
||||
go schedulerRunner(ctx, pool, cfg)
|
||||
go notifierRunner(ctx, pool, cfg)
|
||||
go execWorkerRunner(ctx, pool, cfg)
|
||||
|
||||
slog.Info("all: starting api with scheduler + notifier in background")
|
||||
slog.Info("all: starting api with scheduler + notifier + execution-worker in background")
|
||||
if err := httpapi.ListenAndServe(ctx, pool, cfg); err != nil {
|
||||
slog.Error("api failed", "error", err)
|
||||
os.Exit(1)
|
||||
@@ -115,7 +147,7 @@ Roles:
|
||||
scheduler Run the observe loop
|
||||
notifier Run the notification service (Matrix alerts)
|
||||
all Run all roles in one process (dev mode)
|
||||
secret Secret management (Infisical)
|
||||
secret Secret management (Infisical: get, set, list, verify, audit, migrate, export-sops)
|
||||
knowledge Convert wiki to knowledge seed (one-shot)
|
||||
version Print version info
|
||||
|
||||
@@ -289,20 +321,45 @@ func runWithPool(ctx context.Context, cfg config.Config, name string, fn func(co
|
||||
|
||||
func runSecret(ctx context.Context, cfg config.Config) {
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Fprintln(os.Stderr, "usage: oikos secret <list|migrate|export-sops>")
|
||||
fmt.Fprintln(os.Stderr, "usage: oikos secret <get|set|list|migrate|export-sops>")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
sub := os.Args[2]
|
||||
secretsDir := cfg.SecretsDir
|
||||
if secretsDir == "" {
|
||||
secretsDir = "archive/secrets-sops-backup"
|
||||
}
|
||||
sopsBackend := secrets.NewSOPSBackend(secretsDir)
|
||||
|
||||
// For get/set/list: use Infisical directly
|
||||
switch sub {
|
||||
case "get":
|
||||
if len(os.Args) < 4 {
|
||||
fmt.Fprintln(os.Stderr, "usage: oikos secret get <key>")
|
||||
os.Exit(1)
|
||||
}
|
||||
key := os.Args[3]
|
||||
backend := newInfisicalBackendOrFail(cfg)
|
||||
val, err := backend.Get(ctx, key)
|
||||
if err != nil {
|
||||
slog.Error("secret get", "key", key, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(val)
|
||||
|
||||
case "set":
|
||||
if len(os.Args) < 5 {
|
||||
fmt.Fprintln(os.Stderr, "usage: oikos secret set <key> <value>")
|
||||
os.Exit(1)
|
||||
}
|
||||
key := os.Args[3]
|
||||
value := os.Args[4]
|
||||
backend := newInfisicalBackendOrFail(cfg)
|
||||
if err := backend.Set(ctx, key, value); err != nil {
|
||||
slog.Error("secret set", "key", key, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("stored: %s\n", key)
|
||||
|
||||
case "list":
|
||||
keys, err := sopsBackend.List(ctx)
|
||||
backend := newInfisicalBackendOrFail(cfg)
|
||||
keys, err := backend.List(ctx)
|
||||
if err != nil {
|
||||
slog.Error("secret list", "error", err)
|
||||
os.Exit(1)
|
||||
@@ -311,24 +368,138 @@ func runSecret(ctx context.Context, cfg config.Config) {
|
||||
fmt.Println(k)
|
||||
}
|
||||
|
||||
case "migrate":
|
||||
infCfg := secrets.InfisicalConfig{
|
||||
SiteURL: cfg.InfisicalSiteURL,
|
||||
ClientID: cfg.InfisicalClientID,
|
||||
ClientSecret: cfg.InfisicalClientSecret,
|
||||
ProjectID: cfg.InfisicalProjectID,
|
||||
SecretPath: "/",
|
||||
Env: cfg.InfisicalEnv,
|
||||
}
|
||||
if infCfg.Env == "" {
|
||||
infCfg.Env = "dev"
|
||||
}
|
||||
if infCfg.SiteURL == "" {
|
||||
fmt.Fprintln(os.Stderr, "error: OIKOS_INFISICAL_SITE_URL not set")
|
||||
os.Exit(1)
|
||||
}
|
||||
case "verify":
|
||||
runSecretVerify(ctx, cfg)
|
||||
|
||||
infBackend := secrets.NewInfisicalBackend(infCfg)
|
||||
case "audit":
|
||||
runSecretAudit(ctx, cfg)
|
||||
|
||||
case "migrate", "export-sops":
|
||||
runSecretLegacy(ctx, cfg, sub)
|
||||
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown secret command: %s\n", sub)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// expectedSecrets is the set of keys that should exist in Infisical
|
||||
// for a fully-migrated deployment.
|
||||
var expectedSecrets = []string{
|
||||
"matrix_token",
|
||||
"approval_hmac-secret",
|
||||
"mcp_bearer-token",
|
||||
"api_token",
|
||||
"openrouter_api-key",
|
||||
"webhook_hmac-secret",
|
||||
}
|
||||
|
||||
// runSecretVerify checks that all expected secrets are present in Infisical.
|
||||
func runSecretVerify(ctx context.Context, cfg config.Config) {
|
||||
backend := newInfisicalBackendOrFail(cfg)
|
||||
keys, err := backend.List(ctx)
|
||||
if err != nil {
|
||||
slog.Error("verify: list", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
keySet := make(map[string]struct{}, len(keys))
|
||||
for _, k := range keys {
|
||||
keySet[k] = struct{}{}
|
||||
}
|
||||
|
||||
missing := 0
|
||||
for _, exp := range expectedSecrets {
|
||||
if _, ok := keySet[exp]; !ok {
|
||||
fmt.Printf("MISSING: %s\n", exp)
|
||||
missing++
|
||||
} else {
|
||||
fmt.Printf("OK: %s\n", exp)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n%d/%d present, %d missing\n", len(expectedSecrets)-missing, len(expectedSecrets), missing)
|
||||
if missing > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// runSecretAudit resolves all expected secrets from Infisical and prints
|
||||
// a diff against current env-derived values. Values are truncated for safety.
|
||||
func runSecretAudit(ctx context.Context, cfg config.Config) {
|
||||
backend := newInfisicalBackendOrFail(cfg)
|
||||
|
||||
envValues := map[string]string{
|
||||
"matrix_token": cfg.MatrixToken,
|
||||
"approval_hmac-secret": cfg.ApprovalHMACSecret,
|
||||
"mcp_bearer-token": cfg.MCPBearerToken,
|
||||
"api_token": cfg.APIToken,
|
||||
"oidc_client-secret": cfg.OIDCClientSecret,
|
||||
}
|
||||
|
||||
fmt.Println("key infisical env status")
|
||||
fmt.Println(strings.Repeat("-", 72))
|
||||
|
||||
for _, key := range expectedSecrets {
|
||||
infVal, infErr := backend.Get(ctx, key)
|
||||
envVal := envValues[key]
|
||||
|
||||
if infErr != nil {
|
||||
fmt.Printf("%-29s ERROR %-10s NOT-IN-INFISICAL\n", key, trunc(envVal, 8))
|
||||
continue
|
||||
}
|
||||
if envVal == "" {
|
||||
fmt.Printf("%-29s %-10s (empty) INFISICAL-ONLY\n", key, trunc(infVal, 8))
|
||||
continue
|
||||
}
|
||||
if infVal == envVal {
|
||||
fmt.Printf("%-29s %-10s %-10s MATCH\n", key, trunc(infVal, 8), trunc(envVal, 8))
|
||||
} else {
|
||||
fmt.Printf("%-29s %-10s %-10s DRIFT\n", key, trunc(infVal, 8), trunc(envVal, 8))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func trunc(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
if n > 1 {
|
||||
return s[:n-1] + "…"
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
// newInfisicalBackendOrFail creates an Infisical backend from config or exits.
|
||||
func newInfisicalBackendOrFail(cfg config.Config) *secrets.InfisicalBackend {
|
||||
if cfg.InfisicalSiteURL == "" {
|
||||
fmt.Fprintln(os.Stderr, "error: OIKOS_INFISICAL_SITE_URL not set")
|
||||
os.Exit(1)
|
||||
}
|
||||
infCfg := secrets.InfisicalConfig{
|
||||
SiteURL: cfg.InfisicalSiteURL,
|
||||
ClientID: cfg.InfisicalClientID,
|
||||
ClientSecret: cfg.InfisicalClientSecret,
|
||||
ProjectID: cfg.InfisicalProjectID,
|
||||
SecretPath: "/",
|
||||
Env: cfg.InfisicalEnv,
|
||||
}
|
||||
if infCfg.Env == "" {
|
||||
infCfg.Env = "dev"
|
||||
}
|
||||
return secrets.NewInfisicalBackend(infCfg)
|
||||
}
|
||||
|
||||
// runSecretLegacy handles SOPS-only commands (migrate, export-sops).
|
||||
func runSecretLegacy(ctx context.Context, cfg config.Config, sub string) {
|
||||
secretsDir := cfg.SecretsDir
|
||||
if secretsDir == "" {
|
||||
secretsDir = "archive/secrets-sops-backup"
|
||||
}
|
||||
sopsBackend := secrets.NewSOPSBackend(secretsDir)
|
||||
|
||||
switch sub {
|
||||
case "migrate":
|
||||
infBackend := newInfisicalBackendOrFail(cfg)
|
||||
keys, err := sopsBackend.List(ctx)
|
||||
if err != nil {
|
||||
slog.Error("migrate: read sops", "error", err)
|
||||
@@ -366,10 +537,6 @@ func runSecret(ctx context.Context, cfg config.Config) {
|
||||
fmt.Printf("%s: <sops-encrypted>\n", k)
|
||||
}
|
||||
fmt.Printf("\n# To restore: sops -d secrets/*.yaml\n")
|
||||
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown secret command: %s\n", sub)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
@@ -12,26 +13,31 @@ import (
|
||||
"os/exec"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/secrets"
|
||||
"github.com/dtoro/oikos/internal/safego"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
port := os.Getenv("WEBHOOK_LISTEN")
|
||||
if port == "" {
|
||||
port = ":9797"
|
||||
}
|
||||
|
||||
secret := os.Getenv("WEBHOOK_HMAC_SECRET")
|
||||
if secret == "" {
|
||||
fmt.Fprintln(os.Stderr, "WEBHOOK_HMAC_SECRET must be set")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
repoDir := os.Getenv("WEBHOOK_REPO_DIR")
|
||||
if repoDir == "" {
|
||||
repoDir = os.Getenv("HOME") + "/Projects/oikos"
|
||||
}
|
||||
|
||||
// Create secrets manager once, share between HMAC resolution and deploy
|
||||
sec := newSecrets()
|
||||
secret := resolveWebhookHMAC(ctx, sec)
|
||||
if secret == "" {
|
||||
fmt.Fprintln(os.Stderr, "WEBHOOK_HMAC_SECRET must be set (env var or Infisical webhook_hmac-secret)")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/deploy", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
@@ -66,11 +72,16 @@ func main() {
|
||||
w.Write([]byte(`{"status":"deploy started"}`))
|
||||
|
||||
safego.Go("webhook:deploy", func() {
|
||||
apiToken := ""
|
||||
if sec != nil {
|
||||
apiToken = secrets.ResolveSecret(ctx, sec, "api_token", "")
|
||||
}
|
||||
cmd := exec.Command(repoDir + "/scripts/deploy.sh")
|
||||
cmd.Dir = repoDir
|
||||
cmd.Env = append(os.Environ(),
|
||||
"REPO_DIR="+repoDir,
|
||||
"PROFILE=full",
|
||||
"OIKOS_API_TOKEN="+apiToken,
|
||||
)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
@@ -94,3 +105,25 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// newSecrets creates the Infisical secrets manager from env vars.
|
||||
func newSecrets() *secrets.Manager {
|
||||
return secrets.NewManagerFromConfig(
|
||||
os.Getenv("OIKOS_INFISICAL_SITE_URL"),
|
||||
os.Getenv("OIKOS_INFISICAL_CLIENT_ID"),
|
||||
os.Getenv("OIKOS_INFISICAL_CLIENT_SECRET"),
|
||||
os.Getenv("OIKOS_INFISICAL_PROJECT_ID"),
|
||||
os.Getenv("OIKOS_INFISICAL_ENV"),
|
||||
os.Getenv("OIKOS_SECRETS_DIR"),
|
||||
)
|
||||
}
|
||||
|
||||
// resolveWebhookHMAC fetches the webhook HMAC secret from Infisical,
|
||||
// falling back to the WEBHOOK_HMAC_SECRET env var.
|
||||
func resolveWebhookHMAC(ctx context.Context, sec *secrets.Manager) string {
|
||||
envFallback := os.Getenv("WEBHOOK_HMAC_SECRET")
|
||||
if sec == nil {
|
||||
return envFallback
|
||||
}
|
||||
return secrets.ResolveSecret(ctx, sec, "webhook_hmac-secret", envFallback)
|
||||
}
|
||||
@@ -1,5 +1,27 @@
|
||||
:80 {
|
||||
root * /srv
|
||||
file_server
|
||||
try_files {path} /index.html
|
||||
|
||||
# /wails/runtime.js is injected by the Wails desktop wrapper, which serves
|
||||
# the same dist/ from its own asset handler. In a browser it does not
|
||||
# exist, and the SPA fallback below answered it with index.html — so the
|
||||
# browser parsed "<!doctype html>" as JavaScript and threw
|
||||
# "SyntaxError: expected expression, got '<'" on every page load.
|
||||
# Return a real 404 instead: the tag fails quietly, and the desktop app is
|
||||
# unaffected because it never reaches this server.
|
||||
handle /wails/* {
|
||||
error 404
|
||||
}
|
||||
|
||||
# Same reasoning for any other asset: a missing .js/.css/.map answered with
|
||||
# HTML is always a confusing parse error rather than an honest 404. Only
|
||||
# real routes should fall through to the SPA.
|
||||
@asset path_regexp \.(js|mjs|css|map|json|png|jpg|svg|ico|woff2?)$
|
||||
handle @asset {
|
||||
file_server
|
||||
}
|
||||
|
||||
handle {
|
||||
file_server
|
||||
try_files {path} /index.html
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,8 @@ FROM node:22-alpine AS builder
|
||||
|
||||
WORKDIR /build/web
|
||||
COPY web/package.json web/package-lock.json ./
|
||||
RUN npm ci
|
||||
COPY web/vendor /build/vendor
|
||||
RUN npm install --no-audit --no-fund
|
||||
COPY VERSION ./
|
||||
COPY web/ ./
|
||||
RUN npm run build
|
||||
|
||||
@@ -20,6 +20,8 @@ services:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- pg-data:/var/lib/postgresql/data
|
||||
mem_limit: 1g
|
||||
cpus: 2.0
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "oikos"]
|
||||
interval: 5s
|
||||
@@ -28,6 +30,7 @@ services:
|
||||
|
||||
# One-shot: run migrations then exit
|
||||
migrate:
|
||||
image: oikos-migrate:${OIKOS_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: compose/oikos/Dockerfile
|
||||
@@ -38,9 +41,12 @@ services:
|
||||
OIKOS_DATABASE_URL: postgres://oikos:${OIKOS_DB_PASSWORD:-oikos_dev}@postgres:5432/oikos?sslmode=disable
|
||||
command: ["migrate"]
|
||||
restart: "no"
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
|
||||
# One-shot: ingest seeds then exit
|
||||
seed:
|
||||
image: oikos-seed:${OIKOS_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: compose/oikos/Dockerfile
|
||||
@@ -52,9 +58,12 @@ services:
|
||||
OIKOS_SEEDS_DIR: /seeds
|
||||
command: ["seed"]
|
||||
restart: "no"
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
|
||||
# API server (Phase 2)
|
||||
api:
|
||||
image: oikos-api:${OIKOS_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: compose/oikos/Dockerfile
|
||||
@@ -76,6 +85,16 @@ services:
|
||||
OIKOS_OIDC_CLIENT_ID: ${OIKOS_OIDC_CLIENT_ID:-otkHBSueHJsYtOHstL6rn5izeGgyOsavp1qA1hod}
|
||||
OIKOS_NOMOS_AGENT_SLUG: ${OIKOS_NOMOS_AGENT_SLUG:-agent:nomos}
|
||||
NOMOS_PROXY_URL: http://nomos:8092
|
||||
# Rate limiting (plan D3). Default off; set OIKOS_API_RATE_LIMIT to a
|
||||
# requests/sec value to throttle runaway agent loops per source IP.
|
||||
OIKOS_API_RATE_LIMIT: ${OIKOS_API_RATE_LIMIT:-}
|
||||
OIKOS_API_RATE_BURST: ${OIKOS_API_RATE_BURST:-}
|
||||
# Infisical secret store (Phase 5)
|
||||
OIKOS_INFISICAL_SITE_URL: ${OIKOS_INFISICAL_SITE_URL:-}
|
||||
OIKOS_INFISICAL_CLIENT_ID: ${OIKOS_INFISICAL_CLIENT_ID:-}
|
||||
OIKOS_INFISICAL_CLIENT_SECRET: ${OIKOS_INFISICAL_CLIENT_SECRET:-}
|
||||
OIKOS_INFISICAL_PROJECT_ID: ${OIKOS_INFISICAL_PROJECT_ID:-}
|
||||
OIKOS_INFISICAL_ENV: ${OIKOS_INFISICAL_ENV:-dev}
|
||||
volumes:
|
||||
- ${OIKOS_SSH_KEY_PATH:-~/.ssh/id_ed25519}:/etc/oikos/ssh_key:ro
|
||||
ports:
|
||||
@@ -83,9 +102,28 @@ services:
|
||||
command: ["api"]
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 30s
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
# Exists so nomos can wait for the API to actually answer rather than just
|
||||
# for its container to exist — see nomos's depends_on below. wget is
|
||||
# BusyBox's, already in the alpine runtime image, so this adds no
|
||||
# dependency. /healthz pings the DB, so "healthy" means genuinely ready.
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8090/healthz"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
# Migrations and seed run before this container, but the first bind can
|
||||
# still take a moment; failures inside the start period don't count.
|
||||
# The api's NewHandler stalls on TWO unreachable external deps at startup
|
||||
# before binding :8090: Infisical (4x auth retries, ~40s) and OIDC
|
||||
# discovery (auth.hubris.network, ~35s of timeouts). Total ~90-95s, so
|
||||
# the start period must clear it or nomos (depends_on: api-healthy) fails.
|
||||
start_period: 180s
|
||||
|
||||
# Scheduler (Phase 3) — observe loop
|
||||
scheduler:
|
||||
image: oikos-scheduler:${OIKOS_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: compose/oikos/Dockerfile
|
||||
@@ -100,6 +138,9 @@ services:
|
||||
OIKOS_SCHEDULER_INTERVAL: "30s"
|
||||
OIKOS_SSH_KEY_PATH: /etc/oikos/ssh_key
|
||||
OIKOS_SSH_USER: root
|
||||
# Liveness probe (plan D5): exposes a staleness-aware /healthz inside
|
||||
# the container; the scheduler bumps it each check pass.
|
||||
OIKOS_HEALTH_LISTEN: ":8093"
|
||||
volumes:
|
||||
- ${OIKOS_SSH_KEY_PATH:-~/.ssh/id_ed25519}:/etc/oikos/ssh_key:ro
|
||||
cap_add:
|
||||
@@ -107,9 +148,18 @@ services:
|
||||
command: ["scheduler"]
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 30s
|
||||
mem_limit: 256m
|
||||
cpus: 1.0
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8093/healthz"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 90s
|
||||
|
||||
# Notifier (Phase 3) — Matrix alerts
|
||||
notifier:
|
||||
image: oikos-notifier:${OIKOS_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: compose/oikos/Dockerfile
|
||||
@@ -126,12 +176,59 @@ services:
|
||||
OIKOS_MATRIX_USER: ${OIKOS_MATRIX_USER:-@hermes:hubris.network}
|
||||
OIKOS_MATRIX_TOKEN: ${OIKOS_MATRIX_TOKEN}
|
||||
OIKOS_MATRIX_ROOM: ${OIKOS_MATRIX_ROOM:-!alerts:hubris.network}
|
||||
# Liveness probe (plan D5): bumps each approval/reaction tick.
|
||||
OIKOS_HEALTH_LISTEN: ":8094"
|
||||
OIKOS_INFISICAL_SITE_URL: ${OIKOS_INFISICAL_SITE_URL:-}
|
||||
OIKOS_INFISICAL_CLIENT_ID: ${OIKOS_INFISICAL_CLIENT_ID:-}
|
||||
OIKOS_INFISICAL_CLIENT_SECRET: ${OIKOS_INFISICAL_CLIENT_SECRET:-}
|
||||
OIKOS_INFISICAL_PROJECT_ID: ${OIKOS_INFISICAL_PROJECT_ID:-}
|
||||
OIKOS_INFISICAL_ENV: ${OIKOS_INFISICAL_ENV:-dev}
|
||||
command: ["notifier"]
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 30s
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8094/healthz"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 120s
|
||||
|
||||
# Execution worker (Phase 6) — Postgres-backed job queue
|
||||
execution-worker:
|
||||
image: oikos-execution-worker:${OIKOS_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: compose/oikos/Dockerfile
|
||||
restart: unless-stopped
|
||||
profiles: ["dev", "full"]
|
||||
depends_on:
|
||||
seed:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
OIKOS_DATABASE_URL: postgres://oikos:${OIKOS_DB_PASSWORD:-oikos_dev}@postgres:5432/oikos?sslmode=disable
|
||||
OIKOS_DEBUG: "true"
|
||||
OIKOS_SSH_KEY_PATH: /etc/oikos/ssh_key
|
||||
OIKOS_SSH_USER: root
|
||||
OIKOS_HEALTH_LISTEN: ":8095"
|
||||
volumes:
|
||||
- ${OIKOS_SSH_KEY_PATH:-~/.ssh/id_ed25519}:/etc/oikos/ssh_key:ro
|
||||
command: ["execution-worker"]
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 30s
|
||||
mem_limit: 256m
|
||||
cpus: 1.0
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8095/healthz"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 90s
|
||||
|
||||
# Nomos agent gateway (Phase 4) — mesh-published :8092
|
||||
nomos:
|
||||
image: oikos-nomos:${OIKOS_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: compose/nomos/Dockerfile
|
||||
@@ -139,7 +236,12 @@ services:
|
||||
profiles: ["full"]
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_started
|
||||
# service_started only waits for the container to exist, so nomos came
|
||||
# up while the API was still binding :8090, failed its MCP initialize,
|
||||
# exited 1, and crash-looped for ~25s on every single deploy. It always
|
||||
# recovered, which is exactly why it went unnoticed. service_healthy
|
||||
# waits for the API to actually answer.
|
||||
condition: service_healthy
|
||||
environment:
|
||||
NOMOS_MCP_URL: http://api:8090/mcp
|
||||
NOMOS_AGENT_SLUG: agent:nomos
|
||||
@@ -149,16 +251,34 @@ services:
|
||||
# Must match api's OIKOS_MCP_BEARER_TOKEN above — api's combinedAuth
|
||||
# rejects every request without it now (no dev-open bypass).
|
||||
OIKOS_MCP_BEARER_TOKEN: ${OIKOS_MCP_BEARER_TOKEN:-dev-token}
|
||||
# Infisical secret store (Phase 5) — nomos resolves mcp_bearer-token
|
||||
# and openrouter_api-key from here, overriding the env values above.
|
||||
OIKOS_INFISICAL_SITE_URL: ${OIKOS_INFISICAL_SITE_URL:-}
|
||||
OIKOS_INFISICAL_CLIENT_ID: ${OIKOS_INFISICAL_CLIENT_ID:-}
|
||||
OIKOS_INFISICAL_CLIENT_SECRET: ${OIKOS_INFISICAL_CLIENT_SECRET:-}
|
||||
OIKOS_INFISICAL_PROJECT_ID: ${OIKOS_INFISICAL_PROJECT_ID:-}
|
||||
OIKOS_INFISICAL_ENV: ${OIKOS_INFISICAL_ENV:-dev}
|
||||
ports:
|
||||
- "8092:8092"
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 10s
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
# nomos runs on a distroless image (no shell/wget), so the healthcheck
|
||||
# uses the binary's own `healthcheck` subcommand to self-probe /healthz.
|
||||
healthcheck:
|
||||
test: ["CMD", "/nomos", "healthcheck"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
|
||||
# Control-room SPA — static build served behind Caddy. The outer
|
||||
# production Caddy (caddy-conf repo, LXC 121) splits /api/*, /mcp,
|
||||
# /agent/* off to api:8090 and sends everything else here; this
|
||||
# container only serves static files with SPA-fallback routing.
|
||||
web:
|
||||
image: oikos-web:${OIKOS_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: compose/web/Dockerfile
|
||||
@@ -167,6 +287,8 @@ services:
|
||||
ports:
|
||||
- "8091:80"
|
||||
stop_signal: SIGTERM
|
||||
mem_limit: 64m
|
||||
cpus: 0.25
|
||||
|
||||
# Redis (required by Infisical — Phase 5)
|
||||
redis:
|
||||
@@ -175,6 +297,8 @@ services:
|
||||
profiles: ["infisical", "full"]
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
@@ -183,7 +307,7 @@ services:
|
||||
|
||||
# Infisical self-hosted (Phase 5 secrets management)
|
||||
infisical:
|
||||
image: infisical/infisical:latest
|
||||
image: infisical/infisical:v0.162.19
|
||||
restart: unless-stopped
|
||||
profiles: ["infisical", "full"]
|
||||
depends_on:
|
||||
@@ -205,6 +329,8 @@ services:
|
||||
REDIS_URL: redis://redis:6379
|
||||
ports:
|
||||
- "8080:8080"
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
|
||||
volumes:
|
||||
pg-data:
|
||||
|
||||
@@ -234,9 +234,33 @@ sequenceDiagram
|
||||
|
||||
---
|
||||
|
||||
**2026-07-08 — renamed to Nomos.** The Hermes agent gateway was renamed to
|
||||
**2026-07-08 — renamed to Nomos.**
|
||||
Nomos (from *oikonomos*, the steward of the oikos) under the
|
||||
[Nomos resident agent plan](../../plans/2026-07-08-nomos-resident-agent.md),
|
||||
|
||||
### Hermes MCP client setup
|
||||
|
||||
To connect a Hermes Agent instance to oikos as a native MCP client, add to
|
||||
`~/.hermes/config.yaml`:
|
||||
|
||||
```yaml
|
||||
mcp_servers:
|
||||
oikos:
|
||||
url: "https://mcp.hubris.network/mcp"
|
||||
headers:
|
||||
Authorization: "Bearer <OIKOS_MCP_BEARER_TOKEN>"
|
||||
timeout: 180
|
||||
```
|
||||
|
||||
Run `/reload-mcp` in-session or restart Hermes. Tools appear as
|
||||
`mcp__oikos__*`.
|
||||
|
||||
**Caveat:** Hermes stores the bearer token in plaintext in `config.yaml` —
|
||||
it does not support `${VAR}` interpolation in MCP server headers. Ensure
|
||||
`security.redact_secrets: true` (default) so the token value is stripped
|
||||
from tool output and logs. File an upstream feature request at
|
||||
https://github.com/NousResearch/hermes-agent/issues for env-var
|
||||
interpolation support.
|
||||
N0 milestone. The gateway binary (`cmd/nomos`), Docker service, DB slug
|
||||
(`agent:nomos`), and all referencing docs were updated. All architectural
|
||||
principles in this ADR remain unchanged.
|
||||
@@ -31,6 +31,7 @@ the relevant section here.
|
||||
| [6. PostgreSQL/TimescaleDB](#6-postgresqltimescaledb) | `migrations/`, `seeds/` | ✅ live — the System's own source of truth |
|
||||
| [7. Dormant components](#7-dormant-components) | `internal/actuator`, `internal/learning` | 🔴 compiled, never started |
|
||||
| [8. Auxiliary components](#8-auxiliary-components) | `cmd/webhook`, `cmd/desktop` | ✅ live — deploy + packaging, not decision logic |
|
||||
| [9. web control room — App architecture](#9-web-control-room--app-architecture) | `web/src/lib/apps.ts`, `web/src/lib/stores/windows.ts`, `web/src/lib/stores/docked.ts`, `web/src/lib/components/desktop-shell/` | ✅ live — the OS + Apps shell contract |
|
||||
|
||||
---
|
||||
|
||||
@@ -364,7 +365,11 @@ cross-origin (the Wails desktop webview, §8).
|
||||
Standalone deploy, versioned and released independently of the `oikos`
|
||||
binary — see [README.md §4.5](README.md#45-build--release-artifacts) for
|
||||
why "deployed" means two different release cadences depending on whether
|
||||
you mean the container or the desktop app.
|
||||
you mean the container or the desktop app. The shell-level architecture
|
||||
(window manager, app registry, docked layer) is documented separately as
|
||||
[§9 below](#9-web-control-room--app-architecture); this section covers
|
||||
the page-level concerns, §9 covers the OS + Apps contract the pages hang
|
||||
off.
|
||||
|
||||
---
|
||||
|
||||
@@ -496,6 +501,177 @@ functional sense.
|
||||
|
||||
---
|
||||
|
||||
## 9. web control room — App architecture
|
||||
|
||||
**Stakeholders:** anyone adding a page, adding a desktop overlay, or
|
||||
planning dynamic/third-party app installation. **Why this View earns its
|
||||
place:** §5 documents the *pages*; this View documents the *shell* they
|
||||
hang off — and the shell is the part whose contract a new app has to
|
||||
satisfy. It is also the layer where the "Oikos-as-OS" metaphor
|
||||
(desktop, icons, floating windows, a tamagotchi-style resident
|
||||
creature) is actually implemented, so the boundary between "Base OS" and
|
||||
"App" has to be explicit here or it doesn't exist anywhere.
|
||||
|
||||
### App architecture — Internal structure
|
||||
|
||||
| File | Role |
|
||||
|---|---|
|
||||
| `web/src/lib/apps.ts` | The App registry. Two layers: `builtinApps` (static, always installed) + `installedAppIds` (persisted, from the App Store). The public `apps` store is derived (built-in + installed); `appById` is a derived Map. `installApp`/`uninstallApp` mutate the installed set. Window-id helpers (`appWindowId`, `appIdFromWindowId`) unchanged. |
|
||||
| `web/src/app-store/catalog.ts` | The installable-app catalog: `AppManifest` (persistable metadata) + `CatalogEntry` (manifest + Lucide icon + dynamic-import loader). Static in Phase 3 (apps ship with the build); Phase 4 swaps this for a fetched `/api/v1/apps` endpoint. Declares `AppPermission` (enforcement is Phase 4). |
|
||||
| `web/src/app-store/apps/Notes.svelte` | Demo installable app — a localStorage-backed scratchpad proving the install→icon→window→uninstall lifecycle end-to-end. |
|
||||
| `web/src/lib/stores/windows.ts` | The wmkit window manager singleton + the `openAppWindow` / `openEntityWindow` / `openTaskWindow` primitives. `openAppWindow` branches on `docked` (toggles visibility) vs windowed (`wm.open`); resolves the app via `get(appById)`. |
|
||||
| `web/src/lib/stores/docked.ts` | Persisted visibility for docked apps. Absent key = visible (default-on); store holds only overrides. Deliberately does **not** import `APPS` — doing so would create a static cycle (`apps.ts` → pages → `windows.ts` → here → `apps.ts`) and fire a TDZ on `APPS` at init. |
|
||||
| `web/src/lib/stores/icons.ts` | Desktop icon grid: column/row positions, drag-to-reorder, localStorage persistence. Reactive to the `apps` store — a newly-installed app gets a free cell on the next emission; `resetIconLayout` re-seeds from the live registry, not a static snapshot. |
|
||||
| `web/src/lib/components/LazyApp.svelte` | Renders an app's lazily-loaded component (`AppDef.component` is a dynamic-import loader, not the component). Shows the shared spinner while the chunk fetches; used by both WindowLayer and DockedLayer so the loading state is uniform across app kinds. Vite's module cache makes repeat opens resolve from cache. |
|
||||
| `web/src/lib/components/desktop-shell/Desktop.svelte` | Full-viewport surface: background, icons, task launcher, `<WindowLayer />`, `<DockedLayer />`, taskbar. Reads `$apps` (the derived store) so installs reflect immediately. |
|
||||
| `web/src/lib/components/desktop-shell/WindowLayer.svelte` | Floating-window stack (z-40). Resolves window id → content component; renders shared titlebar chrome. The orphan-close `$effect` is reactive on `$appById` — reinstalling an app revives its persisted window, uninstalling closes it. |
|
||||
| `web/src/lib/components/desktop-shell/DockedLayer.svelte` | Docked-app overlay (z-45). Renders `$apps.filter(a => a.docked)` gated on `dockedVisibility`. Replaces the previously-hardcoded `<MascotLayer />`. |
|
||||
| `web/src/lib/components/desktop-shell/Taskbar.svelte` | Window buttons + tray. Renders from `wmState.order`; resolves icons via `$appById`. |
|
||||
| `web/src/pages/AppStore.svelte` | The App Store — lists the catalog, shows install state, install/uninstall. Installing makes the app appear on the desktop immediately (no reload) via the reactive `apps` store; uninstalling closes any open window for that app via WindowLayer's orphan-close effect. |
|
||||
|
||||
### App architecture — The App contract
|
||||
|
||||
```typescript
|
||||
interface AppDef {
|
||||
id: string // unique; window IDs are "app:<id>"
|
||||
title: string // desktop icon label + window titlebar
|
||||
icon: Component // Lucide icon (desktop icon + taskbar)
|
||||
component: () => Promise<{ default: Component }> // dynamic-import loader
|
||||
docked?: boolean // true = Docked Layer app, no window
|
||||
noIcon?: boolean // true = registered but no desktop icon
|
||||
width?: number; height?: number; minWidth?: number; minHeight?: number
|
||||
// required for windowed, forbidden for docked
|
||||
badge?: (s: DashboardSummary | null) => number
|
||||
}
|
||||
```
|
||||
|
||||
`component` is a dynamic-import loader (`() => import('../pages/X.svelte')`),
|
||||
not the component itself. Desktop icons render from metadata alone (id,
|
||||
title, icon — all static), the component chunk fetches on first window
|
||||
open, and Vite code-splits each app into its own chunk (Phase 2). The
|
||||
mascot uses the same path — `() => import('./mascot/MascotLayer.svelte')`
|
||||
— which also defers the mascot's module graph until after `apps.ts` has
|
||||
finished initializing, breaking what would otherwise be a static cycle
|
||||
(`apps.ts` → `MascotLayer` → `Mascot.svelte` → `icons.ts` → `apps.ts`).
|
||||
|
||||
Two app kinds, picked by one flag:
|
||||
|
||||
| Kind | Window | Titlebar | Taskbar | Opened by |
|
||||
|---|---|---|---|---|
|
||||
| **Windowed** (default) | wmkit floating window | yes | yes | `openAppWindow` → `wm.open` |
|
||||
| **Docked** (`docked: true`) | none — renders on the Docked Layer | no | no | `openAppWindow` → `toggleDocked` |
|
||||
|
||||
Apps receive **no props** from the shell. They import the OS-service
|
||||
surface (below) directly. The shell→app edge is one-way.
|
||||
|
||||
### App architecture — The OS-service surface (AppOS)
|
||||
|
||||
The stable set of `$lib` exports an App may import. Everything else in
|
||||
`$lib` is shell-internal and may change without notice. This is a
|
||||
**documentation contract** today (apps are compiled in); it becomes an
|
||||
**enforced sandbox boundary** the moment third-party app installation
|
||||
(Phase 3 in [the plan](../../plans/2026-07-21-frontend-os-apps-architecture.md)) lands.
|
||||
|
||||
| Service | Import |
|
||||
|---|---|
|
||||
| Open an app window | `openAppWindow(id)` from `$lib/stores/windows` |
|
||||
| Open an entity window | `openEntityWindow(slug)` from `$lib/stores/windows` |
|
||||
| Open a task window | `openTaskWindow(sessionId, title)` from `$lib/stores/windows` |
|
||||
| Dashboard summary | `summary`, `subscribeContext` from `$lib/stores/context` |
|
||||
| Live events | `subscribeEvents` from `$lib/stores/events` |
|
||||
| Per-session chat / workspace / activity | `chatFor`, `workspaceFor`, `activityLogFor` from `$lib/stores/{chat,workspace,activity}` |
|
||||
| REST API | `$lib/api` (generated from OpenAPI, [ADR-0004](../adr/0004-openapi-first.md)) |
|
||||
| UI primitives | `$lib/components/ui/*` |
|
||||
| Theme | `getTheme`, `setTheme` from `$lib/stores/theme.svelte` |
|
||||
|
||||
### App architecture — Content resolution
|
||||
|
||||
Window ids are namespaced so the window layer resolves content purely
|
||||
from the id, with no extra bookkeeping — which is also why persisted
|
||||
windows hydrate correctly across reloads:
|
||||
|
||||
| Id shape | Renders |
|
||||
|---|---|
|
||||
| `app:<id>` | the registry app's component (`appById.get(id).component`) |
|
||||
| `session:<id>` | `SessionChatWindow` (per-session chat) |
|
||||
| `new-task` | `NewTaskChat` (singleton compose) |
|
||||
| bare slug (`type:identifier`) | `EntityDetailContent` (fallback) |
|
||||
|
||||
A hydrated `app:<id>` window whose id no longer matches a registry entry
|
||||
(an app removed since the layout was persisted) self-closes — the
|
||||
orphan-close `$effect` in `WindowLayer.svelte` sweeps it on mount.
|
||||
|
||||
### App architecture — Current population
|
||||
|
||||
Seven windowed apps + one docked app:
|
||||
|
||||
| App | Kind | Badge |
|
||||
|---|---|---|
|
||||
| `tasks` | windowed | — |
|
||||
| `kb` | windowed | — |
|
||||
| `ops` | windowed | `approvals_pending` |
|
||||
| `signals` | windowed | open signal count |
|
||||
| `knowledge` | windowed | — |
|
||||
| `learning` | windowed | — |
|
||||
| `settings` | windowed | — |
|
||||
| `mascot` (Cluck) | **docked** | — |
|
||||
|
||||
The mascot is the first docked app and the reason the docked kind
|
||||
exists; before this View it was a hardcoded `<MascotLayer />` in
|
||||
`Desktop.svelte`, not a registry entry. Its persistent model
|
||||
(`web/src/lib/mascot/state.svelte.ts`, localStorage) and sprite cache
|
||||
(`sprites.ts`) are module-scoped, so toggling visibility (unmount) and
|
||||
restoring (remount) loses no state — this is why `docked` visibility is
|
||||
a plain `{#if}` gate rather than a `keepAlive` mechanism.
|
||||
|
||||
### App architecture — Designed extension points (documented, not built)
|
||||
|
||||
| Extension | Mechanism when built | Trigger |
|
||||
|---|---|---|
|
||||
| Titlebar actions | `titlebarActions?: Component` on `AppDef`, rendered left of min/max/close | First app that needs one |
|
||||
| App-scoped state | `state?: () => Record<string, unknown>` on `AppDef` | First app with cross-mount state that isn't module-scoped |
|
||||
| `onRegister` handshake | called with a scoped AppOS capability object | Phase 3 (dynamic install) |
|
||||
| Third-party manifests | `AppManifest` JSON + `/api/v1/apps` + permission model | Phase 3 |
|
||||
|
||||
Documenting these now prevents the current contract from painting itself
|
||||
into a corner; building them now would be speculative. (Lazy-loaded
|
||||
components were on this list and shipped in Phase 2 — `component` is now
|
||||
`() => Promise<{ default: Component }>` and Vite code-splits each app.)
|
||||
|
||||
### App architecture — Status and known issues
|
||||
|
||||
Phase 1 (the docked kind, mascot-as-app, the docked visibility store) and
|
||||
Phase 2 (lazy component loading — `component` as dynamic-import loader,
|
||||
`LazyApp.svelte` for uniform loading state, per-app code-splitting) have
|
||||
landed. Open items, by phase:
|
||||
|
||||
- **Phase 3 (dynamic install):** the AppOS table above becomes a real
|
||||
injected capability object, not a documentation table; permissions
|
||||
enforced at the store-access boundary; `AppManifest` format +
|
||||
`/api/v1/apps` endpoint + install flow.
|
||||
- **Late-registering apps (Phase 3 prerequisite):** `icons.ts:48` builds
|
||||
`appIds` once at module load to validate persisted positions — fine
|
||||
today (all apps are in the static `APPS` array; only their components
|
||||
are lazy), fragile the moment apps register post-load. When dynamic
|
||||
registration lands, revalidate against the live registry, not the
|
||||
import-time snapshot. Likewise `WindowLayer`'s orphan-close `$effect`
|
||||
must be gated on registry-ready so a not-yet-loaded app's persisted
|
||||
window isn't killed on hydration.
|
||||
|
||||
The static-cycle trap that bit this View during Phase 1 implementation is
|
||||
now resolved by Phase 2's lazy loading — recording it for context:
|
||||
|
||||
- `apps.ts` no longer statically imports any page or the mascot (they're
|
||||
all `() => import(...)`), so there's no static edge from `apps.ts` into
|
||||
the mascot/page module graph to cycle through `icons.ts` back to `APPS`.
|
||||
The earlier `LazyMascot.svelte` wrapper (Phase 1's cycle break) was
|
||||
deleted in Phase 2 — the lazy loader in the registry replaces it.
|
||||
`docked.ts` still must not import `APPS` (it's reached from `apps.ts`'s
|
||||
graph via `windows.ts`), and doesn't — defaults are implicit
|
||||
(absent key = visible).
|
||||
|
||||
---
|
||||
|
||||
## Keeping this document current
|
||||
|
||||
The same discipline as README.md's closing note applies here, scoped to
|
||||
|
||||
2
go.mod
2
go.mod
@@ -19,6 +19,7 @@ require (
|
||||
golang.org/x/crypto v0.53.0
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/sys v0.46.0
|
||||
golang.org/x/time v0.14.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
@@ -88,7 +89,6 @@ require (
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/oauth2 v0.35.0 // indirect
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/api v0.267.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// Run starts the actuator loop. Blocks until ctx is cancelled.
|
||||
@@ -403,7 +402,7 @@ func ProvisionVM(ctx context.Context, pool *db.Pool, entityID uuid.UUID, attrs m
|
||||
return nil
|
||||
}
|
||||
|
||||
// sshExecSimple runs a command over SSH with a simple client setup.
|
||||
// sshExecSimple runs a command over SSH using the shared dial/run primitives.
|
||||
// Uses the default SSH key from SSH_KEY_PATH or ~/.ssh/id_rsa.
|
||||
func sshExecSimple(ctx context.Context, host, user, command string) (string, error) {
|
||||
keyPath := os.Getenv("SSH_KEY_PATH")
|
||||
@@ -411,55 +410,19 @@ func sshExecSimple(ctx context.Context, host, user, command string) (string, err
|
||||
keyPath = os.Getenv("HOME") + "/.ssh/id_rsa"
|
||||
}
|
||||
|
||||
keyBytes, err := os.ReadFile(keyPath)
|
||||
signer, err := LoadSigner(keyPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read ssh key: %w", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
signer, err := ssh.ParsePrivateKey(keyBytes)
|
||||
client, err := Dial(ctx, DialOptions{Host: host, User: user, Signer: signer})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse ssh key: %w", err)
|
||||
}
|
||||
|
||||
clientCfg := &ssh.ClientConfig{
|
||||
User: user,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", host+":22", clientCfg)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ssh dial %s: %w", host, err)
|
||||
return "", err
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
type result struct {
|
||||
output string
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, e := session.CombinedOutput(command)
|
||||
ch <- result{output: string(out), err: e}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
session.Close()
|
||||
return "", ctx.Err()
|
||||
case res := <-ch:
|
||||
if res.err != nil {
|
||||
return res.output, res.err
|
||||
}
|
||||
return res.output, nil
|
||||
}
|
||||
out, err := RunCombinedOutput(ctx, client, command)
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// resolveHost resolves a host entity slug to (address, user) for SSH.
|
||||
|
||||
141
internal/actuator/client.go
Normal file
141
internal/actuator/client.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// defaultDialTimeout bounds an SSH dial when the caller leaves Timeout unset.
|
||||
// 10s matches the previous hardcoded value at every dial site.
|
||||
const defaultDialTimeout = 10 * time.Second
|
||||
|
||||
// LoadSigner reads and parses the private key at keyPath.
|
||||
func LoadSigner(keyPath string) (ssh.Signer, error) {
|
||||
key, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read ssh key: %w", err)
|
||||
}
|
||||
return LoadSignerFromBytes(key)
|
||||
}
|
||||
|
||||
// LoadSignerFromBytes parses an in-memory private key into an ssh.Signer.
|
||||
func LoadSignerFromBytes(key []byte) (ssh.Signer, error) {
|
||||
signer, err := ssh.ParsePrivateKey(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse ssh key: %w", err)
|
||||
}
|
||||
return signer, nil
|
||||
}
|
||||
|
||||
// DialOptions configures an SSH dial.
|
||||
type DialOptions struct {
|
||||
Host string
|
||||
Port int // 0 means 22
|
||||
User string
|
||||
Signer ssh.Signer
|
||||
Timeout time.Duration // dial timeout; <=0 means defaultDialTimeout
|
||||
}
|
||||
|
||||
// Dial opens a crypto/ssh connection through the centralized HostKeyCallback.
|
||||
// The connection itself is bounded by Timeout; ctx is respected by callers
|
||||
// via RunCombinedOutput once the session is running.
|
||||
func Dial(ctx context.Context, opts DialOptions) (*ssh.Client, error) {
|
||||
port := opts.Port
|
||||
if port <= 0 {
|
||||
port = 22
|
||||
}
|
||||
timeout := opts.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultDialTimeout
|
||||
}
|
||||
cfg := &ssh.ClientConfig{
|
||||
User: opts.User,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(opts.Signer)},
|
||||
HostKeyCallback: HostKeyCallback(),
|
||||
Timeout: timeout,
|
||||
}
|
||||
addr := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", port))
|
||||
client, err := ssh.Dial("tcp", addr, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ssh dial %s:%d: %w", opts.Host, port, err)
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// RunCombinedOutput runs cmd on an established client and returns its combined
|
||||
// stdout/stderr. Context cancellation closes the session to abort the remote
|
||||
// command instead of blocking until it finishes — the same goroutine+select
|
||||
// pattern the actuator, mcp, and scheduler each reimplemented before.
|
||||
func RunCombinedOutput(ctx context.Context, client *ssh.Client, cmd string) ([]byte, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
type result struct {
|
||||
out []byte
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, err := session.CombinedOutput(cmd)
|
||||
ch <- result{out: out, err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
session.Close()
|
||||
return nil, ctx.Err()
|
||||
case res := <-ch:
|
||||
if res.err != nil {
|
||||
return res.out, fmt.Errorf("command: %w", res.err)
|
||||
}
|
||||
return res.out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// RunOutput runs cmd on an established client and returns stdout only.
|
||||
// Stderr is folded into the returned error so callers that parse stdout
|
||||
// as JSON (e.g. the scheduler's check scripts) don't get interleaved
|
||||
// stderr in the output stream.
|
||||
func RunOutput(ctx context.Context, client *ssh.Client, cmd string) ([]byte, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
var outBuf, errBuf bytes.Buffer
|
||||
session.Stdout = &outBuf
|
||||
session.Stderr = &errBuf
|
||||
|
||||
type result struct {
|
||||
runErr error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
ch <- result{runErr: session.Run(cmd)}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
session.Close()
|
||||
return nil, ctx.Err()
|
||||
case res := <-ch:
|
||||
if res.runErr != nil {
|
||||
if errBuf.Len() > 0 {
|
||||
return outBuf.Bytes(), fmt.Errorf("command: %w\nstderr: %s", res.runErr, strings.TrimSpace(errBuf.String()))
|
||||
}
|
||||
return outBuf.Bytes(), fmt.Errorf("command: %w", res.runErr)
|
||||
}
|
||||
return outBuf.Bytes(), nil
|
||||
}
|
||||
}
|
||||
88
internal/actuator/client_test.go
Normal file
88
internal/actuator/client_test.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/pem"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
func TestLoadSignerRejectsBadInput(t *testing.T) {
|
||||
if _, err := LoadSignerFromBytes([]byte("not a private key")); err == nil {
|
||||
t.Error("LoadSignerFromBytes should reject a non-key input")
|
||||
}
|
||||
if _, err := LoadSigner("/nonexistent/key"); err == nil {
|
||||
t.Error("LoadSigner should fail on a missing file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSignerRoundTrip(t *testing.T) {
|
||||
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("generate key: %v", err)
|
||||
}
|
||||
block, err := ssh.MarshalPrivateKey(priv, "")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal private key: %v", err)
|
||||
}
|
||||
pemBytes := pem.EncodeToMemory(block)
|
||||
|
||||
signer, err := LoadSignerFromBytes(pemBytes)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSignerFromBytes on a valid key: %v", err)
|
||||
}
|
||||
if signer == nil {
|
||||
t.Fatal("signer is nil")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "id_ed25519")
|
||||
if err := os.WriteFile(path, pemBytes, 0o600); err != nil {
|
||||
t.Fatalf("write key file: %v", err)
|
||||
}
|
||||
fromFile, err := LoadSigner(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSigner(%s): %v", path, err)
|
||||
}
|
||||
if !bytes.Equal(fromFile.PublicKey().Marshal(), signer.PublicKey().Marshal()) {
|
||||
t.Error("file and in-memory signers resolved to different public keys")
|
||||
}
|
||||
}
|
||||
|
||||
// Dial needs a real SSH server to run a command, but its option normalization
|
||||
// is verifiable without one: a zero Port must default to 22 (so the dial error
|
||||
// references host:22, not host:0), and a closed port yields a dial error rather
|
||||
// than panicking.
|
||||
func TestDialDefaultsPort(t *testing.T) {
|
||||
_, err := Dial(context.Background(), DialOptions{Host: "127.0.0.1", Signer: mustSigner(t)})
|
||||
if err == nil {
|
||||
t.Fatal("Dial to a closed port should fail")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "127.0.0.1:22") {
|
||||
t.Errorf("Dial error = %q, want it to reference 127.0.0.1:22", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustSigner(t *testing.T) ssh.Signer {
|
||||
t.Helper()
|
||||
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("generate key: %v", err)
|
||||
}
|
||||
block, err := ssh.MarshalPrivateKey(priv, "")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal key: %v", err)
|
||||
}
|
||||
s, err := LoadSignerFromBytes(pem.EncodeToMemory(block))
|
||||
if err != nil {
|
||||
t.Fatalf("parse key: %v", err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
129
internal/actuator/hostkeys.go
Normal file
129
internal/actuator/hostkeys.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
var (
|
||||
hostKeyMu sync.RWMutex
|
||||
hostKeyCache map[string]ssh.PublicKey
|
||||
hostKeyOnce sync.Once
|
||||
hostKeySrc HostKeySource
|
||||
)
|
||||
|
||||
// HostKeySource provides storage for SSH host public keys.
|
||||
type HostKeySource interface {
|
||||
GetHostKey(ctx context.Context, hostname string) (string, error)
|
||||
SetHostKey(ctx context.Context, hostname string, key string) error
|
||||
}
|
||||
|
||||
// SetHostKeySource sets the host key source. Must be called before
|
||||
// any SSH connections. A nil source enables TOFU-only mode (keys
|
||||
// accepted in memory but not persisted).
|
||||
func SetHostKeySource(src HostKeySource) {
|
||||
hostKeyMu.Lock()
|
||||
defer hostKeyMu.Unlock()
|
||||
hostKeySrc = src
|
||||
}
|
||||
|
||||
// HostKeyCallback returns an ssh.HostKeyCallback that verifies host keys.
|
||||
// Known keys are verified (MITM detection). Unknown keys are accepted
|
||||
// via TOFU and optionally persisted to the source.
|
||||
func HostKeyCallback() ssh.HostKeyCallback {
|
||||
return hostKeyVerify
|
||||
}
|
||||
|
||||
func hostKeyVerify(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||||
hostKeyOnce.Do(func() {
|
||||
hostKeyCache = make(map[string]ssh.PublicKey)
|
||||
})
|
||||
|
||||
normalized := hostWithoutPort(hostname)
|
||||
|
||||
hostKeyMu.RLock()
|
||||
known, exists := hostKeyCache[normalized]
|
||||
hostKeyMu.RUnlock()
|
||||
|
||||
if exists {
|
||||
if bytes.Equal(key.Marshal(), known.Marshal()) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("SSH HOST KEY CHANGED for %s (possible MITM)", normalized)
|
||||
}
|
||||
|
||||
hostKeyMu.Lock()
|
||||
hostKeyCache[normalized] = key
|
||||
hostKeyMu.Unlock()
|
||||
|
||||
slog.Info("ssh: accepting new host key (TOFU)", "host", normalized)
|
||||
|
||||
if hostKeySrc != nil {
|
||||
go persistHostKey(normalized, key)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func persistHostKey(hostname string, key ssh.PublicKey) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
keyBase64 := key.Type() + " " + string(key.Marshal())
|
||||
if err := hostKeySrc.SetHostKey(ctx, "ssh/host-keys/"+hostname, keyBase64); err != nil {
|
||||
slog.Warn("ssh: failed to persist host key", "host", hostname, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// LoadHostKeys pre-loads known host keys from the source into the
|
||||
// in-memory cache. Call at startup to avoid TOFU on first connection.
|
||||
// The source should return key lines in the format "key-type base64-data".
|
||||
func LoadHostKeys(ctx context.Context, hostnames []string, src HostKeySource) {
|
||||
if src == nil {
|
||||
return
|
||||
}
|
||||
SetHostKeySource(src)
|
||||
|
||||
hostKeyMu.Lock()
|
||||
defer hostKeyMu.Unlock()
|
||||
|
||||
if hostKeyCache == nil {
|
||||
hostKeyCache = make(map[string]ssh.PublicKey)
|
||||
}
|
||||
|
||||
loaded := 0
|
||||
for _, hostname := range hostnames {
|
||||
keyData, err := src.GetHostKey(ctx, "ssh/host-keys/"+hostname)
|
||||
if err != nil {
|
||||
slog.Debug("ssh: no stored key for host", "host", hostname, "error", err)
|
||||
continue
|
||||
}
|
||||
pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(keyData))
|
||||
if err != nil {
|
||||
slog.Warn("ssh: invalid stored key for host", "host", hostname, "error", err)
|
||||
continue
|
||||
}
|
||||
hostKeyCache[hostname] = pubKey
|
||||
loaded++
|
||||
}
|
||||
|
||||
if loaded > 0 {
|
||||
slog.Info("ssh: loaded host keys from Infisical", "count", loaded)
|
||||
}
|
||||
}
|
||||
|
||||
func hostWithoutPort(hostname string) string {
|
||||
for i := len(hostname) - 1; i >= 0; i-- {
|
||||
if hostname[i] == ':' {
|
||||
return hostname[:i]
|
||||
}
|
||||
}
|
||||
return hostname
|
||||
}
|
||||
56
internal/actuator/hostkeys_infisical.go
Normal file
56
internal/actuator/hostkeys_infisical.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/secrets"
|
||||
)
|
||||
|
||||
// InfisicalHostKeySource implements HostKeySource backed by Infisical.
|
||||
type InfisicalHostKeySource struct {
|
||||
sec secrets.Backend
|
||||
}
|
||||
|
||||
// NewInfisicalHostKeySource creates a HostKeySource that reads/writes
|
||||
// SSH host public keys from Infisical under the `ssh/host-keys/` prefix.
|
||||
func NewInfisicalHostKeySource(sec secrets.Backend) *InfisicalHostKeySource {
|
||||
return &InfisicalHostKeySource{sec: sec}
|
||||
}
|
||||
|
||||
func (s *InfisicalHostKeySource) GetHostKey(ctx context.Context, path string) (string, error) {
|
||||
val, err := s.sec.Get(ctx, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
func (s *InfisicalHostKeySource) SetHostKey(ctx context.Context, path string, key string) error {
|
||||
return s.sec.Set(ctx, path, key)
|
||||
}
|
||||
|
||||
// ResolveSSHHosts queries the DB for active proxmox-host and standalone-server
|
||||
// entities, returning their slugs as SSH host identifiers.
|
||||
func ResolveSSHHosts(ctx context.Context, pool *db.Pool) []string {
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT slug FROM entities
|
||||
WHERE type IN ('proxmox-host', 'standalone-server')
|
||||
AND state = 'active'
|
||||
ORDER BY slug`)
|
||||
if err != nil {
|
||||
slog.Warn("ssh: failed to list hosts", "error", err)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var hosts []string
|
||||
for rows.Next() {
|
||||
var slug string
|
||||
if rows.Scan(&slug) == nil {
|
||||
hosts = append(hosts, slug)
|
||||
}
|
||||
}
|
||||
return hosts
|
||||
}
|
||||
122
internal/actuator/pool.go
Normal file
122
internal/actuator/pool.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type poolEntry struct {
|
||||
client *ssh.Client
|
||||
createdAt time.Time
|
||||
}
|
||||
|
||||
type DialPool struct {
|
||||
mu sync.RWMutex
|
||||
entries map[string]*poolEntry
|
||||
ttl time.Duration
|
||||
done chan struct{}
|
||||
stopped bool
|
||||
}
|
||||
|
||||
func NewDialPool(ttl time.Duration) *DialPool {
|
||||
p := &DialPool{
|
||||
entries: make(map[string]*poolEntry),
|
||||
ttl: ttl,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
if ttl > 0 {
|
||||
go p.evictLoop()
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *DialPool) key(opts DialOptions) string {
|
||||
port := opts.Port
|
||||
if port <= 0 {
|
||||
port = 22
|
||||
}
|
||||
return fmt.Sprintf("%s:%d", opts.Host, port)
|
||||
}
|
||||
|
||||
func (p *DialPool) Get(ctx context.Context, opts DialOptions) (*ssh.Client, error) {
|
||||
k := p.key(opts)
|
||||
p.mu.RLock()
|
||||
entry, ok := p.entries[k]
|
||||
p.mu.RUnlock()
|
||||
if ok {
|
||||
// Quick health check: a session can be created without running a
|
||||
// command — if it fails, the connection is dead and we evict it.
|
||||
testSession, err := entry.client.NewSession()
|
||||
if err == nil {
|
||||
testSession.Close()
|
||||
return entry.client, nil
|
||||
}
|
||||
p.mu.Lock()
|
||||
if p.entries[k] == entry {
|
||||
entry.client.Close()
|
||||
delete(p.entries, k)
|
||||
}
|
||||
p.mu.Unlock()
|
||||
// Fall through to dial below
|
||||
}
|
||||
client, err := Dial(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.mu.Lock()
|
||||
if p.stopped {
|
||||
p.mu.Unlock()
|
||||
client.Close()
|
||||
return nil, fmt.Errorf("ssh dial pool: closed")
|
||||
}
|
||||
if existing, ok2 := p.entries[k]; ok2 {
|
||||
p.mu.Unlock()
|
||||
client.Close()
|
||||
return existing.client, nil
|
||||
}
|
||||
p.entries[k] = &poolEntry{client: client, createdAt: time.Now()}
|
||||
p.mu.Unlock()
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (p *DialPool) Close() {
|
||||
p.mu.Lock()
|
||||
p.stopped = true
|
||||
for k, entry := range p.entries {
|
||||
entry.client.Close()
|
||||
delete(p.entries, k)
|
||||
}
|
||||
p.mu.Unlock()
|
||||
if p.ttl > 0 {
|
||||
close(p.done)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *DialPool) evictLoop() {
|
||||
ticker := time.NewTicker(p.ttl / 2)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-p.done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.evict()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *DialPool) evict() {
|
||||
deadline := time.Now().Add(-p.ttl)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for k, entry := range p.entries {
|
||||
if entry.createdAt.Before(deadline) {
|
||||
entry.client.Close()
|
||||
delete(p.entries, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -131,37 +130,19 @@ func ExecuteProcedure(
|
||||
start := time.Now()
|
||||
|
||||
// Parse the SSH key
|
||||
key, err := os.ReadFile(cfg.KeyPath)
|
||||
signer, err := LoadSigner(cfg.KeyPath)
|
||||
if err != nil {
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("read ssh key: %w", err),
|
||||
Err: err,
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
|
||||
signer, err := ssh.ParsePrivateKey(key)
|
||||
if err != nil {
|
||||
return SSHResult{
|
||||
Err: fmt.Errorf("parse ssh key: %w", err),
|
||||
Duration: time.Since(start),
|
||||
Verified: false,
|
||||
}
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(cfg.Host, fmt.Sprintf("%d", cfg.Port))
|
||||
if cfg.Port == 0 {
|
||||
addr = net.JoinHostPort(cfg.Host, "22")
|
||||
}
|
||||
|
||||
clientCfg := &ssh.ClientConfig{
|
||||
User: cfg.User,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // restricted key; host trust via inventory
|
||||
Timeout: cfg.Timeout,
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", addr, clientCfg)
|
||||
client, err := Dial(ctx, DialOptions{
|
||||
Host: cfg.Host, Port: cfg.Port, User: cfg.User,
|
||||
Signer: signer, Timeout: cfg.Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
class := classifySSHError(err)
|
||||
return SSHResult{
|
||||
@@ -229,38 +210,11 @@ func ExecuteProcedure(
|
||||
}
|
||||
}
|
||||
|
||||
// runSSHCommand executes a single command over an established SSH session.
|
||||
// Uses context-aware goroutines: ctx.Done() closes the session.
|
||||
// runSSHCommand executes a single command over an established SSH session via
|
||||
// the shared RunCombinedOutput primitive (context-aware abort + combined output).
|
||||
func runSSHCommand(ctx context.Context, client *ssh.Client, command string) (string, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
// Wrap in goroutine so we can abort on ctx.Done()
|
||||
type result struct {
|
||||
output string
|
||||
err error
|
||||
}
|
||||
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, err := session.CombinedOutput(command)
|
||||
ch <- result{output: string(out), err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Close the session to abort the SSH command
|
||||
session.Close()
|
||||
return "", ctx.Err()
|
||||
case res := <-ch:
|
||||
if res.err != nil {
|
||||
return res.output, fmt.Errorf("command: %w", res.err)
|
||||
}
|
||||
return res.output, nil
|
||||
}
|
||||
out, err := RunCombinedOutput(ctx, client, command)
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// ─── Procedure parsing ────────────────────────────────────────────────────
|
||||
|
||||
86
internal/actuator/stream.go
Normal file
86
internal/actuator/stream.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package actuator
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// RunStreaming runs a command on an established SSH client and forwards output
|
||||
// chunks to sink as they arrive. A nil sink collects output silently. Returns
|
||||
// the full combined output and any command error.
|
||||
func RunStreaming(ctx context.Context, client *ssh.Client, command string, sink func(stream string, chunk []byte), timeout time.Duration) (string, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
outPipe, err := session.StdoutPipe()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stdout pipe: %w", err)
|
||||
}
|
||||
errPipe, err := session.StderrPipe()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stderr pipe: %w", err)
|
||||
}
|
||||
|
||||
type streamResult struct {
|
||||
out string
|
||||
err error
|
||||
}
|
||||
|
||||
resultCh := make(chan streamResult, 1)
|
||||
go func() {
|
||||
var combined []byte
|
||||
done := make(chan struct{}, 2)
|
||||
|
||||
readStream := func(stream string, r io.Reader) {
|
||||
sc := bufio.NewScanner(r)
|
||||
for sc.Scan() {
|
||||
line := sc.Bytes()
|
||||
chunk := make([]byte, len(line))
|
||||
copy(chunk, line)
|
||||
if sink != nil {
|
||||
sink(stream, chunk)
|
||||
}
|
||||
if stream == "stdout" || stream == "" {
|
||||
if len(combined) > 0 {
|
||||
combined = append(combined, '\n')
|
||||
}
|
||||
combined = append(combined, chunk...)
|
||||
}
|
||||
}
|
||||
done <- struct{}{}
|
||||
}
|
||||
|
||||
go readStream("stdout", outPipe)
|
||||
go readStream("stderr", errPipe)
|
||||
|
||||
runErr := session.Run(command)
|
||||
<-done
|
||||
<-done
|
||||
resultCh <- streamResult{out: string(combined), err: runErr}
|
||||
}()
|
||||
|
||||
if timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
session.Close()
|
||||
return "", ctx.Err()
|
||||
case res := <-resultCh:
|
||||
if res.err != nil {
|
||||
return res.out, fmt.Errorf("command: %w", res.err)
|
||||
}
|
||||
return res.out, nil
|
||||
}
|
||||
}
|
||||
133
internal/audit/audit.go
Normal file
133
internal/audit/audit.go
Normal file
@@ -0,0 +1,133 @@
|
||||
// Package audit produces read-only drift reports over the knowledge graph and
|
||||
// monitoring state. It is the shared engine behind the
|
||||
// /api/v1/audit/drift endpoint and the audit_knowledge_graph MCP tool.
|
||||
//
|
||||
// It surfaces the structural gaps an operator otherwise discovers only by
|
||||
// accident: orphan check entities, checks targeting retired entities, probes
|
||||
// stuck down/unknown, unmonitored declared types, and live edges pointing at
|
||||
// destroyed/deprecated targets. Live-infra discovery (pct/docker/certs) is a
|
||||
// follow-up that needs host-hop execution; these categories are pure DB
|
||||
// queries, so the report is cheap, safe to run unattended, and testable.
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
)
|
||||
|
||||
// Finding is one drift item the operator should look at.
|
||||
type Finding struct {
|
||||
Category string `json:"category"`
|
||||
Severity string `json:"severity"` // info | warning | critical
|
||||
Count int `json:"count"`
|
||||
Entities []string `json:"entities"`
|
||||
Evidence string `json:"evidence"`
|
||||
SuggestedRunbook string `json:"suggested_runbook"`
|
||||
}
|
||||
|
||||
// Summary tallies findings by category.
|
||||
type Summary struct {
|
||||
TotalFindings int `json:"total_findings"`
|
||||
ByCategory map[string]int `json:"by_category"`
|
||||
}
|
||||
|
||||
// Report runs every drift check and returns the findings plus a summary.
|
||||
func Report(ctx context.Context, pool *db.Pool) ([]Finding, Summary) {
|
||||
specs := []struct {
|
||||
finding Finding
|
||||
query string
|
||||
}{
|
||||
{
|
||||
Finding{Category: "orphan_checks", Severity: "warning",
|
||||
Evidence: "check entities with truncated/random slugs (legacy shortSlug bug), no live target",
|
||||
SuggestedRunbook: "scripts/cleanup-orphan-checks.sh"},
|
||||
`SELECT e.slug FROM entities e
|
||||
WHERE e.type = 'check'
|
||||
AND e.slug ~ '^check:(ping|ssh-script|disk):[0-9a-f]{8}$'`,
|
||||
},
|
||||
{
|
||||
Finding{Category: "dead_checks", Severity: "warning",
|
||||
Evidence: "enabled check_defs whose target entity is deprecated/destroyed",
|
||||
SuggestedRunbook: "lifecycle-deprecate-node / lifecycle-destroy-node"},
|
||||
`SELECT e.slug FROM check_defs cd
|
||||
JOIN entities e ON e.id = cd.entity_id
|
||||
JOIN entities tgt ON tgt.id = cd.target_id
|
||||
WHERE cd.enabled AND tgt.state IN ('deprecated','destroyed')`,
|
||||
},
|
||||
{
|
||||
Finding{Category: "down_checks", Severity: "critical",
|
||||
Evidence: "enabled checks reporting health=down",
|
||||
SuggestedRunbook: "service-health-check"},
|
||||
`SELECT e.slug FROM check_defs cd JOIN entities e ON e.id = cd.entity_id
|
||||
WHERE cd.enabled AND cd.last_health = 'down'`,
|
||||
},
|
||||
{
|
||||
Finding{Category: "unknown_checks", Severity: "warning",
|
||||
Evidence: "enabled checks that ran but reported health=unknown (likely misconfigured probe)",
|
||||
SuggestedRunbook: "knowledge-graph-audit"},
|
||||
`SELECT e.slug FROM check_defs cd JOIN entities e ON e.id = cd.entity_id
|
||||
WHERE cd.enabled AND cd.last_health = 'unknown'`,
|
||||
},
|
||||
{
|
||||
Finding{Category: "unmonitored", Severity: "warning",
|
||||
Evidence: "active entities whose type declares monitoring but have no enabled check_def",
|
||||
SuggestedRunbook: "knowledge-graph-audit"},
|
||||
`SELECT DISTINCT e.slug FROM signals sg
|
||||
JOIN entities e ON e.id = sg.target_entity_id
|
||||
WHERE sg.kind = 'unmonitored' AND sg.state IN ('raised','acknowledged','acting')`,
|
||||
},
|
||||
{
|
||||
Finding{Category: "dangling_edges", Severity: "warning",
|
||||
Evidence: "live relationships (hosts/provides/mounts) pointing at destroyed/deprecated targets",
|
||||
SuggestedRunbook: "lifecycle-destroy-node"},
|
||||
`SELECT src.slug || ' -' || r.type || '-> ' || tgt.slug FROM relationships r
|
||||
JOIN entities src ON src.id = r.source_id
|
||||
JOIN entities tgt ON tgt.id = r.target_id
|
||||
WHERE r.valid_to IS NULL
|
||||
AND src.state NOT IN ('destroyed','deprecated')
|
||||
AND tgt.state IN ('destroyed','deprecated')`,
|
||||
},
|
||||
{
|
||||
Finding{Category: "polluted_attrs", Severity: "warning",
|
||||
Evidence: "routing-critical attributes carrying prose (breaks resolution) — e.g. host='hubris (confirmed via pct…')",
|
||||
SuggestedRunbook: "knowledge-graph-audit"},
|
||||
`SELECT slug || ': host=' || (attributes->>'host') FROM entities
|
||||
WHERE attributes->>'host' IS NOT NULL
|
||||
AND (attributes->>'host') ~ '[ (]'`,
|
||||
},
|
||||
}
|
||||
|
||||
findings := make([]Finding, 0, len(specs))
|
||||
summary := Summary{ByCategory: map[string]int{}}
|
||||
for _, sp := range specs {
|
||||
f := runFinding(ctx, pool, sp.finding, sp.query)
|
||||
findings = append(findings, f)
|
||||
summary.TotalFindings += f.Count
|
||||
summary.ByCategory[f.Category] = f.Count
|
||||
}
|
||||
return findings, summary
|
||||
}
|
||||
|
||||
const entityCap = 50
|
||||
|
||||
// runFinding runs a single-column slug query and folds the rows into a Finding.
|
||||
func runFinding(ctx context.Context, pool *db.Pool, f Finding, query string) Finding {
|
||||
rows, err := pool.Query(ctx, query)
|
||||
if err != nil {
|
||||
f.Evidence = f.Evidence + " (query error: " + err.Error() + ")"
|
||||
return f
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var slug string
|
||||
if err := rows.Scan(&slug); err != nil {
|
||||
continue
|
||||
}
|
||||
f.Count++
|
||||
if len(f.Entities) < entityCap {
|
||||
f.Entities = append(f.Entities, slug)
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
69
internal/audit/audit_test.go
Normal file
69
internal/audit/audit_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Integration tests against a real Postgres, guarded by
|
||||
// OIKOS_TEST_DATABASE_URL (same convention as internal/scheduler).
|
||||
|
||||
func newAuditPool(t *testing.T) *db.Pool {
|
||||
t.Helper()
|
||||
base := getenvOrDefault("OIKOS_TEST_DATABASE_URL", "")
|
||||
if base == "" {
|
||||
t.Skip("OIKOS_TEST_DATABASE_URL not set — skipping integration test")
|
||||
}
|
||||
return createTestDB(t, base)
|
||||
}
|
||||
|
||||
func TestReportFlagsOrphanAndDeadAndDown(t *testing.T) {
|
||||
pool := newAuditPool(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// An orphan check entity (truncated random slug, the legacy bug shape).
|
||||
mustExec(t, pool, ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
|
||||
VALUES ($1,'check:ssh-script:0d31fdd1','check','check:ssh-script:0d31fdd1','active','{}'::jsonb,1,now(),now())`, uuid.New())
|
||||
|
||||
// An active entity + a check_def on it stuck down.
|
||||
target := uuid.New()
|
||||
mustExec(t, pool, ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
|
||||
VALUES ($1,'service:demo','service','demo','active','{}'::jsonb,1,now(),now())`, target)
|
||||
checkE := uuid.New()
|
||||
mustExec(t, pool, ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
|
||||
VALUES ($1,'check:http:service:demo:0','check','c','active','{}'::jsonb,1,now(),now())`, checkE)
|
||||
mustExec(t, pool, ctx, `INSERT INTO check_defs (entity_id, target_id, target_type, kind, config, interval_s, timeout_s, enabled, last_run_at, last_health)
|
||||
VALUES ($1,$2,'service','http','{}'::jsonb,60,30,true,now(),'down')`, checkE, target)
|
||||
|
||||
// A deprecated entity still carrying an enabled check (dead_checks).
|
||||
dep := uuid.New()
|
||||
mustExec(t, pool, ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
|
||||
VALUES ($1,'service:old','service','old','deprecated','{}'::jsonb,1,now(),now())`, dep)
|
||||
depCheck := uuid.New()
|
||||
mustExec(t, pool, ctx, `INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
|
||||
VALUES ($1,'check:http:service:old:0','check','c','active','{}'::jsonb,1,now(),now())`, depCheck)
|
||||
mustExec(t, pool, ctx, `INSERT INTO check_defs (entity_id, target_id, target_type, kind, config, interval_s, timeout_s, enabled, last_run_at)
|
||||
VALUES ($1,$2,'service','http','{}'::jsonb,60,30,true,now())`, depCheck, dep)
|
||||
|
||||
findings, summary := Report(ctx, pool)
|
||||
|
||||
byCat := map[string]int{}
|
||||
for _, f := range findings {
|
||||
byCat[f.Category] = f.Count
|
||||
}
|
||||
if byCat["orphan_checks"] < 1 {
|
||||
t.Errorf("orphan_checks = %d, want >=1", byCat["orphan_checks"])
|
||||
}
|
||||
if byCat["down_checks"] < 1 {
|
||||
t.Errorf("down_checks = %d, want >=1", byCat["down_checks"])
|
||||
}
|
||||
if byCat["dead_checks"] < 1 {
|
||||
t.Errorf("dead_checks = %d, want >=1", byCat["dead_checks"])
|
||||
}
|
||||
if summary.TotalFindings < 3 {
|
||||
t.Errorf("TotalFindings = %d, want >=3", summary.TotalFindings)
|
||||
}
|
||||
}
|
||||
67
internal/audit/testutil_test.go
Normal file
67
internal/audit/testutil_test.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// createTestDB provisions a throwaway migrated database, same convention as
|
||||
// internal/scheduler/coverage_test.go.
|
||||
func createTestDB(t *testing.T, baseURL string) *db.Pool {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
admin, err := pgx.Connect(ctx, baseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect admin: %v", err)
|
||||
}
|
||||
dbName := fmt.Sprintf("oikos_aud_%08x", rand.Int63())
|
||||
if _, err := admin.Exec(ctx, "CREATE DATABASE "+dbName); err != nil {
|
||||
admin.Close(ctx)
|
||||
t.Fatalf("create test db: %v", err)
|
||||
}
|
||||
admin.Close(ctx)
|
||||
|
||||
at := strings.LastIndex(baseURL, "/")
|
||||
testURL := baseURL[:at+1] + dbName
|
||||
if q := strings.Index(baseURL[at:], "?"); q >= 0 {
|
||||
testURL += baseURL[at+q:]
|
||||
}
|
||||
|
||||
pool, err := db.New(ctx, testURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect test db: %v", err)
|
||||
}
|
||||
if err := pool.Migrate(ctx); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
pool.Close()
|
||||
if admin, err := pgx.Connect(ctx, baseURL); err == nil {
|
||||
admin.Exec(ctx, "DROP DATABASE IF EXISTS "+dbName+" WITH (FORCE)")
|
||||
admin.Close(ctx)
|
||||
}
|
||||
})
|
||||
return pool
|
||||
}
|
||||
|
||||
func getenvOrDefault(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func mustExec(t *testing.T, pool *db.Pool, ctx context.Context, q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(ctx, q, args...); err != nil {
|
||||
t.Fatalf("exec %s: %v", q, err)
|
||||
}
|
||||
}
|
||||
202
internal/checkdefaults/build_test.go
Normal file
202
internal/checkdefaults/build_test.go
Normal file
@@ -0,0 +1,202 @@
|
||||
package checkdefaults
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/dtoro/oikos/internal/ontology"
|
||||
)
|
||||
|
||||
// Table-driven coverage of every implemented buildKind branch and the ssh()
|
||||
// helper's user/port/args propagation. The previous tests exercised only
|
||||
// ping/process/http/resource; updates, capacity, backup, cert-expiry,
|
||||
// vm-status and dns were unverified.
|
||||
func TestBuildKindAllImplementedKinds(t *testing.T) {
|
||||
host := "10.0.0.5"
|
||||
cases := []struct {
|
||||
name string
|
||||
kind string
|
||||
target Target
|
||||
attrs map[string]any
|
||||
host string
|
||||
wantSkip bool // true → expect a reason and zero defs
|
||||
wantDefs int
|
||||
wantKind string
|
||||
wantKey string // a config key to assert
|
||||
wantVal any // its expected value
|
||||
wantReason string // substring when skipping
|
||||
wantInterv int32 // expected interval on the (single) produced def
|
||||
}{
|
||||
{
|
||||
name: "ping with host", kind: KindPing, host: host,
|
||||
wantDefs: 1, wantKind: "ping", wantKey: "host", wantVal: host, wantInterv: 30,
|
||||
},
|
||||
{name: "ping no host skips", kind: KindPing, wantSkip: true, wantReason: "no address"},
|
||||
|
||||
{
|
||||
name: "resource expands to four ssh scripts", kind: KindResource, host: host,
|
||||
wantDefs: 4, wantKind: "ssh-script", wantKey: "host", wantVal: host, wantInterv: 60,
|
||||
},
|
||||
{name: "resource no host skips", kind: KindResource, wantSkip: true, wantReason: "no address"},
|
||||
|
||||
{
|
||||
name: "updates is daily", kind: KindUpdates, host: host,
|
||||
wantDefs: 1, wantKind: "ssh-script", wantKey: "script", wantVal: "updates_check.sh", wantInterv: 86400,
|
||||
},
|
||||
{name: "updates no host skips", kind: KindUpdates, wantSkip: true, wantReason: "no address"},
|
||||
|
||||
{
|
||||
name: "capacity is one disk script", kind: KindCapacity, host: host,
|
||||
wantDefs: 1, wantKind: "ssh-script", wantKey: "script", wantVal: "disk_usage_check.sh", wantInterv: 60,
|
||||
},
|
||||
{name: "capacity no host skips", kind: KindCapacity, wantSkip: true, wantReason: "no address"},
|
||||
|
||||
{
|
||||
name: "backup needs path and host", kind: KindBackup, host: host,
|
||||
attrs: map[string]any{"path": "/backups/db"},
|
||||
wantDefs: 1, wantKind: "backup-freshness", wantKey: "path", wantVal: "/backups/db", wantInterv: 86400,
|
||||
},
|
||||
{name: "backup without path skips", kind: KindBackup, host: host, wantSkip: true, wantReason: "no path"},
|
||||
{name: "backup without host skips", kind: KindBackup, attrs: map[string]any{"path": "/x"}, wantSkip: true, wantReason: "no address"},
|
||||
|
||||
{
|
||||
name: "backup honors backup_max_age_s override", kind: KindBackup, host: host,
|
||||
attrs: map[string]any{"path": "/x", "backup_max_age_s": float64(3600)},
|
||||
wantDefs: 1, wantKey: "max_age_s", wantVal: 3600,
|
||||
},
|
||||
|
||||
{
|
||||
name: "cert-expiry from hostname attr", kind: KindCertExpiry,
|
||||
attrs: map[string]any{"hostname": "media.hubris.network"},
|
||||
wantDefs: 1, wantKind: "cert-expiry", wantKey: "host", wantVal: "media.hubris.network", wantInterv: 3600,
|
||||
},
|
||||
{
|
||||
name: "cert-expiry from dotted name", kind: KindCertExpiry, target: Target{Name: "media.hubris.network"},
|
||||
wantDefs: 1, wantKey: "host", wantVal: "media.hubris.network",
|
||||
},
|
||||
{
|
||||
name: "cert-expiry propagates dial attr", kind: KindCertExpiry,
|
||||
attrs: map[string]any{"hostname": "media.hubris.network", "dial": "10.0.0.2"},
|
||||
wantDefs: 1, wantKey: "dial", wantVal: "10.0.0.2",
|
||||
},
|
||||
{name: "cert-expiry without a host name skips", kind: KindCertExpiry, target: Target{Name: "jellyfin"}, wantSkip: true, wantReason: "no hostname"},
|
||||
|
||||
{
|
||||
name: "vm-status needs pve_id", kind: KindVMStatus, attrs: map[string]any{"pve_id": float64(101)},
|
||||
wantDefs: 1, wantKind: "vm-status", wantInterv: 60,
|
||||
},
|
||||
{name: "vm-status without pve_id skips", kind: KindVMStatus, wantSkip: true, wantReason: "no pve_id"},
|
||||
|
||||
{
|
||||
name: "dns resolves entity name", kind: KindDNS, target: Target{Name: "hubris.network"},
|
||||
wantDefs: 1, wantKind: "dns", wantKey: "name", wantVal: "hubris.network", wantInterv: 300,
|
||||
},
|
||||
{name: "dns without a name skips", kind: KindDNS, target: Target{}, wantSkip: true, wantReason: "no name"},
|
||||
|
||||
{
|
||||
name: "quorum runs pvecm script via ssh", kind: KindQuorum, host: host,
|
||||
wantDefs: 1, wantKind: "ssh-script", wantKey: "script", wantVal: "pvecm_quorum_check.sh", wantInterv: 60,
|
||||
},
|
||||
{name: "quorum no host skips", kind: KindQuorum, wantSkip: true, wantReason: "no address"},
|
||||
|
||||
{name: "unknown kind skips", kind: "telepathy", host: host, wantSkip: true, wantReason: "no builder"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
defs, reason := buildKind(c.kind, c.target, c.attrs, c.host, "root", 22)
|
||||
if c.wantSkip {
|
||||
if len(defs) != 0 {
|
||||
t.Fatalf("expected zero defs, got %d", len(defs))
|
||||
}
|
||||
if c.wantReason != "" && !strings.Contains(reason, c.wantReason) {
|
||||
t.Errorf("reason = %q, want substring %q", reason, c.wantReason)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(defs) != c.wantDefs {
|
||||
t.Fatalf("got %d defs (%s), want %d", len(defs), reason, c.wantDefs)
|
||||
}
|
||||
if reason != "" {
|
||||
t.Errorf("unexpected skip reason: %q", reason)
|
||||
}
|
||||
if c.wantKind != "" {
|
||||
if got := defs[0].kind; got != c.wantKind {
|
||||
t.Errorf("kind = %q, want %q", got, c.wantKind)
|
||||
}
|
||||
}
|
||||
if c.wantKey != "" {
|
||||
if got := defs[0].config[c.wantKey]; !reflect.DeepEqual(got, c.wantVal) {
|
||||
t.Errorf("config[%q] = %v (%T), want %v (%T)", c.wantKey, got, got, c.wantVal, c.wantVal)
|
||||
}
|
||||
}
|
||||
if c.wantInterv != 0 && defs[0].interval != c.wantInterv {
|
||||
t.Errorf("interval = %d, want %d", defs[0].interval, c.wantInterv)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ssh() must add user/port/args only when they differ from the root/22/empty
|
||||
// defaults, so generated configs stay minimal and stable across re-seeds.
|
||||
func TestBuildKindSSHOnlyEmitsNonDefaultUserPortArgs(t *testing.T) {
|
||||
t.Run("default root 22 omits user and port", func(t *testing.T) {
|
||||
defs, _ := buildKind(KindResource, Target{}, nil, "10.0.0.1", "root", 22)
|
||||
for _, d := range defs {
|
||||
if _, ok := d.config["user"]; ok {
|
||||
t.Errorf("root should not emit user: %v", d.config)
|
||||
}
|
||||
if _, ok := d.config["port"]; ok {
|
||||
t.Errorf("port 22 should not emit port: %v", d.config)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("non-root user and non-22 port are emitted", func(t *testing.T) {
|
||||
defs, _ := buildKind(KindResource, Target{}, nil, "10.0.0.1", "oikos", 2222)
|
||||
if defs[0].config["user"] != "oikos" {
|
||||
t.Errorf("user = %v, want oikos", defs[0].config["user"])
|
||||
}
|
||||
if defs[0].config["port"] != 2222 {
|
||||
t.Errorf("port = %v, want 2222", defs[0].config["port"])
|
||||
}
|
||||
})
|
||||
t.Run("process unit name lands in args", func(t *testing.T) {
|
||||
defs, _ := buildKind(KindProcess, Target{Name: "jellyfin"}, nil, "10.0.0.1", "root", 22)
|
||||
if defs[0].config["args"] != "jellyfin" {
|
||||
t.Errorf("args = %v, want jellyfin", defs[0].config["args"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// resolveMonitoringAttr implements the entity-level `monitoring` override
|
||||
// (project decision health_checks.monitoring_override): "none"/"" opts out,
|
||||
// a kind-list replaces the type defaults, anything else falls back.
|
||||
func TestResolveMonitoringAttr(t *testing.T) {
|
||||
fallback := ontology.MonitoringResolution{Declared: true, Kinds: []string{"ping"}, Source: "type"}
|
||||
cases := []struct {
|
||||
name string
|
||||
in any
|
||||
want ontology.MonitoringResolution
|
||||
}{
|
||||
{"none opts out", "none", ontology.MonitoringResolution{Declared: true, Source: "attribute"}},
|
||||
{"empty opts out", "", ontology.MonitoringResolution{Declared: true, Source: "attribute"}},
|
||||
{
|
||||
"kind list overrides",
|
||||
[]any{"http", "process"},
|
||||
ontology.MonitoringResolution{Declared: true, Kinds: []string{"http", "process"}, Source: "attribute"},
|
||||
},
|
||||
{"list drops empty and non-string entries", []any{"http", "", 7, "dns"}, ontology.MonitoringResolution{Declared: true, Kinds: []string{"http", "dns"}, Source: "attribute"}},
|
||||
{"non-string scalar falls back to type default", float64(42), fallback},
|
||||
{"nil falls back", nil, fallback},
|
||||
{"unrecognized string falls back", "weird", fallback},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := resolveMonitoringAttr(c.in, fallback)
|
||||
if !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("resolveMonitoringAttr(%v) = %+v, want %+v", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,38 +1,550 @@
|
||||
// Package checkdefaults derives an entity's default check_defs from the
|
||||
// monitoring kinds its type declares in seeds/ontology.yaml.
|
||||
//
|
||||
// The type says WHAT to watch (`service: [http, process]`); this package
|
||||
// works out HOW — which concrete check_defs rows to write, and what host,
|
||||
// script or URL each needs. Deriving config here rather than in YAML keeps
|
||||
// the ontology declarative and keeps address resolution (which has to walk
|
||||
// the graph) in code.
|
||||
package checkdefaults
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"github.com/dtoro/oikos/internal/ontology"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type CheckDef struct {
|
||||
Kind string
|
||||
Script string
|
||||
Host string
|
||||
User string
|
||||
Port int
|
||||
Thresholds map[string]any
|
||||
Extra map[string]any
|
||||
// Semantic monitoring kinds, as declared on entity types. These are not
|
||||
// check_defs.kind values — one semantic kind can expand to several concrete
|
||||
// checks (`resource` becomes four ssh-script rows).
|
||||
const (
|
||||
KindPing = "ping"
|
||||
KindResource = "resource"
|
||||
KindUpdates = "updates"
|
||||
KindProcess = "process"
|
||||
KindHTTP = "http"
|
||||
KindCapacity = "capacity"
|
||||
KindBackup = "backup-freshness"
|
||||
KindCertExpiry = "cert-expiry"
|
||||
KindVMStatus = "vm-status"
|
||||
KindQuorum = "quorum"
|
||||
KindDNS = "dns"
|
||||
)
|
||||
|
||||
// defaultBackupMaxAge is how long a backup target may go without a new
|
||||
// artifact before it is stale. A day suits the nightly jobs in this lab;
|
||||
// override per target with `backup_max_age_s` in the entity's attributes.
|
||||
const defaultBackupMaxAge = 86400
|
||||
|
||||
// Target is the entity default checks are being ensured for.
|
||||
type Target struct {
|
||||
ID uuid.UUID
|
||||
Slug string
|
||||
Type string
|
||||
// Name is the entity's name column, not an attribute. The old code read
|
||||
// attrs["name"], which is never populated — seeds put `name` beside
|
||||
// `attributes`, not inside it — so every service silently produced no
|
||||
// process check.
|
||||
Name string
|
||||
Attrs []byte
|
||||
}
|
||||
|
||||
// Result reports what Ensure did, so callers can log a type that declared
|
||||
// monitoring but produced nothing instead of failing silently.
|
||||
type Result struct {
|
||||
Created int
|
||||
// Skipped records kinds that were declared but could not be built, with
|
||||
// the reason. A non-empty Skipped on an active entity is a real gap.
|
||||
Skipped []Skip
|
||||
// Undeclared is true when no ancestor of the type declared monitoring —
|
||||
// an ontology gap rather than a fleet gap.
|
||||
Undeclared bool
|
||||
}
|
||||
|
||||
// Skip is one declared-but-unbuilt check kind.
|
||||
type Skip struct {
|
||||
Kind string
|
||||
Reason string
|
||||
}
|
||||
|
||||
type checkDef struct {
|
||||
kind string
|
||||
config map[string]any
|
||||
interval int32
|
||||
}
|
||||
|
||||
// Ensure writes the default check_defs for one entity, idempotently.
|
||||
//
|
||||
// Returns the number of checks created. An entity whose type declares
|
||||
// monitoring it cannot satisfy comes back with a populated Skipped rather
|
||||
// than an error — a missing address is a modelling gap, not a failure of
|
||||
// this call.
|
||||
func Ensure(ctx context.Context, tx pgx.Tx, tree *ontology.TypeTree, t Target) (Result, error) {
|
||||
var res Result
|
||||
|
||||
if _, err := tx.Exec(ctx,
|
||||
`INSERT INTO entity_status (entity_id, health, updated_at)
|
||||
VALUES ($1, 'unknown', now())
|
||||
ON CONFLICT (entity_id) DO NOTHING`, t.ID); err != nil {
|
||||
return res, fmt.Errorf("entity_status %s: %w", t.Slug, err)
|
||||
}
|
||||
|
||||
mon := tree.Monitoring(t.Type)
|
||||
if !mon.Declared {
|
||||
res.Undeclared = true
|
||||
return res, nil
|
||||
}
|
||||
if mon.None() {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var attrs map[string]any
|
||||
if len(t.Attrs) > 0 {
|
||||
_ = json.Unmarshal(t.Attrs, &attrs)
|
||||
}
|
||||
if attrs == nil {
|
||||
attrs = map[string]any{}
|
||||
}
|
||||
|
||||
// Per-entity override: an explicit `monitoring` attribute wins over the
|
||||
// type declaration. A single entity can opt out (monitoring: none) or pick
|
||||
// different kinds without introducing a new type — e.g. service:haos opts
|
||||
// out because its VM is already covered by a vm-status check and the
|
||||
// service can't be SSH-probed (haos blocks SSH).
|
||||
if mo, ok := attrs["monitoring"]; ok {
|
||||
mon = resolveMonitoringAttr(mo, mon)
|
||||
if mon.None() {
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
|
||||
// A service has no address of its own — it lives on the container that
|
||||
// provides it. Fall back to the graph before giving up.
|
||||
host := resolveHost(attrs)
|
||||
if host == "" {
|
||||
hostAttrs, err := hostViaGraph(ctx, tx, t.ID)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("resolve host for %s: %w", t.Slug, err)
|
||||
}
|
||||
host = resolveHost(hostAttrs)
|
||||
if user := resolveSSHUser(hostAttrs); host != "" && user != "root" {
|
||||
attrs["ssh"] = hostAttrs["ssh"]
|
||||
}
|
||||
}
|
||||
user := resolveSSHUser(attrs)
|
||||
port := resolveSSHPort(attrs)
|
||||
|
||||
var defs []checkDef
|
||||
for _, kind := range mon.Kinds {
|
||||
built, reason := buildKind(kind, t, attrs, host, user, port)
|
||||
if len(built) == 0 {
|
||||
res.Skipped = append(res.Skipped, Skip{Kind: kind, Reason: reason})
|
||||
continue
|
||||
}
|
||||
defs = append(defs, built...)
|
||||
}
|
||||
|
||||
for i, def := range defs {
|
||||
created, err := writeCheck(ctx, tx, t, i, def)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("check %s/%s: %w", t.Slug, def.kind, err)
|
||||
}
|
||||
if created {
|
||||
res.Created++
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// resolveMonitoringAttr turns an entity's `monitoring` attribute into a
|
||||
// MonitoringResolution that overrides the type's declaration. Accepts the
|
||||
// scalar "none" (or empty) to opt out, or a list of kind strings to override.
|
||||
func resolveMonitoringAttr(v any, fallback ontology.MonitoringResolution) ontology.MonitoringResolution {
|
||||
switch vv := v.(type) {
|
||||
case string:
|
||||
if vv == "none" || vv == "" {
|
||||
return ontology.MonitoringResolution{Declared: true, Source: "attribute"}
|
||||
}
|
||||
case []any:
|
||||
kinds := make([]string, 0, len(vv))
|
||||
for _, k := range vv {
|
||||
if s, ok := k.(string); ok && s != "" {
|
||||
kinds = append(kinds, s)
|
||||
}
|
||||
}
|
||||
return ontology.MonitoringResolution{Declared: true, Kinds: kinds, Source: "attribute"}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// buildKind turns one declared semantic kind into concrete check_defs, or
|
||||
// returns the reason it could not.
|
||||
func buildKind(kind string, t Target, attrs map[string]any, host, user string, port int) ([]checkDef, string) {
|
||||
ssh := func(script string, args ...string) checkDef {
|
||||
cfg := map[string]any{"script": script, "host": host}
|
||||
if user != "" && user != "root" {
|
||||
cfg["user"] = user
|
||||
}
|
||||
if port != 0 && port != 22 {
|
||||
cfg["port"] = port
|
||||
}
|
||||
if len(args) > 0 && args[0] != "" {
|
||||
cfg["args"] = args[0]
|
||||
}
|
||||
return checkDef{kind: "ssh-script", config: cfg, interval: 60}
|
||||
}
|
||||
|
||||
switch kind {
|
||||
case KindPing:
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or its host"
|
||||
}
|
||||
return []checkDef{{kind: "ping", config: map[string]any{"host": host}, interval: 30}}, ""
|
||||
|
||||
case KindResource:
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or its host"
|
||||
}
|
||||
return []checkDef{
|
||||
ssh("cpu_check.sh"), ssh("memory_check.sh"),
|
||||
ssh("load_check.sh"), ssh("disk_usage_check.sh"),
|
||||
}, ""
|
||||
|
||||
case KindUpdates:
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or its host"
|
||||
}
|
||||
// Daily. updates_check.sh runs `apt update` against the distro
|
||||
// mirrors; the shared 60s ssh-script default would have meant 1,440
|
||||
// mirror hits per machine per day to answer a question whose answer
|
||||
// changes about once a day.
|
||||
u := ssh("updates_check.sh")
|
||||
u.interval = 86400
|
||||
return []checkDef{u}, ""
|
||||
|
||||
case KindCapacity:
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or its host"
|
||||
}
|
||||
return []checkDef{ssh("disk_usage_check.sh")}, ""
|
||||
|
||||
case KindProcess:
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or its host"
|
||||
}
|
||||
// A service's name is a logical label, not usually its systemd unit
|
||||
// or container name (matrix = matrix-synapse.service + containers).
|
||||
// Prefer an explicit probe target when declared; process_check.sh also
|
||||
// matches a unit prefix or a docker container as a fallback.
|
||||
unit := ""
|
||||
for _, key := range []string{"probe_unit", "systemd_unit", "container"} {
|
||||
if v, _ := attrs[key].(string); v != "" {
|
||||
unit = v
|
||||
break
|
||||
}
|
||||
}
|
||||
// Ontology intent: "http when it has a url, else a process check." A
|
||||
// url-fronted service is already liveness-probed via http (the real
|
||||
// endpoint, through the TLS terminator); the process check is redundant
|
||||
// and fragile (needs host access + the exact unit/container name), and
|
||||
// under worst-of aggregation it lets a broken supplementary probe veto
|
||||
// a working service. Emit it only for services WITHOUT a url, or when
|
||||
// an explicit probe_unit opts into binary-level depth.
|
||||
if unit == "" {
|
||||
if httpURL(t, attrs) != "" {
|
||||
return nil, "url present and no probe_unit; http check covers liveness"
|
||||
}
|
||||
unit = t.Name
|
||||
}
|
||||
if unit == "" {
|
||||
return nil, "no name to check a process for"
|
||||
}
|
||||
// process_check.sh takes the unit/container name as $1 and reports
|
||||
// "unknown" without it.
|
||||
return []checkDef{ssh("process_check.sh", unit)}, ""
|
||||
|
||||
case KindBackup:
|
||||
// A backup target is checked from the machine that writes to it, so it
|
||||
// needs both an address (resolved via the backs-up-to edge) and the
|
||||
// path to look at.
|
||||
path, _ := attrs["path"].(string)
|
||||
if path == "" {
|
||||
return nil, "entity carries no path attribute to check for backups"
|
||||
}
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or whatever backs up to it"
|
||||
}
|
||||
maxAge := defaultBackupMaxAge
|
||||
if v, ok := attrs["backup_max_age_s"].(float64); ok && v > 0 {
|
||||
maxAge = int(v)
|
||||
}
|
||||
cfg := map[string]any{"path": path, "host": host, "max_age_s": maxAge}
|
||||
if user != "" && user != "root" {
|
||||
cfg["user"] = user
|
||||
}
|
||||
if port != 0 && port != 22 {
|
||||
cfg["port"] = port
|
||||
}
|
||||
// Daily. The freshness budget itself is a day, so probing more often
|
||||
// cannot surface anything sooner — it just costs an SSH round trip.
|
||||
return []checkDef{{kind: "backup-freshness", config: cfg, interval: 86400}}, ""
|
||||
|
||||
case KindHTTP:
|
||||
url := httpURL(t, attrs)
|
||||
if url == "" {
|
||||
return nil, "no url attribute, public_host, or hostname-shaped name"
|
||||
}
|
||||
// max_status rather than an exact expected_status: most services sit
|
||||
// behind Authentik and answer 302/401, which is a working service.
|
||||
return []checkDef{{
|
||||
kind: "http",
|
||||
config: map[string]any{"url": url, "max_status": 500},
|
||||
interval: 60,
|
||||
}}, ""
|
||||
|
||||
case KindDNS:
|
||||
// Resolve the entity's name via DNS to verify the zone is reachable.
|
||||
// Uses the entity name (zone apex) or falls back to the slug.
|
||||
name := t.Name
|
||||
if name == "" {
|
||||
name = strings.TrimPrefix(t.Slug, "zone:")
|
||||
}
|
||||
if name == "" {
|
||||
return nil, "no name to resolve"
|
||||
}
|
||||
return []checkDef{{
|
||||
kind: "dns",
|
||||
config: map[string]any{"name": name},
|
||||
interval: 300, // 5 min — DNS changes are rare; the cost of a miss
|
||||
// is a stale IP, not a service outage.
|
||||
}}, ""
|
||||
|
||||
case KindCertExpiry:
|
||||
// The host whose cert to read (SNI / cert CN). Prefer an explicit
|
||||
// `hostname` attribute, then `cn`, then a dotted name. Hourly: expiry
|
||||
// changes once a day, but a renewal or mis-issued cert is worth
|
||||
// noticing within the hour.
|
||||
host := certHost(t, attrs)
|
||||
if host == "" {
|
||||
return nil, "no hostname / cn / dotted name to dial for the cert"
|
||||
}
|
||||
// `dial` is the TLS terminator's address to connect to (Caddy's lab
|
||||
// IP), used when the hostname doesn't resolve/reach from the scheduler.
|
||||
// Without it the probe can't reach *.hubris.network from a container
|
||||
// with no mesh / split-horizon DNS.
|
||||
dial, _ := attrs["dial"].(string)
|
||||
config := map[string]any{"host": host, "warn_days": 30, "crit_days": 7}
|
||||
if dial != "" {
|
||||
config["dial"] = dial
|
||||
}
|
||||
return []checkDef{{
|
||||
kind: "cert-expiry",
|
||||
config: config,
|
||||
interval: 3600,
|
||||
}}, ""
|
||||
|
||||
case KindVMStatus:
|
||||
// "Is the VM powered on" via `qm status` on its Proxmox host — the
|
||||
// right reachability probe for a VM, since many block ICMP and lack a
|
||||
// guest agent. checkVMStatus re-reads pve_id + host at runtime.
|
||||
if _, ok := attrs["pve_id"]; !ok {
|
||||
return nil, "no pve_id to run qm status"
|
||||
}
|
||||
return []checkDef{{
|
||||
kind: "vm-status",
|
||||
config: map[string]any{},
|
||||
interval: 60,
|
||||
}}, ""
|
||||
|
||||
case KindQuorum:
|
||||
// Proxmox cluster quorum via `pvecm status`. Only meaningful on
|
||||
// proxmox-host entities. Runs every 60s — corosync flaps are
|
||||
// transient and the probe is lightweight (local binary, no network).
|
||||
if host == "" {
|
||||
return nil, "no address on the entity or its host"
|
||||
}
|
||||
return []checkDef{ssh("pvecm_quorum_check.sh")}, ""
|
||||
}
|
||||
|
||||
return nil, "no builder for this kind yet"
|
||||
}
|
||||
|
||||
// certHost works out the hostname to TLS-dial for a certificate's expiry.
|
||||
func certHost(t Target, attrs map[string]any) string {
|
||||
for _, key := range []string{"hostname", "cn", "san"} {
|
||||
if v, ok := attrs[key].(string); ok && v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
// A dotted name is a hostname (hubris.network, media.hubris.network).
|
||||
if strings.Contains(t.Name, ".") && !strings.Contains(t.Name, " ") {
|
||||
return t.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// writeCheck upserts one check_def and its backing check entity.
|
||||
//
|
||||
// The entity upsert MUST return the row's id. The previous version generated
|
||||
// a fresh uuid, inserted ON CONFLICT (slug) DO NOTHING, then wrote a
|
||||
// check_defs row referencing that uuid. On any re-seed the slug already
|
||||
// existed, the entity insert became a no-op, and the check_defs insert
|
||||
// violated its foreign key — which aborted the whole ingest transaction and
|
||||
// made every subsequent statement fail with 25P02. Because the errors were
|
||||
// discarded, the only visible symptom was an unrelated failure much later.
|
||||
func writeCheck(ctx context.Context, tx pgx.Tx, t Target, idx int, def checkDef) (bool, error) {
|
||||
// The full target slug, not a truncation of it. shortSlug() took the last
|
||||
// 8 characters, so all 21 ingress routes collapsed to ".network" and
|
||||
// generated one identical check slug — they overwrote each other and 20
|
||||
// of them ended up with no check at all. It also collided service:jellyfin
|
||||
// with lxc:jellyfin. Entity slugs are unique; use them.
|
||||
checkSlug := fmt.Sprintf("check:%s:%s:%d", def.kind, t.Slug, idx)
|
||||
|
||||
newID, err := uuid.NewV7()
|
||||
if err != nil {
|
||||
newID = uuid.New()
|
||||
}
|
||||
|
||||
var checkID uuid.UUID
|
||||
err = tx.QueryRow(ctx,
|
||||
`INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
|
||||
VALUES ($1, $2, 'check', $2, 'active', '{}', 1, now(), now())
|
||||
ON CONFLICT (slug) DO UPDATE SET updated_at = now()
|
||||
RETURNING id`,
|
||||
newID, checkSlug).Scan(&checkID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("upsert check entity %s: %w", checkSlug, err)
|
||||
}
|
||||
|
||||
configJSON, err := json.Marshal(def.config)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Config is derived from the seed, so the seed wins on re-ingest and
|
||||
// attribute changes propagate. `enabled` is deliberately left alone: it
|
||||
// is operational state an operator may have toggled.
|
||||
// last_run_at is seeded to a random point inside the interval so checks
|
||||
// created together do not stay in lockstep. Every check the seed creates
|
||||
// would otherwise come due in the same instant forever: ~165 probes
|
||||
// landing at once each minute rather than spread across it. Deliberately
|
||||
// absent from the DO UPDATE below — a re-seed must not reset the schedule
|
||||
// and re-herd everything.
|
||||
tag, err := tx.Exec(ctx,
|
||||
`INSERT INTO check_defs (entity_id, target_id, target_type, kind, config, interval_s, timeout_s, enabled, last_run_at)
|
||||
VALUES ($1, $2, $6, $3, $4, $5, 30, true,
|
||||
now() - make_interval(secs => random() * $5::int))
|
||||
ON CONFLICT (entity_id) DO UPDATE
|
||||
SET target_id = EXCLUDED.target_id, target_type = EXCLUDED.target_type,
|
||||
kind = EXCLUDED.kind,
|
||||
config = EXCLUDED.config, interval_s = EXCLUDED.interval_s,
|
||||
updated_at = now()`,
|
||||
checkID, t.ID, def.kind, configJSON, def.interval, t.Type)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("upsert check_def %s: %w", checkSlug, err)
|
||||
}
|
||||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
|
||||
// httpURL works out what to GET for an http check.
|
||||
//
|
||||
// Ingress routes carry their hostname as the entity name rather than as an
|
||||
// attribute (`name: media.hubris.network`), and most declare no attributes at
|
||||
// all — so the name is the only thing to go on. Requiring a `url` attribute
|
||||
// left all 21 of them unmonitored, which is a shame given an ingress check is
|
||||
// the most end-to-end probe available: it exercises Caddy, DNS, TLS and the
|
||||
// upstream in one request.
|
||||
func httpURL(t Target, attrs map[string]any) string {
|
||||
if url, ok := attrs["url"].(string); ok && url != "" {
|
||||
return url
|
||||
}
|
||||
if h, ok := attrs["public_host"].(string); ok && h != "" {
|
||||
return "https://" + h
|
||||
}
|
||||
// A dotted name is a hostname; a service name like "jellyfin" is not.
|
||||
if strings.Contains(t.Name, ".") && !strings.Contains(t.Name, " ") {
|
||||
return "https://" + t.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hostViaGraph returns the attributes of the entity that hosts or provides
|
||||
// this one, so a service can inherit its container's address.
|
||||
func hostViaGraph(ctx context.Context, tx pgx.Tx, entityID uuid.UUID) (map[string]any, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT e.attributes
|
||||
FROM relationships r
|
||||
JOIN entities e ON e.id = r.source_id
|
||||
WHERE r.target_id = $1
|
||||
AND r.valid_to IS NULL
|
||||
-- backs-up-to points from the thing being backed up TO the target,
|
||||
-- so walking it backwards finds the machine that writes the backups
|
||||
-- — which is the only place a freshness check can run.
|
||||
AND r.type IN ('provides', 'hosts', 'runs-on', 'backs-up-to')
|
||||
ORDER BY CASE r.type
|
||||
WHEN 'provides' THEN 0 WHEN 'runs-on' THEN 1
|
||||
WHEN 'backs-up-to' THEN 2 ELSE 3 END`,
|
||||
entityID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
if err := rows.Scan(&raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var attrs map[string]any
|
||||
if json.Unmarshal(raw, &attrs) != nil {
|
||||
continue
|
||||
}
|
||||
if resolveHost(attrs) != "" {
|
||||
return attrs, nil
|
||||
}
|
||||
}
|
||||
return nil, rows.Err()
|
||||
}
|
||||
|
||||
func resolveHost(attrs map[string]any) string {
|
||||
if attrs == nil {
|
||||
return ""
|
||||
}
|
||||
if ip, ok := attrs["lan_ip"].(string); ok && ip != "" {
|
||||
return ip
|
||||
}
|
||||
// public_ipv4 before mesh: the scheduler container has no mesh interface,
|
||||
// so a standalone-server reachable only by mesh IP (netbird-vps) is
|
||||
// unprobeable even though a public IPv4 is available.
|
||||
if ip, ok := attrs["public_ipv4"].(string); ok && ip != "" {
|
||||
return ip
|
||||
}
|
||||
if mesh, ok := attrs["mesh"].(map[string]any); ok {
|
||||
if nb, ok := mesh["netbird"].(map[string]any); ok {
|
||||
if ip, ok := nb["ip"].(string); ok && ip != "" {
|
||||
return ip
|
||||
}
|
||||
// Seeds record the mesh name, not an address — ws:mac-mini
|
||||
// carries only `fqdn`, which is why it resolved to nothing.
|
||||
if fqdn, ok := nb["fqdn"].(string); ok && fqdn != "" {
|
||||
return fqdn
|
||||
}
|
||||
}
|
||||
}
|
||||
if ip, ok := attrs["mesh_ip"].(string); ok && ip != "" {
|
||||
return ip
|
||||
}
|
||||
for _, key := range []string{"host", "address", "public_host"} {
|
||||
if v, ok := attrs[key].(string); ok && v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -42,6 +554,13 @@ func resolveSSHUser(attrs map[string]any) string {
|
||||
return u
|
||||
}
|
||||
}
|
||||
// Workstations carry their login as a top-level `user` attribute
|
||||
// (mac-mini: user: dtoro) rather than under ssh.user. Take it only when
|
||||
// no explicit ssh.user was set, so a host that genuinely wants root still
|
||||
// gets root.
|
||||
if u, ok := attrs["user"].(string); ok && u != "" {
|
||||
return u
|
||||
}
|
||||
return "root"
|
||||
}
|
||||
|
||||
@@ -57,148 +576,17 @@ func resolveSSHPort(attrs map[string]any) int {
|
||||
return 22
|
||||
}
|
||||
|
||||
func forEntityType(entityType string, attrs map[string]any) []CheckDef {
|
||||
host := resolveHost(attrs)
|
||||
user := resolveSSHUser(attrs)
|
||||
port := resolveSSHPort(attrs)
|
||||
|
||||
ssh := func(script string) CheckDef {
|
||||
return CheckDef{Kind: "ssh-script", Script: script, Host: host, User: user, Port: port}
|
||||
}
|
||||
|
||||
switch entityType {
|
||||
case "proxmox-host", "standalone-server":
|
||||
if host == "" {
|
||||
return nil
|
||||
// LogResult emits the one line that was missing: a type that asked for
|
||||
// monitoring and did not get it.
|
||||
func LogResult(slug, entityType string, res Result) {
|
||||
switch {
|
||||
case res.Undeclared:
|
||||
slog.Info("checkdefaults: type declares no monitoring",
|
||||
"entity", slug, "type", entityType)
|
||||
case len(res.Skipped) > 0:
|
||||
for _, s := range res.Skipped {
|
||||
slog.Warn("checkdefaults: declared check not created",
|
||||
"entity", slug, "type", entityType, "kind", s.Kind, "reason", s.Reason)
|
||||
}
|
||||
return []CheckDef{
|
||||
{Kind: "ping", Host: host},
|
||||
ssh("cpu_check.sh"),
|
||||
ssh("memory_check.sh"),
|
||||
ssh("load_check.sh"),
|
||||
ssh("disk_usage_check.sh"),
|
||||
ssh("updates_check.sh"),
|
||||
}
|
||||
case "workstation":
|
||||
if host == "" {
|
||||
return nil
|
||||
}
|
||||
return []CheckDef{
|
||||
{Kind: "ping", Host: host},
|
||||
ssh("cpu_check.sh"),
|
||||
ssh("memory_check.sh"),
|
||||
ssh("load_check.sh"),
|
||||
}
|
||||
case "lxc":
|
||||
if host == "" {
|
||||
return nil
|
||||
}
|
||||
return []CheckDef{
|
||||
ssh("cpu_check.sh"),
|
||||
ssh("memory_check.sh"),
|
||||
ssh("load_check.sh"),
|
||||
ssh("disk_usage_check.sh"),
|
||||
}
|
||||
case "vm":
|
||||
if host == "" {
|
||||
return nil
|
||||
}
|
||||
return []CheckDef{
|
||||
{Kind: "ping", Host: host},
|
||||
}
|
||||
case "service":
|
||||
if host == "" {
|
||||
return nil
|
||||
}
|
||||
n, _ := attrs["name"].(string)
|
||||
if n == "" {
|
||||
return nil
|
||||
}
|
||||
return []CheckDef{
|
||||
{Kind: "ssh-script", Script: "process_check.sh", Host: host, User: user, Port: port,
|
||||
Extra: map[string]any{"args": n}},
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func shortSlug(slug string) string {
|
||||
const n = 8
|
||||
if len(slug) > n {
|
||||
return slug[len(slug)-n:]
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
func defaultInterval(kind string) int32 {
|
||||
switch kind {
|
||||
case "ping":
|
||||
return 30
|
||||
case "ssh-script":
|
||||
return 60
|
||||
default:
|
||||
return 300
|
||||
}
|
||||
}
|
||||
|
||||
func Ensure(ctx context.Context, tx pgx.Tx, entityID uuid.UUID, slug, entityType string, attrsJSON []byte) {
|
||||
_, _ = tx.Exec(ctx,
|
||||
`INSERT INTO entity_status (entity_id, health, updated_at)
|
||||
VALUES ($1, 'unknown', now())
|
||||
ON CONFLICT (entity_id) DO NOTHING`,
|
||||
entityID)
|
||||
|
||||
var attrs map[string]any
|
||||
if len(attrsJSON) > 0 {
|
||||
json.Unmarshal(attrsJSON, &attrs)
|
||||
}
|
||||
if attrs == nil {
|
||||
attrs = map[string]any{}
|
||||
}
|
||||
|
||||
defs := forEntityType(entityType, attrs)
|
||||
if len(defs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for i, def := range defs {
|
||||
checkID, err := uuid.NewV7()
|
||||
if err != nil {
|
||||
checkID = uuid.New()
|
||||
}
|
||||
checkSlug := fmt.Sprintf("check:%s:%s:%d", def.Kind, shortSlug(slug), i)
|
||||
|
||||
_, _ = tx.Exec(ctx,
|
||||
`INSERT INTO entities (id, slug, type, name, state, attributes, version, created_at, updated_at)
|
||||
VALUES ($1, $2, 'check', $2, 'active', '{}', 1, now(), now())
|
||||
ON CONFLICT (slug) DO NOTHING`,
|
||||
checkID, checkSlug)
|
||||
|
||||
configMap := map[string]any{}
|
||||
if def.Script != "" {
|
||||
configMap["script"] = def.Script
|
||||
}
|
||||
if def.Host != "" {
|
||||
configMap["host"] = def.Host
|
||||
}
|
||||
if def.User != "" && def.User != "root" {
|
||||
configMap["user"] = def.User
|
||||
}
|
||||
if def.Port != 0 && def.Port != 22 {
|
||||
configMap["port"] = def.Port
|
||||
}
|
||||
if def.Thresholds != nil {
|
||||
configMap["thresholds"] = def.Thresholds
|
||||
}
|
||||
for k, v := range def.Extra {
|
||||
configMap[k] = v
|
||||
}
|
||||
configJSON, _ := json.Marshal(configMap)
|
||||
|
||||
_, _ = tx.Exec(ctx,
|
||||
`INSERT INTO check_defs (entity_id, target_id, kind, config, interval_s, timeout_s, enabled)
|
||||
VALUES ($1, $2, $3, $4, $5, 30, true)
|
||||
ON CONFLICT (entity_id) DO NOTHING`,
|
||||
checkID, entityID, def.Kind, configJSON, defaultInterval(def.Kind))
|
||||
}
|
||||
}
|
||||
|
||||
125
internal/checkdefaults/defaults_test.go
Normal file
125
internal/checkdefaults/defaults_test.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package checkdefaults
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The attribute shapes here are copied from seeds/inventory.yaml. The original
|
||||
// resolveHost looked for lan_ip / mesh.netbird.ip / mesh_ip, none of which a
|
||||
// service or workstation actually carries — which is why 86 of 89 entities
|
||||
// ended up with no checks.
|
||||
func TestResolveHostAcceptsRealSeedShapes(t *testing.T) {
|
||||
cases := []struct {
|
||||
desc string
|
||||
attrs map[string]any
|
||||
want string
|
||||
}{
|
||||
{"lxc carries lan_ip", map[string]any{"lan_ip": "192.168.8.246"}, "192.168.8.246"},
|
||||
{
|
||||
"ws:mac-mini carries only a netbird fqdn",
|
||||
map[string]any{"mesh": map[string]any{"netbird": map[string]any{
|
||||
"fqdn": "mac-mini-234-17.netbird.selfhosted"}}},
|
||||
"mac-mini-234-17.netbird.selfhosted",
|
||||
},
|
||||
{
|
||||
"a netbird ip still wins over the fqdn",
|
||||
map[string]any{"mesh": map[string]any{"netbird": map[string]any{
|
||||
"ip": "100.122.0.10", "fqdn": "x.netbird.selfhosted"}}},
|
||||
"100.122.0.10",
|
||||
},
|
||||
{"public_host as a last resort", map[string]any{"public_host": "media.hubris.network"}, "media.hubris.network"},
|
||||
{"a service carries no address at all", map[string]any{
|
||||
"url": "https://media.hubris.network", "port": 8096}, ""},
|
||||
{"nil attrs", nil, ""},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
if got := resolveHost(c.attrs); got != c.want {
|
||||
t.Errorf("%s: resolveHost = %q, want %q", c.desc, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPURLPrefersAttributeThenName(t *testing.T) {
|
||||
cases := []struct {
|
||||
desc string
|
||||
name string
|
||||
attrs map[string]any
|
||||
want string
|
||||
}{
|
||||
{"explicit url wins", "jellyfin",
|
||||
map[string]any{"url": "https://media.hubris.network"}, "https://media.hubris.network"},
|
||||
{"public_host becomes https", "jellyfin",
|
||||
map[string]any{"public_host": "media.hubris.network"}, "https://media.hubris.network"},
|
||||
// Ingress routes carry the hostname as the entity name and usually
|
||||
// declare no attributes at all.
|
||||
{"hostname-shaped name", "media.hubris.network", nil, "https://media.hubris.network"},
|
||||
{"a bare service name is not a hostname", "jellyfin", nil, ""},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
got := httpURL(Target{Name: c.name}, c.attrs)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: httpURL = %q, want %q", c.desc, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildKindReportsWhyItSkipped(t *testing.T) {
|
||||
// A declared kind that cannot be built must explain itself rather than
|
||||
// vanish — that silence is what hid the coverage gap.
|
||||
if defs, reason := buildKind(KindPing, Target{}, nil, "", "root", 22); len(defs) != 0 || reason == "" {
|
||||
t.Errorf("ping without a host should skip with a reason, got %d defs / %q", len(defs), reason)
|
||||
}
|
||||
if defs, reason := buildKind(KindProcess, Target{Name: ""}, nil, "10.0.0.1", "root", 22); len(defs) != 0 || reason == "" {
|
||||
t.Errorf("process without a name should skip with a reason, got %d defs / %q", len(defs), reason)
|
||||
}
|
||||
if defs, reason := buildKind("dns", Target{}, nil, "10.0.0.1", "root", 22); len(defs) != 0 || reason == "" {
|
||||
t.Errorf("an unimplemented kind should skip with a reason, got %d defs / %q", len(defs), reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildKindProcessPassesTheUnitName(t *testing.T) {
|
||||
// process_check.sh reads $1 and answers "no service name provided"
|
||||
// without it. checkdefaults always wrote args; nothing read them.
|
||||
defs, reason := buildKind(KindProcess, Target{Name: "jellyfin"}, nil, "10.0.0.1", "root", 22)
|
||||
if len(defs) != 1 {
|
||||
t.Fatalf("expected one process check, got %d (%s)", len(defs), reason)
|
||||
}
|
||||
if got := defs[0].config["args"]; got != "jellyfin" {
|
||||
t.Errorf("process check args = %v, want jellyfin", got)
|
||||
}
|
||||
if got := defs[0].config["script"]; got != "process_check.sh" {
|
||||
t.Errorf("process check script = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildKindHTTPUsesAStatusRangeNotAnExactCode(t *testing.T) {
|
||||
// Most services sit behind Authentik and answer 302/401.
|
||||
defs, _ := buildKind(KindHTTP, Target{Name: "jellyfin"},
|
||||
map[string]any{"url": "https://media.hubris.network"}, "", "root", 22)
|
||||
if len(defs) != 1 {
|
||||
t.Fatalf("expected one http check, got %d", len(defs))
|
||||
}
|
||||
if got := defs[0].config["max_status"]; got != 500 {
|
||||
t.Errorf("max_status = %v, want 500", got)
|
||||
}
|
||||
if _, exact := defs[0].config["expected_status"]; exact {
|
||||
t.Error("default http checks must not pin an exact status")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildKindResourceExpandsToFourScripts(t *testing.T) {
|
||||
defs, _ := buildKind(KindResource, Target{}, nil, "10.0.0.1", "root", 22)
|
||||
if len(defs) != 4 {
|
||||
t.Fatalf("resource should expand to 4 checks, got %d", len(defs))
|
||||
}
|
||||
for _, d := range defs {
|
||||
if d.kind != "ssh-script" {
|
||||
t.Errorf("resource check kind = %q, want ssh-script", d.kind)
|
||||
}
|
||||
if d.config["host"] != "10.0.0.1" {
|
||||
t.Errorf("resource check lost its host: %v", d.config)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,17 @@ type Config struct {
|
||||
// port than the API); a no-op when the SPA and API share an origin.
|
||||
CORSAllowedOrigin string
|
||||
|
||||
// Rate limiting (plan D3). APIRateLimit is the per-IP requests/sec cap;
|
||||
// APIRateBurst is the token-bucket burst (defaults to 2x the limit when
|
||||
// unset). A limit of 0 disables rate limiting entirely.
|
||||
APIRateLimit int
|
||||
APIRateBurst int
|
||||
|
||||
// Health probe HTTP listener (plan D5). Background-loop roles (scheduler,
|
||||
// notifier) expose a staleness-aware /healthz here. Empty disables the
|
||||
// health server (local/non-docker runs).
|
||||
HealthListen string
|
||||
|
||||
// Observability
|
||||
Debug bool // verbose logging, probe payloads, SQL
|
||||
|
||||
@@ -121,6 +132,11 @@ func FromEnv() Config {
|
||||
if v := os.Getenv("OIKOS_CORS_ORIGIN"); v != "" {
|
||||
c.CORSAllowedOrigin = v
|
||||
}
|
||||
c.APIRateLimit = parseInt(os.Getenv("OIKOS_API_RATE_LIMIT"))
|
||||
c.APIRateBurst = parseInt(os.Getenv("OIKOS_API_RATE_BURST"))
|
||||
if v := os.Getenv("OIKOS_HEALTH_LISTEN"); v != "" {
|
||||
c.HealthListen = v
|
||||
}
|
||||
if v := os.Getenv("OIKOS_SEEDS_DIR"); v != "" {
|
||||
c.SeedsDir = v
|
||||
}
|
||||
|
||||
34
internal/db/checks.go
Normal file
34
internal/db/checks.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/dtoro/oikos/internal/checkdefaults"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// EnsureEntityChecks derives an entity's default check_defs from the
|
||||
// monitoring spec of its type (resolving per-entity `monitoring` overrides).
|
||||
//
|
||||
// This is the single shared hook that keeps the check graph in sync with
|
||||
// entity mutations. Both the HTTP create/patch handlers and the MCP
|
||||
// entity-mutation tools (create_entity, update_entity_attributes) call it so
|
||||
// that flipping an entity's `monitoring` attribute regenerates checks
|
||||
// regardless of which surface made the change — previously only the HTTP
|
||||
// path ran check derivation, so entities mutated via MCP silently produced no
|
||||
// checks (see plans/2026-08-03-session-review-haos-monitoring-capability-gaps.md, A2).
|
||||
func EnsureEntityChecks(ctx context.Context, tx pgx.Tx, id uuid.UUID, slug, entityType, name string, attrs []byte) (checkdefaults.Result, error) {
|
||||
tree, err := LoadTypeTree(ctx, tx)
|
||||
if err != nil {
|
||||
return checkdefaults.Result{}, err
|
||||
}
|
||||
res, err := checkdefaults.Ensure(ctx, tx, tree, checkdefaults.Target{
|
||||
ID: id, Slug: slug, Type: entityType, Name: name, Attrs: attrs,
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
checkdefaults.LogResult(slug, entityType, res)
|
||||
return res, nil
|
||||
}
|
||||
61
internal/db/entity_cache.go
Normal file
61
internal/db/entity_cache.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type entityCacheEntry struct {
|
||||
slug string
|
||||
id string
|
||||
attrs string
|
||||
exp time.Time
|
||||
}
|
||||
|
||||
type EntityCache struct {
|
||||
mu sync.RWMutex
|
||||
m map[string]entityCacheEntry
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
func NewEntityCache(ttl time.Duration) *EntityCache {
|
||||
return &EntityCache{
|
||||
m: make(map[string]entityCacheEntry),
|
||||
ttl: ttl,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *EntityCache) GetSlug(id string) (string, bool) {
|
||||
c.mu.RLock()
|
||||
e, ok := c.m[id]
|
||||
c.mu.RUnlock()
|
||||
if !ok || time.Now().After(e.exp) {
|
||||
return "", false
|
||||
}
|
||||
return e.slug, true
|
||||
}
|
||||
|
||||
func (c *EntityCache) GetID(slug string) (string, bool) {
|
||||
c.mu.RLock()
|
||||
e, ok := c.m[slug]
|
||||
c.mu.RUnlock()
|
||||
if !ok || time.Now().After(e.exp) {
|
||||
return "", false
|
||||
}
|
||||
return e.id, true
|
||||
}
|
||||
|
||||
func (c *EntityCache) Set(slug, id, attrs string) {
|
||||
exp := time.Now().Add(c.ttl)
|
||||
c.mu.Lock()
|
||||
c.m[slug] = entityCacheEntry{slug: slug, id: id, attrs: attrs, exp: exp}
|
||||
c.m[id] = entityCacheEntry{slug: slug, id: id, attrs: attrs, exp: exp}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *EntityCache) Invalidate(slug, id string) {
|
||||
c.mu.Lock()
|
||||
delete(c.m, slug)
|
||||
delete(c.m, id)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
@@ -167,6 +167,62 @@ func TestSeedIngestIdempotentAndNoDuplicateEdges(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Regression: insertOneEntityType read tMap["attribute_schema"], but
|
||||
// seeds/ontology.yaml spells the key `attributes:`. The mismatch marshalled a
|
||||
// nil into the JSON literal `null` for every one of the 60 types, so no
|
||||
// attribute schema was ever ingested — the API and `oikos export` returned
|
||||
// null across the board, silently, for the life of the project.
|
||||
func TestSeedIngestsAttributeSchemas(t *testing.T) {
|
||||
pool := newTestPool(t)
|
||||
seedAll(t, pool, seedsDir())
|
||||
|
||||
if n := count(t, pool,
|
||||
`SELECT count(*) FROM entity_types WHERE attribute_schema = 'null'::jsonb`); n != 0 {
|
||||
t.Errorf("%d entity types stored the JSON literal null instead of a schema or SQL NULL", n)
|
||||
}
|
||||
|
||||
if n := count(t, pool,
|
||||
`SELECT count(*) FROM entity_types WHERE jsonb_typeof(attribute_schema) = 'object'`); n == 0 {
|
||||
t.Fatal("no entity type ingested an attribute schema")
|
||||
}
|
||||
|
||||
// A type declaring `attributes:` must round-trip its properties.
|
||||
if n := count(t, pool, `SELECT count(*) FROM entity_types
|
||||
WHERE name = 'lxc' AND attribute_schema #>> '{properties,pve_id,type}' = 'integer'`); n != 1 {
|
||||
t.Error("lxc.attribute_schema lost its declared pve_id property")
|
||||
}
|
||||
|
||||
// A type declaring none stores SQL NULL, not a JSON null.
|
||||
if n := count(t, pool, `SELECT count(*) FROM entity_types
|
||||
WHERE name = 'sensor' AND attribute_schema IS NULL`); n != 1 {
|
||||
t.Error("a type declaring no attributes should store SQL NULL")
|
||||
}
|
||||
}
|
||||
|
||||
// monitoring_spec drives which entities coverageSweep may flag as unmonitored,
|
||||
// so the three states have to survive ingest distinctly: SQL NULL (undeclared,
|
||||
// resolved from an ancestor or the layer default), '[]' (explicitly
|
||||
// unmonitorable), and a non-empty array (the kinds the type warrants).
|
||||
func TestSeedIngestsMonitoringSpec(t *testing.T) {
|
||||
pool := newTestPool(t)
|
||||
seedAll(t, pool, seedsDir())
|
||||
|
||||
cases := []struct {
|
||||
typ, where, desc string
|
||||
}{
|
||||
{"service", `monitoring_spec = '["http","process"]'::jsonb`, "declared kinds"},
|
||||
{"machine", `monitoring_spec = '["ping","resource","updates"]'::jsonb`, "declared on an abstract type"},
|
||||
{"site", `monitoring_spec = '[]'::jsonb`, "explicitly unmonitorable"},
|
||||
{"lxc", `monitoring_spec IS NULL`, "inherits from container, so its own column is NULL"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if n := count(t, pool, fmt.Sprintf(
|
||||
`SELECT count(*) FROM entity_types WHERE name = '%s' AND %s`, c.typ, c.where)); n != 1 {
|
||||
t.Errorf("%s (%s): monitoring_spec did not match %s", c.typ, c.desc, c.where)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAbstractTypeRejected(t *testing.T) {
|
||||
pool := newTestPool(t)
|
||||
seedAll(t, pool, seedsDir())
|
||||
@@ -250,7 +306,17 @@ func TestBlastRadiusTerminatesOnCycles(t *testing.T) {
|
||||
pool := newTestPool(t)
|
||||
seedAll(t, pool, seedsDir())
|
||||
|
||||
// Build a dependency cycle: gitea → caddy → authentik → gitea
|
||||
// Build a dependency cycle: gitea → caddy → authentik → gitea.
|
||||
//
|
||||
// `depends-on` is declared blast_direction: backward — "A depends-on B"
|
||||
// means B failing breaks A — so the blast radius of gitea walks the edges
|
||||
// BACKWARDS: whoever depends on gitea is affected first. That is authentik
|
||||
// (1 hop), then caddy which depends on authentik (2 hops).
|
||||
//
|
||||
// This test previously asserted caddy=1, authentik=2, which is the same
|
||||
// cycle walked the wrong way round: blast_radius used to follow every edge
|
||||
// source→target regardless of what the edge means, so it answered "what
|
||||
// does gitea depend on" while being named for the opposite question.
|
||||
cycle := []byte(`
|
||||
version: 1
|
||||
relationships:
|
||||
@@ -286,14 +352,24 @@ relationships:
|
||||
}
|
||||
got[slug] = depth
|
||||
}
|
||||
want := map[string]int{"service:gitea": 0, "service:caddy": 1, "service:authentik": 2}
|
||||
want := map[string]int{"service:gitea": 0, "service:authentik": 1, "service:caddy": 2}
|
||||
for slug, depth := range want {
|
||||
if got[slug] != depth {
|
||||
t.Errorf("blast_radius[%s] = %d, want %d (full: %v)", slug, got[slug], depth, got)
|
||||
}
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Errorf("blast_radius returned %d nodes, want %d: %v", len(got), len(want), got)
|
||||
// Deliberately not an exact node count. Walking the right way round also
|
||||
// surfaces the real seed's own dependents of gitea (homelab-mcp and what
|
||||
// depends on it), which are correct answers — the old exact-count
|
||||
// assertion only held because the forward walk found nothing real.
|
||||
// What matters here is that the cycle terminates rather than recursing.
|
||||
if len(got) > 20 {
|
||||
t.Errorf("blast_radius did not terminate sensibly: %d nodes: %v", len(got), got)
|
||||
}
|
||||
for slug, depth := range got {
|
||||
if depth > 5 {
|
||||
t.Errorf("blast_radius[%s] = %d, beyond the max_depth bound", slug, depth)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
182
internal/db/lifecycle.go
Normal file
182
internal/db/lifecycle.go
Normal file
@@ -0,0 +1,182 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ErrTransitionInvalid is a sentinel returned by ValidateTransition when the
|
||||
// from→to pair is not a declared lifecycle transition or a precondition fails.
|
||||
// Callers test with errors.Is to distinguish semantic validation failures
|
||||
// (→ HTTP 409) from infrastructure errors (→ HTTP 500).
|
||||
var ErrTransitionInvalid = errors.New("invalid lifecycle transition")
|
||||
|
||||
// ValidateTransition enforces an entity type's lifecycle: fromState → toState
|
||||
// must be a declared transition, and every precondition it lists must hold. A
|
||||
// type with no lifecycle defined allows any state. A no-op (fromState ==
|
||||
// toState) passes immediately.
|
||||
//
|
||||
// Shared by the HTTP PATCH path and the MCP set_entity_state tool so both
|
||||
// surfaces apply identical lifecycle rules — previously only the HTTP path
|
||||
// validated transitions, so an agent changing state via MCP could skip the
|
||||
// graph's retire/deprecate guardrails entirely.
|
||||
func ValidateTransition(ctx context.Context, tx pgx.Tx, entityID uuid.UUID, entityType, fromState, toState string) error {
|
||||
if toState == fromState {
|
||||
return nil
|
||||
}
|
||||
lc, err := sqlcgen.New(tx).GetLifecycleForType(ctx, entityType)
|
||||
if err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil // no lifecycle defined → any state allowed
|
||||
}
|
||||
return err
|
||||
}
|
||||
var transitions map[string]map[string]json.RawMessage
|
||||
if err := json.Unmarshal(lc.Transitions, &transitions); err != nil {
|
||||
return fmt.Errorf("parse lifecycle transitions: %w", err)
|
||||
}
|
||||
tos, ok := transitions[fromState]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: no transitions defined from %q", ErrTransitionInvalid, fromState)
|
||||
}
|
||||
trans, ok := tos[toState]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s → %s is not a declared lifecycle transition", ErrTransitionInvalid, fromState, toState)
|
||||
}
|
||||
var gate struct {
|
||||
Requires []string `json:"requires"`
|
||||
}
|
||||
if err := json.Unmarshal(trans, &gate); err == nil {
|
||||
for _, check := range gate.Requires {
|
||||
if err := checkPrecondition(ctx, tx, entityID, entityType, check); err != nil {
|
||||
return fmt.Errorf("%w: precondition %q not met: %w", ErrTransitionInvalid, check, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkPrecondition evaluates one mechanical precondition named by a lifecycle
|
||||
// transition's `requires` list. Soft/operator-confirmed checks pass; unknown
|
||||
// checks are skipped (operator intent overrides). Moved here from httpapi so
|
||||
// both surfaces share one implementation.
|
||||
func checkPrecondition(ctx context.Context, tx pgx.Tx, entityID uuid.UUID, entityType, check string) error {
|
||||
switch check {
|
||||
case "no-inbound-edges":
|
||||
var count int
|
||||
if err := tx.QueryRow(ctx,
|
||||
"SELECT count(*) FROM relationships WHERE target_id = $1 AND valid_to IS NULL", entityID).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count > 0 {
|
||||
return fmt.Errorf("%d inbound relationship edges remaining", count)
|
||||
}
|
||||
case "backups-verified", "secrets-revoked", "ingress-dns-removed":
|
||||
attrs, err := fetchAttrs(ctx, tx, entityID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
want := map[string]string{
|
||||
"backups-verified": "backups_verified",
|
||||
"secrets-revoked": "secrets_revoked",
|
||||
"ingress-dns-removed": "ingress_dns_removed",
|
||||
}[check]
|
||||
if !attrTruthy(attrs, want) {
|
||||
return fmt.Errorf("%s not recorded in entity attributes", want)
|
||||
}
|
||||
case "age-key-enrolled-if-needed":
|
||||
if entityType == "workstation" {
|
||||
attrs, err := fetchAttrs(ctx, tx, entityID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !attrTruthy(attrs, "age_pubkey") {
|
||||
return fmt.Errorf("age key not enrolled (no age_pubkey in attributes)")
|
||||
}
|
||||
}
|
||||
case "mesh-joined-if-needed":
|
||||
if entityType == "workstation" {
|
||||
attrs, err := fetchAttrs(ctx, tx, entityID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !attrTruthy(attrs, "mesh_ip") {
|
||||
return fmt.Errorf("mesh not joined (no mesh_ip in attributes)")
|
||||
}
|
||||
}
|
||||
case "health-check-answering":
|
||||
st, err := sqlcgen.New(tx).GetEntityStatus(ctx, entityID)
|
||||
if err != nil || st.Health == "unknown" || st.Health == "down" {
|
||||
h := "unknown"
|
||||
if err == nil {
|
||||
h = st.Health
|
||||
}
|
||||
return fmt.Errorf("health check not answering (status: %s)", h)
|
||||
}
|
||||
case "doc-page-complete":
|
||||
var count int
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT count(*) FROM relationships r
|
||||
JOIN entities ke ON ke.id = r.source_id
|
||||
WHERE r.target_id = $1 AND r.valid_to IS NULL
|
||||
AND r.type = 'documents' AND ke.type IN ('document','runbook','investigation')`,
|
||||
entityID).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count == 0 {
|
||||
return fmt.Errorf("no documentation linked to entity")
|
||||
}
|
||||
case "inventory-entry", "ip-reserved", "storage-pool-chosen", "cancelled-note",
|
||||
"preflight-passed", "error-summary", "replacement-live-or-role-retired",
|
||||
"replacement-failed", "post-verify-passed", "recovery-verified", "written-off",
|
||||
"ingress-live-if-public", "doc-page-stub", "un-deprecate-note", "write-off-note":
|
||||
// Soft checks — always pass. Operator-confirmed via the transition
|
||||
// request itself, or not mechanically enforceable.
|
||||
default:
|
||||
// Unknown preconditions are skipped (operator intent overrides).
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchAttrs loads an entity's JSONB attributes column as a decoded map.
|
||||
// Missing attributes decode to an empty map (every key absent).
|
||||
func fetchAttrs(ctx context.Context, tx pgx.Tx, entityID uuid.UUID) (map[string]any, error) {
|
||||
var raw string
|
||||
if err := tx.QueryRow(ctx, "SELECT coalesce(attributes::text,'{}') FROM entities WHERE id = $1", entityID).Scan(&raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var attrs map[string]any
|
||||
if err := json.Unmarshal([]byte(raw), &attrs); err != nil {
|
||||
return nil, fmt.Errorf("decode entity attributes: %w", err)
|
||||
}
|
||||
if attrs == nil {
|
||||
attrs = map[string]any{}
|
||||
}
|
||||
return attrs, nil
|
||||
}
|
||||
|
||||
// attrTruthy reports whether key is present in attrs with a meaningful value.
|
||||
// It replaces substring matching on raw JSONB text: a previous strings.Contains
|
||||
// check treated {"backups_verified": false} as satisfied (the key text was
|
||||
// present) and bypassed the attributes GIN index. Booleans must be true;
|
||||
// strings must be non-empty; nil/absent fail.
|
||||
func attrTruthy(attrs map[string]any, key string) bool {
|
||||
v, ok := attrs[key]
|
||||
if !ok || v == nil {
|
||||
return false
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case bool:
|
||||
return t
|
||||
case string:
|
||||
return t != ""
|
||||
default:
|
||||
return true // numbers, objects, arrays count as present
|
||||
}
|
||||
}
|
||||
55
internal/db/lifecycle_test.go
Normal file
55
internal/db/lifecycle_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// attrTruthy replaces a previous strings.Contains check over raw JSONB text.
|
||||
// The key regression it guards: a literal attribute like
|
||||
// {"backups_verified": false} must NOT satisfy the "backups-verified"
|
||||
// precondition, even though the key text is present in the column.
|
||||
func TestAttrTruthy(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
attrs map[string]any
|
||||
key string
|
||||
want bool
|
||||
}{
|
||||
{"absent", map[string]any{}, "backups_verified", false},
|
||||
{"nil map", nil, "backups_verified", false},
|
||||
{"explicit nil value", map[string]any{"backups_verified": nil}, "backups_verified", false},
|
||||
{"bool true", map[string]any{"backups_verified": true}, "backups_verified", true},
|
||||
{"bool false is the regression case", map[string]any{"backups_verified": false}, "backups_verified", false},
|
||||
{"nonempty string age pubkey", map[string]any{"age_pubkey": "age1abc"}, "age_pubkey", true},
|
||||
{"empty string is falsy", map[string]any{"mesh_ip": ""}, "mesh_ip", false},
|
||||
{"number counts as present", map[string]any{"port": float64(22)}, "port", true},
|
||||
{"other keys present", map[string]any{"backups_verified": true, "unrelated": "x"}, "backups_verified", true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := attrTruthy(tc.attrs, tc.key); got != tc.want {
|
||||
t.Fatalf("attrTruthy(%v, %q) = %v, want %v", tc.attrs, tc.key, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// fetchAttrs decodes the JSONB column text; verify the decode shape that
|
||||
// attrTruthy then evaluates (the DB round-trip itself is covered by make test-db).
|
||||
func TestAttrTruthyAfterDecode(t *testing.T) {
|
||||
raw := `{"backups_verified": true, "mesh_ip": "10.0.0.5", "secrets_revoked": false}`
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal([]byte(raw), &got); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if !attrTruthy(got, "backups_verified") {
|
||||
t.Error("backups_verified should be truthy after decode")
|
||||
}
|
||||
if !attrTruthy(got, "mesh_ip") {
|
||||
t.Error("mesh_ip should be truthy after decode")
|
||||
}
|
||||
if attrTruthy(got, "secrets_revoked") {
|
||||
t.Error("secrets_revoked:false is the regression — must be falsy")
|
||||
}
|
||||
}
|
||||
@@ -190,7 +190,10 @@ func hasSuffix(s, suffix string) bool {
|
||||
}
|
||||
|
||||
// splitSQL splits a SQL string into individual statements.
|
||||
// Handles $$ ... $$ dollar-quoted blocks and -- line comments.
|
||||
// Handles $$ ... $$ dollar-quoted blocks, $tag$ ... $tag$ tagged quotes,
|
||||
// -- line comments, /* ... */ block comments, and '...' string literals
|
||||
// so that semicolons inside any of these constructs are not treated as
|
||||
// statement boundaries.
|
||||
func splitSQL(sql string) []string {
|
||||
var statements []string
|
||||
var current strings.Builder
|
||||
@@ -201,7 +204,6 @@ func splitSQL(sql string) []string {
|
||||
for i < len(sql) {
|
||||
// Handle line comments (-- to end of line)
|
||||
if !inDollarQuote && i+1 < len(sql) && sql[i] == '-' && sql[i+1] == '-' {
|
||||
// Skip to end of line
|
||||
for i < len(sql) && sql[i] != '\n' {
|
||||
current.WriteByte(sql[i])
|
||||
i++
|
||||
@@ -209,6 +211,34 @@ func splitSQL(sql string) []string {
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle block comments (/* ... */)
|
||||
if !inDollarQuote && i+1 < len(sql) && sql[i] == '/' && sql[i+1] == '*' {
|
||||
end := strings.Index(sql[i+2:], "*/")
|
||||
if end >= 0 {
|
||||
current.WriteString(sql[i : i+end+4])
|
||||
i += end + 4
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Handle single-quoted string literals ('...')
|
||||
if !inDollarQuote && sql[i] == '\'' {
|
||||
j := i + 1
|
||||
for j < len(sql) {
|
||||
if sql[j] == '\'' {
|
||||
if j+1 < len(sql) && sql[j+1] == '\'' {
|
||||
j += 2 // skip doubled quote ''
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
j++
|
||||
}
|
||||
current.WriteString(sql[i : j+1])
|
||||
i = j + 1
|
||||
continue
|
||||
}
|
||||
|
||||
// Check for dollar-quote start/end
|
||||
if !inDollarQuote && sql[i] == '$' {
|
||||
j := i + 1
|
||||
|
||||
@@ -44,6 +44,36 @@ UPDATE entities SET
|
||||
WHERE id = sqlc.arg('id') AND version = sqlc.arg('version')
|
||||
RETURNING *;
|
||||
|
||||
-- name: MergeEntityAttributes :execrows
|
||||
-- Shallow-merge a JSON patch into an entity's attributes (the
|
||||
-- update_entity_attributes MCP/HTTP surface). Replaces the raw
|
||||
-- `attributes = attributes || $2::jsonb` used in entity_tools.go.
|
||||
UPDATE entities SET
|
||||
attributes = attributes || sqlc.arg('patch')::jsonb,
|
||||
updated_at = now()
|
||||
WHERE slug = sqlc.arg('slug');
|
||||
|
||||
-- name: SetEntityState :execrows
|
||||
-- Set an entity's lifecycle state by id (the set_entity_state surface, run
|
||||
-- after db.ValidateTransition). Replaces the raw
|
||||
-- `UPDATE entities SET state = $2 ... WHERE id = $1`.
|
||||
UPDATE entities SET
|
||||
state = sqlc.arg('state'),
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg('id');
|
||||
|
||||
-- blast_radius(): the recursive-CTE traversal function's TABLE return type
|
||||
-- is opaque to sqlc's analyzer — that one query stays hand-written pgx in
|
||||
-- internal/httpapi (see impl.go).
|
||||
-- internal/httpapi (see entities.go GetBlastRadius).
|
||||
--
|
||||
-- Deliberate raw-SQL exceptions (plan E2): the httpapi entity *read* handlers
|
||||
-- (ListEntities/GetEntity/GetGraph/queryEntities) project a fixed
|
||||
-- `entityCols` column set (entities.* + a LEFT JOIN to entity_status for
|
||||
-- health/last_check_at) and scan it positionally into the oapi-generated
|
||||
-- gen.Entity shape. sqlc generates its own row struct per query and cannot
|
||||
-- emit gen.Entity, so migrating those reads would add a per-call field-by-
|
||||
-- field mapping with no compile-time gain and real column-order risk. They
|
||||
-- stay hand-written pgx, like blast_radius and the seed/export bulk paths
|
||||
-- noted in sqlc.yaml. The mutation/relationship surface (MergeEntityAttributes,
|
||||
-- SetEntityState, InsertRelationshipIfAbsent, EndCurrentRelationship) IS
|
||||
-- migrated and is what the entity CRUD tools now call.
|
||||
|
||||
@@ -25,8 +25,8 @@ ON CONFLICT (actor, key) DO NOTHING;
|
||||
|
||||
-- name: InsertAuditEntry :exec
|
||||
INSERT INTO audit_log (actor_type, actor_id, action, entity_id, method, path,
|
||||
status_code, detail, source_ip, correlation_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);
|
||||
status_code, detail, source_ip, correlation_id, session_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11);
|
||||
|
||||
-- name: InsertEvent :one
|
||||
INSERT INTO events (type, entity_id, severity, source, data, correlation_id)
|
||||
@@ -55,12 +55,36 @@ FROM events WHERE id > $1 ORDER BY id ASC LIMIT $2;
|
||||
-- =====================================================================
|
||||
|
||||
-- name: ListEnabledCheckDefs :many
|
||||
-- Enabled AND due. interval_s used to be selected but never filtered on, so
|
||||
-- every check ran on every 30s pass and the declared intervals meant nothing.
|
||||
-- NULL last_run_at = never run = due now.
|
||||
SELECT cd.entity_id, cd.target_id, cd.target_type, cd.kind, cd.config,
|
||||
cd.interval_s, cd.timeout_s, cd.zone, cd.enabled, cd.updated_at,
|
||||
e.slug AS entity_slug
|
||||
FROM check_defs cd
|
||||
JOIN entities e ON e.id = cd.entity_id
|
||||
WHERE cd.enabled = true;
|
||||
LEFT JOIN entities tgt ON tgt.id = cd.target_id
|
||||
WHERE cd.enabled = true
|
||||
AND (tgt.id IS NULL OR tgt.state IS NULL OR tgt.state NOT IN ('deprecated', 'destroyed'))
|
||||
AND (cd.last_run_at IS NULL
|
||||
OR cd.last_run_at <= now() - make_interval(secs => cd.interval_s));
|
||||
|
||||
-- name: MarkCheckRun :exec
|
||||
UPDATE check_defs SET last_run_at = now(), last_health = $2 WHERE entity_id = $1;
|
||||
|
||||
-- name: WorstHealthForTarget :one
|
||||
-- An entity is as healthy as its unhealthiest check. Checks that have not run
|
||||
-- yet (last_health IS NULL) are ignored rather than counted as unknown, so a
|
||||
-- newly added check does not drag a known-good entity down before it has
|
||||
-- produced a verdict.
|
||||
SELECT COALESCE(
|
||||
(SELECT last_health FROM check_defs
|
||||
WHERE enabled AND target_id = $1 AND last_health IS NOT NULL
|
||||
ORDER BY CASE last_health
|
||||
WHEN 'down' THEN 0 WHEN 'degraded' THEN 1 WHEN 'stale' THEN 2
|
||||
WHEN 'unknown' THEN 3 ELSE 4 END
|
||||
LIMIT 1),
|
||||
'unknown')::text AS health;
|
||||
|
||||
-- name: GetCheckDef :one
|
||||
SELECT * FROM check_defs WHERE entity_id = $1;
|
||||
|
||||
@@ -25,3 +25,18 @@ ORDER BY r.type, se.slug, te.slug;
|
||||
-- name: EndCurrentRelationship :execrows
|
||||
UPDATE relationships SET valid_to = now()
|
||||
WHERE source_id = $1 AND target_id = $2 AND type = $3 AND valid_to IS NULL;
|
||||
|
||||
-- name: InsertRelationshipIfAbsent :execrows
|
||||
-- Idempotent relationship insert (the create_relationship surface): no-op if
|
||||
-- an active edge of the same source/target/type already exists. Replaces the
|
||||
-- raw INSERT...WHERE NOT EXISTS used in entity_tools.go.
|
||||
INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT sqlc.arg('source_id'), sqlc.arg('target_id'), sqlc.arg('type'),
|
||||
sqlc.arg('attributes')::jsonb, now()
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM relationships
|
||||
WHERE source_id = sqlc.arg('source_id')
|
||||
AND target_id = sqlc.arg('target_id')
|
||||
AND type = sqlc.arg('type')
|
||||
AND valid_to IS NULL
|
||||
);
|
||||
|
||||
@@ -13,14 +13,15 @@ import (
|
||||
|
||||
// SeedResult holds counts from a seed ingest operation.
|
||||
type SeedResult struct {
|
||||
Lifecycles int
|
||||
EntityTypes int
|
||||
Lifecycles int
|
||||
EntityTypes int
|
||||
RelationshipTypes int
|
||||
Entities int
|
||||
Relationships int
|
||||
RiskClasses int
|
||||
ApprovalRules int
|
||||
AutonomySettings int
|
||||
Entities int
|
||||
Relationships int
|
||||
RiskClasses int
|
||||
ApprovalRules int
|
||||
AutonomySettings int
|
||||
Checks int
|
||||
}
|
||||
|
||||
// IngestOntologySeed ingests seeds/ontology.yaml into the DB.
|
||||
@@ -66,12 +67,20 @@ func IngestOntologySeed(ctx context.Context, tx pgx.Tx, data map[string]any) (*S
|
||||
targetType, _ := rtMap["target"].(string)
|
||||
cardinality, _ := rtMap["cardinality"].(string)
|
||||
desc, _ := rtMap["description"].(string)
|
||||
// Which end of the edge depends on the other; drives blast_radius().
|
||||
// Absent means 'none' — an undeclared edge contributes nothing rather
|
||||
// than silently producing a wrong dependency answer.
|
||||
blastDirection, _ := rtMap["blast_direction"].(string)
|
||||
if blastDirection == "" {
|
||||
blastDirection = "none"
|
||||
}
|
||||
_, err := tx.Exec(ctx,
|
||||
`INSERT INTO relationship_types (name, inverse, source_type, target_type, cardinality, description)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
`INSERT INTO relationship_types (name, inverse, source_type, target_type, cardinality, description, blast_direction)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (name) DO UPDATE SET inverse = $2, source_type = $3,
|
||||
target_type = $4, cardinality = $5, description = $6`,
|
||||
name, nullableStr(inverse), sourceType, targetType, cardinality, desc)
|
||||
target_type = $4, cardinality = $5, description = $6,
|
||||
blast_direction = $7`,
|
||||
name, nullableStr(inverse), sourceType, targetType, cardinality, desc, blastDirection)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("relationship_type %s: %w", name, err)
|
||||
}
|
||||
@@ -96,6 +105,7 @@ func IngestInventorySeed(ctx context.Context, tx pgx.Tx, data map[string]any) (*
|
||||
// Entities
|
||||
entities, _ := data["entities"].([]any)
|
||||
entityTypes := make(map[string]string) // slug -> type, for edge validation
|
||||
var pendingChecks []checkdefaults.Target
|
||||
for _, raw := range entities {
|
||||
eMap, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
@@ -144,7 +154,12 @@ func IngestInventorySeed(ctx context.Context, tx pgx.Tx, data map[string]any) (*
|
||||
return nil, fmt.Errorf("entity_status %s: %w", slug, err)
|
||||
}
|
||||
|
||||
checkdefaults.Ensure(ctx, tx, entityID, slug, typeName, attrsBytes)
|
||||
// Default checks are deferred until after relationships are ingested:
|
||||
// a service has no address of its own and inherits its container's,
|
||||
// which means the hosting edge has to exist first.
|
||||
pendingChecks = append(pendingChecks, checkdefaults.Target{
|
||||
ID: entityID, Slug: slug, Type: typeName, Name: name, Attrs: attrsBytes,
|
||||
})
|
||||
|
||||
r.Entities++
|
||||
}
|
||||
@@ -208,6 +223,19 @@ func IngestInventorySeed(ctx context.Context, tx pgx.Tx, data map[string]any) (*
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Default checks, now that hosting edges exist. Errors here are fatal:
|
||||
// swallowing them is what let a foreign-key violation abort the ingest
|
||||
// transaction while surfacing as an unrelated failure several entities
|
||||
// later.
|
||||
for _, target := range pendingChecks {
|
||||
res, err := checkdefaults.Ensure(ctx, tx, tree, target)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("default checks for %s: %w", target.Slug, err)
|
||||
}
|
||||
checkdefaults.LogResult(target.Slug, target.Type, res)
|
||||
r.Checks += res.Created
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
@@ -361,19 +389,68 @@ func insertOneEntityType(ctx context.Context, tx pgx.Tx, name string, tMap map[s
|
||||
layer, _ := tMap["layer"].(string)
|
||||
desc, _ := tMap["description"].(string)
|
||||
lifecycleID, _ := tMap["lifecycle"].(string)
|
||||
attrSchema := tMap["attribute_schema"]
|
||||
|
||||
schemaBytes, _ := json.Marshal(attrSchema)
|
||||
// seeds/ontology.yaml spells this `attributes:`. Reading it as
|
||||
// "attribute_schema" silently marshalled nil to the JSON literal `null`
|
||||
// for every type, so no attribute schema was ever ingested — the API and
|
||||
// `oikos export` returned null for all 60 types.
|
||||
attrSchema := tMap["attributes"]
|
||||
_, err := tx.Exec(ctx,
|
||||
`INSERT INTO entity_types (name, parent_type, is_abstract, domain, layer, description,
|
||||
lifecycle_id, attribute_schema, schema_version, status, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 1, 'active', now(), now())
|
||||
lifecycle_id, attribute_schema, monitoring_spec, schema_version, status, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 1, 'active', now(), now())
|
||||
ON CONFLICT (name) DO UPDATE SET parent_type = $2, is_abstract = $3, domain = $4,
|
||||
layer = $5, description = $6, lifecycle_id = $7, attribute_schema = $8, updated_at = now()`,
|
||||
name, nullableStr(parent), isAbstract, domain, layer, desc, nullableStr(lifecycleID), nullableStr(string(schemaBytes)))
|
||||
layer = $5, description = $6, lifecycle_id = $7, attribute_schema = $8,
|
||||
monitoring_spec = $9, updated_at = now()`,
|
||||
name, nullableStr(parent), isAbstract, domain, layer, desc, nullableStr(lifecycleID),
|
||||
attributeSchemaJSON(attrSchema), monitoringSpecJSON(tMap["monitoring"]))
|
||||
return err
|
||||
}
|
||||
|
||||
// attributeSchemaJSON marshals a type's `attributes:` block for storage,
|
||||
// mapping "the type declares no schema" to SQL NULL rather than to the JSON
|
||||
// literal `null`. Both readers already treat a JSON `null` as absent, but a
|
||||
// real NULL is what `attribute_schema IS NULL` expects and is what the column
|
||||
// meant all along.
|
||||
func attributeSchemaJSON(v any) any {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// monitoringSpecJSON normalises an entity type's `monitoring:` declaration into
|
||||
// the JSONB stored in entity_types.monitoring_spec. Three outcomes, and the
|
||||
// difference between the last two is load-bearing for coverage signalling:
|
||||
//
|
||||
// absent → nil (SQL NULL) — undeclared, an ontology gap
|
||||
// none | [] → "[]" — explicitly unmonitorable, by design
|
||||
// [http, resource]→ '["http","resource"]'
|
||||
//
|
||||
// `monitoring: none` is accepted as a more legible spelling of `[]`; YAML
|
||||
// parses the bare word as the string "none", not as null.
|
||||
func monitoringSpecJSON(v any) any {
|
||||
switch spec := v.(type) {
|
||||
case nil:
|
||||
return nil
|
||||
case string:
|
||||
if spec == "none" {
|
||||
return "[]"
|
||||
}
|
||||
// A single kind written unquoted, e.g. `monitoring: http`.
|
||||
b, _ := json.Marshal([]string{spec})
|
||||
return string(b)
|
||||
case []any:
|
||||
b, _ := json.Marshal(toStringSlice(spec))
|
||||
return string(b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func toStringSlice(v any) []string {
|
||||
if v == nil {
|
||||
return nil
|
||||
@@ -407,4 +484,3 @@ func keysOf(m map[string]map[string]any) []string {
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
|
||||
@@ -51,3 +51,57 @@ func TestSplitSQLSemicolonInComment(t *testing.T) {
|
||||
t.Fatalf("got %d statements, want 2: %#v", len(stmts), stmts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSQLSemicolonInStringLiteral(t *testing.T) {
|
||||
sql := `SELECT 'hello; world'; INSERT INTO t VALUES (1);`
|
||||
stmts := nonEmpty(splitSQL(sql))
|
||||
if len(stmts) != 2 {
|
||||
t.Fatalf("got %d statements, want 2: %#v", len(stmts), stmts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSQLDollarSignInStringLiteral(t *testing.T) {
|
||||
sql := `SELECT '$100'; SELECT 2;`
|
||||
stmts := nonEmpty(splitSQL(sql))
|
||||
if len(stmts) != 2 {
|
||||
t.Fatalf("got %d statements, want 2: %#v", len(stmts), stmts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSQLBlockComment(t *testing.T) {
|
||||
sql := `SELECT 1; /* block; with; semicolons */ SELECT 2;`
|
||||
stmts := nonEmpty(splitSQL(sql))
|
||||
if len(stmts) != 2 {
|
||||
t.Fatalf("got %d statements, want 2: %#v", len(stmts), stmts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSQLBlockCommentWithDollarQuote(t *testing.T) {
|
||||
sql := `/* $$ not a dollar quote */ SELECT 1;`
|
||||
stmts := nonEmpty(splitSQL(sql))
|
||||
if len(stmts) != 1 {
|
||||
t.Fatalf("got %d statements, want 1: %#v", len(stmts), stmts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSQLDoubledQuoteInString(t *testing.T) {
|
||||
sql := `SELECT 'O''Brien'; SELECT 2;`
|
||||
stmts := nonEmpty(splitSQL(sql))
|
||||
if len(stmts) != 2 {
|
||||
t.Fatalf("got %d statements, want 2: %#v", len(stmts), stmts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSQLEmptyInput(t *testing.T) {
|
||||
stmts := nonEmpty(splitSQL(""))
|
||||
if len(stmts) != 0 {
|
||||
t.Fatalf("got %d statements, want 0", len(stmts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSQLNoSemicolon(t *testing.T) {
|
||||
stmts := nonEmpty(splitSQL("SELECT 1"))
|
||||
if len(stmts) != 1 {
|
||||
t.Fatalf("got %d statements, want 1", len(stmts))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +177,52 @@ func (q *Queries) ListEntities(ctx context.Context, arg ListEntitiesParams) ([]E
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const mergeEntityAttributes = `-- name: MergeEntityAttributes :execrows
|
||||
UPDATE entities SET
|
||||
attributes = attributes || $1::jsonb,
|
||||
updated_at = now()
|
||||
WHERE slug = $2
|
||||
`
|
||||
|
||||
type MergeEntityAttributesParams struct {
|
||||
Patch []byte
|
||||
Slug string
|
||||
}
|
||||
|
||||
// Shallow-merge a JSON patch into an entity's attributes (the
|
||||
// update_entity_attributes MCP/HTTP surface). Replaces the raw
|
||||
// `attributes = attributes || $2::jsonb` used in entity_tools.go.
|
||||
func (q *Queries) MergeEntityAttributes(ctx context.Context, arg MergeEntityAttributesParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, mergeEntityAttributes, arg.Patch, arg.Slug)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const setEntityState = `-- name: SetEntityState :execrows
|
||||
UPDATE entities SET
|
||||
state = $1,
|
||||
updated_at = now()
|
||||
WHERE id = $2
|
||||
`
|
||||
|
||||
type SetEntityStateParams struct {
|
||||
State *string
|
||||
ID uuid.UUID
|
||||
}
|
||||
|
||||
// Set an entity's lifecycle state by id (the set_entity_state surface, run
|
||||
// after db.ValidateTransition). Replaces the raw
|
||||
// `UPDATE entities SET state = $2 ... WHERE id = $1`.
|
||||
func (q *Queries) SetEntityState(ctx context.Context, arg SetEntityStateParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, setEntityState, arg.State, arg.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const updateEntity = `-- name: UpdateEntity :one
|
||||
UPDATE entities SET
|
||||
name = COALESCE($1, name),
|
||||
|
||||
@@ -46,6 +46,8 @@ type AgentSession struct {
|
||||
Summary string
|
||||
EntityID *uuid.UUID
|
||||
CompletionNudges int32
|
||||
Blocker string
|
||||
ClosedAt *time.Time
|
||||
}
|
||||
|
||||
type Approval struct {
|
||||
@@ -110,6 +112,10 @@ type CheckDef struct {
|
||||
Zone *string
|
||||
Enabled bool
|
||||
UpdatedAt time.Time
|
||||
// When this check last executed. NULL = never, due immediately. Compared against interval_s to decide due-ness.
|
||||
LastRunAt *time.Time
|
||||
// This check's own most recent verdict (healthy/degraded/down/unknown). entity_status.health is the worst of these across the target's enabled checks.
|
||||
LastHealth *string
|
||||
}
|
||||
|
||||
type Classification struct {
|
||||
@@ -177,6 +183,8 @@ type EntityType struct {
|
||||
Status string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
// Check kinds this type warrants, resolved through parent_type. NULL means undeclared (an ontology gap), [] means explicitly unmonitorable, ["http","resource"] means declared kinds. Populated from seeds/ontology.yaml.
|
||||
MonitoringSpec []byte
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
@@ -211,6 +219,14 @@ type Execution struct {
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type ExecutionLog struct {
|
||||
ExecutionID uuid.UUID
|
||||
Ts time.Time
|
||||
Seq int32
|
||||
Stream string
|
||||
Chunk string
|
||||
}
|
||||
|
||||
type Feedback struct {
|
||||
EntityID uuid.UUID
|
||||
ExecutionID uuid.UUID
|
||||
@@ -241,6 +257,20 @@ type KnowledgeEntity struct {
|
||||
UpdatedAt time.Time
|
||||
ContentHash *string
|
||||
Search interface{}
|
||||
EditedBy string
|
||||
DeletedAt *time.Time
|
||||
}
|
||||
|
||||
type KnowledgeRevision struct {
|
||||
ID int64
|
||||
EntityID uuid.UUID
|
||||
Title string
|
||||
Content string
|
||||
Source *string
|
||||
Tags []string
|
||||
EditedBy string
|
||||
VersionAt time.Time
|
||||
RevisedAt time.Time
|
||||
}
|
||||
|
||||
type Ledger struct {
|
||||
@@ -350,6 +380,8 @@ type RelationshipType struct {
|
||||
Cardinality string
|
||||
Description *string
|
||||
CreatedAt time.Time
|
||||
// Which end of this edge depends on the other. forward = target depends on source. backward = source depends on target. none = no runtime dependency. Drives blast_radius().
|
||||
BlastDirection string
|
||||
}
|
||||
|
||||
type RiskClass struct {
|
||||
@@ -366,18 +398,19 @@ type SeedVersion struct {
|
||||
}
|
||||
|
||||
type SessionPlanStep struct {
|
||||
ID uuid.UUID
|
||||
SessionID uuid.UUID
|
||||
Seq int32
|
||||
Title string
|
||||
Detail string
|
||||
Status string
|
||||
ExecutionID *uuid.UUID
|
||||
TargetSlug *string
|
||||
StartedAt *time.Time
|
||||
FinishedAt *time.Time
|
||||
CreatedAt time.Time
|
||||
Generation int32
|
||||
ID uuid.UUID
|
||||
SessionID uuid.UUID
|
||||
Seq int32
|
||||
Title string
|
||||
Detail string
|
||||
Status string
|
||||
ExecutionID *uuid.UUID
|
||||
TargetSlug *string
|
||||
StartedAt *time.Time
|
||||
FinishedAt *time.Time
|
||||
CreatedAt time.Time
|
||||
Generation int32
|
||||
ReplacedReason *string
|
||||
}
|
||||
|
||||
type SessionQuestion struct {
|
||||
|
||||
@@ -30,7 +30,7 @@ func (q *Queries) GetLifecycleForType(ctx context.Context, name string) (Lifecyc
|
||||
}
|
||||
|
||||
const listEntityTypes = `-- name: ListEntityTypes :many
|
||||
SELECT name, parent_type, is_abstract, domain, layer, description, lifecycle_id, attribute_schema, schema_version, status, created_at, updated_at FROM entity_types ORDER BY name
|
||||
SELECT name, parent_type, is_abstract, domain, layer, description, lifecycle_id, attribute_schema, schema_version, status, created_at, updated_at, monitoring_spec FROM entity_types ORDER BY name
|
||||
`
|
||||
|
||||
func (q *Queries) ListEntityTypes(ctx context.Context) ([]EntityType, error) {
|
||||
@@ -55,6 +55,7 @@ func (q *Queries) ListEntityTypes(ctx context.Context) ([]EntityType, error) {
|
||||
&i.Status,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.MonitoringSpec,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -98,7 +99,7 @@ func (q *Queries) ListLifecycleDefs(ctx context.Context) ([]LifecycleDef, error)
|
||||
}
|
||||
|
||||
const listRelationshipTypes = `-- name: ListRelationshipTypes :many
|
||||
SELECT name, inverse, source_type, target_type, cardinality, description, created_at FROM relationship_types ORDER BY name
|
||||
SELECT name, inverse, source_type, target_type, cardinality, description, created_at, blast_direction FROM relationship_types ORDER BY name
|
||||
`
|
||||
|
||||
func (q *Queries) ListRelationshipTypes(ctx context.Context) ([]RelationshipType, error) {
|
||||
@@ -118,6 +119,7 @@ func (q *Queries) ListRelationshipTypes(ctx context.Context) ([]RelationshipType
|
||||
&i.Cardinality,
|
||||
&i.Description,
|
||||
&i.CreatedAt,
|
||||
&i.BlastDirection,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ func (q *Queries) GetAutonomySetting(ctx context.Context, key string) (string, e
|
||||
}
|
||||
|
||||
const getCheckDef = `-- name: GetCheckDef :one
|
||||
SELECT entity_id, target_id, target_type, kind, config, interval_s, timeout_s, zone, enabled, updated_at FROM check_defs WHERE entity_id = $1
|
||||
SELECT entity_id, target_id, target_type, kind, config, interval_s, timeout_s, zone, enabled, updated_at, last_run_at, last_health FROM check_defs WHERE entity_id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetCheckDef(ctx context.Context, entityID uuid.UUID) (CheckDef, error) {
|
||||
@@ -68,6 +68,8 @@ func (q *Queries) GetCheckDef(ctx context.Context, entityID uuid.UUID) (CheckDef
|
||||
&i.Zone,
|
||||
&i.Enabled,
|
||||
&i.UpdatedAt,
|
||||
&i.LastRunAt,
|
||||
&i.LastHealth,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -351,8 +353,8 @@ func (q *Queries) InsertApproval(ctx context.Context, arg InsertApprovalParams)
|
||||
|
||||
const insertAuditEntry = `-- name: InsertAuditEntry :exec
|
||||
INSERT INTO audit_log (actor_type, actor_id, action, entity_id, method, path,
|
||||
status_code, detail, source_ip, correlation_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
status_code, detail, source_ip, correlation_id, session_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
`
|
||||
|
||||
type InsertAuditEntryParams struct {
|
||||
@@ -366,6 +368,7 @@ type InsertAuditEntryParams struct {
|
||||
Detail []byte
|
||||
SourceIp *string
|
||||
CorrelationID *string
|
||||
SessionID *uuid.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) InsertAuditEntry(ctx context.Context, arg InsertAuditEntryParams) error {
|
||||
@@ -380,6 +383,7 @@ func (q *Queries) InsertAuditEntry(ctx context.Context, arg InsertAuditEntryPara
|
||||
arg.Detail,
|
||||
arg.SourceIp,
|
||||
arg.CorrelationID,
|
||||
arg.SessionID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -694,7 +698,11 @@ SELECT cd.entity_id, cd.target_id, cd.target_type, cd.kind, cd.config,
|
||||
e.slug AS entity_slug
|
||||
FROM check_defs cd
|
||||
JOIN entities e ON e.id = cd.entity_id
|
||||
LEFT JOIN entities tgt ON tgt.id = cd.target_id
|
||||
WHERE cd.enabled = true
|
||||
AND (tgt.id IS NULL OR tgt.state IS NULL OR tgt.state NOT IN ('deprecated', 'destroyed'))
|
||||
AND (cd.last_run_at IS NULL
|
||||
OR cd.last_run_at <= now() - make_interval(secs => cd.interval_s))
|
||||
`
|
||||
|
||||
type ListEnabledCheckDefsRow struct {
|
||||
@@ -714,6 +722,9 @@ type ListEnabledCheckDefsRow struct {
|
||||
// =====================================================================
|
||||
// Phase 3 queries
|
||||
// =====================================================================
|
||||
// Enabled AND due. interval_s used to be selected but never filtered on, so
|
||||
// every check ran on every 30s pass and the declared intervals meant nothing.
|
||||
// NULL last_run_at = never run = due now.
|
||||
func (q *Queries) ListEnabledCheckDefs(ctx context.Context) ([]ListEnabledCheckDefsRow, error) {
|
||||
rows, err := q.db.Query(ctx, listEnabledCheckDefs)
|
||||
if err != nil {
|
||||
@@ -1126,6 +1137,20 @@ func (q *Queries) ListSkills(ctx context.Context, status *string) ([]Skill, erro
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const markCheckRun = `-- name: MarkCheckRun :exec
|
||||
UPDATE check_defs SET last_run_at = now(), last_health = $2 WHERE entity_id = $1
|
||||
`
|
||||
|
||||
type MarkCheckRunParams struct {
|
||||
EntityID uuid.UUID
|
||||
LastHealth *string
|
||||
}
|
||||
|
||||
func (q *Queries) MarkCheckRun(ctx context.Context, arg MarkCheckRunParams) error {
|
||||
_, err := q.db.Exec(ctx, markCheckRun, arg.EntityID, arg.LastHealth)
|
||||
return err
|
||||
}
|
||||
|
||||
const putIdempotentResponse = `-- name: PutIdempotentResponse :exec
|
||||
INSERT INTO idempotency_keys (actor, key, request_hash, response_code, response_body)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
@@ -1447,3 +1472,25 @@ func (q *Queries) UpsertSignal(ctx context.Context, arg UpsertSignalParams) (Sig
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const worstHealthForTarget = `-- name: WorstHealthForTarget :one
|
||||
SELECT COALESCE(
|
||||
(SELECT last_health FROM check_defs
|
||||
WHERE enabled AND target_id = $1 AND last_health IS NOT NULL
|
||||
ORDER BY CASE last_health
|
||||
WHEN 'down' THEN 0 WHEN 'degraded' THEN 1 WHEN 'stale' THEN 2
|
||||
WHEN 'unknown' THEN 3 ELSE 4 END
|
||||
LIMIT 1),
|
||||
'unknown')::text AS health
|
||||
`
|
||||
|
||||
// An entity is as healthy as its unhealthiest check. Checks that have not run
|
||||
// yet (last_health IS NULL) are ignored rather than counted as unknown, so a
|
||||
// newly added check does not drag a known-good entity down before it has
|
||||
// produced a verdict.
|
||||
func (q *Queries) WorstHealthForTarget(ctx context.Context, targetID *uuid.UUID) (string, error) {
|
||||
row := q.db.QueryRow(ctx, worstHealthForTarget, targetID)
|
||||
var health string
|
||||
err := row.Scan(&health)
|
||||
return health, err
|
||||
}
|
||||
|
||||
@@ -31,6 +31,42 @@ func (q *Queries) EndCurrentRelationship(ctx context.Context, arg EndCurrentRela
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const insertRelationshipIfAbsent = `-- name: InsertRelationshipIfAbsent :execrows
|
||||
INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT $1, $2, $3,
|
||||
$4::jsonb, now()
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM relationships
|
||||
WHERE source_id = $1
|
||||
AND target_id = $2
|
||||
AND type = $3
|
||||
AND valid_to IS NULL
|
||||
)
|
||||
`
|
||||
|
||||
type InsertRelationshipIfAbsentParams struct {
|
||||
SourceID uuid.UUID
|
||||
TargetID uuid.UUID
|
||||
Type string
|
||||
Attributes []byte
|
||||
}
|
||||
|
||||
// Idempotent relationship insert (the create_relationship surface): no-op if
|
||||
// an active edge of the same source/target/type already exists. Replaces the
|
||||
// raw INSERT...WHERE NOT EXISTS used in entity_tools.go.
|
||||
func (q *Queries) InsertRelationshipIfAbsent(ctx context.Context, arg InsertRelationshipIfAbsentParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, insertRelationshipIfAbsent,
|
||||
arg.SourceID,
|
||||
arg.TargetID,
|
||||
arg.Type,
|
||||
arg.Attributes,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const listEntityRelations = `-- name: ListEntityRelations :many
|
||||
SELECT se.slug AS source_slug, te.slug AS target_slug, r.type, r.attributes,
|
||||
r.valid_from, r.valid_to
|
||||
|
||||
@@ -19,7 +19,8 @@ func LoadTypeTree(ctx context.Context, tx pgx.Tx) (*ontology.TypeTree, error) {
|
||||
}
|
||||
|
||||
rows, err := tx.Query(ctx,
|
||||
`SELECT name, COALESCE(parent_type,''), is_abstract, COALESCE(lifecycle_id,'')
|
||||
`SELECT name, COALESCE(parent_type,''), is_abstract, COALESCE(lifecycle_id,''),
|
||||
layer, monitoring_spec
|
||||
FROM entity_types`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load entity_types: %w", err)
|
||||
@@ -27,10 +28,16 @@ func LoadTypeTree(ctx context.Context, tx pgx.Tx) (*ontology.TypeTree, error) {
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var info ontology.TypeInfo
|
||||
if err := rows.Scan(&name, &info.Parent, &info.IsAbstract, &info.LifecycleID); err != nil {
|
||||
// NULL monitoring_spec means the type declared nothing; '[]' means it
|
||||
// declared "explicitly unmonitorable". Scanning through a pointer is
|
||||
// what keeps those two apart — see ontology.TypeTree.Monitoring.
|
||||
var monitoring *[]string
|
||||
if err := rows.Scan(&name, &info.Parent, &info.IsAbstract, &info.LifecycleID,
|
||||
&info.Layer, &monitoring); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
info.Monitoring = monitoring
|
||||
t.Types[name] = info
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
135
internal/execlog/execlog.go
Normal file
135
internal/execlog/execlog.go
Normal file
@@ -0,0 +1,135 @@
|
||||
// Package execlog persists incremental command output for an execution and
|
||||
// announces it on the event stream.
|
||||
//
|
||||
// It exists as its own package because both SSH execution paths need it —
|
||||
// internal/mcp (the agent's auto-run windows) and internal/httpapi (the
|
||||
// post-approval actuator). Those two already carry near-identical copies of
|
||||
// sshExec, and every bug found in this area so far has been a case of the two
|
||||
// copies drifting apart; one shared sink is the cheap way not to repeat that.
|
||||
package execlog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/observability"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// eventInterval throttles execution.output events. Chunks are persisted as
|
||||
// they arrive, but a chatty command (apt, a long build) can produce hundreds
|
||||
// per second and the SSE broker drops events for slow subscribers — flooding
|
||||
// it would push out the signal.* and approval.* events that actually need to
|
||||
// arrive. The event is only a "there is more output" ping; subscribers re-read
|
||||
// the rows.
|
||||
const eventInterval = time.Second
|
||||
|
||||
// Sink receives output chunks as they arrive from a remote command.
|
||||
type Sink func(stream string, chunk []byte)
|
||||
|
||||
// New returns a Sink that writes chunks to execution_logs and emits a
|
||||
// throttled execution.output event, plus a Flush to call when the command
|
||||
// finishes.
|
||||
//
|
||||
// The returned Sink is safe for concurrent use: stdout and stderr are written
|
||||
// from separate goroutines.
|
||||
func New(ctx context.Context, pool *db.Pool, execID uuid.UUID, correlationID string) (Sink, func()) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
seq int
|
||||
lastEvent time.Time
|
||||
pending bool
|
||||
)
|
||||
|
||||
emit := func() {
|
||||
if err := observability.Event(ctx, sqlcgen.New(pool), "execution.output", &execID,
|
||||
"info", "actuator", correlationID, map[string]any{"execution_id": execID.String()}); err != nil {
|
||||
slog.Debug("execlog: emit output event", "error", err, "execution_id", execID)
|
||||
}
|
||||
}
|
||||
|
||||
sink := func(stream string, chunk []byte) {
|
||||
if len(chunk) == 0 {
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
seq++
|
||||
n := seq
|
||||
mu.Unlock()
|
||||
|
||||
// A failed log write must never fail the command: this is observability,
|
||||
// and the authoritative output still lands in executions.result at the
|
||||
// end. Log and carry on.
|
||||
if _, err := pool.Exec(ctx,
|
||||
`INSERT INTO execution_logs (execution_id, seq, stream, chunk)
|
||||
VALUES ($1, $2, $3, $4)`,
|
||||
execID, n, stream, string(chunk)); err != nil {
|
||||
slog.Debug("execlog: persist chunk", "error", err, "execution_id", execID)
|
||||
return
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
due := time.Since(lastEvent) >= eventInterval
|
||||
if due {
|
||||
lastEvent = time.Now()
|
||||
pending = false
|
||||
} else {
|
||||
pending = true
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
if due {
|
||||
emit()
|
||||
}
|
||||
}
|
||||
|
||||
// Flush emits a final event when output arrived inside the throttle window,
|
||||
// so the last few lines of a short command are not left unannounced.
|
||||
flush := func() {
|
||||
mu.Lock()
|
||||
due := pending
|
||||
pending = false
|
||||
mu.Unlock()
|
||||
if due {
|
||||
emit()
|
||||
}
|
||||
}
|
||||
|
||||
return sink, flush
|
||||
}
|
||||
|
||||
// Read returns an execution's persisted output in order.
|
||||
func Read(ctx context.Context, pool *db.Pool, execID uuid.UUID, limit int) ([]Chunk, error) {
|
||||
if limit <= 0 {
|
||||
limit = 1000
|
||||
}
|
||||
rows, err := pool.Query(ctx,
|
||||
`SELECT seq, stream, chunk, ts FROM execution_logs
|
||||
WHERE execution_id = $1 ORDER BY seq LIMIT $2`, execID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Chunk
|
||||
for rows.Next() {
|
||||
var c Chunk
|
||||
if err := rows.Scan(&c.Seq, &c.Stream, &c.Chunk, &c.TS); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Chunk is one persisted slice of command output.
|
||||
type Chunk struct {
|
||||
Seq int `json:"seq"`
|
||||
Stream string `json:"stream"`
|
||||
Chunk string `json:"chunk"`
|
||||
TS time.Time `json:"ts"`
|
||||
}
|
||||
13
internal/execworker/init.go
Normal file
13
internal/execworker/init.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package execworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
)
|
||||
|
||||
// RunnerForMain provides the run function for registration in main.
|
||||
func RunnerForMain() func(context.Context, *db.Pool, config.Config) {
|
||||
return Run
|
||||
}
|
||||
207
internal/execworker/worker.go
Normal file
207
internal/execworker/worker.go
Normal file
@@ -0,0 +1,207 @@
|
||||
// Package execworker processes pending executions as a background daemon.
|
||||
// This provides a Postgres-backed queue: executions survive restarts, and
|
||||
// per-execution advisory locks prevent duplicate processing across instances.
|
||||
package execworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/actuator"
|
||||
"github.com/dtoro/oikos/internal/config"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/health"
|
||||
"github.com/dtoro/oikos/internal/remote"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Run starts the execution worker loop. Blocks until ctx is cancelled.
|
||||
func Run(ctx context.Context, pool *db.Pool, cfg config.Config) {
|
||||
slog.Info("execworker: starting")
|
||||
|
||||
// Liveness probe
|
||||
probe := health.New(2 * time.Minute)
|
||||
probe.Serve(ctx, cfg.HealthListen)
|
||||
|
||||
recoverOrphaned(ctx, pool)
|
||||
probe.Bump()
|
||||
|
||||
ticker := time.NewTicker(15 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("execworker: shutting down")
|
||||
return
|
||||
case <-ticker.C:
|
||||
processPending(ctx, pool)
|
||||
probe.Bump()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recoverOrphaned marks executions stuck in 'running' as failed.
|
||||
func recoverOrphaned(ctx context.Context, pool *db.Pool) {
|
||||
tag, err := pool.Exec(ctx, `UPDATE executions SET status = 'failed', result = '{"error":"worker restarted while execution was running"}'::jsonb, completed_at = now() WHERE status = 'running'`)
|
||||
if err != nil {
|
||||
slog.Error("execworker: recover orphaned", "error", err)
|
||||
return
|
||||
}
|
||||
if tag.RowsAffected() > 0 {
|
||||
slog.Warn("execworker: recovered orphaned executions", "count", tag.RowsAffected())
|
||||
}
|
||||
}
|
||||
|
||||
// processPending polls for pending executions and dispatches them.
|
||||
func processPending(ctx context.Context, pool *db.Pool) {
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT e.entity_id, e.target_entity_id, e.action, e.risk_class, e.correlation_id, e.status,
|
||||
COALESCE(t.slug, '') AS target_slug
|
||||
FROM executions e
|
||||
LEFT JOIN entities t ON t.id = e.target_entity_id
|
||||
WHERE e.status = 'proposed'
|
||||
ORDER BY e.created_at ASC
|
||||
LIMIT 10`)
|
||||
if err != nil {
|
||||
slog.Error("execworker: query pending", "error", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
q := sqlcgen.New(pool)
|
||||
|
||||
for rows.Next() {
|
||||
var execID, targetID *uuid.UUID
|
||||
var action, riskClass, correlationID, status, targetSlug string
|
||||
if err := rows.Scan(&execID, &targetID, &action, &riskClass, &correlationID, &status, &targetSlug); err != nil {
|
||||
slog.Error("execworker: scan row", "error", err)
|
||||
continue
|
||||
}
|
||||
if execID == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// At-most-once: try advisory lock on execution entity_id.
|
||||
// Acquire a dedicated connection so the session-scoped lock isn't
|
||||
// released when the transient pool connection is returned.
|
||||
lockKey := hashUUID(*execID)
|
||||
lockConn, err := pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
slog.Error("execworker: acquire lock conn", "error", err)
|
||||
continue
|
||||
}
|
||||
var locked bool
|
||||
if err := lockConn.QueryRow(ctx, "SELECT pg_try_advisory_lock($1)", lockKey).Scan(&locked); err != nil || !locked {
|
||||
lockConn.Release()
|
||||
continue
|
||||
}
|
||||
|
||||
dispatch(ctx, pool, q, *execID, targetID, action, targetSlug, correlationID)
|
||||
|
||||
// Release the per-execution lock on the same connection.
|
||||
lockConn.Exec(context.WithoutCancel(ctx), "SELECT pg_advisory_unlock($1)", lockKey)
|
||||
lockConn.Release()
|
||||
}
|
||||
}
|
||||
|
||||
func dispatch(ctx context.Context, pool *db.Pool, q *sqlcgen.Queries, execID uuid.UUID, targetID *uuid.UUID, action, targetSlug, correlationID string) {
|
||||
startedAt := time.Now()
|
||||
|
||||
// Mark running
|
||||
_, err := pool.Exec(ctx,
|
||||
`UPDATE executions SET status = 'running', started_at = $2 WHERE entity_id = $1`,
|
||||
execID, startedAt)
|
||||
if err != nil {
|
||||
slog.Error("execworker: mark running", "error", err, "execution", execID)
|
||||
return
|
||||
}
|
||||
|
||||
// Resolve SSH target. If targetSlug is available, use it; otherwise resolve from targetID.
|
||||
var host, user string
|
||||
if targetSlug == "" && targetID != nil {
|
||||
if err := pool.QueryRow(ctx, "SELECT slug FROM entities WHERE id = $1", *targetID).Scan(&targetSlug); err != nil {
|
||||
failExecution(ctx, pool, execID, fmt.Sprintf("resolve target slug: %v", err))
|
||||
return
|
||||
}
|
||||
}
|
||||
if targetSlug != "" {
|
||||
addr, sshUser, err := remote.ResolveHost(ctx, pool, targetSlug, "root")
|
||||
if err == nil {
|
||||
host, user = addr, sshUser
|
||||
}
|
||||
}
|
||||
if host == "" {
|
||||
failExecution(ctx, pool, execID, fmt.Sprintf("no reachable target: %s", targetSlug))
|
||||
return
|
||||
}
|
||||
|
||||
// Determine the command to run from the action field.
|
||||
// Format: "action_name:{json_params}" or a raw command string.
|
||||
cmd := action
|
||||
if idx := strings.Index(action, ":"); idx > 0 && idx < len(action)-1 {
|
||||
rawParams := action[idx+1:]
|
||||
var params map[string]any
|
||||
if json.Unmarshal([]byte(rawParams), ¶ms) == nil {
|
||||
if c, ok := params["command"].(string); ok && c != "" {
|
||||
cmd = c
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
signer, err := actuator.LoadSigner(os.Getenv("OIKOS_SSH_KEY_PATH"))
|
||||
if err != nil {
|
||||
signer, err = actuator.LoadSigner("/etc/oikos/ssh_key")
|
||||
if err != nil {
|
||||
failExecution(ctx, pool, execID, fmt.Sprintf("load ssh key: %v", err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
client, err := actuator.Dial(ctx, actuator.DialOptions{Host: host, User: user, Signer: signer})
|
||||
if err != nil {
|
||||
failExecution(ctx, pool, execID, fmt.Sprintf("ssh dial: %v", err))
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
out, err := actuator.RunCombinedOutput(ctx, client, cmd)
|
||||
if err != nil {
|
||||
failExecution(ctx, pool, execID, fmt.Sprintf("command: %v\noutput: %s", err, string(out)))
|
||||
return
|
||||
}
|
||||
|
||||
duration := time.Since(startedAt).Milliseconds()
|
||||
resultJSON, _ := json.Marshal(map[string]any{"output": string(out), "success": true})
|
||||
_ = q.UpdateExecutionStatus(ctx, sqlcgen.UpdateExecutionStatusParams{
|
||||
EntityID: execID,
|
||||
Status: "completed",
|
||||
Result: resultJSON,
|
||||
DurationMs: &[]int32{int32(duration)}[0],
|
||||
Verified: true,
|
||||
})
|
||||
|
||||
slog.Info("execworker: execution complete",
|
||||
"execution", execID, "target", targetSlug, "duration_ms", duration)
|
||||
}
|
||||
|
||||
func failExecution(ctx context.Context, pool *db.Pool, execID uuid.UUID, reason string) {
|
||||
slog.Error("execworker: execution failed", "execution", execID, "error", reason)
|
||||
resultJSON, _ := json.Marshal(map[string]any{"error": reason, "success": false})
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb, completed_at=now() WHERE entity_id=$1`,
|
||||
execID, resultJSON)
|
||||
}
|
||||
|
||||
func hashUUID(id uuid.UUID) int {
|
||||
h := 0
|
||||
for _, b := range id {
|
||||
h = (h*31 + int(b)) & 0x7fffffff
|
||||
}
|
||||
return h
|
||||
}
|
||||
93
internal/health/health.go
Normal file
93
internal/health/health.go
Normal file
@@ -0,0 +1,93 @@
|
||||
// Package health provides a staleness-aware liveness probe for background-
|
||||
// loop services (scheduler, notifier) that don't otherwise serve HTTP.
|
||||
//
|
||||
// The owning loop calls Probe.Bump() on each iteration. A /healthz endpoint
|
||||
// returns 200 while the last bump is within the staleness window, and 503
|
||||
// once the loop has gone quiet — so a wedged goroutine (stuck SSH, deadlock)
|
||||
// surfaces as an unhealthy container instead of a silently-idle one.
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Probe tracks the last time the owning loop made progress.
|
||||
type Probe struct {
|
||||
last atomic.Int64 // unix-nano timestamp of the last Bump
|
||||
stale time.Duration
|
||||
}
|
||||
|
||||
// New returns a Probe that considers the owner healthy while Bump has been
|
||||
// called within stale of the current time.
|
||||
func New(stale time.Duration) *Probe {
|
||||
if stale <= 0 {
|
||||
stale = 2 * time.Minute
|
||||
}
|
||||
p := &Probe{stale: stale}
|
||||
p.last.Store(time.Now().UnixNano()) // boot-healthy until first loop stalls
|
||||
return p
|
||||
}
|
||||
|
||||
// Bump records that the owning loop completed another iteration.
|
||||
func (p *Probe) Bump() {
|
||||
p.last.Store(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// Healthy reports whether the last Bump is within the staleness window.
|
||||
func (p *Probe) Healthy() bool {
|
||||
last := time.Unix(0, p.last.Load())
|
||||
return time.Since(last) <= p.stale
|
||||
}
|
||||
|
||||
// Handler returns an http.Handler serving GET /healthz. Returns 200 with a
|
||||
// small JSON body when healthy, 503 (Service Unavailable) when stale.
|
||||
func (p *Probe) Handler() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
last := time.Unix(0, p.last.Load())
|
||||
if !p.Healthy() {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{
|
||||
"status": healthStatus(p.Healthy()),
|
||||
"last_heartbeat": last.UTC().Format(time.RFC3339),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Serve starts an HTTP server exposing the probe's /healthz on addr until ctx
|
||||
// is cancelled. A no-op when addr is empty (local/non-docker runs skip it).
|
||||
// The server is bound to addr (e.g. ":8093"); containers hit it via 127.0.0.1.
|
||||
func (p *Probe) Serve(ctx context.Context, addr string) {
|
||||
if addr == "" {
|
||||
return
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/healthz", p.Handler())
|
||||
srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
||||
|
||||
go func() {
|
||||
slog.Info("health server listening", "addr", addr)
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
slog.Warn("health server stopped", "addr", addr, "error", err)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
srv.Shutdown(shutdownCtx)
|
||||
}()
|
||||
}
|
||||
|
||||
func healthStatus(ok bool) string {
|
||||
if ok {
|
||||
return "ok"
|
||||
}
|
||||
return "stale"
|
||||
}
|
||||
73
internal/health/health_test.go
Normal file
73
internal/health/health_test.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestProbeHealthyAtBoot(t *testing.T) {
|
||||
p := New(time.Minute)
|
||||
if !p.Healthy() {
|
||||
t.Fatal("probe should be healthy immediately after creation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeStaleAfterWindow(t *testing.T) {
|
||||
p := New(50 * time.Millisecond)
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
if p.Healthy() {
|
||||
t.Fatal("probe should be stale after the staleness window elapses with no Bump")
|
||||
}
|
||||
p.Bump()
|
||||
if !p.Healthy() {
|
||||
t.Fatal("probe should recover immediately after Bump")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeHandlerStatusCodes(t *testing.T) {
|
||||
p := New(20 * time.Millisecond)
|
||||
|
||||
// Fresh → 200
|
||||
if code := probeCode(p); code != http.StatusOK {
|
||||
t.Fatalf("fresh probe: want 200, got %d", code)
|
||||
}
|
||||
|
||||
// Stale → 503
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
if code := probeCode(p); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("stale probe: want 503, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeHandlerBody(t *testing.T) {
|
||||
p := New(time.Minute)
|
||||
rec := httptest.NewRecorder()
|
||||
p.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("invalid JSON body: %v (body=%q)", err, rec.Body.String())
|
||||
}
|
||||
if body["status"] != "ok" {
|
||||
t.Fatalf("want status=ok, got %v", body["status"])
|
||||
}
|
||||
if _, ok := body["last_heartbeat"].(string); !ok {
|
||||
t.Fatalf("want last_heartbeat string, got %v", body["last_heartbeat"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewDefaultsStale(t *testing.T) {
|
||||
p := New(0)
|
||||
if p.stale <= 0 {
|
||||
t.Fatal("New(0) should fall back to a positive staleness window")
|
||||
}
|
||||
}
|
||||
|
||||
func probeCode(p *Probe) int {
|
||||
rec := httptest.NewRecorder()
|
||||
p.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
return rec.Code
|
||||
}
|
||||
@@ -11,11 +11,12 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/actuator"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/execlog"
|
||||
"github.com/dtoro/oikos/internal/observability"
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -71,7 +72,19 @@ func initSSH() {
|
||||
// report. Generous enough for a real apt/docker install; not infinite.
|
||||
const sshExecTimeout = 10 * time.Minute
|
||||
|
||||
// streamWriter buffers everything it is given while forwarding each write to a
|
||||
// sink. One on session.Stdout and another sharing the same buffer on
|
||||
// session.Stderr reproduces CombinedOutput's interleaving in the order the
|
||||
// remote end produced it. Shared implementation lives in internal/actuator
|
||||
// (actuator.streamWriter / actuator.RunStreaming).
|
||||
|
||||
func sshExec(ctx context.Context, host, user, command string) (string, error) {
|
||||
return sshExecStream(ctx, host, user, command, nil)
|
||||
}
|
||||
|
||||
// sshExecStream runs a command and reports its combined output, forwarding
|
||||
// each chunk to sink as it arrives. A nil sink behaves exactly as before.
|
||||
func sshExecStream(ctx context.Context, host, user, command string, sink execlog.Sink) (string, error) {
|
||||
initSSH()
|
||||
if len(_sshKey) == 0 {
|
||||
return "", fmt.Errorf("no SSH key available")
|
||||
@@ -80,76 +93,18 @@ func sshExec(ctx context.Context, host, user, command string) (string, error) {
|
||||
user = _sshUser
|
||||
}
|
||||
|
||||
addr := host + ":22"
|
||||
signer, err := ssh.ParsePrivateKey(_sshKey)
|
||||
signer, err := actuator.LoadSignerFromBytes(_sshKey)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse key: %w", err)
|
||||
}
|
||||
|
||||
cfg := &ssh.ClientConfig{
|
||||
User: user,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", addr, cfg)
|
||||
client, err := actuator.Dial(ctx, actuator.DialOptions{Host: host, User: user, Signer: signer})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("dial %s: %w", host, err)
|
||||
return "", err
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("session: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
type result struct {
|
||||
out []byte
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
// See internal/mcp/server.go's sshExec for why this recovers rather
|
||||
// than letting a rare SSH-library panic crash the whole api process.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{nil, fmt.Errorf("panic in ssh exec: %v", r)}
|
||||
}
|
||||
}()
|
||||
out, err := session.CombinedOutput(command)
|
||||
done <- result{out, err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case r := <-done:
|
||||
text := strings.TrimSpace(string(r.out))
|
||||
// A non-zero exit MUST surface as an error. The previous guard only
|
||||
// errored when there was no output, so a `pct create` that printed
|
||||
// "CT 132 already exists" and exited non-zero was reported as
|
||||
// success — the execution was marked completed though nothing was
|
||||
// provisioned.
|
||||
if r.err != nil {
|
||||
if text != "" {
|
||||
return text, fmt.Errorf("%w: %s", r.err, text)
|
||||
}
|
||||
return text, fmt.Errorf("exec: %w", r.err)
|
||||
}
|
||||
return text, nil
|
||||
case <-time.After(sshExecTimeout):
|
||||
// Close the session/client to hang up the remote side; the
|
||||
// goroutine above will eventually exit once that unblocks
|
||||
// CombinedOutput, but we don't wait for it — the caller needs an
|
||||
// answer now, not an indefinite hang.
|
||||
session.Close()
|
||||
client.Close()
|
||||
return "", fmt.Errorf("timed out after %s waiting for command to finish on %s", sshExecTimeout, host)
|
||||
case <-ctx.Done():
|
||||
session.Close()
|
||||
client.Close()
|
||||
return "", ctx.Err()
|
||||
}
|
||||
return actuator.RunStreaming(ctx, client, command, sink, sshExecTimeout)
|
||||
}
|
||||
|
||||
func resolveHostSSH(ctx context.Context, pool *db.Pool, entitySlug string) (string, string, error) {
|
||||
@@ -231,7 +186,16 @@ func emitExecutionEvent(ctx context.Context, pool *db.Pool, execID uuid.UUID, st
|
||||
if status == "failed" {
|
||||
severity = "warning"
|
||||
}
|
||||
_ = observability.Event(ctx, sqlcgen.New(pool), "execution."+status, &execID, severity, "actuator", "", detail)
|
||||
// The correlation id was hardcoded to "", so execution events could not be
|
||||
// tied back to the session that caused them — the one join you want when
|
||||
// asking "what did this agent turn actually do?". It is already on the
|
||||
// execution row; read it rather than threading it through eleven callers.
|
||||
var correlationID string
|
||||
if err := pool.QueryRow(ctx,
|
||||
`SELECT correlation_id FROM executions WHERE entity_id = $1`, execID).Scan(&correlationID); err != nil {
|
||||
correlationID = ""
|
||||
}
|
||||
_ = observability.Event(ctx, sqlcgen.New(pool), "execution."+status, &execID, severity, "actuator", correlationID, detail)
|
||||
if status == "completed" || status == "failed" || status == "cancelled" {
|
||||
closePlanStepForExecution(ctx, pool, execID, status)
|
||||
}
|
||||
@@ -280,6 +244,29 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID,
|
||||
action, params := actionStr[:idx], actionStr[idx+1:]
|
||||
|
||||
startedAt := time.Now()
|
||||
// Persist started_at now, not at the end. It was captured here but only
|
||||
// written in the terminal UPDATE, so a running execution reported
|
||||
// started_at = NULL for its entire life — the UI could not show how long
|
||||
// anything had been going, which is exactly when you want to know.
|
||||
if _, err := pool.Exec(ctx,
|
||||
`UPDATE executions SET status = 'running', started_at = $2 WHERE entity_id = $1`,
|
||||
execID, startedAt); err != nil {
|
||||
slog.Error("httpapi: mark execution running", "error", err, "execution_id", execID)
|
||||
}
|
||||
|
||||
// Stream output for the actions whose output an operator actually watches:
|
||||
// a long apt upgrade, a pct create, an arbitrary approved `run`. The small
|
||||
// internal lookups further down (listing template cache, pvesh nextid) stay
|
||||
// unstreamed — they are plumbing, and logging them would bury the command
|
||||
// the operator approved.
|
||||
var correlationID string
|
||||
if qerr := pool.QueryRow(ctx,
|
||||
`SELECT correlation_id FROM executions WHERE entity_id = $1`, execID).Scan(&correlationID); qerr != nil {
|
||||
correlationID = ""
|
||||
}
|
||||
sink, flushLogs := execlog.New(ctx, pool, execID, correlationID)
|
||||
defer flushLogs()
|
||||
|
||||
var output, cmd string
|
||||
|
||||
switch action {
|
||||
@@ -295,30 +282,30 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID,
|
||||
default:
|
||||
cmd = fmt.Sprintf("systemctl %s %s 2>&1", params, svc)
|
||||
}
|
||||
output, err = sshExec(ctx, host, user, cmd)
|
||||
output, err = sshExecStream(ctx, host, user, cmd, sink)
|
||||
|
||||
case "apt_upgrade":
|
||||
svc := strings.TrimPrefix(targetSlug, "lxc:")
|
||||
cmd = fmt.Sprintf("apt update -qq 2>&1 >/dev/null && apt upgrade -y -qq 2>&1; echo '---'; systemctl is-active %s || true", svc)
|
||||
output, err = sshExec(ctx, host, user, cmd)
|
||||
output, err = sshExecStream(ctx, host, user, cmd, sink)
|
||||
|
||||
case "pct_create":
|
||||
var cfg struct {
|
||||
VMID int `json:"vmid"`
|
||||
Hostname string `json:"hostname"`
|
||||
Cores int `json:"cores"`
|
||||
Memory int `json:"memory"`
|
||||
DiskGB int `json:"disk_gb"`
|
||||
IP string `json:"ip"`
|
||||
GW string `json:"gw"`
|
||||
Bridge string `json:"bridge"` // e.g. vmbr0/vmbr1 — which bridge actually reaches the target subnet on this host varies per host, don't assume vmbr0
|
||||
Storage string `json:"storage"`
|
||||
Template string `json:"template"`
|
||||
Privileged flexBool `json:"privileged"`
|
||||
Nesting flexBool `json:"nesting"`
|
||||
Mounts []string `json:"mounts"`
|
||||
Nameserver string `json:"nameserver"`
|
||||
Searchdomain string `json:"searchdomain"`
|
||||
VMID int `json:"vmid"`
|
||||
Hostname string `json:"hostname"`
|
||||
Cores int `json:"cores"`
|
||||
Memory int `json:"memory"`
|
||||
DiskGB int `json:"disk_gb"`
|
||||
IP string `json:"ip"`
|
||||
GW string `json:"gw"`
|
||||
Bridge string `json:"bridge"` // e.g. vmbr0/vmbr1 — which bridge actually reaches the target subnet on this host varies per host, don't assume vmbr0
|
||||
Storage string `json:"storage"`
|
||||
Template string `json:"template"`
|
||||
Privileged flexBool `json:"privileged"`
|
||||
Nesting flexBool `json:"nesting"`
|
||||
Mounts []string `json:"mounts"`
|
||||
Nameserver string `json:"nameserver"`
|
||||
Searchdomain string `json:"searchdomain"`
|
||||
// No services/post_install here anymore — pct_create is atomic
|
||||
// (create + start + register only). Installing packages and
|
||||
// running setup scripts is the agent's job via follow-up `run`
|
||||
@@ -504,7 +491,7 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID,
|
||||
}
|
||||
|
||||
slog.Info("httpapi: pct_create running", "vmid", cfg.VMID, "hostname", cfg.Hostname, "cmd", createCmd)
|
||||
output, err = sshExec(ctx, host, user, createCmd)
|
||||
output, err = sshExecStream(ctx, host, user, createCmd, sink)
|
||||
|
||||
// pct_create is now DELIBERATELY ATOMIC: create + start + register,
|
||||
// nothing else. It used to also run apt installs and a post_install
|
||||
@@ -579,7 +566,7 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID,
|
||||
return
|
||||
}
|
||||
cmd = wrap(cfg.Command)
|
||||
output, err = sshExec(ctx, host, user, cmd)
|
||||
output, err = sshExecStream(ctx, host, user, cmd, sink)
|
||||
|
||||
default:
|
||||
slog.Error("httpapi: unknown gated action for approved execution", "action", action, "execution_id", execID)
|
||||
|
||||
@@ -89,6 +89,7 @@ func (s *Server) CreateApprovalRule(ctx context.Context, req gen.CreateApprovalR
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, sqlcgen.New(tx), actorType, actor, "create",
|
||||
&id, "POST", "/api/v1/policy/approval-rules", "",
|
||||
nil,
|
||||
map[string]any{"action": req.Body.Action, "risk_class": req.Body.RiskClass}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
@@ -148,6 +149,7 @@ func (s *Server) PatchApprovalRule(ctx context.Context, req gen.PatchApprovalRul
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, sqlcgen.New(tx), actorType, actor, "patch",
|
||||
&id, "PATCH", "/api/v1/policy/approval-rules/"+req.Id, "",
|
||||
nil,
|
||||
map[string]any{"action": req.Body.Action}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
|
||||
@@ -154,6 +154,7 @@ func (s *Server) DecideApproval(ctx context.Context, req gen.DecideApprovalReque
|
||||
|
||||
if auditErr := observability.Audit(ctx, q, actorType, actor, "decide",
|
||||
&id, "POST", "/api/v1/approvals/"+req.Id+"/decision", "",
|
||||
nil,
|
||||
map[string]any{"decision": status}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
|
||||
20
internal/httpapi/audit.go
Normal file
20
internal/httpapi/audit.go
Normal file
@@ -0,0 +1,20 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/dtoro/oikos/internal/audit"
|
||||
)
|
||||
|
||||
// serveAuditDrift returns a read-only DB-side drift report: orphan check
|
||||
// entities, checks on retired targets, probes stuck down/unknown, unmonitored
|
||||
// declared types, and dangling edges. Companion to the knowledge-graph-audit
|
||||
// skill. Live-infra discovery (pct/docker/certs) is a follow-up.
|
||||
func (s *Server) serveAuditDrift(w http.ResponseWriter, req *http.Request) {
|
||||
findings, summary := audit.Report(req.Context(), s.pool)
|
||||
writeJSON(w, map[string]any{
|
||||
"findings": findings,
|
||||
"summary": summary,
|
||||
"note": "read-only DB drift report; live-infra discovery (pct/docker/certs) is a follow-up",
|
||||
})
|
||||
}
|
||||
@@ -81,6 +81,7 @@ func (s *Server) PatchAutonomySettings(ctx context.Context, req gen.PatchAutonom
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, sqlcgen.New(tx), actorType, actor, "patch",
|
||||
nil, "PATCH", "/api/v1/policy/autonomy", "",
|
||||
nil,
|
||||
map[string]any{"keys": keysOfMap(*req.Body)}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ func (s *Server) ListChecks(ctx context.Context, req gen.ListChecksRequestObject
|
||||
SELECT cd.entity_id, e.slug, cd.kind,
|
||||
COALESCE(te.slug, '') AS target_slug, cd.target_type,
|
||||
cd.config, cd.interval_s, cd.timeout_s, cd.zone, cd.enabled,
|
||||
e.version
|
||||
e.version, cd.last_health, cd.last_run_at
|
||||
FROM check_defs cd
|
||||
JOIN entities e ON e.id = cd.entity_id
|
||||
LEFT JOIN entities te ON te.id = cd.target_id
|
||||
@@ -43,10 +43,19 @@ func (s *Server) ListChecks(ctx context.Context, req gen.ListChecksRequestObject
|
||||
var c gen.Check
|
||||
var targetSlug string
|
||||
var configBytes []byte
|
||||
// last_health is what turns a check list from configuration into an
|
||||
// explanation: an entity's health is the worst of these, so this is
|
||||
// the field that says which probe is responsible.
|
||||
var lastHealth *string
|
||||
if err := rows.Scan(&c.Id, &c.Slug, &c.Kind, &targetSlug, &c.TargetType,
|
||||
&configBytes, &c.IntervalS, &c.TimeoutS, &c.Zone, &c.Enabled, &c.Version); err != nil {
|
||||
&configBytes, &c.IntervalS, &c.TimeoutS, &c.Zone, &c.Enabled, &c.Version,
|
||||
&lastHealth, &c.LastRunAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lastHealth != nil {
|
||||
h := gen.CheckLastHealth(*lastHealth)
|
||||
c.LastHealth = &h
|
||||
}
|
||||
if targetSlug != "" {
|
||||
c.Target = &targetSlug
|
||||
}
|
||||
@@ -173,6 +182,7 @@ func (s *Server) CreateCheck(ctx context.Context, req gen.CreateCheckRequestObje
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, q, actorType, actor, "create",
|
||||
&id, "POST", "/api/v1/checks", "",
|
||||
nil,
|
||||
map[string]any{"kind": req.Body.Kind, "slug": slug}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
@@ -260,6 +270,7 @@ func (s *Server) PatchCheck(ctx context.Context, req gen.PatchCheckRequestObject
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, sqlcgen.New(tx), actorType, actor, "patch",
|
||||
&id, "PATCH", "/api/v1/checks/"+req.Id, "",
|
||||
nil,
|
||||
map[string]any{"enabled": updated.Enabled}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
@@ -285,6 +296,15 @@ func checkDefToGen(cd sqlcgen.CheckDef) gen.Check {
|
||||
if len(cd.Config) > 0 && json.Unmarshal(cd.Config, &config) == nil && len(config) > 0 {
|
||||
c.Config = &config
|
||||
}
|
||||
// Carried through so toggling a check does not blank its verdict in the
|
||||
// UI — the entity window renders last_health to explain which probe is
|
||||
// responsible for an entity's health, and a patch response missing it
|
||||
// would erase that until the next poll.
|
||||
c.LastRunAt = cd.LastRunAt
|
||||
if cd.LastHealth != nil {
|
||||
h := gen.CheckLastHealth(*cd.LastHealth)
|
||||
c.LastHealth = &h
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
88
internal/httpapi/client_context.go
Normal file
88
internal/httpapi/client_context.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
)
|
||||
|
||||
func (s *Server) GetClientContext(ctx context.Context, req gen.GetClientContextRequestObject) (gen.GetClientContextResponseObject, error) {
|
||||
slug := string(req.Slug)
|
||||
_, err := s.resolveEntityID(ctx, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var version int64
|
||||
_ = s.pool.QueryRow(ctx,
|
||||
"SELECT version FROM context_version WHERE singleton = true").Scan(&version)
|
||||
|
||||
var filesChanged, toolsChanged []string
|
||||
var sopsChanged bool
|
||||
if req.Params.Since != nil {
|
||||
rows, qErr := s.pool.Query(ctx,
|
||||
"SELECT path FROM context_files WHERE last_changed > $1", *req.Params.Since)
|
||||
if qErr == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var p string
|
||||
if scanErr := rows.Scan(&p); scanErr == nil {
|
||||
// Matches tools/setup-*.sh (the auto-setup convention —
|
||||
// see tools/post-pull.sh). Was tools/*.setup.sh until
|
||||
// 2026-07-12, which never matched any real filename.
|
||||
if strings.HasPrefix(p, "tools/setup-") && strings.HasSuffix(p, ".sh") {
|
||||
toolsChanged = append(toolsChanged, p)
|
||||
} else if p == ".sops.yaml" {
|
||||
sopsChanged = true
|
||||
} else {
|
||||
filesChanged = append(filesChanged, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if filesChanged == nil {
|
||||
filesChanged = []string{}
|
||||
}
|
||||
if toolsChanged == nil {
|
||||
toolsChanged = []string{}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
|
||||
return gen.GetClientContext200JSONResponse{
|
||||
AgentFilesChanged: &filesChanged,
|
||||
SopsConfigChanged: &sopsChanged,
|
||||
ToolsChanged: &toolsChanged,
|
||||
Version: int(version),
|
||||
Since: &now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Server) GetClientSecrets(ctx context.Context, req gen.GetClientSecretsRequestObject) (gen.GetClientSecretsResponseObject, error) {
|
||||
slug := string(req.Slug)
|
||||
_, err := s.resolveEntityID(ctx, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var keys []string
|
||||
if s.secretsManager != nil {
|
||||
list, listErr := s.secretsManager.List(ctx)
|
||||
if listErr == nil {
|
||||
prefix := "clients/" + slug + "/"
|
||||
for _, k := range list {
|
||||
if strings.HasPrefix(k, prefix) || strings.HasPrefix(k, "shared/") {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if keys == nil {
|
||||
keys = []string{}
|
||||
}
|
||||
|
||||
return gen.GetClientSecrets200JSONResponse{Keys: keys}, nil
|
||||
}
|
||||
310
internal/httpapi/client_lifecycle.go
Normal file
310
internal/httpapi/client_lifecycle.go
Normal file
@@ -0,0 +1,310 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/domain"
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
"github.com/dtoro/oikos/internal/observability"
|
||||
"github.com/google/uuid"
|
||||
openapi_types "github.com/oapi-codegen/runtime/types"
|
||||
)
|
||||
|
||||
func (s *Server) EnrollClient(ctx context.Context, req gen.EnrollClientRequestObject) (gen.EnrollClientResponseObject, error) {
|
||||
if req.Body == nil {
|
||||
return nil, fmt.Errorf("%w: request body is required", domain.ErrInvalidInput)
|
||||
}
|
||||
|
||||
id, err := s.resolveEntityID(ctx, req.Body.Slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
current, err := sqlcgen.New(s.pool).GetEntityByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s", domain.ErrNotFound, req.Body.Slug)
|
||||
}
|
||||
|
||||
currentState := ""
|
||||
if current.State != nil {
|
||||
currentState = *current.State
|
||||
}
|
||||
if currentState != "planned" && currentState != "provisioning" {
|
||||
return nil, fmt.Errorf("%w: entity %s is in state %q, expected planned or provisioning",
|
||||
domain.ErrInvalidTransition, req.Body.Slug, currentState)
|
||||
}
|
||||
|
||||
meshIP := ""
|
||||
if req.Body.MeshIp != nil {
|
||||
meshIP = *req.Body.MeshIp
|
||||
}
|
||||
if meshIP == "" {
|
||||
return nil, fmt.Errorf("%w: mesh_ip is required for enrollment", domain.ErrInvalidInput)
|
||||
}
|
||||
|
||||
agePubKey, agePrivKey, err := generateAgeKeypair()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("age key generation: %w", err)
|
||||
}
|
||||
|
||||
if s.secretsManager != nil {
|
||||
keyPath := "clients/" + req.Body.Slug + "/age-key"
|
||||
_ = s.secretsManager.Set(ctx, keyPath, agePrivKey)
|
||||
}
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var attrs map[string]any
|
||||
if len(current.Attributes) > 0 {
|
||||
json.Unmarshal(current.Attributes, &attrs)
|
||||
}
|
||||
if attrs == nil {
|
||||
attrs = map[string]any{}
|
||||
}
|
||||
attrs["age_pubkey"] = agePubKey
|
||||
attrs["mesh_ip"] = meshIP
|
||||
attrs["enrolled_at"] = time.Now().UTC().Format(time.RFC3339)
|
||||
if req.Body.Hostname != nil {
|
||||
attrs["hostname"] = *req.Body.Hostname
|
||||
}
|
||||
attrsJSON, _ := json.Marshal(attrs)
|
||||
|
||||
q := sqlcgen.New(tx)
|
||||
provisioning := "provisioning"
|
||||
now := time.Now().UTC()
|
||||
_, err = q.UpdateEntity(ctx, sqlcgen.UpdateEntityParams{
|
||||
State: &provisioning,
|
||||
Attributes: attrsJSON,
|
||||
ID: id,
|
||||
Version: current.Version,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, _ = tx.Exec(ctx,
|
||||
"UPDATE entities SET enrolled_at = $1 WHERE id = $2", now, id)
|
||||
|
||||
_, actor := actorInfo(ctx)
|
||||
entityID := id
|
||||
_ = observability.Audit(ctx, q, "operator", actor, "enroll",
|
||||
&entityID, "POST", "/api/v1/clients/enroll", "",
|
||||
nil,
|
||||
map[string]any{"slug": req.Body.Slug, "mesh_ip": meshIP})
|
||||
_ = observability.Event(ctx, q, "client.enrolled", &entityID,
|
||||
"info", "oikos-api", "",
|
||||
map[string]any{"slug": req.Body.Slug, "type": current.Type})
|
||||
|
||||
if err := ensureDefaultChecks(ctx, tx, id, req.Body.Slug, current.Type, current.Name, attrsJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Store age key in Infisical when backend is available.
|
||||
if s.secretsManager != nil {
|
||||
keyPath := "clients/" + req.Body.Slug + "/age-key"
|
||||
_ = s.secretsManager.Set(ctx, keyPath, agePrivKey)
|
||||
}
|
||||
|
||||
resp := gen.EnrollResponse{
|
||||
AgePublicKey: agePubKey,
|
||||
AgePrivateKey: agePrivKey,
|
||||
}
|
||||
|
||||
return gen.EnrollClient200JSONResponse(resp), nil
|
||||
}
|
||||
|
||||
func (s *Server) ProvisionEntity(ctx context.Context, req gen.ProvisionEntityRequestObject) (gen.ProvisionEntityResponseObject, error) {
|
||||
if req.Body == nil {
|
||||
return nil, fmt.Errorf("%w: request body is required", domain.ErrInvalidInput)
|
||||
}
|
||||
|
||||
hostSlug := req.Body.Host
|
||||
hostID, err := s.resolveEntityID(ctx, hostSlug)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: host %q not found", domain.ErrNotFound, hostSlug)
|
||||
}
|
||||
|
||||
var existingID uuid.UUID
|
||||
err = s.pool.QueryRow(ctx,
|
||||
"SELECT id FROM entities WHERE slug = $1", req.Body.Slug).Scan(&existingID)
|
||||
if err == nil {
|
||||
return nil, fmt.Errorf("%w: entity slug %q already exists", domain.ErrConflict, req.Body.Slug)
|
||||
}
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
entityID := uuid.Must(uuid.NewV7())
|
||||
var attrsJSON []byte
|
||||
if req.Body.Attributes != nil {
|
||||
attrsJSON, _ = json.Marshal(req.Body.Attributes)
|
||||
}
|
||||
if len(attrsJSON) == 0 {
|
||||
attrsJSON = []byte("{}")
|
||||
}
|
||||
|
||||
plannedState := "planned"
|
||||
q := sqlcgen.New(tx)
|
||||
inserted, err := q.InsertEntity(ctx, sqlcgen.InsertEntityParams{
|
||||
ID: entityID,
|
||||
Slug: req.Body.Slug,
|
||||
Type: req.Body.Type,
|
||||
Name: req.Body.Name,
|
||||
State: &plannedState,
|
||||
Attributes: attrsJSON,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
execID := uuid.Must(uuid.NewV7())
|
||||
corrID := "provision_" + entityID.String()[:8]
|
||||
if err := q.InsertExecution(ctx, sqlcgen.InsertExecutionParams{
|
||||
EntityID: entityID,
|
||||
Action: "provision",
|
||||
RiskClass: "config_mutation",
|
||||
CorrelationID: corrID,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("create execution: %w", err)
|
||||
}
|
||||
|
||||
type stepDef struct {
|
||||
order int
|
||||
name string
|
||||
}
|
||||
steps := []stepDef{
|
||||
{1, "validate-constraints"},
|
||||
{2, "create-container"},
|
||||
{3, "configure-network"},
|
||||
{4, "install-services"},
|
||||
{5, "configure-mounts"},
|
||||
{6, "health-check"},
|
||||
}
|
||||
for _, st := range steps {
|
||||
_, err = tx.Exec(ctx,
|
||||
`INSERT INTO provisioning_steps (id, entity_id, execution_id, step_order, step_name)
|
||||
VALUES ($1, $2, $3, $4, $5)`,
|
||||
uuid.Must(uuid.NewV7()), entityID, entityID /* executions PK is entity_id */, st.order, st.name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("insert provisioning step: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = tx.Exec(ctx,
|
||||
`INSERT INTO relationships (source_id, target_id, type)
|
||||
VALUES ($1, $2, 'hosts')`, hostID, entityID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("insert relationship: %w", err)
|
||||
}
|
||||
|
||||
_, actor := actorInfo(ctx)
|
||||
_ = observability.Audit(ctx, q, "operator", actor, "provision",
|
||||
&entityID, "POST", "/api/v1/entities/provision", "",
|
||||
nil,
|
||||
map[string]any{"slug": req.Body.Slug, "host": hostSlug})
|
||||
_ = observability.Event(ctx, q, "entity.provisioned", &entityID,
|
||||
"info", "oikos-api", "",
|
||||
map[string]any{"slug": req.Body.Slug, "type": req.Body.Type, "host": hostSlug})
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entity := sqlcEntityToGen(inserted)
|
||||
return gen.ProvisionEntity201JSONResponse{
|
||||
Body: gen.ProvisionResponse{
|
||||
Entity: entity,
|
||||
ExecutionId: openapi_types.UUID(execID),
|
||||
},
|
||||
Headers: gen.ProvisionEntity201ResponseHeaders{ETag: `"` + strconv.Itoa(int(inserted.Version)) + `"`},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Server) GetProvisionStatus(ctx context.Context, req gen.GetProvisionStatusRequestObject) (gen.GetProvisionStatusResponseObject, error) {
|
||||
slug := string(req.Slug)
|
||||
id, err := s.resolveEntityID(ctx, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var state string
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
"SELECT state FROM entities WHERE id = $1", id).Scan(&state); err != nil {
|
||||
return nil, fmt.Errorf("%w: %s", domain.ErrNotFound, slug)
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT step_name, status, error_message, started_at, finished_at
|
||||
FROM provisioning_steps WHERE entity_id = $1 ORDER BY step_order`, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var provSteps []struct {
|
||||
ErrorMessage *string `json:"error_message"`
|
||||
FinishedAt *time.Time `json:"finished_at"`
|
||||
StartedAt *time.Time `json:"started_at"`
|
||||
Status gen.ProvisionStatusStepsStatus `json:"status"`
|
||||
Step string `json:"step"`
|
||||
}
|
||||
for rows.Next() {
|
||||
var stepName, status string
|
||||
var errMsg *string
|
||||
var started, finished *time.Time
|
||||
if scanErr := rows.Scan(&stepName, &status, &errMsg, &started, &finished); scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
provSteps = append(provSteps, struct {
|
||||
ErrorMessage *string `json:"error_message"`
|
||||
FinishedAt *time.Time `json:"finished_at"`
|
||||
StartedAt *time.Time `json:"started_at"`
|
||||
Status gen.ProvisionStatusStepsStatus `json:"status"`
|
||||
Step string `json:"step"`
|
||||
}{
|
||||
Step: stepName,
|
||||
Status: gen.ProvisionStatusStepsStatus(status),
|
||||
ErrorMessage: errMsg,
|
||||
StartedAt: started,
|
||||
FinishedAt: finished,
|
||||
})
|
||||
}
|
||||
if rows.Err() != nil {
|
||||
return nil, rows.Err()
|
||||
}
|
||||
|
||||
return gen.GetProvisionStatus200JSONResponse{
|
||||
Slug: slug,
|
||||
State: state,
|
||||
Steps: provSteps,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func generateAgeKeypair() (pubKey, privKey string, err error) {
|
||||
seed := make([]byte, 32)
|
||||
if _, err := rand.Read(seed); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
n := new(big.Int).SetBytes(seed)
|
||||
pub := fmt.Sprintf("age1%064x", n)
|
||||
priv := fmt.Sprintf("AGE-SECRET-KEY-1%064x", n)
|
||||
return pub, priv, nil
|
||||
}
|
||||
@@ -3,11 +3,22 @@ package httpapi
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/dtoro/oikos/internal/checkdefaults"
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func ensureDefaultChecks(ctx context.Context, tx pgx.Tx, entityID uuid.UUID, slug, entityType string, attrsJSON []byte) {
|
||||
checkdefaults.Ensure(ctx, tx, entityID, slug, entityType, attrsJSON)
|
||||
// ensureDefaultChecks derives an entity's default checks from the monitoring
|
||||
// kinds its type declares. Thin wrapper over the shared db.EnsureEntityChecks
|
||||
// hook so the HTTP create/patch paths and the MCP entity-mutation tools stay
|
||||
// in lockstep.
|
||||
//
|
||||
// Note the ordering caveat (carried from db.LoadTypeTree / checkdefaults.Ensure):
|
||||
// an entity created through the API usually has no edges yet, so a type whose
|
||||
// address comes from its host (a service) will produce no checks on this pass.
|
||||
// That gap is real and deliberately visible — coverageSweep reports it, and
|
||||
// the next inventory ingest fills it in once the hosting edge exists.
|
||||
func ensureDefaultChecks(ctx context.Context, tx pgx.Tx, entityID uuid.UUID, slug, entityType, name string, attrsJSON []byte) error {
|
||||
_, err := db.EnsureEntityChecks(ctx, tx, entityID, slug, entityType, name, attrsJSON)
|
||||
return err
|
||||
}
|
||||
|
||||
354
internal/httpapi/entities.go
Normal file
354
internal/httpapi/entities.go
Normal file
@@ -0,0 +1,354 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (s *Server) ListEntities(ctx context.Context, req gen.ListEntitiesRequestObject) (gen.ListEntitiesResponseObject, error) {
|
||||
limit := clampLimit(req.Params.Limit)
|
||||
|
||||
// Type filter includes descendants via the parent hierarchy (R3-1).
|
||||
query := `
|
||||
WITH RECURSIVE tt AS (
|
||||
SELECT name FROM entity_types WHERE $1::text IS NULL OR name = $1
|
||||
UNION
|
||||
SELECT et.name FROM entity_types et JOIN tt ON et.parent_type = tt.name
|
||||
WHERE $1::text IS NOT NULL
|
||||
)
|
||||
SELECT ` + entityCols + ` FROM entities e
|
||||
JOIN entity_types et ON et.name = e.type
|
||||
LEFT JOIN entity_status st ON st.entity_id = e.id
|
||||
WHERE e.type IN (SELECT name FROM tt)
|
||||
AND ($2::text IS NULL OR e.state = $2)
|
||||
AND ($3::text IS NULL OR et.domain = $3)
|
||||
AND ($4::text IS NULL OR et.layer = $4)
|
||||
AND ($5::text IS NULL OR e.slug ILIKE '%'||$5||'%' OR e.name ILIKE '%'||$5||'%')
|
||||
AND ($6::text IS NULL OR e.slug > $6)
|
||||
ORDER BY e.slug
|
||||
LIMIT $7`
|
||||
|
||||
rows, err := s.pool.Query(ctx, query,
|
||||
req.Params.Type, req.Params.State, req.Params.Domain, req.Params.Layer,
|
||||
req.Params.Q, req.Params.Cursor, limit+1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []gen.Entity
|
||||
for rows.Next() {
|
||||
e, err := scanEntity(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, e)
|
||||
}
|
||||
if rows.Err() != nil {
|
||||
return nil, rows.Err()
|
||||
}
|
||||
|
||||
var next *string
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
next = &items[len(items)-1].Slug
|
||||
}
|
||||
if items == nil {
|
||||
items = []gen.Entity{}
|
||||
}
|
||||
return gen.ListEntities200JSONResponse{Items: items, NextCursor: next}, nil
|
||||
}
|
||||
|
||||
func (s *Server) GetEntity(ctx context.Context, req gen.GetEntityRequestObject) (gen.GetEntityResponseObject, error) {
|
||||
id, err := s.resolveEntityID(ctx, req.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := scanEntity(s.pool.QueryRow(ctx,
|
||||
"SELECT "+entityCols+" FROM entities e LEFT JOIN entity_status st ON st.entity_id = e.id WHERE e.id = $1", id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gen.GetEntity200JSONResponse{
|
||||
Body: e,
|
||||
Headers: gen.GetEntity200ResponseHeaders{ETag: `"` + strconv.Itoa(e.Version) + `"`},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Server) GetEntityRelations(ctx context.Context, req gen.GetEntityRelationsRequestObject) (gen.GetEntityRelationsResponseObject, error) {
|
||||
id, err := s.resolveEntityID(ctx, req.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dir := "both"
|
||||
if req.Params.Direction != nil {
|
||||
dir = string(*req.Params.Direction)
|
||||
}
|
||||
relType := req.Params.RelType
|
||||
rows, err := sqlcgen.New(s.pool).ListEntityRelations(ctx, sqlcgen.ListEntityRelationsParams{
|
||||
Direction: dir,
|
||||
ID: id,
|
||||
RelType: relType,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := []gen.Relationship{}
|
||||
for _, r := range rows {
|
||||
var attrs *map[string]any
|
||||
if len(r.Attributes) > 0 {
|
||||
var m map[string]any
|
||||
if json.Unmarshal(r.Attributes, &m) == nil && len(m) > 0 {
|
||||
attrs = &m
|
||||
}
|
||||
}
|
||||
validTo := r.ValidTo
|
||||
items = append(items, gen.Relationship{
|
||||
Source: r.SourceSlug,
|
||||
Target: r.TargetSlug,
|
||||
Type: r.Type,
|
||||
Attributes: attrs,
|
||||
ValidFrom: r.ValidFrom,
|
||||
ValidTo: validTo,
|
||||
})
|
||||
}
|
||||
return gen.GetEntityRelations200JSONResponse{Items: items}, nil
|
||||
}
|
||||
|
||||
func (s *Server) GetBlastRadius(ctx context.Context, req gen.GetBlastRadiusRequestObject) (gen.GetBlastRadiusResponseObject, error) {
|
||||
id, err := s.resolveEntityID(ctx, req.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
depth := 3
|
||||
if req.Params.Depth != nil {
|
||||
depth = *req.Params.Depth
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT `+entityCols+`, b.depth
|
||||
FROM blast_radius($1, $2) b
|
||||
JOIN entities e ON e.id = b.entity_id
|
||||
LEFT JOIN entity_status st ON st.entity_id = e.id
|
||||
ORDER BY b.depth, e.slug`, id, depth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
resp := gen.GetBlastRadius200JSONResponse{Items: []struct {
|
||||
Depth int `json:"depth"`
|
||||
Entity gen.Entity `json:"entity"`
|
||||
}{}}
|
||||
for rows.Next() {
|
||||
var e gen.Entity
|
||||
var state *string
|
||||
var attrsJSON []byte
|
||||
var maint *time.Time
|
||||
var health *string
|
||||
var lastCheckAt *time.Time
|
||||
var d int
|
||||
if err := rows.Scan(&e.Id, &e.Slug, &e.Type, &e.Name, &state, &attrsJSON,
|
||||
&maint, &e.Version, &e.CreatedAt, &e.UpdatedAt, &health, &lastCheckAt, &d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.State = state
|
||||
e.MaintenanceUntil = maint
|
||||
if health != nil {
|
||||
h := gen.EntityHealth(*health)
|
||||
e.Health = &h
|
||||
}
|
||||
e.LastCheckAt = lastCheckAt
|
||||
var attrs map[string]any
|
||||
if len(attrsJSON) > 0 && json.Unmarshal(attrsJSON, &attrs) == nil && len(attrs) > 0 {
|
||||
e.Attributes = &attrs
|
||||
}
|
||||
resp.Items = append(resp.Items, struct {
|
||||
Depth int `json:"depth"`
|
||||
Entity gen.Entity `json:"entity"`
|
||||
}{Depth: d, Entity: e})
|
||||
}
|
||||
return resp, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Server) GetGraph(ctx context.Context, req gen.GetGraphRequestObject) (gen.GetGraphResponseObject, error) {
|
||||
depth := 2
|
||||
if req.Params.Depth != nil {
|
||||
depth = *req.Params.Depth
|
||||
}
|
||||
|
||||
var nodes []gen.Entity
|
||||
var err error
|
||||
truncated := false
|
||||
|
||||
// pgx can't infer the array element type from a nil *[]string (the
|
||||
// param is absent from the request, not an empty list), so dereference
|
||||
// to a plain []string first — nil there still encodes as SQL NULL, but
|
||||
// pgx has a concrete type to work with.
|
||||
var relTypes []string
|
||||
if req.Params.RelType != nil {
|
||||
relTypes = *req.Params.RelType
|
||||
}
|
||||
|
||||
if req.Params.Root != nil && *req.Params.Root != "" {
|
||||
rootID, rerr := s.resolveEntityID(ctx, *req.Params.Root)
|
||||
if rerr != nil {
|
||||
return nil, rerr
|
||||
}
|
||||
nodes, err = s.queryEntities(ctx, `
|
||||
SELECT `+entityCols+`
|
||||
FROM blast_radius($1, $2, $3) b JOIN entities e ON e.id = b.entity_id
|
||||
LEFT JOIN entity_status st ON st.entity_id = e.id
|
||||
ORDER BY e.slug`, rootID, depth, relTypes)
|
||||
} else {
|
||||
// Whole-graph view: pick the most-connected entities first so the
|
||||
// graph shows actual topology, not just whatever sorts first
|
||||
// alphabetically. Without this the cap fills with exec:* rows and
|
||||
// drops every host/lxc/service/vm — and every edge those entities
|
||||
// connect — because edges require both endpoints in the node set.
|
||||
// Exclude the cognition transactional types (execution/task): they
|
||||
// are audit records rather than topology, and at ~380 rows they
|
||||
// consumed most of the old 500-node cap.
|
||||
nodes, err = s.queryEntities(ctx, `
|
||||
SELECT `+entityCols+`
|
||||
FROM entities e
|
||||
LEFT JOIN entity_status st ON st.entity_id = e.id
|
||||
WHERE e.type NOT IN ('execution','task')
|
||||
AND e.id IN (
|
||||
SELECT e2.id FROM entities e2
|
||||
LEFT JOIN relationships r ON r.valid_to IS NULL
|
||||
AND (r.source_id = e2.id OR r.target_id = e2.id)
|
||||
WHERE e2.type NOT IN ('execution','task')
|
||||
GROUP BY e2.id
|
||||
ORDER BY count(r.type) DESC, e2.slug
|
||||
LIMIT $1
|
||||
)
|
||||
ORDER BY e.slug`,
|
||||
graphNodeCap+1)
|
||||
if err == nil && len(nodes) > graphNodeCap {
|
||||
nodes = nodes[:graphNodeCap]
|
||||
truncated = true
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ids := make([]uuid.UUID, len(nodes))
|
||||
for i, n := range nodes {
|
||||
ids[i] = uuid.UUID(n.Id)
|
||||
}
|
||||
edgeRows, err := sqlcgen.New(s.pool).ListGraphEdges(ctx, sqlcgen.ListGraphEdgesParams{
|
||||
Ids: ids,
|
||||
RelTypes: relTypes,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
edges := []gen.Relationship{}
|
||||
for _, r := range edgeRows {
|
||||
var attrs *map[string]any
|
||||
if len(r.Attributes) > 0 {
|
||||
var m map[string]any
|
||||
if json.Unmarshal(r.Attributes, &m) == nil && len(m) > 0 {
|
||||
attrs = &m
|
||||
}
|
||||
}
|
||||
validTo := r.ValidTo
|
||||
edges = append(edges, gen.Relationship{
|
||||
Source: r.SourceSlug,
|
||||
Target: r.TargetSlug,
|
||||
Type: r.Type,
|
||||
Attributes: attrs,
|
||||
ValidFrom: r.ValidFrom,
|
||||
ValidTo: validTo,
|
||||
})
|
||||
}
|
||||
|
||||
resp := gen.GetGraph200JSONResponse{Nodes: nodes, Edges: edges}
|
||||
if truncated {
|
||||
resp.Truncated = &truncated
|
||||
}
|
||||
|
||||
if req.Params.Include != nil {
|
||||
for _, inc := range *req.Params.Include {
|
||||
if inc == gen.Status {
|
||||
health, herr := s.entityHealthByID(ctx, ids)
|
||||
if herr != nil {
|
||||
return nil, herr
|
||||
}
|
||||
resp.Health = &health
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// entityHealthByID returns entity_status.health keyed by entity id, for the
|
||||
// given id set (used by GetGraph's include=status).
|
||||
func (s *Server) entityHealthByID(ctx context.Context, ids []uuid.UUID) (map[string]gen.GraphViewHealth, error) {
|
||||
health := make(map[string]gen.GraphViewHealth, len(ids))
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT entity_id, health FROM entity_status WHERE entity_id = ANY($1)`, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
var h string
|
||||
if err := rows.Scan(&id, &h); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
health[id.String()] = gen.GraphViewHealth(h)
|
||||
}
|
||||
return health, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Server) queryEntities(ctx context.Context, query string, args ...any) ([]gen.Entity, error) {
|
||||
rows, err := s.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []gen.Entity{}
|
||||
for rows.Next() {
|
||||
e, err := scanEntity(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, e)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
// sqlcEntityToGen converts a sqlcgen.Entity to a gen.Entity.
|
||||
func sqlcEntityToGen(e sqlcgen.Entity) gen.Entity {
|
||||
out := gen.Entity{
|
||||
Id: e.ID,
|
||||
Slug: e.Slug,
|
||||
Type: e.Type,
|
||||
Name: e.Name,
|
||||
State: e.State,
|
||||
Version: int(e.Version),
|
||||
CreatedAt: e.CreatedAt,
|
||||
UpdatedAt: e.UpdatedAt,
|
||||
}
|
||||
if e.MaintenanceUntil != nil {
|
||||
out.MaintenanceUntil = e.MaintenanceUntil
|
||||
}
|
||||
if len(e.Attributes) > 0 {
|
||||
var attrs map[string]any
|
||||
if json.Unmarshal(e.Attributes, &attrs) == nil && len(attrs) > 0 {
|
||||
out.Attributes = &attrs
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
290
internal/httpapi/entity_mutations.go
Normal file
290
internal/httpapi/entity_mutations.go
Normal file
@@ -0,0 +1,290 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/domain"
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
"github.com/dtoro/oikos/internal/observability"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (s *Server) CreateEntity(ctx context.Context, req gen.CreateEntityRequestObject) (gen.CreateEntityResponseObject, error) {
|
||||
if req.Body == nil {
|
||||
return nil, fmt.Errorf("%w: request body is required", domain.ErrInvalidInput)
|
||||
}
|
||||
|
||||
// Check idempotency if a key was provided. The idempotency scope is the
|
||||
// calling actor, so replays are per-caller.
|
||||
actorType, actorLabel := actorInfo(ctx)
|
||||
actor := actorLabel
|
||||
var bodyHash string
|
||||
if req.Params.IdempotencyKey != nil && *req.Params.IdempotencyKey != "" {
|
||||
key := *req.Params.IdempotencyKey
|
||||
q := sqlcgen.New(s.pool)
|
||||
cached, err := q.GetIdempotentResponse(ctx, sqlcgen.GetIdempotentResponseParams{
|
||||
Actor: actor,
|
||||
Key: key,
|
||||
})
|
||||
if err == nil {
|
||||
// Verify the request body hasn't changed.
|
||||
bodyJSON, _ := json.Marshal(req.Body)
|
||||
bodyHash = fmt.Sprintf("%x", sha256.Sum256(bodyJSON))
|
||||
if cached.RequestHash != bodyHash {
|
||||
return nil, fmt.Errorf("%w: idempotency key %s used with different request body", domain.ErrConflict, key)
|
||||
}
|
||||
// Replay the cached response.
|
||||
if cached.ResponseCode != nil && *cached.ResponseCode == 201 {
|
||||
var entity gen.Entity
|
||||
if len(cached.ResponseBody) > 0 {
|
||||
if err := json.Unmarshal(cached.ResponseBody, &entity); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal cached response: %w", err)
|
||||
}
|
||||
}
|
||||
return gen.CreateEntity201JSONResponse{
|
||||
Body: entity,
|
||||
Headers: gen.CreateEntity201ResponseHeaders{ETag: `"` + strconv.Itoa(entity.Version) + `"`},
|
||||
}, nil
|
||||
}
|
||||
// Forward cached error response.
|
||||
return gen.CreateEntitydefaultApplicationProblemPlusJSONResponse{
|
||||
Body: gen.Problem{Status: int(*cached.ResponseCode), Title: "replayed error"},
|
||||
StatusCode: int(*cached.ResponseCode),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
id, err := uuid.NewV7()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slug := req.Body.Slug
|
||||
if slug == "" {
|
||||
slug = req.Body.Type + ":" + req.Body.Name
|
||||
}
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
q := sqlcgen.New(tx)
|
||||
|
||||
// Validate type exists and is NOT abstract.
|
||||
var isAbstract bool
|
||||
if err := tx.QueryRow(ctx, `SELECT is_abstract FROM entity_types WHERE name = $1`, req.Body.Type).Scan(&isAbstract); err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, fmt.Errorf("%w: entity type %q", domain.ErrNotFound, req.Body.Type)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if isAbstract {
|
||||
return nil, fmt.Errorf("%w: %s", domain.ErrAbstractType, req.Body.Type)
|
||||
}
|
||||
|
||||
// Get default state from lifecycle.
|
||||
var defaultState *string
|
||||
var lcDefault string
|
||||
if err := tx.QueryRow(ctx, `SELECT ld.default_state FROM lifecycle_defs ld
|
||||
JOIN entity_types et ON et.lifecycle_id = ld.id
|
||||
WHERE et.name = $1`, req.Body.Type).Scan(&lcDefault); err == nil {
|
||||
defaultState = &lcDefault
|
||||
}
|
||||
|
||||
state := req.Body.State
|
||||
if state == nil && defaultState != nil {
|
||||
state = defaultState
|
||||
}
|
||||
|
||||
// attributes is NOT NULL; the column default only applies when omitted,
|
||||
// not when an explicit NULL is bound — so default to an empty object.
|
||||
attrsJSON := []byte("{}")
|
||||
if req.Body.Attributes != nil {
|
||||
attrsJSON, _ = json.Marshal(req.Body.Attributes)
|
||||
}
|
||||
|
||||
// Insert the entity.
|
||||
inserted, err := q.InsertEntity(ctx, sqlcgen.InsertEntityParams{
|
||||
ID: id,
|
||||
Slug: slug,
|
||||
Type: req.Body.Type,
|
||||
Name: req.Body.Name,
|
||||
State: state,
|
||||
Attributes: attrsJSON,
|
||||
})
|
||||
if err != nil {
|
||||
// Duplicate slug.
|
||||
if strings.Contains(err.Error(), "unique") || strings.Contains(err.Error(), "duplicate") {
|
||||
return nil, fmt.Errorf("%w: slug %q already exists", domain.ErrAlreadyExists, slug)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert sqlcgen.Entity → gen.Entity.
|
||||
entity := sqlcEntityToGen(inserted)
|
||||
|
||||
// Cache idempotent response.
|
||||
if req.Params.IdempotencyKey != nil && *req.Params.IdempotencyKey != "" {
|
||||
respBody, _ := json.Marshal(entity)
|
||||
code := int32(201)
|
||||
if bodyHash == "" {
|
||||
bodyJSON, _ := json.Marshal(req.Body)
|
||||
bodyHash = fmt.Sprintf("%x", sha256.Sum256(bodyJSON))
|
||||
}
|
||||
if putErr := q.PutIdempotentResponse(ctx, sqlcgen.PutIdempotentResponseParams{
|
||||
Actor: actor,
|
||||
Key: *req.Params.IdempotencyKey,
|
||||
RequestHash: bodyHash,
|
||||
ResponseCode: &code,
|
||||
ResponseBody: respBody,
|
||||
}); putErr != nil {
|
||||
return nil, putErr
|
||||
}
|
||||
}
|
||||
|
||||
// Audit.
|
||||
entityID := inserted.ID
|
||||
if auditErr := observability.Audit(ctx, q, actorType, actor, "create",
|
||||
&entityID, "POST", "/api/v1/entities", "",
|
||||
nil,
|
||||
map[string]any{"type": req.Body.Type, "slug": slug}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
|
||||
if eventErr := observability.Event(ctx, q, "entity.created", &entityID,
|
||||
"info", "oikos-api", "",
|
||||
map[string]any{"slug": slug, "type": req.Body.Type}); eventErr != nil {
|
||||
return nil, eventErr
|
||||
}
|
||||
|
||||
if err := ensureDefaultChecks(ctx, tx, inserted.ID, slug, req.Body.Type, inserted.Name, attrsJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return gen.CreateEntity201JSONResponse{
|
||||
Body: entity,
|
||||
Headers: gen.CreateEntity201ResponseHeaders{ETag: `"` + strconv.Itoa(entity.Version) + `"`},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Server) PatchEntity(ctx context.Context, req gen.PatchEntityRequestObject) (gen.PatchEntityResponseObject, error) {
|
||||
if req.Body == nil {
|
||||
return nil, fmt.Errorf("%w: request body is required", domain.ErrInvalidInput)
|
||||
}
|
||||
|
||||
id, err := s.resolveEntityID(ctx, req.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse If-Match header (quoted version string).
|
||||
ifMatch := strings.Trim(req.Params.IfMatch, `"`)
|
||||
expectedVersion, err := strconv.Atoi(ifMatch)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: invalid If-Match header %q", domain.ErrInvalidInput, req.Params.IfMatch)
|
||||
}
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
// Get current entity for version check + lifecycle validation.
|
||||
current, err := sqlcgen.New(tx).GetEntityByID(ctx, id)
|
||||
if err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, fmt.Errorf("%w: %s", domain.ErrNotFound, req.Id)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if int(current.Version) != expectedVersion {
|
||||
return nil, fmt.Errorf("%w: expected version %d, current version %d",
|
||||
domain.ErrConflict, expectedVersion, current.Version)
|
||||
}
|
||||
|
||||
// Validate lifecycle transition if state is being changed.
|
||||
if req.Body.State != nil && *req.Body.State != "" {
|
||||
fromState := ""
|
||||
if current.State != nil {
|
||||
fromState = *current.State
|
||||
}
|
||||
if err := db.ValidateTransition(ctx, tx, id, current.Type, fromState, *req.Body.State); err != nil {
|
||||
if errors.Is(err, db.ErrTransitionInvalid) {
|
||||
return nil, fmt.Errorf("%w: %v", domain.ErrInvalidTransition, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Check idempotency (note: the spec doesn't define Idempotency-Key for PATCH,
|
||||
// but we handle it if the generated code ever adds it).
|
||||
// For now, no idempotency check on PATCH.
|
||||
|
||||
// Marshal attributes if provided.
|
||||
var attrsJSON []byte
|
||||
if req.Body.Attributes != nil {
|
||||
attrsJSON, _ = json.Marshal(req.Body.Attributes)
|
||||
}
|
||||
|
||||
// Perform the update via sqlcgen.
|
||||
q := sqlcgen.New(tx)
|
||||
updated, err := q.UpdateEntity(ctx, sqlcgen.UpdateEntityParams{
|
||||
Name: req.Body.Name,
|
||||
State: req.Body.State,
|
||||
Attributes: attrsJSON,
|
||||
SetMaintenance: req.Body.MaintenanceUntil != nil,
|
||||
MaintenanceUntil: req.Body.MaintenanceUntil,
|
||||
ID: id,
|
||||
Version: int32(expectedVersion),
|
||||
})
|
||||
if err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
// Version mismatch or entity not found.
|
||||
return nil, fmt.Errorf("%w: entity was modified concurrently", domain.ErrConflict)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entity := sqlcEntityToGen(updated)
|
||||
|
||||
// Audit.
|
||||
patchActorType, patchActor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, q, patchActorType, patchActor, "patch",
|
||||
&id, "PATCH", "/api/v1/entities/"+req.Id, "",
|
||||
nil,
|
||||
map[string]any{"version": expectedVersion}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
|
||||
if eventErr := observability.Event(ctx, q, "entity.updated", &id,
|
||||
"info", "oikos-api", "",
|
||||
map[string]any{"slug": entity.Slug, "type": entity.Type, "version": updated.Version}); eventErr != nil {
|
||||
return nil, eventErr
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.entityCache.Invalidate(entity.Slug, entity.Id.String())
|
||||
|
||||
return gen.PatchEntity200JSONResponse{
|
||||
Body: entity,
|
||||
Headers: gen.PatchEntity200ResponseHeaders{ETag: `"` + strconv.Itoa(entity.Version) + `"`},
|
||||
}, nil
|
||||
}
|
||||
@@ -67,6 +67,7 @@ func (s *Server) CreateEntityType(ctx context.Context, req gen.CreateEntityTypeR
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, sqlcgen.New(tx), actorType, actor, "create",
|
||||
nil, "POST", "/api/v1/ontology/entity-types", "",
|
||||
nil,
|
||||
map[string]any{"name": req.Body.Name, "domain": req.Body.Domain}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
@@ -148,6 +149,7 @@ func (s *Server) PatchEntityType(ctx context.Context, req gen.PatchEntityTypeReq
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, sqlcgen.New(tx), actorType, actor, "patch",
|
||||
nil, "PATCH", "/api/v1/ontology/entity-types/"+req.Name, "",
|
||||
nil,
|
||||
map[string]any{"status": req.Body.Status}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
|
||||
60
internal/httpapi/events.go
Normal file
60
internal/httpapi/events.go
Normal file
@@ -0,0 +1,60 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
)
|
||||
|
||||
func (s *Server) QueryEvents(ctx context.Context, req gen.QueryEventsRequestObject) (gen.QueryEventsResponseObject, error) {
|
||||
limit := clampLimit(req.Params.Limit)
|
||||
var eventType, entityID, severity, correlationID *string
|
||||
if req.Params.Type != nil {
|
||||
eventType = req.Params.Type
|
||||
}
|
||||
if req.Params.EntityId != nil {
|
||||
entityID = req.Params.EntityId
|
||||
}
|
||||
if req.Params.Severity != nil {
|
||||
severity = req.Params.Severity
|
||||
}
|
||||
if req.Params.CorrelationId != nil {
|
||||
correlationID = req.Params.CorrelationId
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id, ts, type, entity_id::text, severity, source, data, correlation_id
|
||||
FROM events
|
||||
WHERE ($1::text IS NULL OR type = $1)
|
||||
AND ($2::text IS NULL OR entity_id::text = $2)
|
||||
AND ($3::text IS NULL OR severity = $3)
|
||||
AND ($4::text IS NULL OR correlation_id = $4)
|
||||
AND ($5::timestamptz IS NULL OR ts >= $5)
|
||||
AND ($6::timestamptz IS NULL OR ts <= $6)
|
||||
ORDER BY ts DESC
|
||||
LIMIT $7`,
|
||||
eventType, entityID, severity, correlationID, req.Params.From, req.Params.To, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := []gen.Event{}
|
||||
for rows.Next() {
|
||||
var e gen.Event
|
||||
var dataBytes []byte
|
||||
var entID, corrID *string
|
||||
if err := rows.Scan(&e.Id, &e.Ts, &e.Type, &entID, &e.Severity, &e.Source, &dataBytes, &corrID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.EntityId = entID
|
||||
e.CorrelationId = corrID
|
||||
var data map[string]any
|
||||
if json.Unmarshal(dataBytes, &data) == nil {
|
||||
e.Data = &data
|
||||
}
|
||||
items = append(items, e)
|
||||
}
|
||||
return gen.QueryEvents200JSONResponse{Items: items}, rows.Err()
|
||||
}
|
||||
55
internal/httpapi/execution_logs.go
Normal file
55
internal/httpapi/execution_logs.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/dtoro/oikos/internal/execlog"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// serveExecutionLogs returns an execution's streamed command output.
|
||||
//
|
||||
// Registered as a carve-out rather than through the OpenAPI codegen for the
|
||||
// same reason as /activity/recent: it is a recency-ordered projection with no
|
||||
// schema type yet. Without this the execution_logs rows would be write-only —
|
||||
// which is the exact shape of the bugs this whole change set has been about.
|
||||
func (s *Server) serveExecutionLogs(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
rawID := chi.URLParam(req, "id")
|
||||
execID, err := uuid.Parse(rawID)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid execution id", rawID)
|
||||
return
|
||||
}
|
||||
|
||||
limit := 1000
|
||||
if l := req.URL.Query().Get("limit"); l != "" {
|
||||
if n, perr := strconv.Atoi(l); perr == nil && n > 0 && n <= 5000 {
|
||||
limit = n
|
||||
}
|
||||
}
|
||||
|
||||
chunks, err := execlog.Read(ctx, s.pool, execID, limit)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Also hand back the concatenation, since that is what a caller tailing
|
||||
// output actually wants to render.
|
||||
var combined strings.Builder
|
||||
for _, c := range chunks {
|
||||
combined.WriteString(c.Chunk)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"items": chunks,
|
||||
"combined": combined.String(),
|
||||
})
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/domain"
|
||||
@@ -15,8 +17,22 @@ import (
|
||||
|
||||
// ─── Executions ────────────────────────────────────────────────────────
|
||||
|
||||
// ListExecutions returns executions newest-first.
|
||||
//
|
||||
// The target/action/correlation_id filters are declared in the OpenAPI spec and
|
||||
// generated into the request struct, but were never bound — so
|
||||
// `GET /executions?target=<id>` silently returned the first page of the whole
|
||||
// fleet. Ordering was by target slug, which is neither useful for a history
|
||||
// view nor unique enough to paginate on: several executions share a target, so
|
||||
// a slug cursor could skip or repeat rows.
|
||||
func (s *Server) ListExecutions(ctx context.Context, req gen.ListExecutionsRequestObject) (gen.ListExecutionsResponseObject, error) {
|
||||
limit := clampLimit(req.Params.Limit)
|
||||
|
||||
cursorTime, cursorID, err := parseExecutionCursor(req.Params.Cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT e.entity_id, e.classification_id::text, e.signal_entity_id::text,
|
||||
e.target_entity_id, e.action, e.risk_class,
|
||||
@@ -27,10 +43,18 @@ func (s *Server) ListExecutions(ctx context.Context, req gen.ListExecutionsReque
|
||||
FROM executions e
|
||||
JOIN entities te ON te.id = e.target_entity_id
|
||||
WHERE ($1::text IS NULL OR e.status = $1)
|
||||
AND ($2::text IS NULL OR te.slug > $2)
|
||||
ORDER BY te.slug
|
||||
LIMIT $3`,
|
||||
req.Params.Status, req.Params.Cursor, limit+1)
|
||||
-- target accepts a slug or a uuid: the SPA passes an entity id,
|
||||
-- while a human poking the API reaches for the slug.
|
||||
AND ($2::text IS NULL OR te.slug = $2 OR e.target_entity_id::text = $2)
|
||||
-- the run tool encodes action as "run:{json}", so match the verb too
|
||||
AND ($3::text IS NULL OR e.action = $3 OR split_part(e.action, ':', 1) = $3)
|
||||
AND ($4::text IS NULL OR e.correlation_id = $4)
|
||||
AND ($5::timestamptz IS NULL
|
||||
OR (e.created_at, e.entity_id) < ($5::timestamptz, $6::uuid))
|
||||
ORDER BY e.created_at DESC, e.entity_id DESC
|
||||
LIMIT $7`,
|
||||
req.Params.Status, req.Params.Target, req.Params.Action, req.Params.CorrelationId,
|
||||
cursorTime, cursorID, limit+1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -64,7 +88,9 @@ func (s *Server) ListExecutions(ctx context.Context, req gen.ListExecutionsReque
|
||||
var next *string
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
next = &items[len(items)-1].Slug
|
||||
last := items[len(items)-1]
|
||||
cursor := formatExecutionCursor(last.CreatedAt, last.Id)
|
||||
next = &cursor
|
||||
}
|
||||
if items == nil {
|
||||
items = []gen.Execution{}
|
||||
@@ -72,6 +98,32 @@ func (s *Server) ListExecutions(ctx context.Context, req gen.ListExecutionsReque
|
||||
return gen.ListExecutions200JSONResponse{Items: items, NextCursor: next}, nil
|
||||
}
|
||||
|
||||
// Executions are ordered by (created_at DESC, entity_id DESC), so the cursor
|
||||
// has to carry both — created_at alone is not unique, and paginating on a
|
||||
// non-unique key drops or repeats rows at page boundaries.
|
||||
func formatExecutionCursor(createdAt time.Time, id uuid.UUID) string {
|
||||
return createdAt.UTC().Format(time.RFC3339Nano) + "," + id.String()
|
||||
}
|
||||
|
||||
func parseExecutionCursor(cursor *string) (*time.Time, *uuid.UUID, error) {
|
||||
if cursor == nil || *cursor == "" {
|
||||
return nil, nil, nil
|
||||
}
|
||||
rawTime, rawID, ok := strings.Cut(*cursor, ",")
|
||||
if !ok {
|
||||
return nil, nil, domain.ErrInvalidInput
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339Nano, rawTime)
|
||||
if err != nil {
|
||||
return nil, nil, domain.ErrInvalidInput
|
||||
}
|
||||
id, err := uuid.Parse(rawID)
|
||||
if err != nil {
|
||||
return nil, nil, domain.ErrInvalidInput
|
||||
}
|
||||
return &t, &id, nil
|
||||
}
|
||||
|
||||
func (s *Server) GetExecution(ctx context.Context, req gen.GetExecutionRequestObject) (gen.GetExecutionResponseObject, error) {
|
||||
id, err := s.resolveEntityID(ctx, req.Id)
|
||||
if err != nil {
|
||||
@@ -188,6 +240,7 @@ func (s *Server) RequestExecution(ctx context.Context, req gen.RequestExecutionR
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, q, actorType, actor, "create",
|
||||
&id, "POST", "/api/v1/executions", "",
|
||||
nil,
|
||||
map[string]any{"action": req.Body.Action, "target": req.Body.Target}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
@@ -255,6 +308,7 @@ func (s *Server) CancelExecution(ctx context.Context, req gen.CancelExecutionReq
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, q, actorType, actor, "cancel",
|
||||
&id, "POST", "/api/v1/executions/"+req.Id+"/cancel", "",
|
||||
nil,
|
||||
map[string]any{"status": "cancelled"}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
|
||||
64
internal/httpapi/executions_cursor_test.go
Normal file
64
internal/httpapi/executions_cursor_test.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// The cursor carries both created_at and entity_id because executions are
|
||||
// ordered by the pair. created_at alone is not unique — several executions can
|
||||
// share a millisecond — and paginating on a non-unique key silently drops or
|
||||
// repeats rows at page boundaries. The previous cursor was the target slug,
|
||||
// which is far less unique still: every execution against the same host shares
|
||||
// it.
|
||||
func TestExecutionCursorRoundTrips(t *testing.T) {
|
||||
created := time.Date(2026, 7, 28, 9, 15, 30, 123456789, time.UTC)
|
||||
id := uuid.MustParse("018f3a2b-0000-7000-8000-000000000042")
|
||||
|
||||
cursor := formatExecutionCursor(created, id)
|
||||
|
||||
gotTime, gotID, err := parseExecutionCursor(&cursor)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if !gotTime.Equal(created) {
|
||||
t.Errorf("time round-trip: got %v, want %v", gotTime, created)
|
||||
}
|
||||
if *gotID != id {
|
||||
t.Errorf("id round-trip: got %v, want %v", *gotID, id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionCursorNanosecondsSurvive(t *testing.T) {
|
||||
// Truncating to seconds would make the cursor ambiguous for executions
|
||||
// started in the same second, which is the normal case for a plan whose
|
||||
// steps run back to back.
|
||||
a := time.Date(2026, 7, 28, 9, 15, 30, 1, time.UTC)
|
||||
b := time.Date(2026, 7, 28, 9, 15, 30, 2, time.UTC)
|
||||
id := uuid.New()
|
||||
|
||||
if formatExecutionCursor(a, id) == formatExecutionCursor(b, id) {
|
||||
t.Error("cursors one nanosecond apart must not collide")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionCursorRejectsGarbage(t *testing.T) {
|
||||
empty := ""
|
||||
tm, id, err := parseExecutionCursor(&empty)
|
||||
if err != nil || tm != nil || id != nil {
|
||||
t.Errorf("empty cursor should mean 'no cursor', got %v/%v/%v", tm, id, err)
|
||||
}
|
||||
|
||||
if tm, id, err := parseExecutionCursor(nil); err != nil || tm != nil || id != nil {
|
||||
t.Errorf("nil cursor should mean 'no cursor', got %v/%v/%v", tm, id, err)
|
||||
}
|
||||
|
||||
for _, bad := range []string{"nonsense", "2026-07-28T09:15:30Z", "notatime,018f3a2b-0000-7000-8000-000000000042", "2026-07-28T09:15:30Z,notauuid"} {
|
||||
b := bad
|
||||
if _, _, err := parseExecutionCursor(&b); err == nil {
|
||||
t.Errorf("cursor %q should have been rejected", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
80
internal/httpapi/fleet_health.go
Normal file
80
internal/httpapi/fleet_health.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
)
|
||||
|
||||
func (s *Server) GetFleetHealth(ctx context.Context, req gen.GetFleetHealthRequestObject) (gen.GetFleetHealthResponseObject, error) {
|
||||
resp := gen.GetFleetHealth200JSONResponse{}
|
||||
resp.Entities = []struct {
|
||||
Health gen.HealthSummaryEntitiesHealth `json:"health"`
|
||||
LastCheckAt *time.Time `json:"last_check_at"`
|
||||
Slug string `json:"slug"`
|
||||
Trend *gen.HealthSummaryEntitiesTrend `json:"trend"`
|
||||
Type string `json:"type"`
|
||||
}{}
|
||||
|
||||
// Exclude 'check' entities (internal probes) — only entities actually
|
||||
// being monitored should count toward fleet health.
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT e.slug, e.type, st.health, st.last_check_at
|
||||
FROM entity_status st JOIN entities e ON e.id = st.entity_id
|
||||
WHERE e.type <> 'check'
|
||||
ORDER BY e.slug`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
stale := 0
|
||||
for rows.Next() {
|
||||
var slug, typ, health string
|
||||
var lastCheck *time.Time
|
||||
if err := rows.Scan(&slug, &typ, &health, &lastCheck); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch health {
|
||||
case "healthy":
|
||||
resp.Summary.Healthy++
|
||||
case "degraded":
|
||||
resp.Summary.Degraded++
|
||||
case "down":
|
||||
resp.Summary.Down++
|
||||
case "stale":
|
||||
stale++
|
||||
default:
|
||||
resp.Summary.Unknown++
|
||||
}
|
||||
resp.Entities = append(resp.Entities, struct {
|
||||
Health gen.HealthSummaryEntitiesHealth `json:"health"`
|
||||
LastCheckAt *time.Time `json:"last_check_at"`
|
||||
Slug string `json:"slug"`
|
||||
Trend *gen.HealthSummaryEntitiesTrend `json:"trend"`
|
||||
Type string `json:"type"`
|
||||
}{
|
||||
Health: gen.HealthSummaryEntitiesHealth(health),
|
||||
LastCheckAt: lastCheck,
|
||||
Slug: slug,
|
||||
Type: typ,
|
||||
})
|
||||
}
|
||||
if stale > 0 {
|
||||
resp.Summary.Stale = &stale
|
||||
}
|
||||
return resp, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Server) ExportSeeds(ctx context.Context, req gen.ExportSeedsRequestObject) (gen.ExportSeedsResponseObject, error) {
|
||||
exports, err := db.ExportToYAML(ctx, s.pool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gen.ExportSeeds200JSONResponse{
|
||||
Ontology: string(exports["ontology.yaml"]),
|
||||
Inventory: string(exports["inventory.yaml"]),
|
||||
Policy: string(exports["policy.yaml"]),
|
||||
}, nil
|
||||
}
|
||||
@@ -86,6 +86,14 @@ const (
|
||||
CheckKindTcp CheckKind = "tcp"
|
||||
)
|
||||
|
||||
// Defines values for CheckLastHealth.
|
||||
const (
|
||||
CheckLastHealthDegraded CheckLastHealth = "degraded"
|
||||
CheckLastHealthDown CheckLastHealth = "down"
|
||||
CheckLastHealthHealthy CheckLastHealth = "healthy"
|
||||
CheckLastHealthUnknown CheckLastHealth = "unknown"
|
||||
)
|
||||
|
||||
// Defines values for CheckCreateKind.
|
||||
const (
|
||||
CheckCreateKindCertExpiry CheckCreateKind = "cert-expiry"
|
||||
@@ -273,10 +281,10 @@ const (
|
||||
|
||||
// Defines values for TrendDirection.
|
||||
const (
|
||||
Degrading TrendDirection = "degrading"
|
||||
Improving TrendDirection = "improving"
|
||||
Stable TrendDirection = "stable"
|
||||
Unknown TrendDirection = "unknown"
|
||||
TrendDirectionDegrading TrendDirection = "degrading"
|
||||
TrendDirectionImproving TrendDirection = "improving"
|
||||
TrendDirectionStable TrendDirection = "stable"
|
||||
TrendDirectionUnknown TrendDirection = "unknown"
|
||||
)
|
||||
|
||||
// Defines values for ListApprovalsParamsStatus.
|
||||
@@ -462,7 +470,13 @@ type Check struct {
|
||||
Id openapi_types.UUID `json:"id"`
|
||||
IntervalS int `json:"interval_s"`
|
||||
Kind CheckKind `json:"kind"`
|
||||
Slug string `json:"slug"`
|
||||
|
||||
// LastHealth This check's own most recent verdict. An entity's health is the worst of these across its enabled checks, so this is what explains *why* an entity is degraded. Null until the check first runs.
|
||||
LastHealth *CheckLastHealth `json:"last_health"`
|
||||
|
||||
// LastRunAt When this check last executed. Null = never run.
|
||||
LastRunAt *time.Time `json:"last_run_at"`
|
||||
Slug string `json:"slug"`
|
||||
|
||||
// Target Entity slug (instance-scoped)
|
||||
Target *string `json:"target"`
|
||||
@@ -477,6 +491,9 @@ type Check struct {
|
||||
// CheckKind defines model for Check.Kind.
|
||||
type CheckKind string
|
||||
|
||||
// CheckLastHealth This check's own most recent verdict. An entity's health is the worst of these across its enabled checks, so this is what explains *why* an entity is degraded. Null until the check first runs.
|
||||
type CheckLastHealth string
|
||||
|
||||
// CheckCreate defines model for CheckCreate.
|
||||
type CheckCreate struct {
|
||||
Config *map[string]interface{} `json:"config,omitempty"`
|
||||
@@ -599,13 +616,10 @@ type EnrollResponse struct {
|
||||
AgePublicKey string `json:"age_public_key"`
|
||||
|
||||
// InfisicalClientId Infisical UniversalAuth client ID
|
||||
InfisicalClientId string `json:"infisical_client_id"`
|
||||
InfisicalClientId *string `json:"infisical_client_id,omitempty"`
|
||||
|
||||
// InfisicalClientSecret Infisical UniversalAuth client secret
|
||||
InfisicalClientSecret string `json:"infisical_client_secret"`
|
||||
|
||||
// MachineIdentityToken Infisical machine identity access token
|
||||
MachineIdentityToken *string `json:"machine_identity_token,omitempty"`
|
||||
InfisicalClientSecret *string `json:"infisical_client_secret,omitempty"`
|
||||
}
|
||||
|
||||
// Entity defines model for Entity.
|
||||
@@ -8208,8 +8222,8 @@ func (sh *strictHandler) GetTrends(w http.ResponseWriter, r *http.Request, entit
|
||||
// Base64 encoded, gzipped, json marshaled Swagger object
|
||||
var swaggerSpec = []string{
|
||||
|
||||
"H4sIAAAAAAAC/+x963IbudXgq6C4WzXUpCn6MpPsyLU/FFljO7FjraXJt6mRiwK7D0mM0EAPgKbEuFyV",
|
||||
"X/sAW3nCPMlXuHY3iSabF1nOVP7YkhqNBs4FOPfzqZfyvOAMmJK9k0+9GeAMhPnx/ApP9f8ZyFSQQhHO",
|
||||
"H4sIAAAAAAAC/+x963IbudXgq6C4WzXUpCn6MpPsyPX9UGSN7cSOtZYm36ZGLgrsPiQxQgM9AJoS43JV",
|
||||
"fu0DbOUJ8yRbuHY3iSabF1nOVP7YkhqNBs4FOPfzqZfyvOAMmJK9k0+9GeAMhPnx/ApP9f8ZyFSQQhHO",
|
||||
"eie9DyB5KVJAcxCScIYmXKA3k8E7rNJZL+nJdAY51u+pRQG9k55UgrBp7/Pnz0mvwALnoNwHzkohuVj9",
|
||||
"xPsC/1oCSs1jNBE8RxgVAuaElxIJkAVnEr6RiMG9GtlhvaRH9Lu/liAWvaTHcK4/Hh62LyvpnTNF1OJN",
|
||||
"trqSn3568xJxgSQtp6gPx9NjdDPjUp3MyrEg8ubIf7bAalZ9lWS9pCfg15IIyHonSpTQZQWXtIwA3D5r",
|
||||
@@ -8231,152 +8245,154 @@ var swaggerSpec = []string{
|
||||
"diBKFRddr287eFn6MrdgL+npL2JlVeaFVOCVvKykLZDdRZgChQmtbaWCwGHEphzUjHebwmjKncQeY/YY",
|
||||
"kaLbaHNMjlKeQUe55wCSTIXZwLhxKrOEeAlK6RlXSO3WauZwj/NCr7k3pXyM6bGm4BFOVexwK61SsJX0",
|
||||
"MMe0jLNj91Pq1ujxdqb159DZDNLb1c2mnE1IxO7yV0yJ1XP0Uetuvwi9arTWybAm/Ha8F/TWxBzTkYxT",
|
||||
"87L0NFOq0POk+t+MyFt9AYNQA3Mla3Bkgkw0lgorgUg5G9itxcWMNqlGYTGFDeJHnzCpMEthYM7IrNOF",
|
||||
"aSduuZDd7Poh6ut/t5qZ5MC1/hMH5Rq6Snp/56yL1rFGcnJUUkNofUUVtXQg1LabsiLXdbQYzDxtelmT",
|
||||
"5sLw3z95knydFLiJhtZTQtjg0+j+PObXY7qO5Fa8XXgb4S5o24SnyLWxjt4/xxappREycTah3SSx1J+k",
|
||||
"K0PGFEs1EjgjVhsiCvK4ROX+gIXAi7gYcRA9uuMR7JTOkUFTBixddxCwMh9b6Fd2qhhiBaQ8z4EZPT6A",
|
||||
"dV8dVPBSwbIYPLC3ck0UnnHaolQaq11nY88toZ0HV9y6wxEaF5rtbpsWwSVS2ah9Wp/CGWcK7lWE4o0B",
|
||||
"aEIoyJG1QkSsChdYzSTiE2RGI33pidKsGJk3kZphhfzryRaEL4mjtuYHr0gOUuG8MNqeVvIZ3CtUcEqR",
|
||||
"hh1IjfBuTCB5IS1pT9t3eCVKQGSCjvXo4wXO9XdSUmjYydrOYjY+TruAzowbfnssQZXFsZztDrPaNb6k",
|
||||
"zHPGFWckRalFd3C/OKZNNsmTay9mQ0iXkAqw4vqK2CxXl/SGTYgkKaZImheRHoawsaGSMQWkOFIzIlFq",
|
||||
"Zt8CDquSsIwu+yWWszHHIrusjMJLLOBUdDnyZqvoZWOUMwJyNF6MtIJjyBZnGdFbxfSiMefq60t2Oivm",
|
||||
"GdOw1ECBDI0XyM6btGiG7uP+0j/wt53qtPrpuT4hhNvw0lT6mZ9pXKa3oOxk3w9ywkoFyF/hCcq5VJqp",
|
||||
"9Bv6oqzjuokQO1H3ay4Y2DdQt5vXvxCjlmVeC7baA6HdTxfHfGIAg559N4shYgaYqoh0lcFU4AxarAEZ",
|
||||
"v2uR+O18i/hDqTCNINysj4+lxqnZB6cZ6BMaW2+0EY6+kQjuC0g1LaTYChQxwbNkt6xldUuY80tNqs26",
|
||||
"nVWzxNBpr3uLO5iDcAasXdHHC2CozzgbCJCcziE7cn7AVXz6z62samlrK5wdO2kC8uNbSiJnWJx2G+wc",
|
||||
"g9g5E5zSD+6OXaG1GZfK+6easDlNVYkp8gPMlTcDBGY+wqYox+mMsCgD5yBnzrbUnPTSRtvo5+jNhREG",
|
||||
"tIBqzq+5NVFYsalVqdoUTnInTxjcDSguFC+ONtqbzLTr4OZiMGJy1qgQZI4VjG5jsR+nr84Hl+dnH86v",
|
||||
"Bn8+/9vg+PjYbJdyfXlmkIpF0bZXM3c5piSNT42n8FTPZ8doGjVTX76/uKxJOXHjjLu+R/Z+drJw2x3/",
|
||||
"EyNagsD0tFQzd6WjNy87zWzlg61nd6/FiMrS28gTzMh4Y9d9wL1RkZiVU5B9cRNpLGEhWUF5HJztoIiT",
|
||||
"mYoHFiglyLhUjXOsem0X5bG6blqvAMiQHZWguxkwx/AGdESinDOiuHX0BdtJh4PcXz4fD+d8Moq5uZsc",
|
||||
"CJpbMou3SzNBL40ttikaG9eWY32RMMxSGJVMWZv/blP5I7f1mKus1bUIvV6Lab5jdEGbbWknY/d2jjdn",
|
||||
"bHL3odl9NUeDnBvLaeeYVtdbg2/abOB4irXOa+hbf+AbicKLIxdQFPn0Rqy1Y6e5kpfWgCettgSIkgmk",
|
||||
"i5TqhTjj3mhJdVjF45KuWEplPL9IizPB/wuV+bnbXdhEUjsCLuLRg6cKUTDcxoLIMCFAM4lyt8JCgASm",
|
||||
"jnvJFrh7B8KEtM1XcNgFcV+Ec+OovjJWpArDVjFAfSUwk0ZorfYUF1daEHDlyKAFhqN6lGR9QX+6fP8X",
|
||||
"dOlBtdF+13g5su2Ma+BGHxE58nQYNwdTvABRN/7loLC9QAW2JqlSaKxM+RyEQZ9R9qaMtEbSBEB3NfO1",
|
||||
"IrTAQl/ent02GxcNTEdrnTKrkTUmMAjM/VkISE3Q48e9Dlx3ujrEeCg30RFW8nEtfW08ZUcrwbwPQTnB",
|
||||
"3zHBVEYdQCuUdEgS2plk1h+3cTytR0iLP+Yg+NiVNqNH1NxF7C77jbaPrsAKP2BsRd2KUKMd3kt6d1h4",
|
||||
"E70gSsvzcQ+EUWnjBk7ZXaAKISxB8LOGgWOBiYRsi8AJd3/XjAluiVHSCrGLW/nOavHIm0N0nCmj6/i0",
|
||||
"4dPr/BbXYFN7Bpk+kN9u63jrzo4+gfOItHR5SyhF9inqr8pM9ok7LI5iEpMAaU7c/T18D+ihs4NrN+Nm",
|
||||
"wK6T1MW+1BOJm3WRu83AWRt5tDaQ1tn6zPGj+XiyqP1sB08wsdEXennZiJfGEq5vOGr/Lrj+YTTG6a37",
|
||||
"Tf84cu993MflWVtIRLLbOho3hOFu6wwNx1ergbM6xaqTVYDB9nqOirAElm1XZ53Gl1jRmpONczDnJggc",
|
||||
"Musi6/PCGq2PesnW4Ur1ZL6Nl4Oba20c3SuBi9lfCdytwhCyKTQDINal2nxwCJQzUsRcMJUdqs1sv6Nx",
|
||||
"KRKZGXGTkQwNrssnT55DsHU5jdTavAhLaZnB/7Y0acxHzkMN0Zg5xrMtgOPsfRGwKFGy1Oc8xZ3Z+lMo",
|
||||
"xYVZ1IyoiP96WcbkNpDaYjCG9tcGBq2+VO+6aGxwyYUQ8HkovK1Y93Y8jlvjsAQ0I8BIrs9in6Gll+wC",
|
||||
"vZT5QMMntaudba2xxcGwiwdTtqHqP27DFrfhMuQdACu/XBTuf2b8jmq+eU0i10pXOzVht5CNoly0MS5E",
|
||||
"YHbbKXCrXahhpChil8hrMp1RMp1p1Jg8Xh9h0omvbOx451hzRRSFNTuu+DDjaZnbsBFRsjHnt8YaNAep",
|
||||
"yLQte2qzvdkuIIbkt17Vf6mP7FWOqptio4aKrN0WuCW2FYicaClip5eDNTGiDXyaCJ6foE+Kn6BPDlTy",
|
||||
"BP3McA7ZwLBqgo6Pjz9+/vx5o3ub+LQpc6+sGKtr64jB+x0oQdJLEA7Ckctm0aZ45ebdlihCSsuiTkkC",
|
||||
"3/WS3tOZ/qclctBIg+suNjyfdmK/LfJBc3zfacqcsE7jtrEwhPyEpRIY+A5ZWCCfd7Dhs8vCpex0b4VL",
|
||||
"d514dGUGRUMqFlYhcFQQcF4hMraICxsJu51xoygoAdkei90Mq21C878IlZwhyu9AoDEvWZZoia2wQSQw",
|
||||
"t+/ZFOLh970ISptj4reyVuFKsXbINo7UYCDYS9wqKlivPPu1xAIzRVhbZPgWuaizRcHVDCT5u8098Iv3",
|
||||
"Kc9LBktzgYQxH9tTwNdBczd3Z4OSarqvh1SDlFYwX1OL14Vx1so2LN9eS3lptfxBIbiI3BQ/EqDZwGT0",
|
||||
"1cJxkB2O+t89e3bUHuVn3Hzx47lNc16CnZ0hjO9yqvh8nQ20E0s12CSUBK9DD495qU7GVMtjteCBUpDN",
|
||||
"mrf5zFp3y4VWPeRaG8Y6p/a7Ny8TlHIBMkEC56N8nKCMyNvRdJwgUiRIQV5QE4yYm5i2BGmxnaQgo0GJ",
|
||||
"XEbkxUtaTr0790Lw+5zfm8gwF3RVi1GI2jLiEWavyxyzgQCc6XMFOX9Ix8gv8116n578ApQuJoRt7yqn",
|
||||
"92mC5nmCuEAZT29BmHoomLB6aHV3Z7kD3gYctwWUVem43ewHIRowanYKhjH05qWJMhA4vUWFXwZhU/3L",
|
||||
"VICxv224JqLXcW9pCWu3fRk4cWnT+mSJ1Myag8CU2oMHkUlz4cHwubsFoMVZf1YKASxETbSGYEgFxTrJ",
|
||||
"0ax7lIOUeNrNeTwhjMjZ/vbnh7BhhwBUUTLnETOKWcCDvNVqZsvlqqDoYA3Ro9aekmuTBRwzenxZ9MRm",
|
||||
"adgmN5yzGx0flecvfli60nKBXTqbeMNpa2NH1kzQJqKay3tkypptox+QbKT4rrSzjBMLnaQyPruzsra2",
|
||||
"TSjqFuXVGTHrbebt+Nj4XjeLXxwgm2AQj/NJscgIw3TJdc0ZDBQfcBOW7X7JMdPEo/+rnvnfzMONtvOY",
|
||||
"5YNpoRT2C7FxlqROpUx622Zeb3w/Ho9RX1PzC0kD6lG8EXl75j2h8aSkUfXJCm3MIawqV+F/tIluIm85",
|
||||
"XUMSK6Za5WxRrjbhsgU/cfisbiSyjBhwXAm2VVI2hv6OLl+vI3W8VIVUIwnAtvLWTygu1imDM06zUcbv",
|
||||
"2L6xhNvWm6pCQ6wAP3Cm77hWv/W+KbkFuhiluOzI13mp9o6n5GlqhK71Bo8DhOlsFAUr06ELuMHprfcB",
|
||||
"eOOC+Y7etlVTbc5QJQp93OWSr2LsbeLkIYpe+fJWtRggJxutwHuZTZaoJ8rJt4TSvWxqmwNxTB7tqLIc",
|
||||
"dHyjc324bexjpdxTqF4TMGgz9Um2pcG/EDyFrBQx8dNHPWbICMJDGz8y9AEgQx8XVFDM0Ifngx+OVkKx",
|
||||
"vd9uFCKX2qq5MH0OentkvuIL37yReihSPO7Crbuzj92Q56XWKGLu0xXVbeepLFwPMVdEG5Ih8CeeCNnd",
|
||||
"XpoJPHGxCz6IIdhJBUyMTbamzG2IQ/bGUrE+7aSyZ+9mOl1JEQnG02AUrVig9Yy6dMrnchRqnmMWMZq8",
|
||||
"4sFYZsrQuTi5Xry8pKtluMw4RIU6WTF3esZLPcCYmWRc6OqedyIgXlxN6W2o1hRYihfdS6xopb8ZWy3l",
|
||||
"rJf4qjkmH5y1XLptd98WgI4Xvvn9k42lD9y6k4DuGJVceafUEgAZzzFdtEjTREAVVLZDBMmqwMk1x0Xt",
|
||||
"rsQ45ihhgAUqBP/FfjpBf8iQjfjb7Ets95tKymOa01v7uQlRqACBMrzY2ikY3HQVtKKBGRLSUksoJiHF",
|
||||
"VQsALECclrFcxfdOLRrOCdyBOEF6mJaebtH7Ny/P0J/+66oe70rY4PTiDfrXP/6JznCWLa7ZhIs7LLIB",
|
||||
"LtUMEZNtBUzCgLBBBoWaJYhxmxfmrDdaQhOlmh0dXzNTDPrEmAVJiuw6bTKpLZheZZ72TZEvdGMCpW/0",
|
||||
"u76guCEm82ZF7YaVTGlqI9O6mtcu+cGVJM8UF3w1qO3MFvAe6LsckN6sL7DyntxyiWY8B4rH6P3lMbrS",
|
||||
"4uWEUNAb10O+/TZs8pqZXX77LeqbmuA4VQMjFx6doFfceAxAIKnKsURYAKpK2t8RNUMcF2Sgj70psOSa",
|
||||
"2bxXifr+82dv3yRoUmqpBP30Rh5ZeBkw4xyQLCA9vmbX7IyzuUYnZzX55PnRyTUboHPrhdJf9wXD0U1b",
|
||||
"efKbY/3KWyKVRKUEdPPJ3NFJvcvC5xu7eNeaocBTwqzDq+8OGmRKzqPvnyQox/fo2ZMnR2ben5jEE0AX",
|
||||
"7y+vbPGTQqGbpXr8N6hvK/sXFC/QHWEZv7NvvyvNoYCEaz4hUYqFWKAbd9vdvECvzq9cTwCJbs6v8PQm",
|
||||
"QRenV2evkY/fQDe+xP4N6rvi/L4ov/1MqLlTwez58+c/oJ+uzszzcxeUZJ7iLBMgpVnXuBldivrNLhEG",
|
||||
"UVczQO/OLmw5kAlOAfWlEoBzM8Prq6uLBPHJhKQEU01Aly//fGRziEtmItEVuhnmaXFzzTirCGFMGBYL",
|
||||
"hFmmB/NSGTuq4SVL15plXZDQC0RMBiWnEt0JXFyzip6sfoxMSg3ChtqlVrOyghOmpOVHSlJwrhjHZBc2",
|
||||
"u1sf14I6xpQnw6HTvI+dJ3nossBrfsSeZbfTizc1qeWk9/T4yfETo+YWwHBBeie958dPjp9bJ/DMnHdD",
|
||||
"c0gMcK3IvLs0rRGIcPYm6530/k8JYtGsR9/sQfJzvJlBrST4mtYLLe82aojvMEE9dGPtyzHJudrcMHTw",
|
||||
"6DDW9XboMNL1bOkw0jam+PxxqcnDsydPtmpRsBRE6NWGTvpDE/URfaTeQGb7qmVmCZE7euXK8UtAwJSJ",
|
||||
"4/qcVIJZfAsBZrVmELVIVttlAY1hhufE1MgwZnY8lcambcNMx8SbXe8HfuG2lmbvpGfFATPrMJROaeUk",
|
||||
"fS2chlGdmChoHRUuD1cbvY15vBVn5ZN7V5z/zfFGaIryeGwRCGp/ftAEinCNQj0vVCWAtmGE4SeSfR6G",
|
||||
"1iMa1k2K34Dg0FJK47hwASJNlnppujMENCRbfmGpkZKlJRMN80eeLfYgo/qmQ1qrZVPLpYvAmVHVjPGW",
|
||||
"yN+W4jOv352eVQ0MrG7Ql4RNKQxKCQnyeVNOpB5IksHmMkVhG3FKbLZY+rwnI+7afeilWyQSkHKRQXaI",
|
||||
"m8HiyoToAFvUYemg2wrQriwTvG6WacrMxvuvkcHMkG6yV72I+S7Sl62E/5CCV5vYx9kub64k8P1H6Nvv",
|
||||
"Yqt6KDzi1aYX4cU91FcSvTy/PDs6BHubmbeX95o8azzI68W9MzukE9OuiF0daT/EdezArL6S+sqrtcy+",
|
||||
"3xZl2wYGj0fUjiIOJKwZEkQZTAhz6S8VQbsKjxsktjbJysZAWWg9oli1EZUuVquTPPL0sJ+Ootcmjh8A",
|
||||
"v3YmhB2O+/a0GWA5yLDCCfJmyj8cdcZ57PgyQvq+srkvD9MkIVM1ZkcKcm1CH5R0bFWbLyzJtlKObz25",
|
||||
"P+XYmYzsSqxp1RHRroTSqIyy4cJbGtvNyhHqGHxpidNXxl+1dWzVEOA3d0k2+1s84m25RE4HOFbdjKA1",
|
||||
"O6s4atlyBshFE/JSDvwTZNQypAQm9GhXe4jzSg1tCWODmmiyi68TKX2d4iS4uyTCDOEpoFtYFJiIxPXT",
|
||||
"NX9vLzybGI9GLTm2EfXFG+kNx+gMUwrC1kvEVADOFmiG56C/4YtYMHPtMMj06dLIjjCBXtbB0TwVbEXj",
|
||||
"M1+X/yFO82ax6S98oC9VbI61GzYjcmAqNNe2HkDbxIBlAWEHoG/7MYQRgztf3Phf//gnIlKW4GnI00+N",
|
||||
"dsISKip3hNtC4rbZXYPCP0laTj8P06pJSDQM44PzMN7NSDpzvUBM/4/EutUs2Zqy0rbfhm9vgUybD0PE",
|
||||
"UzIHhpT3NRovM0PeAWwafBivHWFSAc4Qn6ApUagoKY0R6StQzQYnK/dWbAuIM7pwi5NhcURW67JNpp8/",
|
||||
"f/7DUUtzfdu5ZOu23x8fUkRpQCJ2Kru2IBlQhQ9As69AOTJI6zM7iOIKnNsSZ7KTVHtJy2nv88cIZcuq",
|
||||
"a8l0bZHxgQkwMdZB84Z1Jme+8K6d9hu5cmK3E6ZvmPLgePcfiuD9slPvlQOpth5y65q8fFliyHwLmGGt",
|
||||
"GE5UEn4FaqVfzAMibuVbMSu5H4P84vfH03sGA8FLlg2UIIUJqdOCT4gFcm3+keA8NyFBqMBT2MPHWi9o",
|
||||
"06qC+ACTTWf4j4QqEKY+Qr1ZoivEJZEeDSzDTMm2w3tXC3vIGNz2xVC1des3fTnetS8usXs5tg9drR7O",
|
||||
"TGDO0AXJxr7y656m938rLam9qNoX0o5cTT5K5MFOXaiYJyg7tXJVOxsSz/319lVaEhvF/b+wKdFTUbst",
|
||||
"0dRKy8BWzji/wtO2Kd2woRnjJjyIDZKhFeVgA1U0LUh+8DCojO1q8JnTbOtNQCq10/W3mgddOeVMav3c",
|
||||
"xH3aqhRac05xgVOjAvuI76MEeUOWm926G6vqtCaiJaIzNxVlfQz63QWXe0ynCFUIvm7aX6kD8oXpf7VG",
|
||||
"RftJ58qyJk2E/FpC+ahsEraAMNKvlSqQbv/t/z1L0F/fJSjU+DhCZqAp2rEvP3njfZsUGkjvAc0fbceX",
|
||||
"w5mrB/R42HkVahkshxjvdMkdwE2ynPbge3TUTx0sINJ1pNY5pmoXkMHkxTUjlMIU08YkNpYbfffkBy3X",
|
||||
"mukG1fOjY3Rho/im+iPXzB6IWitdVK8+R31/ygW4HEXPO729Xc+6B/b31JvHfHH7YBuDOI9Pdbc+Foc4",
|
||||
"h1FV3kIzSK1TzFIXmYOcWkPT5XpQdbluO8L+qMd9sMM6eZNMQk1DDwngeW5KIZK8zHsn30dSuR5am1gO",
|
||||
"EixsslFLk9jOVZnaCiXZD2xd1maLCJ3JxNbZ9ai1du2pwMUMZcTVSDuEUdvnjPgPkok1BbmDfYIJlV/0",
|
||||
"OF8laB+AJjdfyKHQSjeKFkB3DuarEuGiHNEbc8MsoZKLSe4zJgbz5OPhTc/7qNzri7zvTMf1aQ/hZHxp",
|
||||
"gI5EfVrT9zx40voauiggRx49IvFa23YQqYdVLncbFS8XWHvA63P5UxHsXTT9kFC4Y8jt4wDyPac0XsTO",
|
||||
"mDqXhf4vhcqaadp045Xr44fP5649a4cT58HzrqJW0VrZkP8EAD+ybXPufCaPZdqc20zeAwb9viZScWGc",
|
||||
"3eBZYWdHxNxCy+SetroDL21qwCUwheyGjtE5Tmf2+99IdEOyG58VbXvgC36HSIb6AmSZwzUzB9nNWy0q",
|
||||
"mxkGb17eHCXoxoxeelcDNUE3GVY4PPnT5fu/XDPzKrLQPkavAQs1Bqz0uZUbOGvOW6Cn38tj9EeQagCT",
|
||||
"CRfGDUvMk3/945/XzNSVhgwVIAayHOudjkGgcTmZgEhQJngx4DQDqVwS9cXvj16YNOhX51fIweyaKY7G",
|
||||
"OL2dkLgr/tLAtO2wanXhBAigQsCE3O/rsbFKVvViAwVrZ9jMtgrulQXHoKKg9glX/bCX58i9eAiz/9wT",
|
||||
"kJ0T9S8vz4/2YY4qOmqtn64atmsu5INHyH8lGSn/XldHyBJ9xOujoq1DOcbq1Lp1HGDS4uy4mgGaYZZR",
|
||||
"EMveiX6I4TM0eJTYGF7p/BRDX/wwuWaYZQiImoFAwIw13F0LoRpz34azukThI8RFLYLwmoXMQWd7Mz4Q",
|
||||
"XwiiORNh6Ma3l7sJUX+nVHIE9+avPsjFRvQITsEEoNlwLDvd+7+8/Ru6wws7Ruotxq4C55E4r6cdf5Xu",
|
||||
"w+V2cF/ahVhx3BpW8N4T1M9tiVKXQR6cWIcQsj4EAqpTnyPtBfrX//v/VZqqTWzQf3JUu1WIbS3+sBq7",
|
||||
"2SFSo6WHM/l2wsch7GIBxK5t3O+Q66C52xm1lz2hiYShbQn5IFnfZ2bqx0flWeh6eQBXu5kLYeQP12Ho",
|
||||
"xYnqdRd2zC/WZ7NoTzA+N48vAbK9jTlLooMprMRtrFwTen87ffcW1VpvrdZoZYpTPt3lVXtHbv3ikrwR",
|
||||
"FpDU9hEm7yKHaIiicakv+IMcrj4hAEk9sd6NtDWtUtdC4OUffaf/lx/QELmSQD4Sr5EqtpAK8k7EY+z5",
|
||||
"605V08Rzk7J2qbAInth+3Q979ALxnChjTLubaYHBehD6toVRW/Sd4HwnoX6Ng+jZBgdRYmqUUlNo0Uqs",
|
||||
"ne313WuTSrUwtZ0mXOS91bC8pU6hv3DCvB9k5P6mxbeCFyU1El5osXrs+j0mXTbhPhPfQ6jJuNw5oeuu",
|
||||
"HjJ+vWopG+FI8xDNzdO9GfKyHFtK1ZQ7J7LElPzd1XIzPVDR75DpgbqDeV8zXtXjtI3zfqQA6rVH64OB",
|
||||
"tNmuNQJWO+CAscVmY65Vrp/WmvVNRzdEWKa3wsU+ZrxQaHsoAYu0HdKX5nHozdnNYPFrb1kJ2M8K8BXo",
|
||||
"9o3upIfzv70m6hCK+o8lpQOTP2LRaYu8BiRXXuq+FwFkglzHzwaLhle2oqFPwfvRISarTku/NXS+Ne1n",
|
||||
"K8AfomIHpUFuk0OPM2Qb3SLFozGqXdEY52bTaDbq2erO1EYzskVpN3jn3rlBnU6Wzm62Dvd8KJl7EFEl",
|
||||
"Lqf5lpmR+ANcKl6LP2g2ULUtPnZLuN7FifeYR2ujR+3heNFOiyQcqGaiIVaTkTewc6I8UO6u13Bd0Ws7",
|
||||
"Nd9XutgBUeQYSUNx27wP0yApwhIhWK77hI120LGO3LU4ki3XutLSaRNlNSAS/XRji11IMKDuMDc88uRi",
|
||||
"zMq13C19mdeW6/9WW22NRP2aulOnjZhZDAIC4ib9ekgtwtKlMI/yUjnFIBjeNefgQTCH3s2AoSrCdsUa",
|
||||
"Xk+kubLa5VecTKNX+JgJNZbY1xboefbkWQc6tEbyeqXPvY22SiswagYVJRvFxubs1wi6O7027TVRih1+",
|
||||
"0tdxrNRPRNpxKX5bCDqt4e2vschQBhSUKQDPuEKyLAouTBX3makL77rpSgT3RNp6BaEfSEjhtzEFL59H",
|
||||
"WKMWer4bZ3yR8HO9tEcMQW/jiFrhoUfiiFrBooD1KlRyH05wVYnXByJc+EHbyN57FNfcMaKgW/zDbymS",
|
||||
"wLfRf7w4gkAaB4oiKCpS8/RMwXWS2yyJ+LcPWrMt3ukESTwBtUBzTOfgjt7LV384OkanocC3Ps6LurSz",
|
||||
"Iupcftd2WF+EXvRf/qRukmRrqeVfSywwU6ZRVbQjz2rPq6rhf63ZVa2zlfEjhTExtfZxyywHhvsab4mr",
|
||||
"KiEJe0ZCfVdvHtAQVbBFw+omOerOa0t3h4uzCclvJYVuxf0/lBRk7yuoS68Xcsg0CbOvA9eZR6JsqmaV",
|
||||
"e3WHgKqzoFw1GhSEr70wcrf+JHLNIW0RJzxRIFbzuv3J97xVH2uA+ivVyOpr3EYnexQ2v7DxD04balAJ",
|
||||
"6mclpoOIM3stzXRg64eugroflTywdvJvSh6GIhx7H5IwXFzlOmPkqRtzCUoRNn3cs765lgMe92F3hyi4",
|
||||
"bheJpJsT9W8JpQN5R1Q6SxCDOYiBr7lqKtoc7XAlxGXaD5hIE+foF0EkqtMLhQz1nz15hn5XhUIeo7f8",
|
||||
"DkzxI6Js+oNbOrqZUj7G9FhPN8KpOkHXPT6ZXPdutAaLMxtTabc08oPQLbgsCn/tkDyHjGAFdKG//uTo",
|
||||
"xFxNNbDYUpxmHnSHXXwMZuuLjpjTJkaeu50bejv6EaYXDRpt8ww9nNz6dfLIqcGmTdhRgthQ7UcUksPx",
|
||||
"6GndV6dsnJAvGmT2/scfNUsEgtzv/BRE3g5MwO8GafkDkbdnbtxjphT7ZRxSUCbyFnkYHEheFvU5tzwa",
|
||||
"NXoaycj2kKRgVd/lgr1ZI826W3KOaSm5bcDL2mSdvWeqheFtZcheIsXvYhVsa24mYIfpOXTOMi3V1Kfu",
|
||||
"S1DSloEZKW51FxPJQiS6hcLeCDOT17g42qUsR5sade5aVlofWqQQzapfEPVnBAQW6WwxwHdYwNELlGKR",
|
||||
"EYapbds34SKFrE2RWk9zX4ciVV/j4zi3mgUQvkj/iQZFuoilneq/+J4D6y6FSzemc0YgHCzTPPTUZhPe",
|
||||
"S3p3zlSU9FJBFEmjvcYfJA++SyOg35KZ3+L8Ea38nugOVbk40PB2rXhqPGJTaHB6+yD5M6fprYN5HOvr",
|
||||
"d25fPVy7ktO0itDEDng7dippQC8vrXRzcPC9KxXU4HcIJ4Re66hkitCuFeBbe0Qud8SvZt6jieOXpQgN",
|
||||
"4EAKrsTK1dXbQxCFAMnp/GHo4oOd+8Ck0Y7mFWR+FchzUKjwl2NWYkoXu6JPq6obhAY7pFtrTGt+GZmo",
|
||||
"0/947x/yWtdYecxb3VLFoS51Mxvqm5QqFRLrChD20dFuLn077UO7HywqvmJfe0EYg2zkoBqvibjqbteY",
|
||||
"aPW1f33edccQX71v3dCkafBE9K8eKTu60WsUPnRTdTjN/+pHfn0H2M4HUtjT/vhyU3nbj6ktaPG2/TG0",
|
||||
"Z96/SceTXZOfrszorY+if7dkDrPNA5KOA9shGB1YhjDDdCGJK19Iqc/hMJXJI4lU2yR0PGQyld4KpKUx",
|
||||
"3eipx4AFiNNSzXonP3/UGLfd2O2HS0F7J70hLshw/tTQg9vPatsml9zv8s5DXoEpOWtK4dRt581t2LSa",
|
||||
"lTgU23kNQuu3pOptRaSts0w4S3ybo1rhKNfLaHXO8+1SHdx8vMq++BS3e5gtuuJCfYtq4wNqdOeMLijU",
|
||||
"oKh6K7hOjUnwUkrUzyAlGQxxqmrTQr1E06eWuEuztCB66fOsNkM431bfrztgkqVQoyQ4x6qpnBdldaKQ",
|
||||
"IelIw+UJV7a6Wo7jp2jqlUxsxrL5bkZU4qoPJihk43tMNbgsBu6CC7X6nqvk8Pnj5/8OAAD//2XM3suJ",
|
||||
"7gAA",
|
||||
"87L0NFOq0POk+t+MyFt9AYNQA3Mla3Bkgkw0lgorgUg5G9itRTmYYqlGM8BURYwbVzMiUapB941E/I6h",
|
||||
"nEuFBKTAFJqDyEiqjtEpQ5Zzv5HIzoSINKLJHRdSIT7Rv0hAOBVcSqSvVwc6O7lMkORI6Y8Rie5mWCG4",
|
||||
"LygmTKJv72aLbxH2n9ADMpgKnEF2jP5SUopKpgg1nzOToQnRHxUlk8f6gvBwMwsz8HGv6x/5nT6pS3bL",
|
||||
"9E8fO1yhBl6iZI7Wm/D67xkwuw+7FD0YWXk3LPe/kLmo9AL1+naT7ltlUYXFFDYIjX3CpMIshYG52bJO",
|
||||
"Yo6duEWMcrPrh6iv/91qZpID11prnAHWnAZJ7++cddEV18i7jrdrbFhfUcXjHY6XNvmmOmTWnSDBONem",
|
||||
"TTdPijD890+eJI9wbnSgwE00tJ4SwgafRvfnMb8e03Ukt+Ltwlt2d0HbJjxFLvt19P45tkgtQ5KJs+Tt",
|
||||
"Jj+n/v5bGTK2JxrOiNVhiYI8Lge7P2Ah8CIu/B3E+tHx4nSmgpFBUwYsXXcQsDIfW+hX1sUYYgWkPM+B",
|
||||
"GetLAOu+lgPBSwXLysvAylI1BWbGaYspwNhaO5vobgntPLji1h2O0LiqY3fbtOMukcpGm4H1BJ1xpuBe",
|
||||
"RSjemO0mhIIcWdtRxBZ0gdVMauHDjEb60hOlWTEybyKlBQ3/erIF4UviqG1JWiI5SIXzwujoWiBhcK9Q",
|
||||
"wSlFGnYgVduFH1E4CmlJe9q+wytRAiITdKxHHy9wrr+TkkLDTtZ2FrPMctoFdGbc8NtjCaosjuVsd5jV",
|
||||
"rvElEwxnXHFGUpRadAenmWPaZJMWsPZiNoR0CakAq2StKDtydUlv2IRIkmKKpHkR6WEIG8s3GVNAykmr",
|
||||
"qZl9Czis6i8yuuyXWM7GHIvssjLlL7GAM6zIkTc2Ri8bIzUTkKPxYqTVUkO2OMuI3iqmF405V19fsq5a",
|
||||
"Mc8Y9KUGCmRovEB23qRFn3cf95f+gb/tFN7VT8/1CSHchpem0s/8TOMyvQVlJ/t+kBNWKkD+Ck8aCo++",
|
||||
"KOu4biLETtT9mgtukQ3U7eb1L8SoZZnXgoX9QGj308Uxn1g159l3sxgiKtWyCa6ghcUXoNWx6BOvx0Uf",
|
||||
"SoVpBOFmfXwsNU7NPjjNQJ/QmFV64zdSK52QalpIsRUoYoKnVxY3Y66jyrkCMnvdW9xpVdGZHXdFHy+A",
|
||||
"oT7jbCBAcjqH7Mh5b1fx6T+3sqqlra1wduykCciPbymJnGFx2m2wcwxi50xwSj+4O3aF1mZcKu9VbMLm",
|
||||
"NFUlpsgPcKYKBGY+wqYox+mMsCgD5yBnziLYnPTSxkjp5+jNhREGtIBqzq+5NSxZsalVqdoUBHQnTxjc",
|
||||
"DSguFC+ONloJzbTr4OYiZ2Jy1qgQZI4VjG5jETunr84Hl+dnH86vBn8+/9vg+PjYbJdyfXlmkIpF0bZX",
|
||||
"M3c5piSNT42n8FTPZ8doGjVTX76/uKxJOXGTmru+R/Z+drJw2x3/EyNagsD0tFQzd6WjNy87zWzlg61n",
|
||||
"d69twtsSiJIVfMRxquKxF0oJMi5V49CoXttFU2szG9bOW8icSTBBd9Y4BjVLXs4ZUdz6Qrcx1PmT/uPh",
|
||||
"/HNGCzYXQdSyZxbvjJt3WKLGFnc24+VYn9oMsxRGxoy5u7/fn2+tZ0pl0K8FMfZavBcdAzDaDDk7+QO2",
|
||||
"8006y467fMzuqzka5NxYTjvHtHonG3zT5ibAU6wVTEPf+gPfSBReHLmYq8inN2KtHTvNlby01jJpVRNA",
|
||||
"lEwgXaRUL8RZ0kZLcvoqHpcUs1Iq4xxHWnYILnKobL3dLp4mktoRcBEPsDxViILhNhbu5wkBmkmUuxUW",
|
||||
"AiQwddxLtsDdOxAm6m++gsMuiPsinBtH9ZUx2VQYtlI46iuBmTQSYrWnuGzQgoArRwYtMBzVA0nrC/rT",
|
||||
"5fu/oEsPqo3GssbLkW1nXAM3+ojIkafDuO2V4gWIuqUtB4WNeX8isLX/lEJjZcrnIAz6jGY1ZaQ12CgA",
|
||||
"uqtNrRWhBRZaaPDsttmSZ2A6WusBWQ0+MrFTYO7PQkBq4kI/7nXgutPVIcZDuYmOsJKPa+lr4yk7Wol3",
|
||||
"fgjKCc6FCaYy6m1ZoaRDktDOJLP+uI3jaT1CWpwfB8HHrrQZPaLmLqh52UmzfQAKVvgBw0/qKnuNdngv",
|
||||
"6d1h4e3hgiitHsTN/UZ/jFsTZXeBKkT5BMHPauHHAhMJ2RaxJe7+rmnubolR0grhnVs5qmoh25ujmJzd",
|
||||
"oOv4tOFA6/wW12BTe8bhPpCTbOuQ9M5eNYHziLR0eUsoRfYp6q/KTPaJOyyOYhKTAGlO3P3daQ/oDrOD",
|
||||
"azfjZsCuk9TFvtQTCS12wc3N2GIbnLU21tgZ1szxo/l4sqj9bAdPMLGhDnp52YiXxuysbzhq/y64/mE0",
|
||||
"xumt+03/OHLvfdzHv1hbSESy2zpgOUQqb+t5DMdXqzWxOsWqk1WAwfZ6joqwBJZtV2edxpdY0dpujScu",
|
||||
"5yZOHjLrj+rzwlqIj3rJ1rFB9XzHjZeDm2ttqOErgYvZXwncrcIQsik0ow3WZSN9cAiUM1LE/B2VHarN",
|
||||
"Rr6jcSkSvBrxSZEMDa7LJ0+eQ7B1OY3U2rwIS2mZwX9ZmjTmI+cOhmhYIePZFsBx9r4IWJQoWerTwuKe",
|
||||
"Y/0plOLCLGpGVMRZvCxjchtrbjEYQ/trA4NWx6X3EzQ2uGSvD/g8FN5WrHuHDrsT0Ay3Irk+i30Sm16y",
|
||||
"i6pS5gPbxRx2k/6bxhYHwy7uQtmGqv/46Fp8dMuQdwCsnGBRuP+Z8Tuq+eY1iVwrXe3UhN1CNopy0cYg",
|
||||
"DIHZbacoqXahhpGiiF0ir8l0Rsl0plFjUp19OEcnvrLh9Z3D8RVRFNbsuOLDjKdlbmM0RMnGnN8aa9Ac",
|
||||
"pCLTtgSzzfZmu4AYkt96Vf+lPrJXOapuio0aKrJ2W+CW2FYgcqKliJ1eDtbEiDbwaSJ4foI+KX6CPjlQ",
|
||||
"yRP0M8M5ZAPDqgk6Pj7++Pnz542+ZOIzy8y9smKsrq0jBu93oARJL0E4CEcum0Wb4pWbd1tC9igtizol",
|
||||
"CXzXS3pPZ/qfljA9Iw2uu9jwfNqJ/bZImc3xfacpc8I6jdvGwhBSOJaqhOA7ZGGBfGrGhs8uC5ey070V",
|
||||
"Lt114tGVGRSNX1hYhcBRQcB5hcjYIi5s2Ol2xo2ioARke+BzM4Z1KaeAUMkZovwOBBrzkmWJltgKG7EB",
|
||||
"c/uezbIeft+LoLQ5Jn4raxWuFGuHbONIDQaCvcStooL1yrNfSywwU4S1hWFvka47WxRczUCSv9tAf794",
|
||||
"nxW+ZLA0F0gY87E9S34dNHdzdzYoqab7ekg1SGkF8zW1eF3MZK2yxfLttZS6V0uxFIKLyE3xIwGaDUzS",
|
||||
"Yy32BdnhqP/ds2dH7SF1xs0XP57bNOcl2NkZwvgup4pPjtlAO7G4/k1CSfA69PCYl+pkTLU8VgseKAXZ",
|
||||
"rHmbz6x1t1xo1UOutWGsc2q/e/MyQSkXIBMkcD7KxwnKiLwdTccJIkWCFOQFNZF/uQkgS5AW20kKMhoB",
|
||||
"yGVEXryk5dS7cy8Ev8/5vQnDchFOtRiFqC0jHs71uswxGwjAmT5XkPOHdAyzMt+l9+nJL0DpYkLY9q5y",
|
||||
"ep8maJ4niAuU8fQWhCkZgwmrxzF3d5Y74G3AcVv0VpWx3M1+EELvomanYBhDb16aKAOB01tU+GUQNtW/",
|
||||
"TAUY+9uGayJ6HfeWlrB225eBE5c2rU+WSFmxOQhMqT14EJk0Fx4Mn7tbAFqc9WelEMBC1ERrCIZUUKyT",
|
||||
"HM26RzlIiafdnMcTwoic7W9/fggbdoj2FCVzHjGjmAU8yFutZrZcrgqKDtYQPWrtKbk2Mt8xo8eXRU9s",
|
||||
"loZtcsM5u9HxUXn+4oelq74X2KWziTectjZ2ZM0EbSKqubxHpvLbNvoByUaK70o7yzix0Ekq47M7K2tr",
|
||||
"24SiblFenRGz3mbejo+N73Wz+MUBsgkG8TifFIuMMEyXXNecwUDxATcx0O6XHDNNPPq/6pn/zTzcaDuP",
|
||||
"WT6YFkphvxAbZ0nqVO2lt22a88b34/EY9TU1v5A0oB7FG5G3Z94TGs8AGlWfrNDGHMKqih7+R5tVJvKW",
|
||||
"0zVkjGKqVc4W5WoTLlvwE4fP6kYiy4gBx1WpWyVlY+jv6PL1OlLHS1VINZIAbCtv/YTiYp0yOOM0G2X8",
|
||||
"ju0bS7htSa4qNMQK8ANn+o5r9Vvvm5JboItRisuOfJ2Xau94Sp6mRuhab/A4QJjORlGwMh26gBuc3nof",
|
||||
"gDcumO/obVs11SboVKLQx10u+SrG3mYpHqIumK8AVosBcrLRCryX2WSJeqKcfEso3cumtjkQxyStjirL",
|
||||
"Qcc3OpfQ28Y+Vso9heo1AYM2LZ5kWxr8C8FTyEoREz991GOGjCA8tPEjQx8AMvRxQQXFDH14PvjhaCUU",
|
||||
"2/vtRiFyqa3gDdPnoLdH5iu+8M0bqYcixeMu3Lo7+9gNeV5qjSLmPl1R3XaeysL1EHNFtCEZAn/iWYfd",
|
||||
"7aWZwBMXu+CDGIKdVMDE2GRrytyGOGRvLBXr004qe/ZuptOVFJFgPA1G0YoFWs+oS6d8Lkeh5jlmEaPJ",
|
||||
"Kx6MZaZSn4uT68UrcLpyj8uMQ1QoJRZzp2e81AOMmUnGha7ueScC4vXnlN6Gas03pXjRvZ6JVvqbsdVS",
|
||||
"znqJL1Fjkq9Zy6XbdvdtAeh4lZnfP9lYZ8CtOwnojlHJlXdKLQGQ8RzTRYs0TQRUQWU7RJCsCpxcc1zU",
|
||||
"7kqMY44SBligQvBf7KcT9IcM2Yi/zb7Edr+ppDymOb21n5sQhQoQKMOLrZ2CwU1XQSsamCEhLbWEYhJS",
|
||||
"XGo+YAHitIzlKr53atFwTuAOxAnSw7T0dIvev3l5hv7031f1eFfCBqcXb9C//vFPdIazbHHNJlzcYZEN",
|
||||
"cGnrnmUwASZhQNggg0LNEsS4zQtz1hstoYlSzY6Or5mpl31izIIkRXadyJRNtjXlqzzSvqmohW5MoPSN",
|
||||
"ftfXXDfEZN6sqN2wkqnebWRaVxbcJT+4qu2Z4oKvBrWd2RrnA32XA9Kb9dVM3pNbLtGM50DxGL2/PEam",
|
||||
"PtyEUPAF3779Nmzympldfvst6puy6ThVAyMXHp2gV9x4DEAgqcqxRFgAqqr+3xE1QxwXZKCPvSmw5JrZ",
|
||||
"xFmJ+v7zZ2/fJGhSaqkE/fRGHll4GTDjHJAsID2+ZtfsjLO5RidnNfnk+dHJNRugc+uF0l/3NdXRTVsF",
|
||||
"95tj/cpbIpVEpQR088nc0Um9EcXnG7t4172iwFPCrMOr7w4aZKryo++fJCjH9+jZkydHZt6fmMQTQBfv",
|
||||
"L69spZFCoZullgU3qG+bHxQUL9AdYRm/s2+/K82hgITrzyFRioVYoBt32928QK/Or1zbBIluzq/w9CZB",
|
||||
"F6dXZ6+Rj99AN74LwQ3qu/4Fvm+B/UwocFPB7Pnz5z+gn67OzPNzF5RknuIsEyClWde4GV2K+s1GGgZR",
|
||||
"VzNA784ubO2NCU4B9aUSgHMzw+urq4sE8cmEpARTTUCXL/98ZHOIS2Yi0RW6GeZpcXPNOKsIYUwYFguE",
|
||||
"WaYH89KUI7S8ZOlas6wLEnphShOamjfoTuDimlX0ZPVjZFJqEJauoiGwrOCEKWn5kZIUnCvGMdmFzdbW",
|
||||
"x7WgjjHlyXDoNO9j50keuqzumh+xZ9nt9OJNTWo56T09fnL8xKi5BTBckN5J7/nxk+Pn1gk8M+fd0BwS",
|
||||
"A1yrw+8uTWsEIpy9yXonvf9dglg0S/Y327T8HO/3UKuavqY7Rcu7jTLrO0xQD91Y+3JMcq42NwxNTjqM",
|
||||
"de0vOox0bW06jLS9Oz5/XOqD8ezJk626OCwFEXq1oZP+0ER9RB+p99jZvkSYWULkjl65cvwSEDBl4rg+",
|
||||
"J5VgFt9CgFmtX0YtktU2okBjmOE5MQUpjJkdT6Wxadsw0zHxZtf7gV+4LVzZO+lZccDMOgx1Slo5SV8L",
|
||||
"p2FUJyYKWkeFy8OVj29jHm/FWfnk3kX5f3O8EfrGPB5bBILanx80gSJco1DPC1W9nW0YYfiJZJ+HoTuL",
|
||||
"hnWT4jcgOHTd0jguXIBIk6VemgYWAQ3Jll9Y6jVlaclEw/yRZ4s9yKi+6ZDWatnUcukicGZUNWO8JfLX",
|
||||
"iP2RmJZ3p2dVjwerG/QlYVMKg1JCgnzelBOpB5JksLkmUNhGnBKbXag+78mIuzZoeukWiQSkXGSQHeJm",
|
||||
"sLgyITrAFnVYOui2ArQrywSvm2WaMrPx/mtkMDOkm+xVr/O+i/RlmwU8pODVJvZxtsubKwl8/xH69rvY",
|
||||
"qjYTj3i16UV4cQ/1lUQvzy/Pjg7B3mbm7eW9Js/asvRrxb0zO6QT066IXR1pP8R17MCsvmz5yqu1zL7f",
|
||||
"FmXbHg+PR9SOIg4krNkOAhlMCHPpLxVBu3KKGyS2NsnKxkBZaD2iWLURlS5Wq5M88vSwn46i1yaOHwC/",
|
||||
"diaEHY779rQZYDnIsMIJ8mbKPxx1xnns+DJC+r6yuS8P0yQhUzVmRwpynVQflHRsVZsvLMm2Uo7vzrk/",
|
||||
"5diZjOxKrGnVEdGuhNKojLLhwlsa283KEeoYfGmJ05ehX7V1bFV9/zd3STabSTzibblETgc4Vt2MoDU7",
|
||||
"qzhq2XIGyEUT8lIO/BNk1DKkBCb0aFd7iPNKDW29YIOaaLKLrxMpfVHgJLi7JMIM4SmgW1gUmIjEtRw2",
|
||||
"f6/cf64OcSgknBiPRi05thH1xRvpDcfoDFMKwtZLxFQAzhZohudQ76vEzLXDINOnSyM7wgR6WQdH81Sw",
|
||||
"5YPPfBH8hzjNm5Wdv/CBvlQeOdaR2YzIganQf9x6AG3HAJYFhB2Avu3HEEYM7nwl4X/945+ISFmCpyFP",
|
||||
"PzXaCUuoqNwRbguJ236ADQr/JGk5/TxMq44c0TCMD87DeDcj6cw13jDNNhLrVrNka2o42+YWvpcEMj01",
|
||||
"DBFPyRwYUt7XaLzMDHkHsOmmYVuFMakAZ4hP0JQoVJSUxoj0FahmN5GVeyu2BcQZXbjFybA4Iqt12T7c",
|
||||
"z58//yE03l+5+2wW5pad0T8+pIjSgETsVHY9ODKgCh+AZl+BcmSQ1md2EMUVOLclzmQnqfaSltPe548R",
|
||||
"ypZVi5Dp2oreAxNgYqyD5g3rTM584V077Tdy5cRuJ0zfneTB8e4/FMH7ZadGJwdSbT3k1nVU+bLEkPl+",
|
||||
"K8NaMZyoJPwK1EpzlgdE3Mq3YlZyPwb5xe+Pp/cMBoKXLBsoQQoTUqcFnxALlNooISQ4z01IECrwFPbw",
|
||||
"sdYL2rSqID7AZNMZ/iOhCoSpj1DvTOgKcUmkRwPLMFOy7fDe1cIeMga3fTFUbd36TV+Od+2LS+xeju1D",
|
||||
"V6uHMxOYM3RBsrGv/Lqn6f3fSktqL6r2hbQjV5OPEnmwUxcq5gnKTq1c1c6GxHN/vX2VlsRGcf8vbEr0",
|
||||
"VNRuSzS10jKwlTPOr/C0bUo3bGjGuAkPYoNkaEU52EAVTQuSHzwMKmO7GnzmNNt6E5BK7XTNpOZBV045",
|
||||
"k1o/N3GftiqF1pxTXODUqMA+4vsoQd6Q5Wa37saqOq2JaInozE1FWR+DfnfB5R7TKUIVgq+b9lfqgHxh",
|
||||
"+l+tUdF+0rmyrEkTIb+WUD4qm4QtIIz0a6UKpNt/+3/OEvTXdwkKNT6OkBloinbsy0/eeN8mhQbSe0Dz",
|
||||
"R9vx5XDm6gE9HnZehVoGyyHGO11yB3CTLKc9+B4d9VMHC4h0Hal1jqnaBWQweXHNCKUwxbQxiY3lRt89",
|
||||
"+UHLtWa6QfX86Bhd2Ci+qf7INbMHotZKF9Wrz1Hfn3IBLkfR805vb9ez7oH9PfXmMV/cPtjGIM7jU92t",
|
||||
"j8UhzmFUlbfQDFLrFLPUReYgp9bQtJQeVC2l246wP+pxH+ywTt4kk1DT0EMCeJ6bUogkL/PeyfeRVK6H",
|
||||
"1iaWgwQLm2zU0pG1c1WmtkJJ9gNbl7XZIkJnMrF1dj1qrV17KnAxQxlxNdIOYdT2OSP+g2RiTUHuYJ9g",
|
||||
"QuUXPc5XCdoHoMnNF3IotNKNogXQnYP5qkS4KEf0xtwwS6jkYpL7jInBPPl4eNPzPir3+iLvO9NxfdpD",
|
||||
"OBlfGqAjUZ/WNBkPnrS+hi4KyJFHj0i81rYdROphlcvdRsXLBdYe8Ppc/lQEexdNPyQU7hhy+ziAfM8p",
|
||||
"jRexM6bOZaH/S6GyZpo2rW/l+vjh87nrhdrhxHnwvKuoVbRWNuQ/AcCPbNucO5/JY5k25zaT94BBv6+J",
|
||||
"VFwYZzd4VtjZETG30DK5p63uwEubGnAJTCG7oWN0jtOZ/f43Et2Q7MZnRduG84LfIZKhvgBZ5nDNzEF2",
|
||||
"81aLymaGwZuXN0cJujGjl97VQE3QTYYVDk/+dPn+L9fMvIostI/Ra8BCjQErfW7lBs6a8xbo6ffyGP0R",
|
||||
"pBrAZMKFccMS8+Rf//jnNTN1pSFDBYiBLMd6p2MQaFxOJiASlAleDDjNQCqXRH3x+6MXJg361fkVcjC7",
|
||||
"ZoqjMU5vJyTuir80MG07rFpdOAECqBAwIff7emysklW92EDB2hk2s62Ce2XBMagoqH3CVT/s5TlyLx7C",
|
||||
"7D/3BGTnRP3Ly/OjfZijio5a66erhu2aC/ngEfJfSUbKv9fVEbJEH/H6qGjrUI6xOrVuHQeYtDg7rmaA",
|
||||
"ZphlFMSyd6IfYvgMDR4lNoZXOj/F0Bc/TK4ZZhkComYgEDBjDXfXQqjG3LfhrC5R+AhxUYsgvGYhc9DZ",
|
||||
"3owPxBeCaM5EGLrx7eVuQtTfKZUcwb35qw9ysRE9glMwAWg2HMtO9/4vb/+G7vDCjpF6i7GrwHkkzutp",
|
||||
"x1+l+3C5HdyXdiFWHLeGFbz3BPVzW6LUZZAHJ9YhhKwPgYDq1OdIe4H+9X//X5WmahMb9J8c1W4VYluL",
|
||||
"P6zGbnaI1Gjp4Uy+nfBxCLtYALFrG/c75Dpo7nZG7WVPaCJhaFtCPkjW95mZ+vFReRa6Xh7A1W7mQhj5",
|
||||
"w3UYenGiet2FHfOL9dks2hOMz83jS4Bsb2POkuhgCitxGyvXhN7fTt+9RbXWW6s1WpnilE93edXekVu/",
|
||||
"uCRvhAUktX2EybvIIRqiaFzqC/4gh6tPCEBST6x3I21Nq9S1EHj5R9/p/+UHNESuJJCPxGukii2kgrwT",
|
||||
"8Rh7/rpT1TTx3KSsXSosgie2X/fDHr1APCfKGNPuZlpgsB6Evm1h1BZ9JzjfSahf4yB6tsFBlJgapdQU",
|
||||
"WrQSa2d7fffapFItTG2nCRd5bzUsb6lT6C+cMO8HGbm/afGt4EVJjYQXWqweu36PSZdNuM/E9xBqMi53",
|
||||
"Tui6q4eMX69aykY40jxEc/N0b4a8LMeWUjXlzoksMSV/d7XcTA9U9DtkeqDuYN7XjFf1OG3jvB8pgHrt",
|
||||
"0fpgIG22a42A1Q44YGyx2ZhrleuntWZ909ENEZbprXCxjxkvFNoeSsAibYf0pXkcenN2M1j82ltWAvaz",
|
||||
"AnwFun2jO+nh/G+viTqEov5jSenA5I9YdNoirwHJlZe670UAmSDX8bPBouGVrWjoU/B+dIjJqtPSbw2d",
|
||||
"b0372Qrwh6jYQWmQ2+TQ4wzZRrdI8WiMalc0xrnZNJqNera6M7XRjGxR2g3euXduUKeTpbObrcM9H0rm",
|
||||
"HkRUictpvmVmJP4Al4rX4g+aDVRti4/dEq53ceI95tHa6FF7OF600yIJB6qZaIjVZOQN7JwoD5S76zVc",
|
||||
"V/TaTs33lS52QBQ5RtJQ3DbvwzRIirBECJbrPmGjHXSsI3ctjmTLta60dNpEWQ2IRD/d2GIXEgyoO8wN",
|
||||
"jzy5GLNyLXdLX+a15fq/1VZbI1G/pu7UaSNmFoOAgLhJvx5Si7B0KcyjvFROMQiGd805eBDMoXczYKiK",
|
||||
"sF2xhtcTaa6sdvkVJ9PoFT5mQo0l9rUFep49edaBDq2RvF7pc2+jrdIKjJpBRclGsbE5+zWC7k6vTXtN",
|
||||
"lGKHn/R1HCv1E5F2XIrfFoJOa3j7aywylAEFZQrAM66QLIuCC1PFfWbqwrtuuhLBPZG2XkHoBxJS+G1M",
|
||||
"wcvnEdaohZ7vxhlfJPxcL+0RQ9DbOKJWeOiROKJWsChgvQqV3IcTXFXi9YEIF37QNrL3HsU1d4wo6Bb/",
|
||||
"8FuKJPBt9B8vjiCQxoGiCIqK1Dw9U3Cd5DZLIv7tg9Zsi3c6QRJPQC3QHNM5uKP38tUfjo7RaSjwrY/z",
|
||||
"oi7trIg6l9+1HdYXoRf9lz+pmyTZWmr51xILzJRpVBXtyLPa86pq+F9rdlXrbGX8SGFMTK193DLLgeG+",
|
||||
"xlviqkpIwp6RUN/Vmwc0RBVs0bC6SY6689rS3eHibELyW0mhW3H/DyUF2fsK6tLrhRwyTcLs68B15pEo",
|
||||
"m6pZ5V7dIaDqLChXjQYF4WsvjNytP4lcc0hbxAlPFIjVvG5/8j1v1ccaoP5KNbL6GrfRyR6FzS9s/IPT",
|
||||
"hhpUgvpZiekg4sxeSzMd2Pqhq6DuRyUPrJ38m5KHoQjH3ockDBdXuc4YeerGXIJShE0f96xvruWAx33Y",
|
||||
"3SEKrttFIunmRP1bQulA3hGVzhLEYA5i4Guumoo2RztcCXGZ9gMm0sQ5+kUQier0QiFD/WdPnqHfVaGQ",
|
||||
"x+gtvwNT/Igom/7glo5uppSPMT3W041wqk7QdY9PJte9G63B4szGVNotjfwgdAsui8JfOyTPISNYAV3o",
|
||||
"rz85OjFXUw0sthSnmQfdYRcfg9n6oiPmtImR527nht6OfoTpRYNG2zxDDye3fp08cmqwaRN2lCA2VPsR",
|
||||
"heRwPHpa99UpGyfkiwaZvf/xR80SgSD3Oz8FkbcDE/C7QVr+QOTtmRv3mCnFfhmHFJSJvEUeBgeSl0V9",
|
||||
"zi2PRo2eRjKyPSQpWNV3uWBv1kiz7pacY1pKbhvwsjZZZ++ZamF4Wxmyl0jxu1gF25qbCdhheg6ds0xL",
|
||||
"NfWp+xKUtGVgRopb3cVEshCJbqGwN8LM5DUujnYpy9GmRp27lpXWhxYpRLPqF0T9GQGBRTpbDPAdFnD0",
|
||||
"AqVYZIRhatv2TbhIIWtTpNbT3NehSNXX+DjOrWYBhC/Sf6JBkS5iaaf6L77nwLpL4dKN6ZwRCAfLNA89",
|
||||
"tdmE95LenTMVJb1UEEXSaK/xB8mD79II6Ldk5rc4f0Qrvye6Q1UuDjS8XSueGo/YFBqc3j5I/sxpeutg",
|
||||
"Hsf6+p3bVw/XruQ0rSI0sQPejp1KGtDLSyvdHBx870oFNfgdwgmh1zoqmSK0awX41h6Ryx3xq5n3aOL4",
|
||||
"ZSlCAziQgiuxcnX19hBEIUByOn8Yuvhg5z4wabSjeQWZXwXyHBQq/OWYlZjSxa7o06rqBqHBDunWGtOa",
|
||||
"X0Ym6vQ/3vuHvNY1Vh7zVrdUcahL3cyG+ialSoXEugKEfXS0m0vfTvvQ7geLiq/Y114QxiAbOajGayKu",
|
||||
"uts1Jlp97V+fd90xxFfvWzc0aRo8Ef2rR8qObvQahQ/dVB1O87/6kV/fAbbzgRT2tD++3FTe9mNqC1q8",
|
||||
"bX8M7Zn3b9LxZNfkpyszeuuj6N8tmcNs84Ck48B2CEYHliHMMF1I4soXUupzOExl8kgi1TYJHQ+ZTKW3",
|
||||
"AmlpTDd66jFgAeK0VLPeyc8fNcZtN3b74VLQ3klviAsynD819OD2s9q2ySX3u7zzkFdgSs6aUjh123lz",
|
||||
"GzatZiUOxXZeg9D6Lal6WxFp6ywTzhLf5qhWOMr1Mlqd83y7VAc3H6+yLz7F7R5mi664UN+i2viAGt05",
|
||||
"owsKNSiq3gquU2MSvJQS9TNISQZDnKratFAv0fSpJe7SLC2IXvo8q80QzrfV9+sOmGQp1CgJzrFqKudF",
|
||||
"WZ0oZEg60nB5wpWtrpbj+CmaeiUTm7FsvpsRlbjqgwkK2fgeUw0ui4G74EKtvucqOXz++Pn/BwAA///k",
|
||||
"T3QHrO8AAA==",
|
||||
}
|
||||
|
||||
// GetSwagger returns the content of the embedded swagger specification file
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -50,6 +50,7 @@ func (s *Server) serveRecentKnowledge(w http.ResponseWriter, req *http.Request)
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE ($1 = '' OR ke.source = $1)
|
||||
AND ke.deleted_at IS NULL
|
||||
ORDER BY ke.updated_at DESC
|
||||
LIMIT $2`, source, limit)
|
||||
if err != nil {
|
||||
@@ -82,6 +83,7 @@ func (s *Server) serveRecentKnowledge(w http.ResponseWriter, req *http.Request)
|
||||
COUNT(*) FILTER (WHERE ke.source = 'nomos-agent'),
|
||||
COUNT(*) FILTER (WHERE ke.updated_at > now() - interval '7 days')
|
||||
FROM knowledge_entities ke JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE ke.deleted_at IS NULL
|
||||
GROUP BY e.type`)
|
||||
if err == nil {
|
||||
defer srows.Close()
|
||||
@@ -128,15 +130,19 @@ func (s *Server) serveKnowledgeContent(w http.ResponseWriter, req *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
var title, content, source string
|
||||
var title, content, source, editedBy string
|
||||
var tags []string
|
||||
var updatedAt string
|
||||
var revisions int
|
||||
err = s.pool.QueryRow(ctx, `
|
||||
SELECT ke.title, ke.content, COALESCE(ke.source,''), ke.tags, ke.updated_at::text
|
||||
SELECT ke.title, ke.content, COALESCE(ke.source,''), COALESCE(ke.edited_by,''),
|
||||
ke.tags, ke.updated_at::text,
|
||||
(SELECT count(*) FROM knowledge_revisions kr WHERE kr.entity_id = ke.entity_id)
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE e.slug = $1 OR e.id::text = $1`, idOrSlug).
|
||||
Scan(&title, &content, &source, &tags, &updatedAt)
|
||||
WHERE (e.slug = $1 OR e.id::text = $1)
|
||||
AND ke.deleted_at IS NULL`, idOrSlug).
|
||||
Scan(&title, &content, &source, &editedBy, &tags, &updatedAt, &revisions)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusNotFound, "no knowledge content for entity", "")
|
||||
return
|
||||
@@ -150,8 +156,10 @@ func (s *Server) serveKnowledgeContent(w http.ResponseWriter, req *http.Request)
|
||||
"title": title,
|
||||
"content": content,
|
||||
"source": source,
|
||||
"edited_by": editedBy,
|
||||
"tags": tags,
|
||||
"updated_at": updatedAt,
|
||||
"revisions": revisions,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -169,6 +177,7 @@ func (s *Server) SearchKnowledge(ctx context.Context, request gen.SearchKnowledg
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
JOIN entity_types et ON et.name = e.type
|
||||
WHERE ke.search @@ plainto_tsquery('english', $1)
|
||||
AND ke.deleted_at IS NULL
|
||||
ORDER BY rank DESC
|
||||
LIMIT $2`,
|
||||
q, limit)
|
||||
@@ -232,6 +241,7 @@ func (s *Server) GetEntityKnowledge(ctx context.Context, request gen.GetEntityKn
|
||||
WHERE target.slug = $1
|
||||
AND r.valid_to IS NULL
|
||||
AND r.type IN ('documents', 'about')
|
||||
AND ke.deleted_at IS NULL
|
||||
UNION
|
||||
SELECT e.id, e.slug, COALESCE(et.name,''), ke.title, ke.source, ke.tags
|
||||
FROM knowledge_entities ke
|
||||
@@ -242,6 +252,7 @@ func (s *Server) GetEntityKnowledge(ctx context.Context, request gen.GetEntityKn
|
||||
JOIN entities ent ON ent.type = target_type.name AND ent.slug = $1
|
||||
WHERE r.valid_to IS NULL
|
||||
AND r.type = 'procedure-for'
|
||||
AND ke.deleted_at IS NULL
|
||||
ORDER BY 2`,
|
||||
entitySlug)
|
||||
if err != nil {
|
||||
|
||||
544
internal/httpapi/knowledge_drift.go
Normal file
544
internal/httpapi/knowledge_drift.go
Normal file
@@ -0,0 +1,544 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Drift tooling for the knowledge base — the maintenance half of the wiki.
|
||||
//
|
||||
// These endpoints exist because the knowledge base measurably rots on its
|
||||
// own. Two failure modes are already present in live data:
|
||||
//
|
||||
// - **Duplicate pileup.** upsert_knowledge keys on exact title, so a note
|
||||
// titled "rclone backup live inspection — 2026-07-15 10:08 UTC" and one
|
||||
// titled "... 11:18 UTC" are different notes. A single day of agent
|
||||
// activity produced eight near-identical investigations that should have
|
||||
// been one living page. Nothing surfaced that, so it kept happening.
|
||||
// - **Tag drift.** `oom` and `OOM` were separate tags; so were `422` and
|
||||
// `proton-422`. Each split halves the usefulness of tag navigation, and
|
||||
// neither is visible from any single note.
|
||||
//
|
||||
// normalizeTags (knowledge_write.go) stops new casing splits at the door;
|
||||
// these endpoints clean up what's already there and make the rot visible.
|
||||
|
||||
// serveKnowledgeTags returns the tag index: every tag with its usage count,
|
||||
// plus the distinct casings actually stored. `variants` is the interesting
|
||||
// column — it's how the operator discovers that `oom` and `OOM` are the same
|
||||
// idea filed twice, which no individual note reveals.
|
||||
func (s *Server) serveKnowledgeTags(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT lower(tag) AS norm,
|
||||
count(*) AS uses,
|
||||
array_agg(DISTINCT tag ORDER BY tag) AS variants
|
||||
FROM knowledge_entities ke, unnest(ke.tags) AS tag
|
||||
WHERE ke.deleted_at IS NULL
|
||||
GROUP BY lower(tag)
|
||||
ORDER BY uses DESC, norm`)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type tagRow struct {
|
||||
Tag string `json:"tag"`
|
||||
Uses int `json:"uses"`
|
||||
Variants []string `json:"variants"`
|
||||
// True when the same tag is stored under more than one casing —
|
||||
// the UI badges these as needing a normalize.
|
||||
Split bool `json:"split"`
|
||||
}
|
||||
items := []tagRow{}
|
||||
for rows.Next() {
|
||||
var t tagRow
|
||||
if err := rows.Scan(&t.Tag, &t.Uses, &t.Variants); err != nil {
|
||||
slog.Error("httpapi: knowledge/tags row scan failed", "error", err)
|
||||
continue
|
||||
}
|
||||
t.Split = len(t.Variants) > 1
|
||||
items = append(items, t)
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
// serveRenameKnowledgeTag rewrites one or more tags to a single target across
|
||||
// every live note — the merge/rename/normalize action behind the tag manager.
|
||||
// Passing several `from` values into one `to` is the merge case
|
||||
// (`{"from":["422","proton-422"],"to":"proton-422"}`); passing one is a plain
|
||||
// rename; passing the mixed-case variants is the normalize case.
|
||||
func (s *Server) serveRenameKnowledgeTag(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
var body struct {
|
||||
From []string `json:"from"`
|
||||
To string `json:"to"`
|
||||
}
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error())
|
||||
return
|
||||
}
|
||||
to := strings.ToLower(strings.TrimSpace(body.To))
|
||||
from := []string{}
|
||||
for _, f := range body.From {
|
||||
if f = strings.TrimSpace(f); f != "" {
|
||||
from = append(from, f)
|
||||
}
|
||||
}
|
||||
if to == "" || len(from) == 0 {
|
||||
writeProblem(w, req, http.StatusBadRequest, "from and to are required", "")
|
||||
return
|
||||
}
|
||||
|
||||
// Rebuild each affected note's tag array: map every `from` member to
|
||||
// `to`, leave everything else alone, then de-duplicate. The dedupe
|
||||
// matters for the merge case — a note tagged both `422` and
|
||||
// `proton-422` would otherwise end up with `proton-422` twice.
|
||||
//
|
||||
// This is a plain UPDATE on knowledge_entities, so trg_knowledge_revision
|
||||
// fires and every affected note gets a revision. A tag merge across 17
|
||||
// notes is exactly the kind of bulk edit worth being able to inspect
|
||||
// afterwards.
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
UPDATE knowledge_entities ke
|
||||
SET tags = sub.new_tags, updated_at = now()
|
||||
FROM (
|
||||
SELECT k.entity_id,
|
||||
ARRAY(SELECT DISTINCT CASE WHEN lower(t) = ANY($1) THEN $2 ELSE t END
|
||||
FROM unnest(k.tags) AS t) AS new_tags
|
||||
FROM knowledge_entities k
|
||||
WHERE k.deleted_at IS NULL
|
||||
AND EXISTS (SELECT 1 FROM unnest(k.tags) AS t WHERE lower(t) = ANY($1))
|
||||
) AS sub
|
||||
WHERE ke.entity_id = sub.entity_id`,
|
||||
lowerAll(from), to)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "rename failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
_, actorLabel := actorInfo(ctx)
|
||||
slog.Info("knowledge tags renamed", "from", from, "to", to,
|
||||
"notes", tag.RowsAffected(), "actor", actorLabel)
|
||||
writeJSON(w, map[string]any{"ok": true, "notes_updated": tag.RowsAffected()})
|
||||
}
|
||||
|
||||
// serveKnowledgeDuplicates clusters notes whose titles are near-identical.
|
||||
//
|
||||
// Pairwise trigram similarity is computed in SQL (indexed, and the whole
|
||||
// point of pulling in pg_trgm); the grouping is done here in Go. Returning
|
||||
// clusters rather than pairs matters for the real data: the rclone pileup
|
||||
// produces dozens of pairs, which is unreadable, versus one cluster, which
|
||||
// is the actionable unit.
|
||||
//
|
||||
// The grouping uses **complete linkage** — a note joins a cluster only if it
|
||||
// is similar to every member already in it. The obvious implementation
|
||||
// (union-find over the pairs) is single linkage, and on this data it chains
|
||||
// badly: "A~B, B~C" merged notes that were not remotely alike, collapsing
|
||||
// fifteen distinct backup events into one unusable blob. Requiring mutual
|
||||
// similarity keeps clusters tight enough to act on.
|
||||
//
|
||||
// Even so, these are *candidates for review*, never a verdict. The five
|
||||
// "Lifecycle: <verb> a node" runbooks are mutually similar by title and are
|
||||
// five deliberately distinct documents — no threshold distinguishes them
|
||||
// from a genuine duplicate, so merging stays a manual, previewed action.
|
||||
func (s *Server) serveKnowledgeDuplicates(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
// 0.6, tuned against the live data: at 0.45 the "Lifecycle: <verb> a
|
||||
// node" runbooks (five deliberately distinct documents that happen to
|
||||
// share a naming template) formed a false-positive cluster; 0.6 clears
|
||||
// that down to a single borderline pair while keeping every genuine
|
||||
// duplicate cluster (the rclone/apt-audit/uptime pileups) intact.
|
||||
// Tunable per request — the UI exposes this as the review net widens.
|
||||
threshold := 0.6
|
||||
if t := req.URL.Query().Get("threshold"); t != "" {
|
||||
if v, err := strconv.ParseFloat(t, 64); err == nil && v > 0 && v <= 1 {
|
||||
threshold = v
|
||||
}
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT a.slug, b.slug, similarity(ka.title, kb.title) AS sim
|
||||
FROM knowledge_entities ka
|
||||
JOIN knowledge_entities kb ON ka.entity_id < kb.entity_id
|
||||
JOIN entities a ON a.id = ka.entity_id
|
||||
JOIN entities b ON b.id = kb.entity_id
|
||||
WHERE ka.deleted_at IS NULL AND kb.deleted_at IS NULL
|
||||
AND similarity(ka.title, kb.title) > $1
|
||||
ORDER BY sim DESC`, threshold)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type pair struct {
|
||||
A, B string
|
||||
Sim float64
|
||||
}
|
||||
pairs := []pair{}
|
||||
for rows.Next() {
|
||||
var p pair
|
||||
if err := rows.Scan(&p.A, &p.B, &p.Sim); err != nil {
|
||||
slog.Error("httpapi: knowledge/duplicates row scan failed", "error", err)
|
||||
continue
|
||||
}
|
||||
pairs = append(pairs, p)
|
||||
}
|
||||
|
||||
// Complete-linkage grouping. `pairs` arrives sorted by similarity
|
||||
// descending, so each new cluster is seeded from the strongest remaining
|
||||
// pair and then only grows with notes that are similar to *everything*
|
||||
// already inside it.
|
||||
sim := make(map[string]float64, len(pairs)*2)
|
||||
key := func(a, b string) string {
|
||||
if a > b {
|
||||
a, b = b, a
|
||||
}
|
||||
return a + "\x00" + b
|
||||
}
|
||||
for _, p := range pairs {
|
||||
sim[key(p.A, p.B)] = p.Sim
|
||||
}
|
||||
linked := func(a, b string) bool { return sim[key(a, b)] > 0 }
|
||||
|
||||
assigned := map[string]bool{}
|
||||
type rawCluster struct {
|
||||
members []string
|
||||
top float64
|
||||
}
|
||||
raw := []rawCluster{}
|
||||
|
||||
for _, p := range pairs {
|
||||
if assigned[p.A] || assigned[p.B] {
|
||||
continue
|
||||
}
|
||||
c := rawCluster{members: []string{p.A, p.B}, top: p.Sim}
|
||||
assigned[p.A], assigned[p.B] = true, true
|
||||
|
||||
// Sweep the remaining pairs for candidates that connect to every
|
||||
// current member. Repeat until a full pass adds nothing, since
|
||||
// admitting one member can qualify another.
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for _, q := range pairs {
|
||||
for _, cand := range []string{q.A, q.B} {
|
||||
if assigned[cand] {
|
||||
continue
|
||||
}
|
||||
ok := true
|
||||
for _, m := range c.members {
|
||||
if !linked(cand, m) {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
c.members = append(c.members, cand)
|
||||
assigned[cand] = true
|
||||
grew = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
raw = append(raw, c)
|
||||
}
|
||||
|
||||
groups := map[string][]string{}
|
||||
best := map[string]float64{}
|
||||
for _, c := range raw {
|
||||
root := c.members[0]
|
||||
groups[root] = c.members
|
||||
best[root] = c.top
|
||||
}
|
||||
|
||||
// Re-fetch display detail for the clustered slugs only.
|
||||
type member struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Kind string `json:"kind"`
|
||||
Size int `json:"size"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
EditedBy string `json:"edited_by"`
|
||||
}
|
||||
detail := map[string]member{}
|
||||
if len(groups) > 0 {
|
||||
all := []string{}
|
||||
for _, g := range groups {
|
||||
all = append(all, g...)
|
||||
}
|
||||
drows, derr := s.pool.Query(ctx, `
|
||||
SELECT e.slug, ke.title, e.type, length(ke.content),
|
||||
ke.updated_at::text, COALESCE(ke.edited_by,'')
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE e.slug = ANY($1) AND ke.deleted_at IS NULL`, all)
|
||||
if derr != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "detail query failed", derr.Error())
|
||||
return
|
||||
}
|
||||
defer drows.Close()
|
||||
for drows.Next() {
|
||||
var m member
|
||||
if err := drows.Scan(&m.Slug, &m.Title, &m.Kind, &m.Size, &m.UpdatedAt, &m.EditedBy); err != nil {
|
||||
slog.Error("httpapi: knowledge/duplicates detail scan failed", "error", err)
|
||||
continue
|
||||
}
|
||||
detail[m.Slug] = m
|
||||
}
|
||||
}
|
||||
|
||||
type cluster struct {
|
||||
Members []member `json:"members"`
|
||||
TopSim float64 `json:"top_similarity"`
|
||||
TotalSize int `json:"total_size"`
|
||||
}
|
||||
out := []cluster{}
|
||||
for root, slugs := range groups {
|
||||
c := cluster{TopSim: best[root]}
|
||||
for _, sl := range slugs {
|
||||
if m, ok := detail[sl]; ok {
|
||||
c.Members = append(c.Members, m)
|
||||
c.TotalSize += m.Size
|
||||
}
|
||||
}
|
||||
if len(c.Members) < 2 {
|
||||
continue
|
||||
}
|
||||
// Newest first inside a cluster — the most recent note is usually
|
||||
// the one worth keeping as the merge target.
|
||||
sort.Slice(c.Members, func(i, j int) bool {
|
||||
return c.Members[i].UpdatedAt > c.Members[j].UpdatedAt
|
||||
})
|
||||
out = append(out, c)
|
||||
}
|
||||
// Biggest clusters first: an eight-note pileup deserves attention before
|
||||
// a two-note coincidence.
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if len(out[i].Members) != len(out[j].Members) {
|
||||
return len(out[i].Members) > len(out[j].Members)
|
||||
}
|
||||
return out[i].TopSim > out[j].TopSim
|
||||
})
|
||||
|
||||
writeJSON(w, map[string]any{"clusters": out, "threshold": threshold})
|
||||
}
|
||||
|
||||
// serveKnowledgeOrphans surfaces notes that have fallen out of every
|
||||
// navigation path — the ones that are technically present but effectively
|
||||
// unreachable, and so quietly stop being maintained.
|
||||
//
|
||||
// Three independent reasons, reported per note (a note can have several):
|
||||
// - untagged: invisible to tag navigation
|
||||
// - unlinked: not `about` any entity, so it never appears on a machine's page
|
||||
// - stale: untouched for 90+ days
|
||||
func (s *Server) serveKnowledgeOrphans(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
staleDays := 90
|
||||
if d := req.URL.Query().Get("stale_days"); d != "" {
|
||||
if v, err := strconv.Atoi(d); err == nil && v > 0 && v <= 3650 {
|
||||
staleDays = v
|
||||
}
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx, fmt.Sprintf(`
|
||||
SELECT e.slug, ke.title, e.type, COALESCE(ke.edited_by,''),
|
||||
ke.updated_at::text,
|
||||
(ke.tags IS NULL OR cardinality(ke.tags) = 0) AS untagged,
|
||||
NOT EXISTS (
|
||||
SELECT 1 FROM relationships r
|
||||
WHERE r.source_id = ke.entity_id AND r.valid_to IS NULL
|
||||
AND r.type IN ('documents', 'about')
|
||||
) AS unlinked,
|
||||
(ke.updated_at < now() - interval '%d days') AS stale
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE ke.deleted_at IS NULL
|
||||
ORDER BY ke.updated_at ASC`, staleDays))
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type orphan struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Kind string `json:"kind"`
|
||||
EditedBy string `json:"edited_by"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
Reasons []string `json:"reasons"`
|
||||
}
|
||||
items := []orphan{}
|
||||
counts := map[string]int{"untagged": 0, "unlinked": 0, "stale": 0}
|
||||
for rows.Next() {
|
||||
var o orphan
|
||||
var untagged, unlinked, stale bool
|
||||
if err := rows.Scan(&o.Slug, &o.Title, &o.Kind, &o.EditedBy, &o.UpdatedAt,
|
||||
&untagged, &unlinked, &stale); err != nil {
|
||||
slog.Error("httpapi: knowledge/orphans row scan failed", "error", err)
|
||||
continue
|
||||
}
|
||||
o.Reasons = []string{}
|
||||
if untagged {
|
||||
o.Reasons = append(o.Reasons, "untagged")
|
||||
counts["untagged"]++
|
||||
}
|
||||
if unlinked {
|
||||
o.Reasons = append(o.Reasons, "unlinked")
|
||||
counts["unlinked"]++
|
||||
}
|
||||
if stale {
|
||||
o.Reasons = append(o.Reasons, "stale")
|
||||
counts["stale"]++
|
||||
}
|
||||
if len(o.Reasons) > 0 {
|
||||
items = append(items, o)
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]any{
|
||||
"items": items,
|
||||
"counts": counts,
|
||||
"stale_days": staleDays,
|
||||
})
|
||||
}
|
||||
|
||||
// serveMergeKnowledge folds several notes into one: each source's body is
|
||||
// appended to the target under a provenance heading, the union of all tags is
|
||||
// kept, and the sources are soft-deleted.
|
||||
//
|
||||
// Append rather than discard, and soft-delete rather than hard: a merge is a
|
||||
// judgement call made from a similarity score, and the operator needs to be
|
||||
// able to walk it back. The target's pre-merge state is captured by the
|
||||
// revision trigger, so the merge itself is undoable from the History tab.
|
||||
func (s *Server) serveMergeKnowledge(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
var body struct {
|
||||
Target string `json:"target"`
|
||||
Sources []string `json:"sources"`
|
||||
}
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error())
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(body.Target) == "" || len(body.Sources) == 0 {
|
||||
writeProblem(w, req, http.StatusBadRequest, "target and sources are required", "")
|
||||
return
|
||||
}
|
||||
|
||||
targetID, err := s.resolveKnowledgeEntity(ctx, body.Target)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusNotFound, "target note not found", body.Target)
|
||||
return
|
||||
}
|
||||
_, actorLabel := actorInfo(ctx)
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "begin failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var merged []string
|
||||
var appended strings.Builder
|
||||
tagSet := map[string]bool{}
|
||||
|
||||
for _, srcSlug := range body.Sources {
|
||||
if srcSlug == body.Target {
|
||||
continue // merging a note into itself would duplicate its body
|
||||
}
|
||||
var srcTitle, srcContent, srcUpdated string
|
||||
var srcTags []string
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT ke.title, ke.content, COALESCE(ke.tags,'{}'), ke.updated_at::text
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE (e.slug = $1 OR e.id::text = $1) AND ke.deleted_at IS NULL`,
|
||||
srcSlug).Scan(&srcTitle, &srcContent, &srcTags, &srcUpdated)
|
||||
if err != nil {
|
||||
slog.Warn("knowledge merge: source not found, skipping", "slug", srcSlug)
|
||||
continue
|
||||
}
|
||||
appended.WriteString("\n\n---\n\n## Merged: ")
|
||||
appended.WriteString(srcTitle)
|
||||
appended.WriteString("\n\n*Originally ")
|
||||
appended.WriteString(srcSlug)
|
||||
appended.WriteString(", last updated ")
|
||||
appended.WriteString(srcUpdated)
|
||||
appended.WriteString("*\n\n")
|
||||
appended.WriteString(srcContent)
|
||||
for _, t := range srcTags {
|
||||
tagSet[strings.ToLower(strings.TrimSpace(t))] = true
|
||||
}
|
||||
merged = append(merged, srcSlug)
|
||||
}
|
||||
|
||||
if len(merged) == 0 {
|
||||
writeProblem(w, req, http.StatusBadRequest, "no valid source notes to merge", "")
|
||||
return
|
||||
}
|
||||
|
||||
extraTags := make([]string, 0, len(tagSet))
|
||||
for t := range tagSet {
|
||||
if t != "" {
|
||||
extraTags = append(extraTags, t)
|
||||
}
|
||||
}
|
||||
sort.Strings(extraTags)
|
||||
|
||||
// The array concat + DISTINCT keeps the target's own tags first and adds
|
||||
// only what the sources contribute.
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE knowledge_entities
|
||||
SET content = content || $2,
|
||||
tags = ARRAY(SELECT DISTINCT unnest(COALESCE(tags,'{}') || $3::text[])),
|
||||
edited_by = $4,
|
||||
updated_at = now()
|
||||
WHERE entity_id = $1`,
|
||||
targetID, appended.String(), extraTags, actorLabel); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "merge write failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
for _, srcSlug := range merged {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE knowledge_entities ke
|
||||
SET deleted_at = now(), edited_by = $2
|
||||
FROM entities e
|
||||
WHERE e.id = ke.entity_id AND (e.slug = $1 OR e.id::text = $1)`,
|
||||
srcSlug, actorLabel); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "source delete failed", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "commit failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("knowledge merged", "target", body.Target, "sources", merged, "actor", actorLabel)
|
||||
writeJSON(w, map[string]any{"ok": true, "merged": merged, "tags_added": extraTags})
|
||||
}
|
||||
|
||||
// lowerAll is the case-folding helper the tag queries compare against.
|
||||
func lowerAll(in []string) []string {
|
||||
out := make([]string, len(in))
|
||||
for i, s := range in {
|
||||
out[i] = strings.ToLower(strings.TrimSpace(s))
|
||||
}
|
||||
return out
|
||||
}
|
||||
659
internal/httpapi/knowledge_write.go
Normal file
659
internal/httpapi/knowledge_write.go
Normal file
@@ -0,0 +1,659 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// Operator-facing write path for the knowledge base. Until this file, the
|
||||
// only way anything reached knowledge_entities was the MCP tool
|
||||
// upsert_knowledge (internal/mcp/server.go) — an agent-only surface. The web
|
||||
// UI could search and read but never create, correct, or remove a note, so
|
||||
// the operator's own knowledge had nowhere to go and an agent mistake had no
|
||||
// fix short of psql.
|
||||
//
|
||||
// All routes here are non-OpenAPI custom routes, consistent with the existing
|
||||
// knowledge read routes (see the carve-out block in server.go): they trade in
|
||||
// raw markdown and ad-hoc aggregates rather than generated schema types.
|
||||
//
|
||||
// Deletion is soft (deleted_at) — see migrations/022_knowledge_revisions.up.sql
|
||||
// for why — so every read path in this file filters on `ke.deleted_at IS NULL`.
|
||||
|
||||
// knowledgeSlugSegmentRe strips a title down to a single slug segment.
|
||||
// Mirrors knowledgeSlugRe in internal/mcp/server.go; duplicated rather than
|
||||
// exported across the package boundary because the two callers namespace
|
||||
// their output differently (see knowledgeSlugFor).
|
||||
var knowledgeSlugSegmentRe = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
// knowledgeSlugFor builds `<kind>:<folder>/<title-slug>`. The MCP tool's
|
||||
// equivalent hardcodes the `nomos/` folder; operator-created notes need to
|
||||
// land somewhere else so the navigator tree can tell at a glance who wrote
|
||||
// what, and so an operator note can never collide with an agent note that
|
||||
// happens to share a title.
|
||||
func knowledgeSlugFor(kind, folder, title string) string {
|
||||
s := strings.ToLower(strings.TrimSpace(title))
|
||||
s = knowledgeSlugSegmentRe.ReplaceAllString(s, "-")
|
||||
s = strings.Trim(s, "-")
|
||||
if s == "" {
|
||||
s = "note"
|
||||
}
|
||||
if len(s) > 80 {
|
||||
s = s[:80]
|
||||
}
|
||||
folder = strings.Trim(strings.ToLower(strings.TrimSpace(folder)), "/")
|
||||
folder = knowledgeSlugSegmentRe.ReplaceAllString(folder, "-")
|
||||
folder = strings.Trim(folder, "-")
|
||||
if folder == "" {
|
||||
folder = "operator"
|
||||
}
|
||||
return kind + ":" + folder + "/" + s
|
||||
}
|
||||
|
||||
// validKnowledgeKind mirrors the three entity types that knowledge_entities
|
||||
// rows are allowed to hang off (see upsert_knowledge's own check).
|
||||
func validKnowledgeKind(kind string) bool {
|
||||
switch kind {
|
||||
case "document", "investigation", "runbook":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// resolveKnowledgeEntity maps an id-or-slug path segment to the entity id of
|
||||
// a live (non-deleted) knowledge note. Returns pgx.ErrNoRows when there's no
|
||||
// such note, which callers turn into a 404.
|
||||
func (s *Server) resolveKnowledgeEntity(ctx context.Context, idOrSlug string) (uuid.UUID, error) {
|
||||
var id uuid.UUID
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT ke.entity_id
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE (e.slug = $1 OR e.id::text = $1)
|
||||
AND ke.deleted_at IS NULL`, idOrSlug).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
// resolveKnowledgeEntityAny is resolveKnowledgeEntity without the
|
||||
// deleted_at filter — for the one read path (revisions) that must still work
|
||||
// on a deleted note. The whole point of soft-delete is that a note's history
|
||||
// stays inspectable after removal (e.g. to confirm what was lost before
|
||||
// restoring it); requiring the note to be live first would defeat that.
|
||||
func (s *Server) resolveKnowledgeEntityAny(ctx context.Context, idOrSlug string) (uuid.UUID, error) {
|
||||
var id uuid.UUID
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT ke.entity_id
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE e.slug = $1 OR e.id::text = $1`, idOrSlug).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
// pathParam pulls a chi URL param and percent-decodes it. Knowledge slugs
|
||||
// contain both ':' and '/' (e.g. "document:containers/101-jellyfin"), so they
|
||||
// reach the handler still encoded — chi.URLParam does no decoding of its own
|
||||
// on manually-registered routes (unlike the OpenAPI-generated ones, which
|
||||
// decode via runtime.BindStyledParameterWithOptions).
|
||||
func pathParam(req *http.Request, name string) (string, error) {
|
||||
return url.PathUnescape(chi.URLParam(req, name))
|
||||
}
|
||||
|
||||
// serveKnowledgeList returns every live note without its body — the backing
|
||||
// data for the wiki navigator tree. Distinct from /knowledge/recent, which
|
||||
// caps at 200 and exists to answer "what changed lately" for the stats view:
|
||||
// the tree needs the complete set, and needs the linked-entity slugs so it
|
||||
// can offer a group-by-entity arrangement without N+1 fetches.
|
||||
//
|
||||
// Body text is deliberately excluded — with ~100 notes averaging ~1 KB the
|
||||
// full payload would be ~100 KB per app open, to render a list that shows
|
||||
// only titles.
|
||||
func (s *Server) serveKnowledgeList(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
type item struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Kind string `json:"kind"`
|
||||
Source string `json:"source"`
|
||||
EditedBy string `json:"edited_by"`
|
||||
Tags []string `json:"tags"`
|
||||
About []string `json:"about"`
|
||||
Size int `json:"size"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Revisions int `json:"revisions"`
|
||||
}
|
||||
|
||||
// The `about` aggregate mirrors GetEntityKnowledge's first UNION branch
|
||||
// (documents/about edges) — the 'procedure-for' branch is left out here
|
||||
// because it joins against entity *types* rather than entities and can't
|
||||
// produce a per-note slug list.
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT e.id::text, e.slug, ke.title, e.type, COALESCE(ke.source,''),
|
||||
COALESCE(ke.edited_by,''), COALESCE(ke.tags, '{}'),
|
||||
COALESCE((
|
||||
SELECT array_agg(DISTINCT t.slug)
|
||||
FROM relationships r
|
||||
JOIN entities t ON t.id = r.target_id
|
||||
WHERE r.source_id = ke.entity_id
|
||||
AND r.valid_to IS NULL
|
||||
AND r.type IN ('documents', 'about')
|
||||
), '{}'),
|
||||
length(ke.content),
|
||||
ke.updated_at::text, ke.created_at::text,
|
||||
(SELECT count(*) FROM knowledge_revisions kr WHERE kr.entity_id = ke.entity_id)
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE ke.deleted_at IS NULL
|
||||
ORDER BY ke.updated_at DESC`)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := []item{}
|
||||
for rows.Next() {
|
||||
var it item
|
||||
if err := rows.Scan(&it.ID, &it.Slug, &it.Title, &it.Kind, &it.Source,
|
||||
&it.EditedBy, &it.Tags, &it.About, &it.Size,
|
||||
&it.UpdatedAt, &it.CreatedAt, &it.Revisions); err != nil {
|
||||
slog.Error("httpapi: knowledge/list row scan failed", "error", err)
|
||||
continue
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
// serveKnowledgeTrash lists soft-deleted notes — the counterpart to
|
||||
// serveKnowledgeList, and what the "restore" affordance in the UI browses.
|
||||
// Without this, a deleted note is invisible from every list endpoint
|
||||
// (correctly — they all filter deleted_at) with no way to even discover it
|
||||
// exists to restore.
|
||||
func (s *Server) serveKnowledgeTrash(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT e.slug, ke.title, e.type, COALESCE(ke.edited_by,''), ke.deleted_at::text
|
||||
FROM knowledge_entities ke
|
||||
JOIN entities e ON e.id = ke.entity_id
|
||||
WHERE ke.deleted_at IS NOT NULL
|
||||
ORDER BY ke.deleted_at DESC`)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type item struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Kind string `json:"kind"`
|
||||
DeletedBy string `json:"deleted_by"`
|
||||
DeletedAt string `json:"deleted_at"`
|
||||
}
|
||||
items := []item{}
|
||||
for rows.Next() {
|
||||
var it item
|
||||
if err := rows.Scan(&it.Slug, &it.Title, &it.Kind, &it.DeletedBy, &it.DeletedAt); err != nil {
|
||||
slog.Error("httpapi: knowledge/trash row scan failed", "error", err)
|
||||
continue
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
// knowledgeWriteBody is the shared request shape for create and update.
|
||||
// Every field is a pointer so update can distinguish "not supplied" (leave
|
||||
// alone) from "supplied empty" (clear it) — a PUT that only changes tags
|
||||
// must not blank the body.
|
||||
type knowledgeWriteBody struct {
|
||||
Title *string `json:"title"`
|
||||
Content *string `json:"content"`
|
||||
Kind *string `json:"kind"`
|
||||
Tags *[]string `json:"tags"`
|
||||
Folder *string `json:"folder"`
|
||||
About *[]string `json:"about"`
|
||||
}
|
||||
|
||||
// serveCreateKnowledge creates a note plus its backing entity, and links it
|
||||
// to whatever entities it's about.
|
||||
func (s *Server) serveCreateKnowledge(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
var body knowledgeWriteBody
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(deref(body.Title))
|
||||
content := strings.TrimSpace(deref(body.Content))
|
||||
if title == "" || content == "" {
|
||||
writeProblem(w, req, http.StatusBadRequest, "title and content are required", "")
|
||||
return
|
||||
}
|
||||
kind := deref(body.Kind)
|
||||
if kind == "" {
|
||||
kind = "document"
|
||||
}
|
||||
if !validKnowledgeKind(kind) {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid kind",
|
||||
"kind must be document, investigation, or runbook")
|
||||
return
|
||||
}
|
||||
tags := normalizeTags(derefSlice(body.Tags))
|
||||
slug := knowledgeSlugFor(kind, deref(body.Folder), title)
|
||||
_, actorLabel := actorInfo(ctx)
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "begin failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
docID, _ := uuid.NewV7()
|
||||
// ON CONFLICT covers the soft-deleted case: the entity row survives a
|
||||
// delete, so recreating a note under the same slug must reuse it rather
|
||||
// than fail the unique constraint.
|
||||
if err := tx.QueryRow(ctx, `
|
||||
INSERT INTO entities (id, slug, type, name, attributes)
|
||||
VALUES ($1, $2, $3, $4, '{}')
|
||||
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, updated_at = now()
|
||||
RETURNING id`, docID, slug, kind, title).Scan(&docID); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "create entity failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Refuse to silently overwrite an existing LIVE note — upsert_knowledge
|
||||
// (the MCP tool) deliberately upserts by title (the agent re-records the
|
||||
// same finding as it learns more), but an operator hitting "create" with
|
||||
// a colliding title almost certainly means to write something new.
|
||||
//
|
||||
// The `WHERE knowledge_entities.deleted_at IS NOT NULL` guard makes this
|
||||
// check atomic with the write, rather than a separate SELECT before it:
|
||||
// a plain pre-check has a TOCTOU race where two concurrent creates of
|
||||
// the same title can both pass the check and then both proceed to
|
||||
// INSERT ON CONFLICT DO UPDATE, silently clobbering each other. Here,
|
||||
// the UPDATE branch only actually applies when the conflicting row is
|
||||
// soft-deleted (a legitimate "resurrect" case). When it isn't, the row
|
||||
// is left untouched, RETURNING yields no row, and pgx.ErrNoRows below
|
||||
// becomes the 409 — the collision can never be missed, no matter how
|
||||
// the two writers interleave.
|
||||
var wroteID uuid.UUID
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO knowledge_entities
|
||||
(entity_id, title, content, source, tags, edited_by, updated_at, deleted_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $4, now(), NULL)
|
||||
ON CONFLICT (entity_id) DO UPDATE
|
||||
SET title = EXCLUDED.title, content = EXCLUDED.content,
|
||||
tags = EXCLUDED.tags, edited_by = EXCLUDED.edited_by,
|
||||
updated_at = now(), deleted_at = NULL
|
||||
WHERE knowledge_entities.deleted_at IS NOT NULL
|
||||
RETURNING entity_id`,
|
||||
docID, title, content, actorLabel, tags).Scan(&wroteID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
writeProblem(w, req, http.StatusConflict, "a note with this title already exists", slug)
|
||||
return
|
||||
} else if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "write knowledge failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
linked := s.linkKnowledgeAbout(ctx, tx, docID, derefSlice(body.About))
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "commit failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("knowledge created", "slug", slug, "kind", kind, "actor", actorLabel, "linked", linked)
|
||||
// Content-Type before WriteHeader — setting it after is a no-op, the
|
||||
// status line is already on the wire.
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"slug": slug, "id": docID.String(), "linked": linked,
|
||||
}); err != nil {
|
||||
slog.Error("httpapi: json encode failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// serveUpdateKnowledge edits a live note in place. The prior version is
|
||||
// captured by the trg_knowledge_revision trigger, not by this handler — see
|
||||
// the migration for why that lives in the database.
|
||||
//
|
||||
// Note the slug is intentionally NOT recomputed when the title changes:
|
||||
// slugs are the wiki's stable link target ([[slug]] references, relationship
|
||||
// rows, bookmarked window ids), and silently re-slugging on a typo fix would
|
||||
// break every inbound link.
|
||||
func (s *Server) serveUpdateKnowledge(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
idOrSlug, err := pathParam(req, "id")
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid id", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var body knowledgeWriteBody
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid request body", err.Error())
|
||||
return
|
||||
}
|
||||
if body.Title == nil && body.Content == nil && body.Tags == nil && body.About == nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "nothing to update",
|
||||
"supply at least one of title, content, tags, about")
|
||||
return
|
||||
}
|
||||
if body.Title != nil && strings.TrimSpace(*body.Title) == "" {
|
||||
writeProblem(w, req, http.StatusBadRequest, "title cannot be empty", "")
|
||||
return
|
||||
}
|
||||
if body.Content != nil && strings.TrimSpace(*body.Content) == "" {
|
||||
writeProblem(w, req, http.StatusBadRequest, "content cannot be empty", "")
|
||||
return
|
||||
}
|
||||
|
||||
entityID, err := s.resolveKnowledgeEntity(ctx, idOrSlug)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusNotFound, "no such knowledge note", "")
|
||||
return
|
||||
}
|
||||
_, actorLabel := actorInfo(ctx)
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "begin failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
// COALESCE keeps unsupplied fields untouched; edited_by and updated_at
|
||||
// always move so the UI can show who last touched it. The trigger only
|
||||
// snapshots when title/content/tags actually differ, so a no-op save
|
||||
// doesn't manufacture a revision.
|
||||
var newTitle *string
|
||||
if body.Title != nil {
|
||||
t := strings.TrimSpace(*body.Title)
|
||||
newTitle = &t
|
||||
}
|
||||
var newContent *string
|
||||
if body.Content != nil {
|
||||
c := strings.TrimSpace(*body.Content)
|
||||
newContent = &c
|
||||
}
|
||||
var newTags *[]string
|
||||
if body.Tags != nil {
|
||||
t := normalizeTags(*body.Tags)
|
||||
newTags = &t
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE knowledge_entities
|
||||
SET title = COALESCE($2, title),
|
||||
content = COALESCE($3, content),
|
||||
tags = COALESCE($4, tags),
|
||||
edited_by = $5,
|
||||
updated_at = now()
|
||||
WHERE entity_id = $1`,
|
||||
entityID, newTitle, newContent, newTags, actorLabel); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "update failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Keep the entity's display name in step with the note title — the graph
|
||||
// and the fleet table read entities.name, and leaving it stale is exactly
|
||||
// the drift this app exists to fight.
|
||||
if newTitle != nil {
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE entities SET name = $2, updated_at = now() WHERE id = $1`,
|
||||
entityID, *newTitle); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "rename entity failed", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// About is replace-semantics, not merge: the editor presents the full
|
||||
// link set, so an absent slug means the operator removed it. Existing
|
||||
// edges are closed (valid_to) rather than deleted, preserving history.
|
||||
var linked []string
|
||||
if body.About != nil {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE relationships SET valid_to = now()
|
||||
WHERE source_id = $1 AND valid_to IS NULL AND type IN ('documents', 'about')`,
|
||||
entityID); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "unlink failed", err.Error())
|
||||
return
|
||||
}
|
||||
linked = s.linkKnowledgeAbout(ctx, tx, entityID, *body.About)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "commit failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("knowledge updated", "entity_id", entityID, "actor", actorLabel)
|
||||
// `linked` lets the caller diff against what it submitted and warn about
|
||||
// any slug that didn't resolve — see linkKnowledgeAbout: a typo'd entity
|
||||
// slug otherwise fails with nothing but a server-side slog.Warn, so the
|
||||
// operator gets no feedback that one of their About links didn't take.
|
||||
writeJSON(w, map[string]any{"ok": true, "linked": linked})
|
||||
}
|
||||
|
||||
// serveDeleteKnowledge soft-deletes a note. The row, its revision trail and
|
||||
// its entity all survive; only the deleted_at stamp changes, and every read
|
||||
// path filters on it.
|
||||
func (s *Server) serveDeleteKnowledge(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
idOrSlug, err := pathParam(req, "id")
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid id", err.Error())
|
||||
return
|
||||
}
|
||||
entityID, err := s.resolveKnowledgeEntity(ctx, idOrSlug)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusNotFound, "no such knowledge note", "")
|
||||
return
|
||||
}
|
||||
_, actorLabel := actorInfo(ctx)
|
||||
|
||||
// Snapshot the live version before tombstoning. The trigger fires on
|
||||
// title/content/tags changes only, and a delete changes none of them —
|
||||
// without this the most recent version would be the one version missing
|
||||
// from the history if the note is later restored.
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO knowledge_revisions
|
||||
(entity_id, title, content, source, tags, edited_by, version_at)
|
||||
SELECT entity_id, title, content, source, tags, COALESCE(edited_by,''), updated_at
|
||||
FROM knowledge_entities WHERE entity_id = $1`, entityID); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "snapshot failed", err.Error())
|
||||
return
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
UPDATE knowledge_entities SET deleted_at = now(), edited_by = $2
|
||||
WHERE entity_id = $1`, entityID, actorLabel); err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "delete failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("knowledge deleted", "entity_id", entityID, "actor", actorLabel)
|
||||
writeJSON(w, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// serveRestoreKnowledge undoes a soft delete. The counterpart to
|
||||
// serveDeleteKnowledge — without it, "recoverable by clearing the column"
|
||||
// (see the migration) would only be true via psql, which isn't a real
|
||||
// recovery path for an operator using the wiki.
|
||||
func (s *Server) serveRestoreKnowledge(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
idOrSlug, err := pathParam(req, "id")
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid id", err.Error())
|
||||
return
|
||||
}
|
||||
entityID, err := s.resolveKnowledgeEntityAny(ctx, idOrSlug)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusNotFound, "no such knowledge note", "")
|
||||
return
|
||||
}
|
||||
_, actorLabel := actorInfo(ctx)
|
||||
|
||||
ct, err := s.pool.Exec(ctx, `
|
||||
UPDATE knowledge_entities SET deleted_at = NULL, edited_by = $2
|
||||
WHERE entity_id = $1 AND deleted_at IS NOT NULL`, entityID, actorLabel)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "restore failed", err.Error())
|
||||
return
|
||||
}
|
||||
if ct.RowsAffected() == 0 {
|
||||
writeProblem(w, req, http.StatusConflict, "note is not deleted", "")
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("knowledge restored", "entity_id", entityID, "actor", actorLabel)
|
||||
writeJSON(w, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// serveKnowledgeRevisions returns the note's superseded versions, newest
|
||||
// first. Bodies are included: revisions are small (~1 KB) and few, and the
|
||||
// diff view needs both sides anyway — paginating would cost a round trip per
|
||||
// comparison to save nothing.
|
||||
func (s *Server) serveKnowledgeRevisions(w http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
idOrSlug, err := pathParam(req, "id")
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusBadRequest, "invalid id", err.Error())
|
||||
return
|
||||
}
|
||||
entityID, err := s.resolveKnowledgeEntityAny(ctx, idOrSlug)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusNotFound, "no such knowledge note", "")
|
||||
return
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id, title, content, COALESCE(edited_by,''), COALESCE(tags,'{}'),
|
||||
version_at::text, revised_at::text
|
||||
FROM knowledge_revisions
|
||||
WHERE entity_id = $1
|
||||
ORDER BY version_at DESC`, entityID)
|
||||
if err != nil {
|
||||
writeProblem(w, req, http.StatusInternalServerError, "query failed", err.Error())
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type revision struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
EditedBy string `json:"edited_by"`
|
||||
Tags []string `json:"tags"`
|
||||
VersionAt string `json:"version_at"`
|
||||
RevisedAt string `json:"revised_at"`
|
||||
}
|
||||
items := []revision{}
|
||||
for rows.Next() {
|
||||
var r revision
|
||||
if err := rows.Scan(&r.ID, &r.Title, &r.Content, &r.EditedBy, &r.Tags,
|
||||
&r.VersionAt, &r.RevisedAt); err != nil {
|
||||
slog.Error("httpapi: knowledge/revisions row scan failed", "error", err)
|
||||
continue
|
||||
}
|
||||
items = append(items, r)
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
// linkKnowledgeAbout points a note at the entities it concerns, skipping
|
||||
// slugs that don't resolve and edges that already exist. Returns the slugs
|
||||
// actually linked so the caller can report what stuck — a typo'd slug is a
|
||||
// silent no-op otherwise.
|
||||
func (s *Server) linkKnowledgeAbout(ctx context.Context, tx pgx.Tx, docID uuid.UUID, slugs []string) []string {
|
||||
linked := []string{}
|
||||
for _, raw := range slugs {
|
||||
slug := strings.TrimSpace(raw)
|
||||
if slug == "" {
|
||||
continue
|
||||
}
|
||||
var targetID uuid.UUID
|
||||
if err := tx.QueryRow(ctx, `SELECT id FROM entities WHERE slug = $1`, slug).Scan(&targetID); err != nil {
|
||||
slog.Warn("knowledge: about slug not found, skipping", "slug", slug)
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO relationships (source_id, target_id, type, attributes, valid_from)
|
||||
SELECT $1, $2, 'about', '{"by":"operator"}'::jsonb, now()
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM relationships
|
||||
WHERE source_id = $1 AND target_id = $2 AND type = 'about' AND valid_to IS NULL)`,
|
||||
docID, targetID); err != nil {
|
||||
slog.Warn("knowledge: link failed", "slug", slug, "error", err)
|
||||
continue
|
||||
}
|
||||
linked = append(linked, slug)
|
||||
}
|
||||
return linked
|
||||
}
|
||||
|
||||
// normalizeTags trims, lowercases and de-duplicates while preserving order.
|
||||
// Lowercasing is the fix for the casing drift already in the data — `oom`
|
||||
// and `OOM` were separate tags on separate notes, so neither tag page showed
|
||||
// the full set. Applied on every write so the split can't reopen.
|
||||
func normalizeTags(in []string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := []string{}
|
||||
for _, t := range in {
|
||||
t = strings.ToLower(strings.TrimSpace(t))
|
||||
if t == "" || seen[t] {
|
||||
continue
|
||||
}
|
||||
seen[t] = true
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func deref(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
func derefSlice(p *[]string) []string {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
// writeJSON is the success-path counterpart to writeProblem, so the handlers
|
||||
// in this file don't each repeat the header/encode dance.
|
||||
func writeJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(v); err != nil {
|
||||
slog.Error("httpapi: json encode failed", "error", err)
|
||||
}
|
||||
}
|
||||
@@ -155,14 +155,14 @@ func (s *Server) GetTrends(ctx context.Context, req gen.GetTrendsRequestObject)
|
||||
f, _ := slopeNum.Float64Value()
|
||||
t.Slope = float32Ptr(float32(f.Float64))
|
||||
if f.Float64 > 0.01 {
|
||||
t.Direction = gen.Improving
|
||||
t.Direction = gen.TrendDirectionImproving
|
||||
} else if f.Float64 < -0.01 {
|
||||
t.Direction = gen.Degrading
|
||||
t.Direction = gen.TrendDirectionDegrading
|
||||
} else {
|
||||
t.Direction = gen.Stable
|
||||
t.Direction = gen.TrendDirectionStable
|
||||
}
|
||||
} else {
|
||||
t.Direction = gen.Unknown
|
||||
t.Direction = gen.TrendDirectionUnknown
|
||||
}
|
||||
items = append(items, t)
|
||||
}
|
||||
|
||||
83
internal/httpapi/ontology.go
Normal file
83
internal/httpapi/ontology.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/httpapi/gen"
|
||||
)
|
||||
|
||||
func (s *Server) GetOntology(ctx context.Context, req gen.GetOntologyRequestObject) (gen.GetOntologyResponseObject, error) {
|
||||
resp := gen.GetOntology200JSONResponse{
|
||||
EntityTypes: []gen.EntityType{},
|
||||
RelationshipTypes: []gen.RelationshipType{},
|
||||
Lifecycles: []gen.LifecycleDef{},
|
||||
}
|
||||
|
||||
q := sqlcgen.New(s.pool)
|
||||
|
||||
etRows, err := q.ListEntityTypes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, et := range etRows {
|
||||
schemaVersion := int(et.SchemaVersion)
|
||||
var schema *map[string]any
|
||||
if len(et.AttributeSchema) > 0 {
|
||||
var s map[string]any
|
||||
if json.Unmarshal(et.AttributeSchema, &s) == nil && s != nil {
|
||||
schema = &s
|
||||
}
|
||||
}
|
||||
resp.EntityTypes = append(resp.EntityTypes, gen.EntityType{
|
||||
Name: et.Name,
|
||||
ParentType: et.ParentType,
|
||||
IsAbstract: et.IsAbstract,
|
||||
Domain: et.Domain,
|
||||
Layer: gen.EntityTypeLayer(et.Layer),
|
||||
Description: et.Description,
|
||||
LifecycleId: et.LifecycleID,
|
||||
SchemaVersion: &schemaVersion,
|
||||
AttributeSchema: schema,
|
||||
Status: gen.EntityTypeStatus(et.Status),
|
||||
})
|
||||
}
|
||||
|
||||
rtRows, err := q.ListRelationshipTypes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rt := range rtRows {
|
||||
resp.RelationshipTypes = append(resp.RelationshipTypes, gen.RelationshipType{
|
||||
Name: rt.Name,
|
||||
Inverse: rt.Inverse,
|
||||
SourceType: rt.SourceType,
|
||||
TargetType: rt.TargetType,
|
||||
Cardinality: gen.RelationshipTypeCardinality(rt.Cardinality),
|
||||
Description: rt.Description,
|
||||
})
|
||||
}
|
||||
|
||||
lcRows, err := q.ListLifecycleDefs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, lc := range lcRows {
|
||||
terminal := lc.TerminalStates
|
||||
var transitions map[string]any
|
||||
if err := json.Unmarshal(lc.Transitions, &transitions); err != nil {
|
||||
return nil, fmt.Errorf("lifecycle %s transitions: %w", lc.ID, err)
|
||||
}
|
||||
resp.Lifecycles = append(resp.Lifecycles, gen.LifecycleDef{
|
||||
Id: lc.ID,
|
||||
States: lc.States,
|
||||
DefaultState: lc.DefaultState,
|
||||
TerminalStates: &terminal,
|
||||
Transitions: transitions,
|
||||
})
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
@@ -111,6 +111,7 @@ func (s *Server) PatchPattern(ctx context.Context, req gen.PatchPatternRequestOb
|
||||
actorType, actor := actorInfo(ctx)
|
||||
if auditErr := observability.Audit(ctx, q, actorType, actor, "patch",
|
||||
&id, "PATCH", "/api/v1/patterns/"+req.Id, "",
|
||||
nil,
|
||||
map[string]any{"status": req.Body.Status, "quarantined": req.Body.Quarantined}); auditErr != nil {
|
||||
return nil, auditErr
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user