Files
oikos/internal/httpapi/executions.go
dtoro 1dca2cfd7a feat(observability): restore monitoring coverage, make gaps visible, stream executions
Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by
discarded errors in checkdefaults:

- writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT
  (slug) DO NOTHING, then wrote a check_defs row referencing it. On any
  re-seed the slug already existed, the entity insert no-oped, and the FK
  violated — aborting the ingest transaction and surfacing as an unrelated
  failure several entities later. Re-seeding has been broken since; prod's
  coverage was frozen at its first successful seed. This is what
  TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting.
- shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed
  to ".network" and overwrote each other; service:jellyfin collided with
  lxc:jellyfin.
- The ssh-script checker never read the `args` config checkdefaults wrote, so
  process_check.sh always ran without its unit name and returned "unknown".

Coverage is now 75/89. Monitoring is declared per entity type in
seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say
it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap.
coverageSweep raises an `unmonitored` signal only where a type declares
monitoring it lacks — 8 real gaps, no false positives.

Also:
- entity_types.attribute_schema was never ingested: the seed loader read
  "attribute_schema" but the YAML says "attributes", so all 60 types stored
  JSON null.
- ListExecutions ignored its declared target/action/correlation_id filters and
  paginated on a non-unique target slug, dropping and repeating rows.
- started_at was captured but only written at terminal state, so a running
  execution reported NULL for its whole life. The three MCP auto-run copies
  wrote no timing at all; they are now one autoRun helper.
- SSH output was buffered to completion and discarded entirely on timeout.
  Both sshExec copies now stream through a shared execlog sink into
  execution_logs, and keep partial output when a command is cancelled.
- executions.correlation_id was a random per-execution uuid that correlated
  nothing; it is now the chat session id, which is what lets the chat tail
  live output.
- reversible_low had no auto-run branch despite policy declaring it
  unattended. Since computeCommandRisk never returns it, the class only arises
  when an agent declares it over a read_only command — so gating it penalised
  candor without adding safety.
- backup-target gains a backup-freshness checker (portable find -mmin, since
  the first target is on macOS), resolving its host by walking backs-up-to
  backwards. The pre-deploy pg_dump is now a tracked backup target.

UI: an Executions section on entity detail with live output tailing, and
streamed output under a running `run` call in the chat timeline.

Migrations 022-024. Ops.svelte and context.ts exclude execution.output from
their refetch triggers, which would otherwise fire once a second per command.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-28 13:51:14 +02:00

320 lines
11 KiB
Go

package httpapi
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/dtoro/oikos/internal/db/sqlcgen"
"github.com/dtoro/oikos/internal/domain"
"github.com/dtoro/oikos/internal/httpapi/gen"
"github.com/dtoro/oikos/internal/observability"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
// ─── Executions ────────────────────────────────────────────────────────
// ListExecutions returns executions newest-first.
//
// The target/action/correlation_id filters are declared in the OpenAPI spec and
// generated into the request struct, but were never bound — so
// `GET /executions?target=<id>` silently returned the first page of the whole
// fleet. Ordering was by target slug, which is neither useful for a history
// view nor unique enough to paginate on: several executions share a target, so
// a slug cursor could skip or repeat rows.
func (s *Server) ListExecutions(ctx context.Context, req gen.ListExecutionsRequestObject) (gen.ListExecutionsResponseObject, error) {
limit := clampLimit(req.Params.Limit)
cursorTime, cursorID, err := parseExecutionCursor(req.Params.Cursor)
if err != nil {
return nil, err
}
rows, err := s.pool.Query(ctx, `
SELECT e.entity_id, e.classification_id::text, e.signal_entity_id::text,
e.target_entity_id, e.action, e.risk_class,
e.approval_id::text, e.agent_id::text, e.skill_id::text,
e.skill_version, e.status, e.result, e.duration_ms,
e.verified, e.correlation_id, e.started_at, e.completed_at, e.created_at,
te.slug
FROM executions e
JOIN entities te ON te.id = e.target_entity_id
WHERE ($1::text IS NULL OR e.status = $1)
-- target accepts a slug or a uuid: the SPA passes an entity id,
-- while a human poking the API reaches for the slug.
AND ($2::text IS NULL OR te.slug = $2 OR e.target_entity_id::text = $2)
-- the run tool encodes action as "run:{json}", so match the verb too
AND ($3::text IS NULL OR e.action = $3 OR split_part(e.action, ':', 1) = $3)
AND ($4::text IS NULL OR e.correlation_id = $4)
AND ($5::timestamptz IS NULL
OR (e.created_at, e.entity_id) < ($5::timestamptz, $6::uuid))
ORDER BY e.created_at DESC, e.entity_id DESC
LIMIT $7`,
req.Params.Status, req.Params.Target, req.Params.Action, req.Params.CorrelationId,
cursorTime, cursorID, limit+1)
if err != nil {
return nil, err
}
defer rows.Close()
items := []gen.Execution{}
for rows.Next() {
var exec gen.Execution
var resultBytes []byte
var targetSlug string
if err := rows.Scan(&exec.Id, &exec.ClassificationId, &exec.SignalId,
&exec.Target, &exec.Action, &exec.RiskClass,
&exec.ApprovalId, &exec.AgentId, &exec.SkillId,
&exec.SkillVersion, &exec.Status, &resultBytes, &exec.DurationMs,
&exec.Verified, &exec.CorrelationId, &exec.StartedAt, &exec.CompletedAt,
&exec.CreatedAt, &targetSlug); err != nil {
return nil, err
}
var result map[string]any
if len(resultBytes) > 0 && json.Unmarshal(resultBytes, &result) == nil {
exec.Result = &result
}
// Target is stored as UUID, but we surface the slug
exec.Slug = targetSlug
items = append(items, exec)
}
if rows.Err() != nil {
return nil, rows.Err()
}
var next *string
if len(items) > limit {
items = items[:limit]
last := items[len(items)-1]
cursor := formatExecutionCursor(last.CreatedAt, last.Id)
next = &cursor
}
if items == nil {
items = []gen.Execution{}
}
return gen.ListExecutions200JSONResponse{Items: items, NextCursor: next}, nil
}
// Executions are ordered by (created_at DESC, entity_id DESC), so the cursor
// has to carry both — created_at alone is not unique, and paginating on a
// non-unique key drops or repeats rows at page boundaries.
func formatExecutionCursor(createdAt time.Time, id uuid.UUID) string {
return createdAt.UTC().Format(time.RFC3339Nano) + "," + id.String()
}
func parseExecutionCursor(cursor *string) (*time.Time, *uuid.UUID, error) {
if cursor == nil || *cursor == "" {
return nil, nil, nil
}
rawTime, rawID, ok := strings.Cut(*cursor, ",")
if !ok {
return nil, nil, domain.ErrInvalidInput
}
t, err := time.Parse(time.RFC3339Nano, rawTime)
if err != nil {
return nil, nil, domain.ErrInvalidInput
}
id, err := uuid.Parse(rawID)
if err != nil {
return nil, nil, domain.ErrInvalidInput
}
return &t, &id, nil
}
func (s *Server) GetExecution(ctx context.Context, req gen.GetExecutionRequestObject) (gen.GetExecutionResponseObject, error) {
id, err := s.resolveEntityID(ctx, req.Id)
if err != nil {
return nil, err
}
var exec gen.Execution
var resultBytes []byte
var targetSlug string
err = s.pool.QueryRow(ctx, `
SELECT e.entity_id, e.classification_id::text, e.signal_entity_id::text,
e.target_entity_id, e.action, e.risk_class,
e.approval_id::text, e.agent_id::text, e.skill_id::text,
e.skill_version, e.status, e.result, e.duration_ms,
e.verified, e.correlation_id, e.started_at, e.completed_at, e.created_at,
te.slug
FROM executions e
JOIN entities te ON te.id = e.target_entity_id
WHERE e.entity_id = $1`, id).
Scan(&exec.Id, &exec.ClassificationId, &exec.SignalId,
&exec.Target, &exec.Action, &exec.RiskClass,
&exec.ApprovalId, &exec.AgentId, &exec.SkillId,
&exec.SkillVersion, &exec.Status, &resultBytes, &exec.DurationMs,
&exec.Verified, &exec.CorrelationId, &exec.StartedAt, &exec.CompletedAt,
&exec.CreatedAt, &targetSlug)
if err != nil {
if err == pgx.ErrNoRows {
return nil, fmt.Errorf("%w: execution %s", domain.ErrNotFound, req.Id)
}
return nil, err
}
var result map[string]any
if len(resultBytes) > 0 && json.Unmarshal(resultBytes, &result) == nil {
exec.Result = &result
}
exec.Slug = targetSlug
return gen.GetExecution200JSONResponse(exec), nil
}
func (s *Server) RequestExecution(ctx context.Context, req gen.RequestExecutionRequestObject) (gen.RequestExecutionResponseObject, error) {
if req.Body == nil {
return nil, fmt.Errorf("%w: request body is required", domain.ErrInvalidInput)
}
id, err := uuid.NewV7()
if err != nil {
return nil, err
}
targetID, err := s.resolveEntityID(ctx, req.Body.Target)
if err != nil {
return nil, err
}
correlationID := uuid.New().String()
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
defer tx.Rollback(ctx)
q := sqlcgen.New(tx)
// Full UUID, not a truncated prefix — an 8-char prefix of a UUIDv7
// collides for real under back-to-back requests since the leading bytes
// encode a millisecond timestamp (observed live via the MCP run tool).
execSlug := "exec:" + id.String()
if _, err := q.InsertEntity(ctx, sqlcgen.InsertEntityParams{
ID: id,
Slug: execSlug,
Type: "execution",
Name: req.Body.Action + " on " + req.Body.Target,
Attributes: []byte("{}"),
}); err != nil {
return nil, err
}
if err := q.InsertExecution(ctx, sqlcgen.InsertExecutionParams{
EntityID: id,
TargetEntityID: &targetID,
Action: req.Body.Action,
RiskClass: "unclassified", // will be classified by classifier
CorrelationID: correlationID,
}); err != nil {
return nil, err
}
// Re-read to get the full record.
var exec gen.Execution
var resultBytes []byte
var targetSlug string
err = tx.QueryRow(ctx, `
SELECT e.entity_id, e.classification_id::text, e.signal_entity_id::text,
e.target_entity_id, e.action, e.risk_class,
e.approval_id::text, e.agent_id::text, e.skill_id::text,
e.skill_version, e.status, e.result, e.duration_ms,
e.verified, e.correlation_id, e.started_at, e.completed_at, e.created_at,
te.slug
FROM executions e
JOIN entities te ON te.id = e.target_entity_id
WHERE e.entity_id = $1`, id).
Scan(&exec.Id, &exec.ClassificationId, &exec.SignalId,
&exec.Target, &exec.Action, &exec.RiskClass,
&exec.ApprovalId, &exec.AgentId, &exec.SkillId,
&exec.SkillVersion, &exec.Status, &resultBytes, &exec.DurationMs,
&exec.Verified, &exec.CorrelationId, &exec.StartedAt, &exec.CompletedAt,
&exec.CreatedAt, &targetSlug)
if err != nil {
return nil, err
}
exec.Slug = targetSlug
actorType, actor := actorInfo(ctx)
if auditErr := observability.Audit(ctx, q, actorType, actor, "create",
&id, "POST", "/api/v1/executions", "",
map[string]any{"action": req.Body.Action, "target": req.Body.Target}); auditErr != nil {
return nil, auditErr
}
if eventErr := observability.Event(ctx, q, "execution.requested", &id,
"info", "oikos-api", "",
map[string]any{"action": req.Body.Action, "target": req.Body.Target}); eventErr != nil {
return nil, eventErr
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
return gen.RequestExecution201JSONResponse(exec), nil
}
func (s *Server) CancelExecution(ctx context.Context, req gen.CancelExecutionRequestObject) (gen.CancelExecutionResponseObject, error) {
id, err := s.resolveEntityID(ctx, req.Id)
if err != nil {
return nil, err
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
defer tx.Rollback(ctx)
q := sqlcgen.New(tx)
if err := q.UpdateExecutionStatus(ctx, sqlcgen.UpdateExecutionStatusParams{
EntityID: id,
Status: "cancelled",
}); err != nil {
if err == pgx.ErrNoRows {
return nil, fmt.Errorf("%w: execution %s", domain.ErrNotFound, req.Id)
}
return nil, err
}
// Re-read.
var exec gen.Execution
var resultBytes []byte
var targetSlug string
err = tx.QueryRow(ctx, `
SELECT e.entity_id, e.classification_id::text, e.signal_entity_id::text,
e.target_entity_id, e.action, e.risk_class,
e.approval_id::text, e.agent_id::text, e.skill_id::text,
e.skill_version, e.status, e.result, e.duration_ms,
e.verified, e.correlation_id, e.started_at, e.completed_at, e.created_at,
te.slug
FROM executions e
JOIN entities te ON te.id = e.target_entity_id
WHERE e.entity_id = $1`, id).
Scan(&exec.Id, &exec.ClassificationId, &exec.SignalId,
&exec.Target, &exec.Action, &exec.RiskClass,
&exec.ApprovalId, &exec.AgentId, &exec.SkillId,
&exec.SkillVersion, &exec.Status, &resultBytes, &exec.DurationMs,
&exec.Verified, &exec.CorrelationId, &exec.StartedAt, &exec.CompletedAt,
&exec.CreatedAt, &targetSlug)
if err != nil {
return nil, err
}
exec.Slug = targetSlug
actorType, actor := actorInfo(ctx)
if auditErr := observability.Audit(ctx, q, actorType, actor, "cancel",
&id, "POST", "/api/v1/executions/"+req.Id+"/cancel", "",
map[string]any{"status": "cancelled"}); auditErr != nil {
return nil, auditErr
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
return gen.CancelExecution200JSONResponse(exec), nil
}