feat(observability): restore monitoring coverage, make gaps visible, stream executions
Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by discarded errors in checkdefaults: - writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT (slug) DO NOTHING, then wrote a check_defs row referencing it. On any re-seed the slug already existed, the entity insert no-oped, and the FK violated — aborting the ingest transaction and surfacing as an unrelated failure several entities later. Re-seeding has been broken since; prod's coverage was frozen at its first successful seed. This is what TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting. - shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed to ".network" and overwrote each other; service:jellyfin collided with lxc:jellyfin. - The ssh-script checker never read the `args` config checkdefaults wrote, so process_check.sh always ran without its unit name and returned "unknown". Coverage is now 75/89. Monitoring is declared per entity type in seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap. coverageSweep raises an `unmonitored` signal only where a type declares monitoring it lacks — 8 real gaps, no false positives. Also: - entity_types.attribute_schema was never ingested: the seed loader read "attribute_schema" but the YAML says "attributes", so all 60 types stored JSON null. - ListExecutions ignored its declared target/action/correlation_id filters and paginated on a non-unique target slug, dropping and repeating rows. - started_at was captured but only written at terminal state, so a running execution reported NULL for its whole life. The three MCP auto-run copies wrote no timing at all; they are now one autoRun helper. - SSH output was buffered to completion and discarded entirely on timeout. Both sshExec copies now stream through a shared execlog sink into execution_logs, and keep partial output when a command is cancelled. - executions.correlation_id was a random per-execution uuid that correlated nothing; it is now the chat session id, which is what lets the chat tail live output. - reversible_low had no auto-run branch despite policy declaring it unattended. Since computeCommandRisk never returns it, the class only arises when an agent declares it over a read_only command — so gating it penalised candor without adding safety. - backup-target gains a backup-freshness checker (portable find -mmin, since the first target is on macOS), resolving its host by walking backs-up-to backwards. The pre-deploy pg_dump is now a tracked backup target. UI: an Executions section on entity detail with live output tailing, and streamed output under a running `run` call in the chat timeline. Migrations 022-024. Ops.svelte and context.ts exclude execution.output from their refetch triggers, which would otherwise fire once a second per command. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -22,6 +22,7 @@ import (
|
||||
|
||||
"github.com/dtoro/oikos/internal/db"
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
"github.com/dtoro/oikos/internal/execlog"
|
||||
"github.com/dtoro/oikos/internal/observability"
|
||||
"github.com/dtoro/oikos/internal/policy"
|
||||
"github.com/google/jsonschema-go/jsonschema"
|
||||
@@ -329,7 +330,37 @@ func initSSH() {
|
||||
// goroutine forever with no way for the caller to ever get an answer.
|
||||
const sshExecTimeout = 10 * time.Minute
|
||||
|
||||
// streamWriter buffers everything it is given while forwarding each write to a
|
||||
// sink. Assigning one to session.Stdout and another (sharing the same buffer)
|
||||
// to session.Stderr reproduces CombinedOutput's interleaving exactly, in the
|
||||
// order the remote end actually produced it — which reading from StdoutPipe
|
||||
// and StderrPipe separately would not guarantee.
|
||||
type streamWriter struct {
|
||||
mu *sync.Mutex
|
||||
buf *bytes.Buffer
|
||||
stream string
|
||||
sink execlog.Sink
|
||||
}
|
||||
|
||||
func (w *streamWriter) Write(p []byte) (int, error) {
|
||||
w.mu.Lock()
|
||||
w.buf.Write(p)
|
||||
w.mu.Unlock()
|
||||
if w.sink != nil {
|
||||
// Copy: the ssh library reuses p after Write returns, and the sink
|
||||
// hands the bytes to a DB call that may outlive this frame.
|
||||
w.sink(w.stream, append([]byte(nil), p...))
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func sshExec(ctx context.Context, host, user, command string) (string, error) {
|
||||
return sshExecStream(ctx, host, user, command, nil)
|
||||
}
|
||||
|
||||
// sshExecStream runs a command and reports its combined output, forwarding
|
||||
// each chunk to sink as it arrives. A nil sink behaves exactly as before.
|
||||
func sshExecStream(ctx context.Context, host, user, command string, sink execlog.Sink) (string, error) {
|
||||
initSSH()
|
||||
if len(sshKey) == 0 {
|
||||
return "", fmt.Errorf("no SSH key available")
|
||||
@@ -363,16 +394,28 @@ func sshExec(ctx context.Context, host, user, command string) (string, error) {
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
type result struct {
|
||||
out []byte
|
||||
err error
|
||||
var (
|
||||
mu sync.Mutex
|
||||
buf bytes.Buffer
|
||||
)
|
||||
session.Stdout = &streamWriter{mu: &mu, buf: &buf, stream: "stdout", sink: sink}
|
||||
session.Stderr = &streamWriter{mu: &mu, buf: &buf, stream: "stderr", sink: sink}
|
||||
|
||||
// collected returns whatever output has arrived so far. Callable while the
|
||||
// command is still running, which is what makes partial output on timeout
|
||||
// possible.
|
||||
collected := func() string {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return strings.TrimSpace(buf.String())
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
// Recovers a panic in CombinedOutput (SSH library internals, rare but
|
||||
// not impossible) and reports it as a failed command instead of
|
||||
// crashing the whole api process — every gated action runs through
|
||||
// this function, so an unrecovered panic here would take down every
|
||||
// Recovers a panic in the SSH library internals (rare but not
|
||||
// impossible) and reports it as a failed command instead of crashing
|
||||
// the whole api process — every gated action runs through this
|
||||
// function, so an unrecovered panic here would take down every
|
||||
// concurrently-running task's execution, not just this one. Without
|
||||
// this, a panic would ALSO silently degrade to "wait out the full
|
||||
// timeout" (done never receives, the select below falls through to
|
||||
@@ -381,36 +424,40 @@ func sshExec(ctx context.Context, host, user, command string) (string, error) {
|
||||
// finds out now, not after sshExecTimeout.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{nil, fmt.Errorf("panic in ssh exec: %v", r)}
|
||||
done <- fmt.Errorf("panic in ssh exec: %v", r)
|
||||
}
|
||||
}()
|
||||
out, err := session.CombinedOutput(command)
|
||||
done <- result{out, err}
|
||||
// Run rather than CombinedOutput so the assigned writers are used;
|
||||
// Run returns only after both streams have been fully drained.
|
||||
done <- session.Run(command)
|
||||
}()
|
||||
|
||||
select {
|
||||
case r := <-done:
|
||||
text := strings.TrimSpace(string(r.out))
|
||||
case err := <-done:
|
||||
text := collected()
|
||||
// A non-zero exit MUST surface as an error — matching the fix
|
||||
// applied to httpapi's sshExec (this copy still had the original
|
||||
// bug: only erroring when there was no output at all, so a command
|
||||
// that failed but printed something was silently reported as
|
||||
// success).
|
||||
if r.err != nil {
|
||||
if err != nil {
|
||||
if text != "" {
|
||||
return text, fmt.Errorf("%w: %s", r.err, text)
|
||||
return text, fmt.Errorf("%w: %s", err, text)
|
||||
}
|
||||
return text, fmt.Errorf("exec: %w", r.err)
|
||||
return text, fmt.Errorf("exec: %w", err)
|
||||
}
|
||||
return text, nil
|
||||
case <-time.After(sshExecTimeout):
|
||||
session.Close()
|
||||
client.Close()
|
||||
return "", fmt.Errorf("timed out after %s waiting for command to finish on %s", sshExecTimeout, host)
|
||||
// Return what the command managed to print before it hung. This used
|
||||
// to return "", discarding everything — so a hung command, the case
|
||||
// where the output matters most, was the one case that left no trace.
|
||||
return collected(), fmt.Errorf("timed out after %s waiting for command to finish on %s", sshExecTimeout, host)
|
||||
case <-ctx.Done():
|
||||
session.Close()
|
||||
client.Close()
|
||||
return "", ctx.Err()
|
||||
return collected(), ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -631,6 +678,56 @@ func resolveProxmoxHostSlug(ctx context.Context, pool *db.Pool, entitySlug, host
|
||||
// fleet's reverse proxy) executed instantly with no approval at all. Routing
|
||||
// every mutating path through the same classifier + approval-queue logic
|
||||
// closes that gap without special-casing each caller.
|
||||
// autoRun resolves a target, runs the command, and finalizes the execution
|
||||
// with full timing.
|
||||
//
|
||||
// The three auto-run windows (read-only, assent, destructive) each carried
|
||||
// their own copy of this logic, and none of them wrote duration_ms, started_at
|
||||
// or completed_at — so every auto-run execution landed in the ledger with no
|
||||
// timing at all, and the Ops "Duration" column was empty for exactly the
|
||||
// executions that run most often.
|
||||
func autoRun(ctx context.Context, pool *db.Pool, id uuid.UUID, targetSlug, command string) (string, error) {
|
||||
startedAt := time.Now()
|
||||
if _, err := pool.Exec(ctx,
|
||||
`UPDATE executions SET status='running', started_at=$2 WHERE entity_id=$1`,
|
||||
id, startedAt); err != nil {
|
||||
slog.Error("mcp: mark execution running", "error", err, "execution_id", id)
|
||||
}
|
||||
|
||||
finalize := func(status string, result []byte) {
|
||||
if _, err := pool.Exec(ctx,
|
||||
`UPDATE executions SET status=$2, result=$3::jsonb, duration_ms=$4,
|
||||
started_at=$5, completed_at=now()
|
||||
WHERE entity_id=$1`,
|
||||
id, status, result, int(time.Since(startedAt).Milliseconds()), startedAt); err != nil {
|
||||
slog.Error("mcp: finalize execution", "error", err, "execution_id", id)
|
||||
}
|
||||
}
|
||||
|
||||
host, user, wrap, err := resolveExecTarget(ctx, pool, targetSlug)
|
||||
if err != nil {
|
||||
finalize("failed", jsonErr("%s", err.Error()))
|
||||
return "", err
|
||||
}
|
||||
|
||||
var correlationID string
|
||||
if qerr := pool.QueryRow(ctx,
|
||||
`SELECT correlation_id FROM executions WHERE entity_id = $1`, id).Scan(&correlationID); qerr != nil {
|
||||
correlationID = ""
|
||||
}
|
||||
sink, flush := execlog.New(ctx, pool, id, correlationID)
|
||||
|
||||
out, err := sshExecStream(ctx, host, user, wrap(command), sink)
|
||||
flush()
|
||||
if err != nil {
|
||||
finalize("failed", jsonErr("%s: %s", err.Error(), out))
|
||||
return out, err
|
||||
}
|
||||
|
||||
finalize("completed", jsonOut(out))
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func classifyAndGate(ctx context.Context, pool *db.Pool, agentID, targetID uuid.UUID, targetSlug, command, purpose, declaredRisk, sessionID string) *mcp.CallToolResult {
|
||||
riskClass := policy.ClassifyCommand(command, declaredRisk)
|
||||
runParams, _ := json.Marshal(map[string]string{"command": command, "purpose": purpose})
|
||||
@@ -686,7 +783,20 @@ func classifyAndGate(ctx context.Context, pool *db.Pool, agentID, targetID uuid.
|
||||
}
|
||||
|
||||
id, _ := uuid.NewV7()
|
||||
correlationID := uuid.New().String()
|
||||
// Correlate the execution to the chat session that asked for it. This was
|
||||
// a fresh random UUID per execution, which correlated nothing — every row
|
||||
// had a unique value, so the correlation_id column and the
|
||||
// ?correlation_id= filter could only ever match one execution.
|
||||
//
|
||||
// Using the session id makes the field mean what it says ("what did this
|
||||
// session do?") and is what lets the chat tail live output: execution
|
||||
// events carry correlation_id, so the UI can match them to the session on
|
||||
// screen without a lookup. Falls back to a random id when there is no
|
||||
// session to scope to, keeping the column non-empty.
|
||||
correlationID := sessionID
|
||||
if correlationID == "" || correlationID == "ephemeral" {
|
||||
correlationID = uuid.New().String()
|
||||
}
|
||||
execName := "run on " + targetSlug + " (" + id.String() + ")"
|
||||
execSlug := "exec:" + targetSlug + ":" + id.String()
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO entities (id, slug, type, name, attributes) VALUES ($1, $2, 'execution', $3, '{}')`,
|
||||
@@ -713,19 +823,29 @@ func classifyAndGate(ctx context.Context, pool *db.Pool, agentID, targetID uuid.
|
||||
id, "task:"+sessionID)
|
||||
}
|
||||
|
||||
if riskClass == policy.RiskReadOnly {
|
||||
host, user, wrap, rerr := resolveExecTarget(ctx, pool, targetSlug)
|
||||
if rerr != nil {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, id, jsonErr("%s", rerr.Error()))
|
||||
return textResult(fmt.Sprintf("resolve target: %v", rerr))
|
||||
}
|
||||
out, xerr := sshExec(ctx, host, user, wrap(command))
|
||||
// read_only and reversible_low both run unattended, as seeds/policy.yaml
|
||||
// and .agents/OIKOS.md declare ("reversible_low — restart, cache clear,
|
||||
// sync pull. Unattended + ledger.").
|
||||
//
|
||||
// reversible_low had no branch here, so it fell through to the gate. That
|
||||
// looked stricter but was actually perverse: computeCommandRisk never
|
||||
// returns reversible_low — the class can ONLY arise when the agent
|
||||
// declares it on a command the classifier already scored read_only
|
||||
// (ClassifyCommand keeps the higher of the two). So an agent that
|
||||
// honestly flagged "this restarts something" got gated, while the same
|
||||
// command with no declaration auto-ran. That penalised candor and gave
|
||||
// the agent a reason to stay quiet.
|
||||
//
|
||||
// Auto-running it is no more permissive than the read_only branch above,
|
||||
// because read_only is the only computed class it can accompany. An
|
||||
// agent still cannot talk a command DOWN: declaring reversible_low on
|
||||
// something computed as config_mutation keeps config_mutation.
|
||||
if riskClass == policy.RiskReadOnly || riskClass == policy.RiskReversibleLow {
|
||||
out, xerr := autoRun(ctx, pool, id, targetSlug, command)
|
||||
if xerr != nil {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, id, jsonErr("%s: %s", xerr.Error(), out))
|
||||
return textResult(fmt.Sprintf("run on %s: ERROR %v\n%s", targetSlug, xerr, out))
|
||||
}
|
||||
pool.Exec(ctx, `UPDATE executions SET status='completed', result=$2::jsonb WHERE entity_id=$1`, id, jsonOut(out))
|
||||
return textResult(fmt.Sprintf("run on %s (read_only, auto): %s", targetSlug, out))
|
||||
return textResult(fmt.Sprintf("run on %s (%s, auto): %s", targetSlug, riskClass, out))
|
||||
}
|
||||
|
||||
// Assent window: if the operator recently approved a plan in this
|
||||
@@ -739,17 +859,10 @@ func classifyAndGate(ctx context.Context, pool *db.Pool, agentID, targetID uuid.
|
||||
// consent. The assent window, opened only on operator approval, is the
|
||||
// sole gate for config_mutation auto-run.)
|
||||
if riskClass == policy.RiskConfigMutation && assentWindowActive(ctx, pool, agentID, sessionID) {
|
||||
host, user, wrap, rerr := resolveExecTarget(ctx, pool, targetSlug)
|
||||
if rerr != nil {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, id, jsonErr("%s", rerr.Error()))
|
||||
return textResult(fmt.Sprintf("resolve target: %v", rerr))
|
||||
}
|
||||
out, xerr := sshExec(ctx, host, user, wrap(command))
|
||||
out, xerr := autoRun(ctx, pool, id, targetSlug, command)
|
||||
if xerr != nil {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, id, jsonErr("%s: %s", xerr.Error(), out))
|
||||
return textResult(fmt.Sprintf("run on %s: ERROR %v\n%s", targetSlug, xerr, out))
|
||||
}
|
||||
pool.Exec(ctx, `UPDATE executions SET status='completed', result=$2::jsonb WHERE entity_id=$1`, id, jsonOut(out))
|
||||
slog.Info("mcp: run auto-executed via assent window", "target", targetSlug, "execution_id", id)
|
||||
return textResult(fmt.Sprintf("run on %s (config_mutation, auto via assent window): %s", targetSlug, out))
|
||||
}
|
||||
@@ -761,17 +874,10 @@ func classifyAndGate(ctx context.Context, pool *db.Pool, agentID, targetID uuid.
|
||||
// operator isn't asked to re-type "I confirm" for every single command
|
||||
// against the thing they just confirmed.
|
||||
if riskClass == policy.RiskDestructive && destructiveWindowActive(ctx, pool, agentID, targetSlug, sessionID) {
|
||||
host, user, wrap, rerr := resolveExecTarget(ctx, pool, targetSlug)
|
||||
if rerr != nil {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, id, jsonErr("%s", rerr.Error()))
|
||||
return textResult(fmt.Sprintf("resolve target: %v", rerr))
|
||||
}
|
||||
out, xerr := sshExec(ctx, host, user, wrap(command))
|
||||
out, xerr := autoRun(ctx, pool, id, targetSlug, command)
|
||||
if xerr != nil {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, id, jsonErr("%s: %s", xerr.Error(), out))
|
||||
return textResult(fmt.Sprintf("run on %s: ERROR %v\n%s", targetSlug, xerr, out))
|
||||
}
|
||||
pool.Exec(ctx, `UPDATE executions SET status='completed', result=$2::jsonb WHERE entity_id=$1`, id, jsonOut(out))
|
||||
slog.Info("mcp: run auto-executed via destructive window", "target", targetSlug, "execution_id", id)
|
||||
return textResult(fmt.Sprintf("run on %s (destructive, auto via confirmed-target window): %s", targetSlug, out))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user