Files
oikos/internal/mcp/sshexec_test.go
dtoro 1dca2cfd7a feat(observability): restore monitoring coverage, make gaps visible, stream executions
Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by
discarded errors in checkdefaults:

- writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT
  (slug) DO NOTHING, then wrote a check_defs row referencing it. On any
  re-seed the slug already existed, the entity insert no-oped, and the FK
  violated — aborting the ingest transaction and surfacing as an unrelated
  failure several entities later. Re-seeding has been broken since; prod's
  coverage was frozen at its first successful seed. This is what
  TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting.
- shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed
  to ".network" and overwrote each other; service:jellyfin collided with
  lxc:jellyfin.
- The ssh-script checker never read the `args` config checkdefaults wrote, so
  process_check.sh always ran without its unit name and returned "unknown".

Coverage is now 75/89. Monitoring is declared per entity type in
seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say
it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap.
coverageSweep raises an `unmonitored` signal only where a type declares
monitoring it lacks — 8 real gaps, no false positives.

Also:
- entity_types.attribute_schema was never ingested: the seed loader read
  "attribute_schema" but the YAML says "attributes", so all 60 types stored
  JSON null.
- ListExecutions ignored its declared target/action/correlation_id filters and
  paginated on a non-unique target slug, dropping and repeating rows.
- started_at was captured but only written at terminal state, so a running
  execution reported NULL for its whole life. The three MCP auto-run copies
  wrote no timing at all; they are now one autoRun helper.
- SSH output was buffered to completion and discarded entirely on timeout.
  Both sshExec copies now stream through a shared execlog sink into
  execution_logs, and keep partial output when a command is cancelled.
- executions.correlation_id was a random per-execution uuid that correlated
  nothing; it is now the chat session id, which is what lets the chat tail
  live output.
- reversible_low had no auto-run branch despite policy declaring it
  unattended. Since computeCommandRisk never returns it, the class only arises
  when an agent declares it over a read_only command — so gating it penalised
  candor without adding safety.
- backup-target gains a backup-freshness checker (portable find -mmin, since
  the first target is on macOS), resolving its host by walking backs-up-to
  backwards. The pre-deploy pg_dump is now a tracked backup target.

UI: an Executions section on entity detail with live output tailing, and
streamed output under a running `run` call in the chat timeline.

Migrations 022-024. Ops.svelte and context.ts exclude execution.output from
their refetch triggers, which would otherwise fire once a second per command.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-28 13:51:14 +02:00

160 lines
4.7 KiB
Go

package mcp
// Streaming tests for sshExecStream against a real SSH endpoint. Guarded by
// OIKOS_SSH_TEST_HOST — skipped when unset. Run with:
//
// OIKOS_SSH_TEST_HOST=localhost OIKOS_SSH_USER=$USER \
// OIKOS_SSH_KEY_PATH=~/.ssh/id_ed25519 go test ./internal/mcp/ -run TestSSHExecStream
//
// These matter because the whole point of the change is behaviour that only
// appears over time: that output arrives *before* the command exits, and that
// a command killed mid-flight still leaves what it printed.
import (
"context"
"os"
"strings"
"sync"
"testing"
"time"
)
func sshTestHost(t *testing.T) string {
t.Helper()
host := os.Getenv("OIKOS_SSH_TEST_HOST")
if host == "" {
t.Skip("OIKOS_SSH_TEST_HOST not set — skipping live SSH test")
}
return host
}
// The core claim: chunks reach the sink while the command is still running,
// not in one lump at the end. A command that prints, sleeps, then prints must
// deliver its first chunk well before it exits.
func TestSSHExecStreamDeliversOutputBeforeExit(t *testing.T) {
host := sshTestHost(t)
var (
mu sync.Mutex
chunks []string
firstA time.Time
)
sink := func(stream string, chunk []byte) {
mu.Lock()
defer mu.Unlock()
if firstA.IsZero() {
firstA = time.Now()
}
chunks = append(chunks, string(chunk))
}
start := time.Now()
out, err := sshExecStream(context.Background(), host, os.Getenv("OIKOS_SSH_USER"),
"echo FIRST; sleep 2; echo SECOND", sink)
elapsed := time.Since(start)
if err != nil {
t.Fatalf("sshExecStream: %v (out=%q)", err, out)
}
mu.Lock()
joined := strings.Join(chunks, "")
firstAt := firstA.Sub(start)
mu.Unlock()
if !strings.Contains(out, "FIRST") || !strings.Contains(out, "SECOND") {
t.Errorf("combined output lost content: %q", out)
}
if !strings.Contains(joined, "FIRST") || !strings.Contains(joined, "SECOND") {
t.Errorf("sink did not receive the full output: %q", joined)
}
if elapsed < 2*time.Second {
t.Fatalf("command returned in %v — the sleep did not run, test is not measuring what it claims", elapsed)
}
// The first chunk must land near the start, not at the end.
if firstAt > elapsed/2 {
t.Errorf("first chunk arrived after %v of a %v command — output is still being buffered to the end",
firstAt, elapsed)
}
}
// stderr must reach the sink too, and land in the combined output, matching
// what CombinedOutput used to return.
func TestSSHExecStreamCapturesBothStreams(t *testing.T) {
host := sshTestHost(t)
var mu sync.Mutex
seen := map[string]bool{}
sink := func(stream string, chunk []byte) {
mu.Lock()
defer mu.Unlock()
seen[stream] = true
}
out, err := sshExecStream(context.Background(), host, os.Getenv("OIKOS_SSH_USER"),
"echo TO_STDOUT; echo TO_STDERR 1>&2", sink)
if err != nil {
t.Fatalf("sshExecStream: %v (out=%q)", err, out)
}
if !strings.Contains(out, "TO_STDOUT") || !strings.Contains(out, "TO_STDERR") {
t.Errorf("combined output missing a stream: %q", out)
}
mu.Lock()
defer mu.Unlock()
if !seen["stdout"] {
t.Error("sink never saw a stdout chunk")
}
if !seen["stderr"] {
t.Error("sink never saw a stderr chunk")
}
}
// A cancelled command used to return "" — everything it had printed was
// thrown away. The hung case is exactly when that output is worth having.
func TestSSHExecStreamKeepsPartialOutputOnCancel(t *testing.T) {
host := sshTestHost(t)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
out, err := sshExecStream(ctx, host, os.Getenv("OIKOS_SSH_USER"),
"echo BEFORE_HANG; sleep 30; echo NEVER", nil)
if err == nil {
t.Fatal("expected a context error for a command that outlives the deadline")
}
if !strings.Contains(out, "BEFORE_HANG") {
t.Errorf("partial output was discarded on cancel: %q", out)
}
if strings.Contains(out, "NEVER") {
t.Errorf("command should not have completed: %q", out)
}
}
// A nil sink must behave exactly as the old CombinedOutput path did.
func TestSSHExecNilSinkStillReturnsOutput(t *testing.T) {
host := sshTestHost(t)
out, err := sshExec(context.Background(), host, os.Getenv("OIKOS_SSH_USER"), "echo PLAIN")
if err != nil {
t.Fatalf("sshExec: %v", err)
}
if out != "PLAIN" {
t.Errorf("out = %q, want %q (output is trimmed)", out, "PLAIN")
}
}
// A non-zero exit must surface as an error while still returning the output.
func TestSSHExecStreamNonZeroExitIsAnError(t *testing.T) {
host := sshTestHost(t)
out, err := sshExecStream(context.Background(), host, os.Getenv("OIKOS_SSH_USER"),
"echo PRINTED_THEN_FAILED; exit 3", nil)
if err == nil {
t.Fatal("a non-zero exit that printed output must still be an error")
}
if !strings.Contains(out, "PRINTED_THEN_FAILED") {
t.Errorf("output lost on failure: %q", out)
}
}