Files
oikos/internal/scheduler/backup_test.go
dtoro 1dca2cfd7a feat(observability): restore monitoring coverage, make gaps visible, stream executions
Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by
discarded errors in checkdefaults:

- writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT
  (slug) DO NOTHING, then wrote a check_defs row referencing it. On any
  re-seed the slug already existed, the entity insert no-oped, and the FK
  violated — aborting the ingest transaction and surfacing as an unrelated
  failure several entities later. Re-seeding has been broken since; prod's
  coverage was frozen at its first successful seed. This is what
  TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting.
- shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed
  to ".network" and overwrote each other; service:jellyfin collided with
  lxc:jellyfin.
- The ssh-script checker never read the `args` config checkdefaults wrote, so
  process_check.sh always ran without its unit name and returned "unknown".

Coverage is now 75/89. Monitoring is declared per entity type in
seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say
it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap.
coverageSweep raises an `unmonitored` signal only where a type declares
monitoring it lacks — 8 real gaps, no false positives.

Also:
- entity_types.attribute_schema was never ingested: the seed loader read
  "attribute_schema" but the YAML says "attributes", so all 60 types stored
  JSON null.
- ListExecutions ignored its declared target/action/correlation_id filters and
  paginated on a non-unique target slug, dropping and repeating rows.
- started_at was captured but only written at terminal state, so a running
  execution reported NULL for its whole life. The three MCP auto-run copies
  wrote no timing at all; they are now one autoRun helper.
- SSH output was buffered to completion and discarded entirely on timeout.
  Both sshExec copies now stream through a shared execlog sink into
  execution_logs, and keep partial output when a command is cancelled.
- executions.correlation_id was a random per-execution uuid that correlated
  nothing; it is now the chat session id, which is what lets the chat tail
  live output.
- reversible_low had no auto-run branch despite policy declaring it
  unattended. Since computeCommandRisk never returns it, the class only arises
  when an agent declares it over a read_only command — so gating it penalised
  candor without adding safety.
- backup-target gains a backup-freshness checker (portable find -mmin, since
  the first target is on macOS), resolving its host by walking backs-up-to
  backwards. The pre-deploy pg_dump is now a tracked backup target.

UI: an Executions section on entity detail with live output tailing, and
streamed output under a running `run` call in the chat timeline.

Migrations 022-024. Ops.svelte and context.ts exclude execution.output from
their refetch triggers, which would otherwise fire once a second per command.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-28 13:51:14 +02:00

102 lines
3.3 KiB
Go

package scheduler
import (
"context"
"os"
"path/filepath"
"testing"
"time"
"github.com/dtoro/oikos/internal/db/sqlcgen"
)
func backupCheckDef(t *testing.T, config string) sqlcgen.ListEnabledCheckDefsRow {
t.Helper()
return sqlcgen.ListEnabledCheckDefsRow{
Kind: "backup-freshness",
Config: []byte(config),
TimeoutS: 15,
}
}
// A misconfigured check must say so rather than quietly reporting healthy —
// "no path configured" and "backup ran fine" must never look the same.
func TestBackupFreshnessRejectsIncompleteConfig(t *testing.T) {
for _, c := range []struct{ desc, config string }{
{"no path", `{"host":"localhost"}`},
{"no host", `{"path":"/tmp"}`},
{"empty", `{}`},
} {
got := checkBackupFreshness(context.Background(), backupCheckDef(t, c.config))
if got.health != "unknown" || got.signalKind != "backup-misconfigured" {
t.Errorf("%s: got health=%q kind=%q, want unknown/backup-misconfigured",
c.desc, got.health, got.signalKind)
}
}
}
// Live probe against a real SSH endpoint. Guarded by OIKOS_SSH_TEST_HOST:
//
// OIKOS_SSH_TEST_HOST=localhost OIKOS_SSH_USER=$USER \
// OIKOS_SSH_KEY_PATH=~/.ssh/id_ed25519 go test ./internal/scheduler/ -run TestBackupFreshnessLive
//
// The probe shell has to work on both GNU and BSD find — the first real target
// is the pre-deploy pg_dump on the macOS mac-mini, so a GNU-only construct
// would fail exactly where it matters.
func TestBackupFreshnessLiveDistinguishesTheFourStates(t *testing.T) {
host := os.Getenv("OIKOS_SSH_TEST_HOST")
if host == "" {
t.Skip("OIKOS_SSH_TEST_HOST not set — skipping live backup probe")
}
sshKeyPath = os.Getenv("OIKOS_SSH_KEY_PATH")
sshUser = os.Getenv("OIKOS_SSH_USER")
dir := t.TempDir()
fresh := filepath.Join(dir, "fresh")
stale := filepath.Join(dir, "stale")
empty := filepath.Join(dir, "empty")
for _, d := range []string{fresh, stale, empty} {
if err := os.Mkdir(d, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(fresh, "dump.sql"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
oldFile := filepath.Join(stale, "dump.sql")
if err := os.WriteFile(oldFile, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
old := time.Now().Add(-72 * time.Hour)
if err := os.Chtimes(oldFile, old, old); err != nil {
t.Fatal(err)
}
cases := []struct {
desc, path, wantHealth, wantKind string
}{
{"recent artifact", fresh, "healthy", ""},
{"artifact older than max_age", stale, "degraded", "backup-stale"},
{"directory exists but is empty", empty, "down", "backup-missing"},
{"directory does not exist", filepath.Join(dir, "nope"), "down", "backup-missing"},
}
for _, c := range cases {
cfg := `{"host":"` + host + `","path":"` + c.path + `","max_age_s":86400}`
got := checkBackupFreshness(context.Background(), backupCheckDef(t, cfg))
if got.health != c.wantHealth || got.signalKind != c.wantKind {
t.Errorf("%s: got health=%q kind=%q evidence=%q, want %q/%q",
c.desc, got.health, got.signalKind, got.evidence, c.wantHealth, c.wantKind)
}
}
}
// A path with a quote in it must not break out of the remote sh command.
func TestShellSingleQuoteEscapes(t *testing.T) {
got := shellSingleQuote(`/tmp/it's; rm -rf /`)
want := `'/tmp/it'\''s; rm -rf /'`
if got != want {
t.Errorf("shellSingleQuote = %s, want %s", got, want)
}
}