feat(observability): restore monitoring coverage, make gaps visible, stream executions
Monitoring coverage was 3 of 89 active entities. Three bugs, each hidden by discarded errors in checkdefaults: - writeCheck generated a fresh uuid, inserted the check entity ON CONFLICT (slug) DO NOTHING, then wrote a check_defs row referencing it. On any re-seed the slug already existed, the entity insert no-oped, and the FK violated — aborting the ingest transaction and surfacing as an unrelated failure several entities later. Re-seeding has been broken since; prod's coverage was frozen at its first successful seed. This is what TestSeedIngestIdempotentAndNoDuplicateEdges had been reporting. - shortSlug truncated to the last 8 chars, so all 21 ingress routes collapsed to ".network" and overwrote each other; service:jellyfin collided with lxc:jellyfin. - The ssh-script checker never read the `args` config checkdefaults wrote, so process_check.sh always ran without its unit name and returned "unknown". Coverage is now 75/89. Monitoring is declared per entity type in seeds/ontology.yaml and resolved through the is-a hierarchy, so a type can say it warrants nothing (site, lan, mesh, cluster) and never be reported as a gap. coverageSweep raises an `unmonitored` signal only where a type declares monitoring it lacks — 8 real gaps, no false positives. Also: - entity_types.attribute_schema was never ingested: the seed loader read "attribute_schema" but the YAML says "attributes", so all 60 types stored JSON null. - ListExecutions ignored its declared target/action/correlation_id filters and paginated on a non-unique target slug, dropping and repeating rows. - started_at was captured but only written at terminal state, so a running execution reported NULL for its whole life. The three MCP auto-run copies wrote no timing at all; they are now one autoRun helper. - SSH output was buffered to completion and discarded entirely on timeout. Both sshExec copies now stream through a shared execlog sink into execution_logs, and keep partial output when a command is cancelled. - executions.correlation_id was a random per-execution uuid that correlated nothing; it is now the chat session id, which is what lets the chat tail live output. - reversible_low had no auto-run branch despite policy declaring it unattended. Since computeCommandRisk never returns it, the class only arises when an agent declares it over a read_only command — so gating it penalised candor without adding safety. - backup-target gains a backup-freshness checker (portable find -mmin, since the first target is on macOS), resolving its host by walking backs-up-to backwards. The pre-deploy pg_dump is now a tracked backup target. UI: an Executions section on entity detail with live output tailing, and streamed output under a running `run` call in the chat timeline. Migrations 022-024. Ops.svelte and context.ts exclude execution.output from their refetch triggers, which would otherwise fire once a second per command. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
101
internal/scheduler/backup_test.go
Normal file
101
internal/scheduler/backup_test.go
Normal file
@@ -0,0 +1,101 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dtoro/oikos/internal/db/sqlcgen"
|
||||
)
|
||||
|
||||
func backupCheckDef(t *testing.T, config string) sqlcgen.ListEnabledCheckDefsRow {
|
||||
t.Helper()
|
||||
return sqlcgen.ListEnabledCheckDefsRow{
|
||||
Kind: "backup-freshness",
|
||||
Config: []byte(config),
|
||||
TimeoutS: 15,
|
||||
}
|
||||
}
|
||||
|
||||
// A misconfigured check must say so rather than quietly reporting healthy —
|
||||
// "no path configured" and "backup ran fine" must never look the same.
|
||||
func TestBackupFreshnessRejectsIncompleteConfig(t *testing.T) {
|
||||
for _, c := range []struct{ desc, config string }{
|
||||
{"no path", `{"host":"localhost"}`},
|
||||
{"no host", `{"path":"/tmp"}`},
|
||||
{"empty", `{}`},
|
||||
} {
|
||||
got := checkBackupFreshness(context.Background(), backupCheckDef(t, c.config))
|
||||
if got.health != "unknown" || got.signalKind != "backup-misconfigured" {
|
||||
t.Errorf("%s: got health=%q kind=%q, want unknown/backup-misconfigured",
|
||||
c.desc, got.health, got.signalKind)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Live probe against a real SSH endpoint. Guarded by OIKOS_SSH_TEST_HOST:
|
||||
//
|
||||
// OIKOS_SSH_TEST_HOST=localhost OIKOS_SSH_USER=$USER \
|
||||
// OIKOS_SSH_KEY_PATH=~/.ssh/id_ed25519 go test ./internal/scheduler/ -run TestBackupFreshnessLive
|
||||
//
|
||||
// The probe shell has to work on both GNU and BSD find — the first real target
|
||||
// is the pre-deploy pg_dump on the macOS mac-mini, so a GNU-only construct
|
||||
// would fail exactly where it matters.
|
||||
func TestBackupFreshnessLiveDistinguishesTheFourStates(t *testing.T) {
|
||||
host := os.Getenv("OIKOS_SSH_TEST_HOST")
|
||||
if host == "" {
|
||||
t.Skip("OIKOS_SSH_TEST_HOST not set — skipping live backup probe")
|
||||
}
|
||||
sshKeyPath = os.Getenv("OIKOS_SSH_KEY_PATH")
|
||||
sshUser = os.Getenv("OIKOS_SSH_USER")
|
||||
|
||||
dir := t.TempDir()
|
||||
fresh := filepath.Join(dir, "fresh")
|
||||
stale := filepath.Join(dir, "stale")
|
||||
empty := filepath.Join(dir, "empty")
|
||||
for _, d := range []string{fresh, stale, empty} {
|
||||
if err := os.Mkdir(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fresh, "dump.sql"), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldFile := filepath.Join(stale, "dump.sql")
|
||||
if err := os.WriteFile(oldFile, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := time.Now().Add(-72 * time.Hour)
|
||||
if err := os.Chtimes(oldFile, old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
desc, path, wantHealth, wantKind string
|
||||
}{
|
||||
{"recent artifact", fresh, "healthy", ""},
|
||||
{"artifact older than max_age", stale, "degraded", "backup-stale"},
|
||||
{"directory exists but is empty", empty, "down", "backup-missing"},
|
||||
{"directory does not exist", filepath.Join(dir, "nope"), "down", "backup-missing"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
cfg := `{"host":"` + host + `","path":"` + c.path + `","max_age_s":86400}`
|
||||
got := checkBackupFreshness(context.Background(), backupCheckDef(t, cfg))
|
||||
if got.health != c.wantHealth || got.signalKind != c.wantKind {
|
||||
t.Errorf("%s: got health=%q kind=%q evidence=%q, want %q/%q",
|
||||
c.desc, got.health, got.signalKind, got.evidence, c.wantHealth, c.wantKind)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A path with a quote in it must not break out of the remote sh command.
|
||||
func TestShellSingleQuoteEscapes(t *testing.T) {
|
||||
got := shellSingleQuote(`/tmp/it's; rm -rf /`)
|
||||
want := `'/tmp/it'\''s; rm -rf /'`
|
||||
if got != want {
|
||||
t.Errorf("shellSingleQuote = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user