From e12954323ceee6857509560cbe67ba09df694334 Mon Sep 17 00:00:00 2001 From: dtoro Date: Mon, 17 Aug 2026 16:49:56 +0200 Subject: [PATCH] feat: structured __renderer envelopes for MCP tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 19 oikos tools now return JSON envelopes with __renderer hints (terminal, service_status, ping, path_report, etc.) carrying structured data alongside the model-facing prose in a `message` field — agent behavior is unchanged while the UI renders native cards. Fixes: ping_service SQL (json||text precedence), timestamptz scan, run:{...} action prefix parsing, get_execution_status target slug join. Also: deploy.sh step 5.5 restarts dsh web after API deploy; watchdog LaunchAgent detects API recovery and kickstarts dsh web for stale MCP sessions. --- internal/mcp/ops_tools.go | 281 +++++++++++------- internal/mcp/server.go | 136 ++++++++- scripts/deploy-plugins.sh | 179 ++++++++--- scripts/deploy.sh | 2 + scripts/network.hubris.dsh-web.plist | 36 +++ .../network.hubris.oikos-api-watchdog.plist | 20 ++ scripts/oikos-api-health-watchdog.sh | 22 ++ 7 files changed, 520 insertions(+), 156 deletions(-) mode change 100644 => 100755 scripts/deploy-plugins.sh create mode 100644 scripts/network.hubris.dsh-web.plist create mode 100644 scripts/network.hubris.oikos-api-watchdog.plist create mode 100755 scripts/oikos-api-health-watchdog.sh diff --git a/internal/mcp/ops_tools.go b/internal/mcp/ops_tools.go index 433d2f35..1208a8cd 100644 --- a/internal/mcp/ops_tools.go +++ b/internal/mcp/ops_tools.go @@ -19,6 +19,23 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" ) +// lxcStateEnvelope runs pct status on the Proxmox host and wraps the output +// in a terminal envelope labelled with the LXC slug. +func lxcStateEnvelope(ctx context.Context, host, user, slug, pveID string) *mcp.CallToolResult { + cmd := fmt.Sprintf("pct status %s --verbose 2>&1 || true", pveID) + out, err := sshExec(ctx, host, user, cmd) + if err != nil { + return textResult(fmt.Sprintf("ssh: %v", err)) + } + return rendererEnvelope("terminal", terminalEnvelope{ + Target: slug, + Command: cmd, + State: "done", + Output: out, + Message: fmt.Sprintf("Resource state of %s from its Proxmox host.", slug), + }) +} + func OpsTools(pool *db.Pool, agentID uuid.UUID, sec ports.Secrets, execSvc *app.ExecutionService) []toolReg { return []toolReg{ // ── request_execution (legacy fixed enum) retired 2026-07-14 ── @@ -137,80 +154,133 @@ func OpsTools(pool *db.Pool, agentID uuid.UUID, sec ports.Secrets, execSvc *app. targets = append(targets, s) } } - results := inspectPathAcrossTargets(ctx, pool, path, targets) - out, _ := json.MarshalIndent(results, "", " ") - return textResult(string(out)), nil + results := inspectPathAcrossTargets(ctx, pool, path, targets) + return rendererEnvelope("path_report", map[string]any{ + "path": path, + "targets": results, + }), nil }}, {tool: &mcp.Tool{Name: "get_execution_status", Description: "Check the status of a requested execution", InputSchema: objSchema( prop{"execution_id", "string", "Execution UUID (from request_execution output)"}, ), - }, handler: func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { - args := argsMap(req) - execID, _ := args["execution_id"].(string) - if execID == "" { - return textResult("execution_id required"), nil + }, handler: func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + args := argsMap(req) + execID, _ := args["execution_id"].(string) + if execID == "" { + return textResult("execution_id required"), nil + } + eid, err := uuid.Parse(execID) + if err != nil { + // Try finding by exec slug prefix + var found uuid.UUID + err2 := pool.QueryRow(ctx, "SELECT entity_id FROM executions WHERE entity_id::text LIKE $1 LIMIT 1", execID+"%").Scan(&found) + if err2 != nil { + return textResult(fmt.Sprintf("execution not found: %s", execID)), nil } - eid, err := uuid.Parse(execID) - if err != nil { - // Try finding by exec slug prefix - var found uuid.UUID - err2 := pool.QueryRow(ctx, "SELECT entity_id FROM executions WHERE entity_id::text LIKE $1 LIMIT 1", execID+"%").Scan(&found) - if err2 != nil { - return textResult(fmt.Sprintf("execution not found: %s", execID)), nil - } - eid = found - } - return queryRows(ctx, pool, ` - SELECT e.entity_id::text, e.action, e.risk_class, e.status, - e.result::text, e.duration_ms, e.started_at::text, - e.completed_at::text, e.correlation_id - FROM executions e - WHERE e.entity_id = $1`, eid), nil - }}, + eid = found + } + rows, err := pool.Query(ctx, ` + SELECT e.entity_id::text, e.action, e.risk_class, e.status, + e.result::text, e.duration_ms, e.started_at::text, + e.completed_at::text, e.correlation_id, + COALESCE(t.slug, '') + FROM executions e + LEFT JOIN entities t ON t.id = e.target_entity_id + WHERE e.entity_id = $1`, eid) + if err != nil { + return textResult(fmt.Sprintf("error: %v", err)), nil + } + defer rows.Close() + if !rows.Next() { + return textResult(fmt.Sprintf("execution not found: %s", execID)), nil + } + var entityID, action, riskClass, status, result, started, completed, correlation, targetSlug string + var duration any + if err := rows.Scan(&entityID, &action, &riskClass, &status, &result, &duration, &started, &completed, &correlation, &targetSlug); err != nil { + return textResult(fmt.Sprintf("error: %v", err)), nil + } + env := executionTerminalEnvelope(action, riskClass, status, result) + env.Target = targetSlug + env.ExecID = entityID + if s, ok := duration.(int64); ok { + env.Message += fmt.Sprintf(" Duration: %dms.", s) + } + return rendererEnvelope("terminal", env), nil + }}, {tool: &mcp.Tool{Name: "tail_log", Description: "Get recent log lines from a service via journalctl", InputSchema: objSchema( prop{"service_slug", "string", "Service entity slug (e.g. lxc:caddy)"}, prop{"lines", "integer", "Number of lines (default 50)"}), - }, handler: func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { - args := argsMap(req) - slug, _ := args["service_slug"].(string) - n := int(getFloat(args, "lines", 50)) - if slug == "" { - return textResult("service_slug is required"), nil - } - host, user, err := resolveHost(ctx, pool, slug) - if err != nil { - return textResult(fmt.Sprintf("resolve host: %v", err)), nil - } - svc := strings.TrimPrefix(slug, "lxc:") - out, err := sshExec(ctx, host, user, fmt.Sprintf("journalctl -u %s -n %d --no-pager 2>&1 || true", svc, n)) - if err != nil { - return textResult(fmt.Sprintf("ssh: %v", err)), nil - } - return textResult(out), nil - }}, + }, handler: func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + args := argsMap(req) + slug, _ := args["service_slug"].(string) + n := int(getFloat(args, "lines", 50)) + if slug == "" { + return textResult("service_slug is required"), nil + } + host, user, err := resolveHost(ctx, pool, slug) + if err != nil { + return textResult(fmt.Sprintf("resolve host: %v", err)), nil + } + svc := strings.TrimPrefix(slug, "lxc:") + cmd := fmt.Sprintf("journalctl -u %s -n %d --no-pager 2>&1 || true", svc, n) + out, err := sshExec(ctx, host, user, cmd) + if err != nil { + return textResult(fmt.Sprintf("ssh: %v", err)), nil + } + return rendererEnvelope("terminal", terminalEnvelope{ + Target: slug, + Command: cmd, + State: "done", + Output: out, + Message: fmt.Sprintf("Last %d log lines of %s.", n, slug), + }), nil + }}, {tool: &mcp.Tool{Name: "get_service_status", Description: "Check systemd service status on a host", InputSchema: objSchema( prop{"service_slug", "string", "Service entity slug (e.g. lxc:caddy)"}), - }, handler: func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { - args := argsMap(req) - slug, _ := args["service_slug"].(string) - if slug == "" { - return textResult("service_slug is required"), nil + }, handler: func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + args := argsMap(req) + slug, _ := args["service_slug"].(string) + if slug == "" { + return textResult("service_slug is required"), nil + } + host, user, err := resolveHost(ctx, pool, slug) + if err != nil { + return textResult(fmt.Sprintf("resolve host: %v", err)), nil + } + svc := strings.TrimPrefix(slug, "lxc:") + out, err := sshExec(ctx, host, user, + fmt.Sprintf("systemctl is-active %s; systemctl is-enabled %s; systemctl show %s -p ActiveEnterTimestamp -p SubState 2>&1 || true", svc, svc, svc)) + if err != nil { + return textResult(fmt.Sprintf("ssh: %v", err)), nil + } + // systemctl output: line 1 is-active, line 2 is-enabled, then + // Key=Value pairs from show. "could not be found" on either of + // the first lines means the unit does not exist. + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + active, enabled := lines[0], "" + if len(lines) > 1 { + enabled = lines[1] + } + status := map[string]string{ + "service": slug, + "active": active, + "enabled": enabled, + } + for _, l := range lines[2:] { + if k, v, ok := strings.Cut(l, "="); ok { + switch k { + case "ActiveEnterTimestamp": + status["since"] = v + case "SubState": + status["sub_state"] = v + } } - host, user, err := resolveHost(ctx, pool, slug) - if err != nil { - return textResult(fmt.Sprintf("resolve host: %v", err)), nil - } - svc := strings.TrimPrefix(slug, "lxc:") - out, err := sshExec(ctx, host, user, - fmt.Sprintf("systemctl is-active %s; systemctl is-enabled %s; systemctl show %s -p ActiveEnterTimestamp -p SubState 2>&1 || true", svc, svc, svc)) - if err != nil { - return textResult(fmt.Sprintf("ssh: %v", err)), nil - } - return textResult(out), nil - }}, + } + return rendererEnvelope("service_status", status), nil + }}, {tool: &mcp.Tool{Name: "get_lxc_state", Description: "Get LXC container resource state from Proxmox host", InputSchema: objSchema( prop{"lxc_slug", "string", "LXC entity slug (e.g. lxc:caddy)"}), @@ -240,29 +310,21 @@ func OpsTools(pool *db.Pool, agentID uuid.UUID, sec ports.Secrets, execSvc *app. if err != nil || hostSlug == "" { return textResult(fmt.Sprintf("cannot resolve Proxmox host for %s", slug)), nil } - var host, user string - host, user, err = resolveHost(ctx, pool, "host:"+hostSlug) - if err != nil { - return textResult(fmt.Sprintf("resolve: %v", err)), nil - } - out, err2 := sshExec(ctx, host, user, fmt.Sprintf("pct status %s --verbose 2>&1 || true", pveID)) - if err2 != nil { - return textResult(fmt.Sprintf("ssh: %v", err2)), nil - } - return textResult(out), nil - } - var hostSlug string - pool.QueryRow(ctx, "SELECT slug FROM entities WHERE id = $1", hostID).Scan(&hostSlug) - host, user, err := resolveHost(ctx, pool, hostSlug) + var host, user string + host, user, err = resolveHost(ctx, pool, "host:"+hostSlug) if err != nil { - return textResult(fmt.Sprintf("resolve host: %v", err)), nil + return textResult(fmt.Sprintf("resolve: %v", err)), nil } - out, err := sshExec(ctx, host, user, fmt.Sprintf("pct status %s --verbose 2>&1 || true", pveID)) - if err != nil { - return textResult(fmt.Sprintf("ssh: %v", err)), nil - } - return textResult(out), nil - }}, + return lxcStateEnvelope(ctx, host, user, slug, pveID), nil + } + var hostSlug string + pool.QueryRow(ctx, "SELECT slug FROM entities WHERE id = $1", hostID).Scan(&hostSlug) + host, user, err := resolveHost(ctx, pool, hostSlug) + if err != nil { + return textResult(fmt.Sprintf("resolve host: %v", err)), nil + } + return lxcStateEnvelope(ctx, host, user, slug, pveID), nil + }}, {tool: &mcp.Tool{Name: "ping_service", Description: "Check if a service is reachable via HTTP — returns scheduler health state plus a live HTTP probe", InputSchema: objSchema(prop{"service_slug", "string", "Service entity slug"}), }, handler: func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { @@ -271,17 +333,17 @@ func OpsTools(pool *db.Pool, agentID uuid.UUID, sec ports.Secrets, execSvc *app. if slug == "" { return textResult("service_slug is required"), nil } - rows, err := pool.Query(ctx, ` - SELECT st.health, st.last_check_at, - COALESCE( - e.attributes->>'url', - CASE WHEN e.attributes->>'public_host' IS NOT NULL - THEN 'https://' || e.attributes->>'public_host' - END - ) AS url - FROM entity_status st - JOIN entities e ON e.id = st.entity_id - WHERE e.slug = $1`, slug) + rows, err := pool.Query(ctx, ` + SELECT st.health, st.last_check_at::text, + COALESCE( + e.attributes->>'url', + CASE WHEN e.attributes->>'public_host' IS NOT NULL + THEN 'https://' || (e.attributes->>'public_host') + END + ) AS url + FROM entity_status st + JOIN entities e ON e.id = st.entity_id + WHERE e.slug = $1`, slug) if err != nil { return textResult(fmt.Sprintf("query error: %v", err)), nil } @@ -289,20 +351,29 @@ func OpsTools(pool *db.Pool, agentID uuid.UUID, sec ports.Secrets, execSvc *app. if !rows.Next() { return textResult(fmt.Sprintf("service not found: %s", slug)), nil } - var health, lastCheck, url string - rows.Scan(&health, &lastCheck, &url) - if url == "" { - return textResult(fmt.Sprintf("health=%s last_check=%s url=no-url (entity has no url or public_host attribute)", health, lastCheck)), nil + var health, lastCheck, url string + if err := rows.Scan(&health, &lastCheck, &url); err != nil { + return textResult(fmt.Sprintf("scan error: %v", err)), nil } - // Live HTTP probe — HEAD request to check current state - code := "n/a" - if resp, err := http.Head(url); err == nil { - resp.Body.Close() - code = fmt.Sprintf("%d", resp.StatusCode) - } else { - code = fmt.Sprintf("err: %v", err) - } - return textResult(fmt.Sprintf("health=%s last_check=%s url=%s http=%s", health, lastCheck, url, code)), nil + if url == "" { + return rendererEnvelope("ping", map[string]string{ + "service": slug, "health": health, "last_check": lastCheck, + "url": "", "http": "", + "note": "entity has no url or public_host attribute", + }), nil + } + // Live HTTP probe — HEAD request to check current state + code := "" + if resp, err := http.Head(url); err == nil { + resp.Body.Close() + code = fmt.Sprintf("%d", resp.StatusCode) + } else { + code = fmt.Sprintf("err: %v", err) + } + return rendererEnvelope("ping", map[string]string{ + "service": slug, "health": health, "last_check": lastCheck, + "url": url, "http": code, + }), nil }}, // ─── Phase 5: operational MCP tools ────────────────────────────── diff --git a/internal/mcp/server.go b/internal/mcp/server.go index fa792b66..6e62529a 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -450,12 +450,86 @@ func annotateJSONResult(result *mcp.CallToolResult, rendererID string) *mcp.Call if err := json.Unmarshal([]byte(tc.Text), &items); err != nil { return result } - wrapper := map[string]any{ + return rendererEnvelope(rendererID, items) +} + +// rendererEnvelope wraps any JSON-serializable payload in the __renderer +// envelope oikos-ui dispatches on. data may be a row list (array) or a single +// structured object; callers that only have prose keep textResult. +func rendererEnvelope(rendererID string, data any) *mcp.CallToolResult { + payload, err := json.MarshalIndent(map[string]any{ "__renderer": rendererID, - "data": items, + "data": data, + }, "", " ") + if err != nil { + return textResult(fmt.Sprintf("%v", data)) } - data, _ := json.MarshalIndent(wrapper, "", " ") - return textResult(string(data)) + return textResult(string(payload)) +} + +// terminalEnvelope is the structured shape of a command-execution result: +// everything a terminal card needs (target, command, combined output, state) +// plus the model-facing prose in Message, so restructuring the result for the +// UI never changes what the agent reads. +type terminalEnvelope struct { + Target string `json:"target"` + Command string `json:"command"` + State string `json:"state"` // queued | started | done | error | refused + Risk string `json:"risk,omitempty"` + ExecID string `json:"exec_id,omitempty"` + Output string `json:"output,omitempty"` + Error string `json:"error,omitempty"` + NeedsApproval bool `json:"needs_approval,omitempty"` + Message string `json:"message"` +} + +// executionTerminalEnvelope builds the terminal envelope for a stored +// execution row: command/purpose from the action JSON, output/error from the +// result JSON, state from the execution status. +func executionTerminalEnvelope(action, riskClass, status, result string) terminalEnvelope { + env := terminalEnvelope{ + Command: action, + Risk: riskClass, + State: "started", + Message: "", + } + var act struct { + Command string `json:"command"` + Purpose string `json:"purpose"` + } + // The stored action prefixes the JSON payload with the tool kind + // (`run:{...}`); parse from the first '{'. + if i := strings.Index(action, "{"); i >= 0 { + if err := json.Unmarshal([]byte(action[i:]), &act); err == nil && act.Command != "" { + env.Command = act.Command + if act.Purpose != "" { + env.Message = "Purpose: " + act.Purpose + "\n" + } + } + } + switch status { + case "completed": + env.State = "done" + case "failed": + env.State = "error" + case "cancelled": + env.State = "refused" + } + var res struct { + Output string `json:"output"` + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(result), &res); err == nil { + env.Output = res.Output + env.Error = res.Error + } else if result != "" { + env.Output = result + } + env.Message += fmt.Sprintf("Execution %s: %s.", status, status) + if env.Error != "" { + env.Message += " Error: " + env.Error + } + return env } // ─── SSH helpers ───────────────────────────────────────────────────────── @@ -650,24 +724,41 @@ func classifyAndGate(ctx context.Context, pool *db.Pool, execSvc *app.ExecutionS return db.NewExecutionLog(ctx, pool, uuid.MustParse(string(execID)), correlationID) }, }) - return renderSubmit(targetSlug, command, res) + return renderSubmit(targetSlug, command, purpose, res) } -// renderSubmit maps an ExecutionSubmitResult onto the agent-facing text, -// preserving the exact pre-service message shapes. -func renderSubmit(targetSlug, command string, res app.ExecutionSubmitResult) *mcp.CallToolResult { +// renderSubmit maps an ExecutionSubmitResult onto the agent-facing result: a +// terminal envelope whose Message keeps the exact pre-service prose (the +// agent's operating instructions live there), with the structured fields the +// terminal card renders. +func renderSubmit(targetSlug, command, purpose string, res app.ExecutionSubmitResult) *mcp.CallToolResult { d := res.Decision + env := terminalEnvelope{ + Target: targetSlug, + Command: command, + State: "unknown", + Message: "", + } + if purpose != "" { + env.Message = "Purpose: " + purpose + "\n" + } switch d.Action { case app.DecisionRefuse: - return textResult(d.Message) + env.State = "refused" + env.Error = d.Message + env.Message += d.Message case app.DecisionQueue: confirmNote := "" if d.RiskClass == policy.RiskDestructive { confirmNote = " This is classified DESTRUCTIVE — flag that clearly to the operator; it needs explicit confirmation, not just a casual \"go ahead\"." } - return textResult(fmt.Sprintf("run on %s requires approval (risk: %s) — execution %s queued.%s Present the command and purpose to the operator and wait; do not re-request.", - targetSlug, d.RiskClass, res.ExecutionID, confirmNote)) + env.State = "queued" + env.Risk = string(d.RiskClass) + env.ExecID = string(res.ExecutionID) + env.NeedsApproval = true + env.Message += fmt.Sprintf("run on %s requires approval (risk: %s) — execution %s queued.%s Present the command and purpose to the operator and wait; do not re-request.", + targetSlug, d.RiskClass, res.ExecutionID, confirmNote) case app.DecisionAuto: label := d.RiskClass @@ -677,6 +768,8 @@ func renderSubmit(targetSlug, command string, res app.ExecutionSubmitResult) *mc case "destructive": label = "destructive" } + env.Risk = string(label) + env.ExecID = string(res.ExecutionID) if res.AsyncStarted { via := "" switch d.AutoViaWindow { @@ -687,11 +780,17 @@ func renderSubmit(targetSlug, command string, res app.ExecutionSubmitResult) *mc default: via = ", async" } - return textResult(fmt.Sprintf("run on %s (%s%s): started — execution %s. Poll with get_execution_status(%s) for result.", - targetSlug, label, via, res.ExecutionID, res.ExecutionID)) + env.State = "started" + env.Message += fmt.Sprintf("run on %s (%s%s): started — execution %s. Poll with get_execution_status(%s) for result.", + targetSlug, label, via, res.ExecutionID, res.ExecutionID) + break } if res.Err != nil { - return textResult(fmt.Sprintf("run on %s: ERROR %v\n%s", targetSlug, res.Err, res.Output)) + env.State = "error" + env.Error = res.Err.Error() + env.Output = res.Output + env.Message += fmt.Sprintf("run on %s: ERROR %v\n%s", targetSlug, res.Err, res.Output) + break } via := ", auto" switch d.AutoViaWindow { @@ -700,9 +799,14 @@ func renderSubmit(targetSlug, command string, res app.ExecutionSubmitResult) *mc case "destructive": via = ", auto via confirmed-target window" } - return textResult(fmt.Sprintf("run on %s (%s%s): %s", targetSlug, label, via, res.Output)) + env.State = "done" + env.Output = res.Output + env.Message += fmt.Sprintf("run on %s (%s%s): %s", targetSlug, label, via, res.Output) + + default: + env.Message += fmt.Sprintf("run on %s: unknown decision %q", targetSlug, d.Action) } - return textResult(fmt.Sprintf("run on %s: unknown decision %q", targetSlug, d.Action)) + return rendererEnvelope("terminal", env) } // autoApprove updates the approval + execution status in the DB to approved, diff --git a/scripts/deploy-plugins.sh b/scripts/deploy-plugins.sh old mode 100644 new mode 100755 index 0a873732..fe77c1a9 --- a/scripts/deploy-plugins.sh +++ b/scripts/deploy-plugins.sh @@ -1,57 +1,166 @@ #!/bin/sh # Oikos-plugins deploy script — triggered by Gitea webhook on push to dtoro/oikos-plugins. -# Runs on mac-mini via launchd unit running cmd/webhook (route: /deploy-plugins). +# Runs on mac-mini via launchd unit network.hubris.oikos-deploy-webhook (route: /deploy-plugins). +# +# Deployment vehicle is the in-tree clone at $DSH_DIR/packages/oikos: the web +# profile symlinks its packages, and their @deepseek-ai peer deps resolve +# through the harness workspace root node_modules. UI and node halves ship as +# committed lib/ artifacts, so no build step runs here. On failure the script +# restores the previous SHAs and notifies via the oikos API (OIKOS_API_TOKEN) +# and Matrix (MATRIX_WEBHOOK_URL) when configured. set -e -REPO_DIR="${REPO_DIR:-$HOME/Projects/oikos}" -PLUGIN_DIR="${PLUGIN_DIR:-$HOME/oikos-plugins}" DSH_DIR="${DSH_DIR:-$HOME/Projects/deepseek-harness}" +PLUGIN_DIR="${PLUGIN_DIR:-$DSH_DIR/packages/oikos}" PROFILE_DIR="${PROFILE_DIR:-$HOME/.dsh/profiles/web}" PORT="${PORT:-3080}" LOCKDIR="${LOCKDIR:-/tmp/oikos-plugins-deploy.lock}" +DSH_BRANCH="${DSH_BRANCH:-master}" +HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:$PORT/}" +RETRIES=${RETRIES:-90} +ROLLBACK_RETRIES=${ROLLBACK_RETRIES:-30} +SLEEP=${SLEEP:-2} +AGENT_LABEL="network.hubris.dsh-web" +UID_N=$(id -u) -acquire_lock() { - if mkdir "$LOCKDIR" 2>/dev/null; then - trap 'rm -rf "$LOCKDIR"' EXIT - return 0 +notify_deploy_failure() { + reason="$1" + echo "NOTIFY: deploy failed — $reason" + if [ -n "${OIKOS_API_TOKEN:-}" ]; then + curl -sf -X POST "http://localhost:8090/api/v1/events" \ + -H "Authorization: Bearer $OIKOS_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"type\":\"deploy.failed\",\"severity\":\"critical\",\"source\":\"webhook\",\"data\":{\"repo\":\"oikos-plugins\",\"reason\":\"$reason\"}}" \ + >/dev/null 2>&1 || true + fi + if [ -n "${MATRIX_WEBHOOK_URL:-}" ]; then + curl -sf -X POST "$MATRIX_WEBHOOK_URL" \ + -H "Content-Type: application/json" \ + -d "{\"msgtype\":\"m.text\",\"body\":\"🚨 oikos-plugins deploy failed: $reason\"}" \ + >/dev/null 2>&1 || true fi - echo "deploy already running, skipping" - exit 0 } -acquire_lock +# Serialize deploys. mkdir is atomic on POSIX (macOS lacks flock); the +# stale-pid check recovers a lock left by a SIGKILLed or rebooted deploy. +if ! mkdir "$LOCKDIR" 2>/dev/null; then + oldpid=$(cat "$LOCKDIR/pid" 2>/dev/null || echo "") + if [ -n "$oldpid" ] && kill -0 "$oldpid" 2>/dev/null; then + echo "deploy already in progress (pid $oldpid) — exiting" + exit 0 + fi + echo "removing stale deploy lock (pid ${oldpid:-?} not running)" + rm -rf "$LOCKDIR" + mkdir "$LOCKDIR" +fi +echo $$ > "$LOCKDIR/pid" +trap 'rc=$?; rm -rf "$LOCKDIR" 2>/dev/null || true; if [ "$_ok" != "1" ] && [ "$_notified" != "1" ]; then notify_deploy_failure "deploy aborted (exit $rc)"; fi' EXIT +_ok=0 +_notified=0 + +# Any HTTP response counts as healthy: this probes liveness (is the port +# serving), not a specific route. +wait_healthy() { + tries="$1" + i=1 + while [ "$i" -le "$tries" ]; do + if curl -s -o /dev/null --max-time 2 "$HEALTH_URL"; then + echo "healthy after $((i * SLEEP))s" + return 0 + fi + sleep "$SLEEP" + i=$((i + 1)) + done + return 1 +} + +restart_dsh() { + if ! launchctl print "gui/$UID_N/$AGENT_LABEL" >/dev/null 2>&1; then + launchctl bootstrap "gui/$UID_N" "$HOME/Library/LaunchAgents/$AGENT_LABEL.plist" + sleep 1 + fi + launchctl kickstart -k "gui/$UID_N/$AGENT_LABEL" +} + echo "=== oikos-plugins deploy started ===" -# 1. Pull latest -if [ ! -d "$PLUGIN_DIR" ]; then - git clone gitea@git-ssh.hubris.network:dtoro/oikos-plugins.git "$PLUGIN_DIR" -fi -cd "$PLUGIN_DIR" -git fetch origin master -git reset --hard origin/master +[ -d "$PLUGIN_DIR/.git" ] || { + echo "ERROR: $PLUGIN_DIR is not a git clone of dtoro/oikos-plugins — refusing" + exit 1 +} -# 2. Symlink packages into dsh profile +PLUGIN_OLD=$(git -C "$PLUGIN_DIR" rev-parse HEAD 2>/dev/null || echo "") +DSH_OLD=$(git -C "$DSH_DIR" rev-parse HEAD 2>/dev/null || echo "") +echo "pre-deploy: plugins=${PLUGIN_OLD:-none} dsh=${DSH_OLD:-none}" + +rollback() { + _notified=1 + echo "=== rolling back ===" + if [ -n "$PLUGIN_OLD" ] && [ "$(git -C "$PLUGIN_DIR" rev-parse HEAD)" != "$PLUGIN_OLD" ]; then + git -C "$PLUGIN_DIR" reset --hard "$PLUGIN_OLD" + fi + if [ -n "$DSH_OLD" ] && [ "$(git -C "$DSH_DIR" rev-parse HEAD)" != "$DSH_OLD" ]; then + git -C "$DSH_DIR" reset --hard "$DSH_OLD" + pnpm -C "$DSH_DIR" install --no-frozen-lockfile >/dev/null 2>&1 || true + git -C "$DSH_DIR" checkout -- pnpm-lock.yaml 2>/dev/null || true + fi + restart_dsh + if wait_healthy "$ROLLBACK_RETRIES"; then + notify_deploy_failure "deploy failed; rolled back to plugins@${PLUGIN_OLD:-?} dsh@${DSH_OLD:-?}" + else + notify_deploy_failure "deploy failed and rollback unhealthy — dsh web DOWN on port $PORT" + fi +} + +# 1. Plugins: advance the in-tree clone to the pushed state. Uncommitted local +# edits mean someone is developing here — never destroy them; deploy the dirty +# tree as-is and say so. +git -C "$PLUGIN_DIR" fetch origin master +if git -C "$PLUGIN_DIR" diff --quiet && git -C "$PLUGIN_DIR" diff --cached --quiet; then + git -C "$PLUGIN_DIR" reset --hard origin/master + echo "plugins at $(git -C "$PLUGIN_DIR" rev-parse --short HEAD)" +else + echo "WARN: $PLUGIN_DIR has uncommitted changes — deploying the dirty tree as-is" +fi + +# 2. Harness: advance by fast-forward only (never discards local commits; +# refuses when diverged). The untracked packages/oikos member is invisible to +# origin/master's lockfile, so install non-frozen and restore the lockfile +# afterwards: node_modules keeps the resolution, the tree stays clean. +git -C "$DSH_DIR" fetch origin "$DSH_BRANCH" +git -C "$DSH_DIR" pull --ff-only origin "$DSH_BRANCH" +pnpm -C "$DSH_DIR" install --no-frozen-lockfile +git -C "$DSH_DIR" checkout -- pnpm-lock.yaml +echo "dsh at $(git -C "$DSH_DIR" rev-parse --short HEAD)" + +# 2b. Rebuild the frontend dist: the web-app bundle serves the gitignored +# apps/web dist through workspace exports, so a harness advance without a +# rebuild keeps serving the previous UI. +pnpm -C "$DSH_DIR" --filter @deepseek-ai/dsh-web-frontend run build >/dev/null +echo "frontend dist rebuilt" + +# 3. Symlink packages into dsh profile for pkg in ui mcp-scope session-summary bundle evals; do - name=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$pkg/package.json')).name)") + name=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$PLUGIN_DIR/$pkg/package.json')).name)") ln -sf "$PLUGIN_DIR/$pkg" "$PROFILE_DIR/node_modules/$name" echo "linked $name" done -# 3. Build UI client bundle (needs dsh workspace for tsdown) -cd "$DSH_DIR" -pnpm install --filter @deepseek-ai/dsh-oikos-ui --frozen-lockfile 2>&1 -cd "$PLUGIN_DIR/ui" -DSH_BUILD_FACE=client npx tsdown --config tsdown.config.ts 2>&1 -echo "UI bundle built" - -# 4. Restart dsh -DASHBOARD_PID=$(pgrep -f 'dsh.*--port.*3080' 2>/dev/null || true) -if [ -n "$DASHBOARD_PID" ]; then - kill "$DASHBOARD_PID" 2>/dev/null || true - sleep 2 +# 4. Patch migration: the oikos overlay used to live only in /tmp (wiped on +# reboot). If the profile patch layer is still empty and the /tmp copy exists, +# move it into the profile so launchd boots need no --patch flag. +if [ -f /tmp/oikos-mcp-patch.yml ] && ! grep -q 'id:' "$PROFILE_DIR/cordis.patch.yml" 2>/dev/null; then + cp /tmp/oikos-mcp-patch.yml "$PROFILE_DIR/cordis.patch.yml" + echo "migrated oikos patch into $PROFILE_DIR/cordis.patch.yml" fi -cd "$DSH_DIR" -nohup pnpm dsh --profile web --patch /tmp/oikos-mcp-patch.yml --port "$PORT" > /tmp/dsh-web.log 2>&1 & -echo "dsh restarted (pid $!)" -echo "=== oikos-plugins deploy complete ===" \ No newline at end of file +# 5. Restart under launchd and health-check +restart_dsh +if ! wait_healthy "$RETRIES"; then + echo "ERROR: health check failed after $((RETRIES * SLEEP))s" + rollback + exit 1 +fi + +_ok=1 +echo "=== oikos-plugins deploy complete ===" diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 19e7a7b3..7c0d9147 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -230,6 +230,8 @@ DOCKER_BUILDKIT=1 docker compose --profile "$PROFILE" build \ # ── 5. Rolling restart ──────────────────────────────────────────────────── echo "[5/8] docker compose up -d" docker compose --profile "$PROFILE" up -d --remove-orphans +echo "[5.5/8] restarting dsh web (stale MCP session after api restart)" +launchctl kickstart -k "gui/$(id -u)/network.hubris.dsh-web" 2>/dev/null || true # ── 6. Prune old image tags — keep the 3 newest per service so rollback ──── # (OIKOS_VERSION=v0.x.y docker compose up) stays available. The repo list diff --git a/scripts/network.hubris.dsh-web.plist b/scripts/network.hubris.dsh-web.plist new file mode 100644 index 00000000..14ffa70a --- /dev/null +++ b/scripts/network.hubris.dsh-web.plist @@ -0,0 +1,36 @@ + + + + + Label + network.hubris.dsh-web + ProgramArguments + + /opt/homebrew/bin/pnpm + dsh + --profile + web + --port + 3080 + + WorkingDirectory + /Users/dtoro/Projects/deepseek-harness + EnvironmentVariables + + HOME + /Users/dtoro + PATH + /Users/dtoro/.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + + RunAtLoad + + KeepAlive + + ThrottleInterval + 10 + StandardOutPath + /Users/dtoro/Library/Logs/dsh-web.log + StandardErrorPath + /Users/dtoro/Library/Logs/dsh-web.log + + diff --git a/scripts/network.hubris.oikos-api-watchdog.plist b/scripts/network.hubris.oikos-api-watchdog.plist new file mode 100644 index 00000000..7c51326f --- /dev/null +++ b/scripts/network.hubris.oikos-api-watchdog.plist @@ -0,0 +1,20 @@ + + + + + Label + network.hubris.oikos-api-watchdog + ProgramArguments + + /Users/dtoro/Projects/deepseek-harness/packages/oikos/scripts/oikos-api-health-watchdog.sh + + StartInterval + 30 + KeepAlive + + StandardOutPath + /Users/dtoro/Library/Logs/oikos-api-watchdog.log + StandardErrorPath + /Users/dtoro/Library/Logs/oikos-api-watchdog.log + + diff --git a/scripts/oikos-api-health-watchdog.sh b/scripts/oikos-api-health-watchdog.sh new file mode 100755 index 00000000..cc1e563a --- /dev/null +++ b/scripts/oikos-api-health-watchdog.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Oikos API health watchdog — if the oikos API restarts while dsh web holds a +# stale MCP session, every tool call fails with "session not found" until dsh +# web itself restarts. This monitor detects the transition and triggers a +# launchd kickstart so the connection recovers automatically. +# +# Runs every 30s via LaunchAgent network.hubris.oikos-api-watchdog. + +set -e + +API_URL="${API_URL:-http://localhost:8090/healthz}" +MARKER="${MARKER:-/tmp/.oikos-api-down}" + +if curl -sf -o /dev/null --max-time 3 "$API_URL"; then + if [ -f "$MARKER" ]; then + rm -f "$MARKER" + echo "oikos API recovered — restarting dsh web" + launchctl kickstart -k "gui/$(id -u)/network.hubris.dsh-web" 2>/dev/null || true + fi +else + touch "$MARKER" +fi \ No newline at end of file