feat: robust provisioning (DNS self-heal) + live execution feedback in chat

Production session provisioned the container but the service never installed:
apt failed with "Temporary failure resolving deb.debian.org" — a static-IP LXC
whose assigned nameserver couldn't resolve. The operator also got zero feedback:
the approval banner just sat there with no running/complete/failed status.

Backend robustness (provisionScript):
- Wait for real DNS/connectivity inside the container before apt, and self-heal
  /etc/resolv.conf to a public resolver (1.1.1.1/8.8.8.8) if the assigned one
  is dead. `set -e` after the gate so apt/post_install failures surface.
- apt-get update/install with Acquire::Retries=3.

Frontend feedback (InlineApproval):
- After approve, poll GET /executions/{id} and show live phase: submitting →
  provisioning… → provisioned successfully / execution failed (with the error).
- add getExecution() to api.ts.

Agent guidance (SOUL.md):
- omit vmid (auto-assigned), prefer dhcp, docker-compose-plugin is not in Debian
  (use docker.io + get.docker.com), end post_install with a health check.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

internal/httpapi/pct_create_test.go

@@ -2,6 +2,7 @@ package httpapi
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 )
 
@@ -87,6 +88,34 @@ func TestJSONErrValidForNastyOutput(t *testing.T) {
 	}
 }
 
+func TestProvisionScript(t *testing.T) {
+	s := provisionScript([]string{"docker.io", "git"}, "echo hi > /root/x")
+	// Network/DNS gate must come before apt.
+	gate := strings.Index(s, "getent hosts")
+	apt := strings.Index(s, "apt-get update")
+	post := strings.Index(s, "echo hi > /root/x")
+	if gate < 0 || apt < 0 || post < 0 {
+		t.Fatalf("missing sections: gate=%d apt=%d post=%d\n%s", gate, apt, post, s)
+	}
+	if !(gate < apt && apt < post) {
+		t.Errorf("wrong ordering: gate=%d apt=%d post=%d", gate, apt, post)
+	}
+	if !strings.Contains(s, "nameserver 1.1.1.1") {
+		t.Error("missing DNS self-heal fallback")
+	}
+	if !strings.Contains(s, "docker.io git") {
+		t.Error("packages not joined into install line")
+	}
+	// No packages: no apt lines, but post_install and gate still present.
+	s2 := provisionScript(nil, "systemctl status foo")
+	if strings.Contains(s2, "apt-get install") {
+		t.Error("apt install should be absent when no packages requested")
+	}
+	if !strings.Contains(s2, "systemctl status foo") || !strings.Contains(s2, "getent hosts") {
+		t.Error("post_install or gate missing in no-package case")
+	}
+}
+
 func TestSanitizePkgs(t *testing.T) {
 	in := []string{"docker.io", "git", "rm -rf /", "curl;wget", "python3-pip", ""}
 	got := sanitizePkgs(in)

internal/httpapi/phase3.go

@@ -368,23 +368,17 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID,
 
 		// Post-create provisioning: install apt packages and run a post_install
 		// script inside the fresh container, so a single approved pct_create
-		// yields a *working service*, not just an empty container. Best-effort
-		// with a boot settle; failures are appended to output and mark the
-		// execution failed so the operator sees exactly which step broke.
+		// yields a *working service*, not just an empty container. The script
+		// waits for real DNS/connectivity and self-heals the resolver first —
+		// a static-IP container with a dead nameserver otherwise fails apt with
+		// "Temporary failure resolving deb.debian.org" and installs nothing.
 		if err == nil && (len(cfg.Services) > 0 || cfg.PostInstall != "") {
-			// Give the container time to boot and (for DHCP) acquire a lease
-			// before apt needs the network.
-			steps := []string{"sleep 10"}
-			if len(cfg.Services) > 0 {
-				pkgs := strings.Join(sanitizePkgs(cfg.Services), " ")
-				steps = append(steps, fmt.Sprintf("pct exec %d -- bash -lc 'apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq %s'", cfg.VMID, pkgs))
-			}
-			if cfg.PostInstall != "" {
-				b64 := base64.StdEncoding.EncodeToString([]byte(cfg.PostInstall))
-				steps = append(steps, fmt.Sprintf("pct exec %d -- bash -lc 'echo %s | base64 -d | bash'", cfg.VMID, b64))
-			}
+			script := provisionScript(sanitizePkgs(cfg.Services), cfg.PostInstall)
+			b64 := base64.StdEncoding.EncodeToString([]byte(script))
+			// sleep on the host so the container is up enough to accept pct exec.
+			cmd := fmt.Sprintf("sleep 4; pct exec %d -- bash -c 'echo %s | base64 -d | bash'", cfg.VMID, b64)
 			var provOut string
-			provOut, err = sshExec(ctx, host, user, strings.Join(steps, " && "))
+			provOut, err = sshExec(ctx, host, user, cmd)
 			output = output + "\n--- post-install ---\n" + provOut
 		}
 
@@ -462,6 +456,36 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID,
 		"execution_id", execID, "action", action, "status", status, "duration_ms", durationMs)
 }
 
+// provisionScript builds the in-container bootstrap run after pct create. It
+// (1) waits for DNS/connectivity and self-heals /etc/resolv.conf with a public
+// resolver if the configured nameserver is dead, (2) installs apt packages with
+// retries, (3) runs the operator's post_install. `set -e` after the network
+// gate means any apt or post_install failure exits non-zero, so sshExec surfaces
+// it and the execution is marked failed with the exact broken step in output.
+func provisionScript(pkgs []string, postInstall string) string {
+	var b strings.Builder
+	b.WriteString("set -o pipefail\n")
+	b.WriteString("probe=deb.debian.org\n")
+	b.WriteString("ok=0\n")
+	b.WriteString("for i in $(seq 1 30); do if getent hosts \"$probe\" >/dev/null 2>&1; then ok=1; break; fi; sleep 2; done\n")
+	// Self-heal: if the assigned resolver can't resolve, fall back to public DNS.
+	b.WriteString("if [ \"$ok\" != 1 ]; then printf 'nameserver 1.1.1.1\\nnameserver 8.8.8.8\\n' > /etc/resolv.conf; ")
+	b.WriteString("for i in $(seq 1 15); do if getent hosts \"$probe\" >/dev/null 2>&1; then ok=1; break; fi; sleep 2; done; fi\n")
+	b.WriteString("if [ \"$ok\" != 1 ]; then echo 'ERROR: container has no DNS/connectivity after ~90s'; exit 1; fi\n")
+	b.WriteString("set -e\n")
+	if len(pkgs) > 0 {
+		b.WriteString("export DEBIAN_FRONTEND=noninteractive\n")
+		b.WriteString("apt-get update -o Acquire::Retries=3 -qq\n")
+		b.WriteString("apt-get install -y -o Acquire::Retries=3 --no-install-recommends -qq " + strings.Join(pkgs, " ") + "\n")
+	}
+	if strings.TrimSpace(postInstall) != "" {
+		b.WriteString("# --- operator post_install ---\n")
+		b.WriteString(postInstall)
+		b.WriteString("\n")
+	}
+	return b.String()
+}
+
 // jsonErr builds a valid {"error": "..."} JSON payload for an execution's
 // result column. Always use this instead of fmt.Sprintf'ing JSON by hand —
 // error text and command output routinely contain quotes/backslashes that

nomos/SOUL.md

@@ -66,6 +66,18 @@ Before calling `request_execution`:
   `git clone && docker compose up -d`). Prefer one pct_create with services+post_install
   over pct_create followed by many pct_exec approvals. Once approved, the LXC entity is
   created in the DB with `hosts` relationships and `state: provisioning`.
+  - **vmid**: omit or set 0 — a free cluster id is assigned automatically. Never reuse an
+    existing container's id.
+  - **networking**: prefer `"ip":"dhcp"` unless the operator needs a fixed address; DHCP
+    yields a working DNS resolver. If you set a static CIDR, the provisioner self-heals DNS
+    to a public resolver when the gateway can't resolve, but DHCP is more reliable.
+  - **Docker**: `docker-compose-plugin` is NOT in Debian's repos — do not put it in
+    `services`. For Docker, put `docker.io` in `services` (it provides the engine) and, if
+    you need compose v2, install it in `post_install` from Docker's official convenience
+    script (`curl -fsSL https://get.docker.com | sh`). Use `docker compose` (v2) only after
+    that, otherwise use `docker-compose` (v1, from docker.io).
+  - **verify**: end `post_install` by confirming the service actually answers (e.g.
+    `curl -fsS http://localhost:<port>/` ), so a green result means it truly works.
 - If `destructive` or `config_mutation`: escalate to operator
 - If `reversible_low` with validated pattern: auto-act allowed
 

web/src/lib/api.ts

@@ -230,6 +230,12 @@ export async function fetchExecutions(status?: string): Promise<Execution[]> {
   return data.items ?? []
 }
 
+export async function getExecution(id: string): Promise<Execution | null> {
+  const res = await fetch(`${API}/executions/${id}`)
+  if (!res.ok) return null
+  return res.json()
+}
+
 export async function cancelExecution(id: string): Promise<Execution | null> {
   const res = await fetch(`${API}/executions/${id}/cancel`, { method: 'POST' })
   if (!res.ok) return null

web/src/lib/components/InlineApproval.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
   import type { PendingApproval } from '$lib/stores/chat'
-  import { decideApproval } from '$lib/api'
+  import { decideApproval, getExecution, type Execution } from '$lib/api'
   import { Button } from '$lib/components/ui/button'
   import { SvelteMap } from 'svelte/reactivity'
   import CheckIcon from '@lucide/svelte/icons/check'
@@ -10,62 +10,87 @@
 
   let { approvals }: { approvals: PendingApproval[] } = $props()
 
-  // Per-execution outcome, keyed by executionId. A resolved entry hides the
-  // action buttons for that approval permanently so the banner clears after a
-  // click and can never re-POST /decision. 'pending' guards against double
-  // submits (the old component reset its state on every prop change, so the
-  // banner never cleared and each click fired another decision).
-  type State = 'pending' | 'approved' | 'denied' | 'failed'
-  const outcome = new SvelteMap<string, State>()
+  // Per-execution UI phase, keyed by executionId. A resolved phase hides the
+  // action buttons permanently so the banner clears after a click and can
+  // never re-POST /decision.
+  type Phase = 'deciding' | 'running' | 'completed' | 'failed' | 'denied'
+  const phase = new SvelteMap<string, Phase>()
+  // Latest execution row (for status/result display), keyed by executionId.
+  const exec = new SvelteMap<string, Execution>()
+
+  const TERMINAL = new Set(['completed', 'failed', 'cancelled', 'denied'])
+
+  function errorText(e: Execution | undefined): string {
+    const r = e?.result as Record<string, unknown> | undefined | null
+    const v = r?.error
+    return typeof v === 'string' && v ? v : 'Execution failed.'
+  }
+
+  // Poll the execution until it reaches a terminal state, so the operator sees
+  // provisioning progress and the final outcome without leaving the chat.
+  async function track(id: string) {
+    for (let i = 0; i < 150; i++) { // ~6min ceiling at 2.5s
+      const e = await getExecution(id)
+      if (e) {
+        exec.set(id, e)
+        if (e.status === 'completed') { phase.set(id, 'completed'); return }
+        if (e.status === 'failed' || e.status === 'cancelled') { phase.set(id, 'failed'); return }
+      }
+      await new Promise((r) => setTimeout(r, 2500))
+    }
+    // Timed out waiting — leave whatever we last saw, mark running-stalled.
+    if (!TERMINAL.has(exec.get(id)?.status ?? '')) phase.set(id, 'running')
+  }
 
   async function decide(approval: PendingApproval, decision: 'approve' | 'deny') {
     const id = approval.executionId
-    const cur = outcome.get(id)
-    if (cur === 'pending' || cur === 'approved' || cur === 'denied') return // no resubmit
-    outcome.set(id, 'pending')
+    const cur = phase.get(id)
+    if (cur && cur !== 'failed') return // no resubmit once decided/in-flight
+    phase.set(id, 'deciding')
     const ok = await decideApproval(id, decision)
-    outcome.set(id, ok ? (decision === 'approve' ? 'approved' : 'denied') : 'failed')
+    if (!ok) { phase.set(id, 'failed'); return }
+    if (decision === 'deny') { phase.set(id, 'denied'); return }
+    phase.set(id, 'running')
+    void track(id)
   }
 </script>
 
 {#each approvals as approval (approval.executionId)}
-  {@const state = outcome.get(approval.executionId)}
-  {#if state === 'approved'}
+  {@const p = phase.get(approval.executionId)}
+  {@const e = exec.get(approval.executionId)}
+  {#if p === 'completed'}
     <div class="my-2 flex items-center gap-2 rounded-lg border border-success/40 bg-success/5 px-3 py-2 text-xs text-success">
-      <CheckIcon class="size-4" />
-      <span>Approved — provisioning. Track progress in the Executions view.</span>
+      <CheckIcon class="size-4 shrink-0" />
+      <span>Provisioned successfully{e?.duration_ms ? ` in ${Math.round(e.duration_ms / 1000)}s` : ''}. See the Executions view for details.</span>
     </div>
-  {:else if state === 'denied'}
-    <div class="my-2 flex items-center gap-2 rounded-lg border border-destructive/40 bg-destructive/5 px-3 py-2 text-xs text-destructive">
-      <XIcon class="size-4" />
-      <span>Denied.</span>
+  {:else if p === 'failed'}
+    <div class="my-2 flex flex-col gap-1 rounded-lg border border-destructive/40 bg-destructive/5 px-3 py-2 text-xs text-destructive">
+      <div class="flex items-center gap-2">
+        <XIcon class="size-4 shrink-0" />
+        <span class="font-medium">Execution failed</span>
+        <Button size="sm" variant="outline" class="ml-auto h-6 px-2 text-xs" onclick={() => decide(approval, 'approve')}>Retry</Button>
       </div>
-  {:else if state === 'failed'}
+      <pre class="whitespace-pre-wrap break-words pl-6 opacity-90">{errorText(e)}</pre>
+    </div>
+  {:else if p === 'denied'}
     <div class="my-2 flex items-center gap-2 rounded-lg border border-destructive/40 bg-destructive/5 px-3 py-2 text-xs text-destructive">
-      <XIcon class="size-4" />
-      <span class="flex-1">Decision failed to send.</span>
-      <Button size="sm" variant="outline" class="h-7 px-2.5 text-xs" onclick={() => { outcome.delete(approval.executionId); decide(approval, 'approve') }}>
-        Retry
-      </Button>
+      <XIcon class="size-4" /><span>Denied.</span>
+    </div>
+  {:else if p === 'running' || p === 'deciding'}
+    <div class="my-2 flex items-center gap-2 rounded-lg border border-warning/40 bg-warning/5 px-3 py-2 text-xs text-muted-foreground">
+      <LoaderCircleIcon class="size-4 shrink-0 animate-spin text-warning" />
+      <span>{p === 'deciding' ? 'Submitting approval…' : `Provisioning ${approval.target}… (this can take a minute)`}</span>
     </div>
   {:else}
     <div class="my-2 flex items-center gap-2 rounded-lg border border-warning/40 bg-warning/5 px-3 py-2">
       <ShieldCheckIcon class="size-4 shrink-0 text-warning" />
-      <span class="flex-1 text-xs text-muted-foreground">
-        {approval.action} on {approval.target} requires approval
-      </span>
-      {#if state === 'pending'}
-        <LoaderCircleIcon class="size-4 animate-spin text-muted-foreground" />
-      {:else}
+      <span class="flex-1 text-xs text-muted-foreground">{approval.action} on {approval.target} requires approval</span>
       <Button size="sm" variant="default" class="h-7 px-2.5 text-xs" onclick={() => decide(approval, 'approve')}>
-          <CheckIcon class="size-3" />
-          <span class="ml-1">Approve</span>
+        <CheckIcon class="size-3" /><span class="ml-1">Approve</span>
       </Button>
       <Button size="sm" variant="outline" class="h-7 px-2.5 text-xs" onclick={() => decide(approval, 'deny')}>
-          <XIcon class="size-3" />
-          <span class="ml-1">Deny</span>
+        <XIcon class="size-3" /><span class="ml-1">Deny</span>
       </Button>
-      {/if}
     </div>
   {/if}
 {/each}