diff --git a/internal/httpapi/pct_create_test.go b/internal/httpapi/pct_create_test.go index 3a7151a..9e41d67 100644 --- a/internal/httpapi/pct_create_test.go +++ b/internal/httpapi/pct_create_test.go @@ -2,6 +2,7 @@ package httpapi import ( "encoding/json" + "strings" "testing" ) @@ -87,6 +88,34 @@ func TestJSONErrValidForNastyOutput(t *testing.T) { } } +func TestProvisionScript(t *testing.T) { + s := provisionScript([]string{"docker.io", "git"}, "echo hi > /root/x") + // Network/DNS gate must come before apt. + gate := strings.Index(s, "getent hosts") + apt := strings.Index(s, "apt-get update") + post := strings.Index(s, "echo hi > /root/x") + if gate < 0 || apt < 0 || post < 0 { + t.Fatalf("missing sections: gate=%d apt=%d post=%d\n%s", gate, apt, post, s) + } + if !(gate < apt && apt < post) { + t.Errorf("wrong ordering: gate=%d apt=%d post=%d", gate, apt, post) + } + if !strings.Contains(s, "nameserver 1.1.1.1") { + t.Error("missing DNS self-heal fallback") + } + if !strings.Contains(s, "docker.io git") { + t.Error("packages not joined into install line") + } + // No packages: no apt lines, but post_install and gate still present. + s2 := provisionScript(nil, "systemctl status foo") + if strings.Contains(s2, "apt-get install") { + t.Error("apt install should be absent when no packages requested") + } + if !strings.Contains(s2, "systemctl status foo") || !strings.Contains(s2, "getent hosts") { + t.Error("post_install or gate missing in no-package case") + } +} + func TestSanitizePkgs(t *testing.T) { in := []string{"docker.io", "git", "rm -rf /", "curl;wget", "python3-pip", ""} got := sanitizePkgs(in) diff --git a/internal/httpapi/phase3.go b/internal/httpapi/phase3.go index 14a5b89..3f77a54 100644 --- a/internal/httpapi/phase3.go +++ b/internal/httpapi/phase3.go @@ -368,23 +368,17 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, // Post-create provisioning: install apt packages and run a post_install // script inside the fresh container, so a single approved pct_create - // yields a *working service*, not just an empty container. Best-effort - // with a boot settle; failures are appended to output and mark the - // execution failed so the operator sees exactly which step broke. + // yields a *working service*, not just an empty container. The script + // waits for real DNS/connectivity and self-heals the resolver first — + // a static-IP container with a dead nameserver otherwise fails apt with + // "Temporary failure resolving deb.debian.org" and installs nothing. if err == nil && (len(cfg.Services) > 0 || cfg.PostInstall != "") { - // Give the container time to boot and (for DHCP) acquire a lease - // before apt needs the network. - steps := []string{"sleep 10"} - if len(cfg.Services) > 0 { - pkgs := strings.Join(sanitizePkgs(cfg.Services), " ") - steps = append(steps, fmt.Sprintf("pct exec %d -- bash -lc 'apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq %s'", cfg.VMID, pkgs)) - } - if cfg.PostInstall != "" { - b64 := base64.StdEncoding.EncodeToString([]byte(cfg.PostInstall)) - steps = append(steps, fmt.Sprintf("pct exec %d -- bash -lc 'echo %s | base64 -d | bash'", cfg.VMID, b64)) - } + script := provisionScript(sanitizePkgs(cfg.Services), cfg.PostInstall) + b64 := base64.StdEncoding.EncodeToString([]byte(script)) + // sleep on the host so the container is up enough to accept pct exec. + cmd := fmt.Sprintf("sleep 4; pct exec %d -- bash -c 'echo %s | base64 -d | bash'", cfg.VMID, b64) var provOut string - provOut, err = sshExec(ctx, host, user, strings.Join(steps, " && ")) + provOut, err = sshExec(ctx, host, user, cmd) output = output + "\n--- post-install ---\n" + provOut } @@ -462,6 +456,36 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, "execution_id", execID, "action", action, "status", status, "duration_ms", durationMs) } +// provisionScript builds the in-container bootstrap run after pct create. It +// (1) waits for DNS/connectivity and self-heals /etc/resolv.conf with a public +// resolver if the configured nameserver is dead, (2) installs apt packages with +// retries, (3) runs the operator's post_install. `set -e` after the network +// gate means any apt or post_install failure exits non-zero, so sshExec surfaces +// it and the execution is marked failed with the exact broken step in output. +func provisionScript(pkgs []string, postInstall string) string { + var b strings.Builder + b.WriteString("set -o pipefail\n") + b.WriteString("probe=deb.debian.org\n") + b.WriteString("ok=0\n") + b.WriteString("for i in $(seq 1 30); do if getent hosts \"$probe\" >/dev/null 2>&1; then ok=1; break; fi; sleep 2; done\n") + // Self-heal: if the assigned resolver can't resolve, fall back to public DNS. + b.WriteString("if [ \"$ok\" != 1 ]; then printf 'nameserver 1.1.1.1\\nnameserver 8.8.8.8\\n' > /etc/resolv.conf; ") + b.WriteString("for i in $(seq 1 15); do if getent hosts \"$probe\" >/dev/null 2>&1; then ok=1; break; fi; sleep 2; done; fi\n") + b.WriteString("if [ \"$ok\" != 1 ]; then echo 'ERROR: container has no DNS/connectivity after ~90s'; exit 1; fi\n") + b.WriteString("set -e\n") + if len(pkgs) > 0 { + b.WriteString("export DEBIAN_FRONTEND=noninteractive\n") + b.WriteString("apt-get update -o Acquire::Retries=3 -qq\n") + b.WriteString("apt-get install -y -o Acquire::Retries=3 --no-install-recommends -qq " + strings.Join(pkgs, " ") + "\n") + } + if strings.TrimSpace(postInstall) != "" { + b.WriteString("# --- operator post_install ---\n") + b.WriteString(postInstall) + b.WriteString("\n") + } + return b.String() +} + // jsonErr builds a valid {"error": "..."} JSON payload for an execution's // result column. Always use this instead of fmt.Sprintf'ing JSON by hand — // error text and command output routinely contain quotes/backslashes that diff --git a/nomos/SOUL.md b/nomos/SOUL.md index c1cff02..4e213d7 100644 --- a/nomos/SOUL.md +++ b/nomos/SOUL.md @@ -66,6 +66,18 @@ Before calling `request_execution`: `git clone && docker compose up -d`). Prefer one pct_create with services+post_install over pct_create followed by many pct_exec approvals. Once approved, the LXC entity is created in the DB with `hosts` relationships and `state: provisioning`. + - **vmid**: omit or set 0 — a free cluster id is assigned automatically. Never reuse an + existing container's id. + - **networking**: prefer `"ip":"dhcp"` unless the operator needs a fixed address; DHCP + yields a working DNS resolver. If you set a static CIDR, the provisioner self-heals DNS + to a public resolver when the gateway can't resolve, but DHCP is more reliable. + - **Docker**: `docker-compose-plugin` is NOT in Debian's repos — do not put it in + `services`. For Docker, put `docker.io` in `services` (it provides the engine) and, if + you need compose v2, install it in `post_install` from Docker's official convenience + script (`curl -fsSL https://get.docker.com | sh`). Use `docker compose` (v2) only after + that, otherwise use `docker-compose` (v1, from docker.io). + - **verify**: end `post_install` by confirming the service actually answers (e.g. + `curl -fsS http://localhost:/` ), so a green result means it truly works. - If `destructive` or `config_mutation`: escalate to operator - If `reversible_low` with validated pattern: auto-act allowed diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 204440f..c518004 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -230,6 +230,12 @@ export async function fetchExecutions(status?: string): Promise { return data.items ?? [] } +export async function getExecution(id: string): Promise { + const res = await fetch(`${API}/executions/${id}`) + if (!res.ok) return null + return res.json() +} + export async function cancelExecution(id: string): Promise { const res = await fetch(`${API}/executions/${id}/cancel`, { method: 'POST' }) if (!res.ok) return null diff --git a/web/src/lib/components/InlineApproval.svelte b/web/src/lib/components/InlineApproval.svelte index 3618d13..24fa7e9 100644 --- a/web/src/lib/components/InlineApproval.svelte +++ b/web/src/lib/components/InlineApproval.svelte @@ -1,6 +1,6 @@ {#each approvals as approval (approval.executionId)} - {@const state = outcome.get(approval.executionId)} - {#if state === 'approved'} + {@const p = phase.get(approval.executionId)} + {@const e = exec.get(approval.executionId)} + {#if p === 'completed'}
- - Approved — provisioning. Track progress in the Executions view. + + Provisioned successfully{e?.duration_ms ? ` in ${Math.round(e.duration_ms / 1000)}s` : ''}. See the Executions view for details.
- {:else if state === 'denied'} -
- - Denied. + {:else if p === 'failed'} +
+
+ + Execution failed + +
+ 
{errorText(e)}
- {:else if state === 'failed'} + {:else if p === 'denied'}
- - Decision failed to send. - + Denied. +
+ {:else if p === 'running' || p === 'deciding'} +
+ + {p === 'deciding' ? 'Submitting approval…' : `Provisioning ${approval.target}… (this can take a minute)`}
{:else}
- - {approval.action} on {approval.target} requires approval - - {#if state === 'pending'} - - {:else} - - - {/if} + {approval.action} on {approval.target} requires approval + +
{/if} {/each}