From 82b0ad22983c460268d79c029c0ad858f4468752 Mon Sep 17 00:00:00 2001 From: dtoro Date: Fri, 10 Jul 2026 11:48:06 +0200 Subject: [PATCH] fix: pct_create fast gateway pre-flight + bridge param (real root cause of TypeType's DNS failures) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigated why the operator couldn't get past "no DNS/connectivity" across multiple retries even after Nomos correctly diagnosed and fixed the gateway (192.168.8.1 -> 192.168.8.2). It still failed. Root cause, confirmed from strong's own documented network topology: on `strong`, vmbr0 physically bridges only to 192.168.178.0/24 — the 192.168.8.0/24 service network is reached via a Fritz!Box static route, not a local bridge. A container attached to vmbr0 can never reach a 192.168.8.x gateway no matter which address in that range is picked; ARP for it just gets silently dropped (matching the earlier hang symptom). The gateway was never the problem — the bridge was. 192.168.8.0/24 is also segmented into /28 blocks each with their own gateway (192.168.8.2 is only the .0-.15 block's gateway), so even a correct bridge with a copy-pasted gateway from a different block would still fail. No amount of retrying with a different gateway guess could have fixed this — the missing fact (which bridge reaches which subnet, and the per-/28 gateway) isn't inferable from the subnet alone. - pct_create gets a `bridge` param (was hardcoded to vmbr0) so a correct bridge can actually be requested once known. - Fast pre-flight: for any static IP, ping the gateway from the target HOST before creating anything. Was: a bad config took a multi-minute hang (or, after last commit's timeout fix, ~2min) before failing. Now: ~2 seconds, with a message that explicitly says not to guess a different gateway in the same subnet — find a real neighbor's config or use DHCP. - SOUL.md: DHCP is now framed as the default, not a fallback; static IP requires finding an existing LXC on the same host in the same /28 and copying its bridge+gateway verbatim — inventing one is explicitly called out as the failure mode that caused this exact incident. - MCP tool schema: pct_create's params description now documents `bridge` and the neighbor-copy rule directly in what the model reads at call time, not just in SOUL.md. Co-Authored-By: Claude Opus 4.8 --- internal/httpapi/phase3.go | 35 +++++++++++++++++++++++++++++++++-- internal/mcp/server.go | 2 +- nomos/SOUL.md | 27 ++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/internal/httpapi/phase3.go b/internal/httpapi/phase3.go index fe1e538..e025167 100644 --- a/internal/httpapi/phase3.go +++ b/internal/httpapi/phase3.go @@ -284,6 +284,7 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, DiskGB int `json:"disk_gb"` IP string `json:"ip"` GW string `json:"gw"` + Bridge string `json:"bridge"` // e.g. vmbr0/vmbr1 — which bridge actually reaches the target subnet on this host varies per host, don't assume vmbr0 Storage string `json:"storage"` Template string `json:"template"` Privileged flexBool `json:"privileged"` @@ -402,10 +403,15 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, nestingFlag = fmt.Sprintf(" --features %s", strings.Join(features, ",")) } + if cfg.Bridge == "" { + cfg.Bridge = "vmbr0" + } + // net0: DHCP when no static IP is given (or ip=="dhcp"). Proxmox // rejects a gateway alongside ip=dhcp, so only add gw for a static IP. - net0 := "name=eth0,bridge=vmbr0," - if cfg.IP == "" || strings.EqualFold(cfg.IP, "dhcp") { + net0 := "name=eth0,bridge=" + cfg.Bridge + "," + isStatic := cfg.IP != "" && !strings.EqualFold(cfg.IP, "dhcp") + if !isStatic { net0 += "ip=dhcp" } else { net0 += "ip=" + cfg.IP @@ -414,6 +420,31 @@ func executeApprovedAction(ctx context.Context, pool *db.Pool, execID uuid.UUID, } } + // Pre-flight: for a static config, ping the gateway from the target + // HOST before spending 5+ minutes creating the container and waiting + // on its network. This is the check that would have caught the real + // TypeType failure immediately instead of after a full provision + // attempt: the gateway wasn't reachable because the chosen bridge + // doesn't carry that subnet on this host at all (on `strong`, vmbr0 + // only reaches 192.168.178.0/24 — a 192.168.8.0/24 gateway is simply + // not on that L2 segment, so ARP for it silently gets nothing, no + // matter which 192.168.8.x address is picked). A 1-2s ping from the + // host itself catches this class of misconfiguration before any + // container work happens. + if isStatic && cfg.GW != "" { + pingOut, pingErr := sshExec(ctx, host, user, fmt.Sprintf("ping -c1 -W2 %s >/dev/null 2>&1 && echo REACHABLE || echo UNREACHABLE", cfg.GW)) + if pingErr != nil || !strings.Contains(pingOut, "REACHABLE") { + msg := fmt.Sprintf( + "gateway %s is not reachable from %s on bridge %s — this almost always means the bridge doesn't carry that subnet on this host (each bridge only reaches the network it's physically wired to). "+ + "Do not retry with a different gateway guess in the same subnet: find an existing LXC on this host with an IP in the same /28 and copy its exact bridge+gateway, or use DHCP instead.", + cfg.GW, targetSlug, cfg.Bridge) + pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, + execID, jsonErr("%s", msg)) + emitExecutionEvent(ctx, pool, execID, "failed", map[string]any{"target": targetSlug, "error": msg}) + return + } + } + templatePath := fmt.Sprintf("/var/lib/vz/template/cache/%s", cfg.Template) createCmd := fmt.Sprintf( "pct create %d %s --hostname %s --cores %d --memory %d --rootfs %s:%d %s --net0 %s%s --start 1", diff --git a/internal/mcp/server.go b/internal/mcp/server.go index 6d6a25a..265dce8 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -272,7 +272,7 @@ func newServer(pool *db.Pool, agentID uuid.UUID) *mcp.Server { InputSchema: objSchema( prop{"target", "string", "Target entity slug. For pct_create this MUST be the Proxmox HOST that will run the container (e.g. host:strong) — NOT the new LXC's name. For restart/systemctl/apt_upgrade/pct_exec use the target service/LXC slug (e.g. lxc:caddy)."}, prop{"action", "string", "Action: restart, systemctl, pct_exec, apt_upgrade, pct_create"}, - prop{"params", "string", "For systemctl: 'enable|disable|reload'. For pct_exec: the shell command. For apt_upgrade: 'audit|upgrade'. For pct_create: a JSON object string with keys: vmid (int, required, unused id), hostname (string, required), cores (int), memory (MB int), disk_gb (int), ip (CIDR e.g. 192.168.8.50/24), gw (gateway ip), storage (default local-lvm), template (optional — omit to auto-pick newest debian on the host), privileged (bool), nesting (bool), mounts ([]string of 'src,mp=/dst'), services ([]string of apt packages to install), post_install (string shell script run inside the container after create). Example: {\"vmid\":150,\"hostname\":\"typetype\",\"cores\":2,\"memory\":2048,\"disk_gb\":16,\"ip\":\"192.168.8.50/24\",\"gw\":\"192.168.8.2\",\"nesting\":true,\"services\":[\"docker.io\",\"git\"],\"post_install\":\"git clone https://github.com/x/y /opt/y && cd /opt/y && docker compose up -d\"}"}, + prop{"params", "string", "For systemctl: 'enable|disable|reload'. For pct_exec: the shell command. For apt_upgrade: 'audit|upgrade'. For pct_create: a JSON object string with keys: vmid (int, required, unused id), hostname (string, required), cores (int), memory (MB int), disk_gb (int), ip (CIDR e.g. 192.168.8.50/24, or omit/\"dhcp\" — DHCP is the safe default, see below), gw (gateway ip, static only), bridge (e.g. vmbr0/vmbr1 — WHICH BRIDGE REACHES WHICH SUBNET IS DIFFERENT PER HOST, never assume vmbr0; see below), storage (default local-lvm), template (optional — omit to auto-pick newest debian on the host), privileged (bool), nesting (bool), mounts ([]string of 'src,mp=/dst'), services ([]string of apt packages to install), post_install (string shell script run inside the container after create). Example: {\"vmid\":150,\"hostname\":\"typetype\",\"cores\":2,\"memory\":2048,\"disk_gb\":16,\"ip\":\"192.168.8.50/24\",\"gw\":\"192.168.8.2\",\"bridge\":\"vmbr1\",\"nesting\":true,\"services\":[\"docker.io\",\"git\"],\"post_install\":\"git clone https://github.com/x/y /opt/y && cd /opt/y && docker compose up -d\"}. STATIC IP RULE: before setting ip/gw/bridge to anything other than DHCP, use list_entities/get_entity_knowledge to find an EXISTING lxc on the SAME host whose IP is in the same /28 block, and copy its exact gw+bridge — do not invent a gateway. If no such neighbor exists, prefer ip:\"dhcp\" (proven to work, gets a real routable address) over guessing; a wrong bridge/gateway pair fails a fast pre-flight ping check now (seconds, not minutes) but is still a wasted turn — better to not guess at all."}, ), }, func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { args := argsMap(req) diff --git a/nomos/SOUL.md b/nomos/SOUL.md index b5108a2..fc7ee9d 100644 --- a/nomos/SOUL.md +++ b/nomos/SOUL.md @@ -100,9 +100,30 @@ Before calling `request_execution`: created in the DB with `hosts` relationships and `state: provisioning`. - **vmid**: omit or set 0 — a free cluster id is assigned automatically. Never reuse an existing container's id. - - **networking**: prefer `"ip":"dhcp"` unless the operator needs a fixed address; DHCP - yields a working DNS resolver. If you set a static CIDR, the provisioner self-heals DNS - to a public resolver when the gateway can't resolve, but DHCP is more reliable. + - **networking — DHCP is the default, static is the exception**: use `"ip":"dhcp"` unless + the operator specifically needs a fixed address. DHCP is proven reliable and always gets + a real, routable IP. **A static IP is not a formula you can compute from the subnet + alone.** Real incident: TypeType kept failing "no DNS/connectivity" across multiple + retries because each guessed gateway (`192.168.8.1`, then `192.168.8.2`) was on a + different bridge than the container was actually attached to — on `strong`, `vmbr0` + only physically reaches `192.168.178.0/24`; `192.168.8.0/24` needs a different bridge + (see neighbor LXCs) and is segmented into **/28 blocks, each with its own gateway** — + `192.168.8.2` is only the gateway for the `.0–.15` block, not the whole `/24`. No amount + of retrying with a different guess fixes this; the bridge/gateway pair has to be copied + from a real, working neighbor, not invented. + - **Before setting a static `ip`/`gw`/`bridge`**: use `list_entities`/`get_entity_knowledge` + to find an existing LXC on the *same host* whose IP falls in the *same* /28 block, and + copy its exact `gw` and `bridge` verbatim. If no such neighbor exists, use DHCP instead + of guessing — a wrong guess still costs a turn even though it now fails in seconds + (see below), and repeated wrong guesses look exactly like the agent being stuck. + - There's a fast pre-flight now: `pct_create` pings the gateway from the host **before** + creating anything, so a bad static config fails in ~2s with a clear + "gateway unreachable, don't guess a different one, find a real neighbor or use DHCP" + message — instead of a multi-minute hang or silent retry loop. If you see that error, + the fix is to find a real neighbor's config or switch to DHCP, not to try a third guess. + - If you set a static CIDR anyway and the *DNS resolver itself* (not the gateway) is the + problem, the provisioner self-heals to a public resolver — but that only helps once the + gateway/bridge are actually correct. - **Docker**: `docker-compose-plugin` is NOT in Debian's repos — do not put it in `services`. For Docker, put `docker.io` in `services` (it provides the engine) and, if you need compose v2, install it in `post_install` from Docker's official convenience